diff --git a/Cargo.lock b/Cargo.lock index d1b13367a..33f0be2ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2321,7 +2321,7 @@ dependencies = [ [[package]] name = "geoengine-datatypes" -version = "0.8.0" +version = "0.9.0" dependencies = [ "arrow", "arrow-array", @@ -2332,6 +2332,7 @@ dependencies = [ "criterion", "fallible-iterator", "float-cmp", + "futures", "gdal", "geo", "geojson", @@ -2356,7 +2357,7 @@ dependencies = [ [[package]] name = "geoengine-expression" -version = "0.8.0" +version = "0.9.0" dependencies = [ "geo", "geoengine-expression-deps", @@ -2385,7 +2386,7 @@ dependencies = [ [[package]] name = "geoengine-macros" -version = "0.8.0" +version = "0.9.0" dependencies = [ "pretty_assertions", "prettyplease", @@ -2396,7 +2397,7 @@ dependencies = [ [[package]] name = "geoengine-operators" -version = "0.8.0" +version = "0.9.0" dependencies = [ "approx", "arrow", @@ -2448,7 +2449,7 @@ dependencies = [ [[package]] name = "geoengine-services" -version = "0.8.0" +version = "0.9.0" dependencies = [ "actix", "actix-files", @@ -2506,6 +2507,7 @@ dependencies = [ "pwhash", "rand 0.9.2", "rayon", + "regex", "reqwest", "serde", "serde_json", @@ -5587,9 +5589,9 @@ dependencies = [ [[package]] name = "schemars" -version = "1.0.5" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1317c3bf3e7df961da95b0a56a172a02abead31276215a0497241a7624b487ce" +checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289" dependencies = [ "dyn-clone", "ref-cast", @@ -5784,7 +5786,7 @@ dependencies = [ "indexmap 1.9.3", "indexmap 2.12.0", "schemars 0.9.0", - "schemars 1.0.5", + "schemars 1.1.0", "serde_core", "serde_json", "serde_with_macros", @@ -6140,9 +6142,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.108" +version = "2.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index e3a36e456..dc9469946 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "0.8.0" +version = "0.9.0" authors = [ "Christian Beilschmidt ", "Johannes Drönner ", @@ -156,6 +156,7 @@ pwhash = "1.0" quote = "1.0" rand = "0.9" rayon = "1.8" +regex = "1.11" reqwest = { version = "0.12", features = [ "json", "multipart", diff --git a/Settings-default.toml b/Settings-default.toml index b76de6c7e..2b9752ba4 100644 --- a/Settings-default.toml +++ b/Settings-default.toml @@ -34,8 +34,6 @@ password = "geoengine" clear_database_on_start = false [raster.tiling_specification] -origin_coordinate_x = 0.0 -origin_coordinate_y = 0.0 tile_shape_pixels_x = 512 tile_shape_pixels_y = 512 diff --git a/Settings-test.toml b/Settings-test.toml index c4f217ddc..dedd81d4a 100644 --- a/Settings-test.toml +++ b/Settings-test.toml @@ -10,8 +10,6 @@ password = "geoengine" raster_data_root_path = "../test_data/raster" # relative to sub crate directory for tests [raster.tiling_specification] -origin_coordinate_x = 0.0 -origin_coordinate_y = 0.0 tile_shape_pixels_x = 512 tile_shape_pixels_y = 512 diff --git a/datatypes/Cargo.toml b/datatypes/Cargo.toml index 737c0c670..ccffffa41 100644 --- a/datatypes/Cargo.toml +++ b/datatypes/Cargo.toml @@ -18,6 +18,7 @@ arrow-schema = { workspace = true } bytes = { workspace = true } chrono = { workspace = true } fallible-iterator = { workspace = true } +futures = { workspace = true } float-cmp = { workspace = true } gdal = { workspace = true } geo = { workspace = true } diff --git a/datatypes/src/error.rs b/datatypes/src/error.rs index a3940d926..5928ac126 100644 --- a/datatypes/src/error.rs +++ b/datatypes/src/error.rs @@ -104,6 +104,11 @@ pub enum Error { i2: TimeInterval, }, + #[snafu(display("Time step must be greater than zero, got: {}", step))] + TimeStepStepMustBeGreaterThanZero { + step: u32, + }, + #[snafu(display( "{} must be larger than {} and {} must be smaller than {}", start.inner(), @@ -294,7 +299,7 @@ pub enum Error { WrongMetadataType, #[snafu(display( - "The conditions ul.x < lr.x && ul.y < lr.y are not met by ul:{} lr:{}", + "The conditions ul.x < lr.x && ul.y > lr.y are not met by ul:{} lr:{}", upper_left_coordinate, lower_right_coordinate ))] @@ -345,6 +350,8 @@ pub enum Error { DuplicateBandInQueryBandSelection, QueryBandSelectionMustNotBeEmpty, + TilingGeoTransformOriginCoordinateMismatch, + TilingGeoTransformResolutionMissmatch, #[snafu(display("Invalid number of suffixes, expected {} found {}", expected, found))] InvalidNumberOfSuffixes { expected: usize, @@ -363,6 +370,11 @@ pub enum Error { expected: usize, found: usize, }, + NoIntersectionWithTargetProjection { + srs_in: SpatialReference, + srs_out: SpatialReference, + bounds: BoundingBox2D, + }, } impl From for Error { diff --git a/datatypes/src/operations/image/colorizer.rs b/datatypes/src/operations/image/colorizer.rs index 85a9ac8ea..be5f01963 100644 --- a/datatypes/src/operations/image/colorizer.rs +++ b/datatypes/src/operations/image/colorizer.rs @@ -46,6 +46,16 @@ impl From for RasterColorizer { } } +impl RasterColorizer { + /// Returns the no data color of this raster colorizer + pub fn no_data_color(&self) -> RgbaColor { + match self { + RasterColorizer::SingleBand { band_colorizer, .. } => band_colorizer.no_data_color(), + RasterColorizer::MultiBand { rgb_params, .. } => rgb_params.no_data_color, + } + } +} + /// The parameters for the RGBA colorizer #[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] diff --git a/datatypes/src/operations/image/mod.rs b/datatypes/src/operations/image/mod.rs index 34690560e..ee08b1969 100644 --- a/datatypes/src/operations/image/mod.rs +++ b/datatypes/src/operations/image/mod.rs @@ -8,4 +8,4 @@ pub use colorizer::{ }; pub use into_lossy::LossyInto; pub use rgba_transmutable::RgbaTransmutable; -pub use to_png::ToPng; +pub use to_png::{ToPng, create_empty_no_data_color_png_bytes}; diff --git a/datatypes/src/operations/image/to_png.rs b/datatypes/src/operations/image/to_png.rs index a7c3619e9..880a93799 100644 --- a/datatypes/src/operations/image/to_png.rs +++ b/datatypes/src/operations/image/to_png.rs @@ -34,6 +34,15 @@ pub trait ToPng { ) -> Result>; } +pub fn create_empty_no_data_color_png_bytes( + width: u32, + height: u32, + no_data_color: RgbaColor, +) -> Result> { + let image_buffer = ImageBuffer::from_pixel(width, height, no_data_color.into()); + image_buffer_to_png_bytes(image_buffer) +} + fn image_buffer_to_png_bytes( image_buffer: ImageBuffer, Vec>, ) -> Result> { @@ -121,9 +130,7 @@ where ) -> Result> { // TODO: use PNG color palette once it is available - let image_buffer = ImageBuffer::from_pixel(width, height, no_data_color.into()); - - image_buffer_to_png_bytes(image_buffer) + create_empty_no_data_color_png_bytes(width, height, no_data_color) } } diff --git a/datatypes/src/operations/reproject.rs b/datatypes/src/operations/reproject.rs index a203e0eba..704909fd0 100644 --- a/datatypes/src/operations/reproject.rs +++ b/datatypes/src/operations/reproject.rs @@ -1,10 +1,13 @@ use crate::{ - error::{self}, + error, primitives::{ - AxisAlignedRectangle, Coordinate2D, Line, MultiLineString, MultiLineStringAccess, - MultiLineStringRef, MultiPoint, MultiPointAccess, MultiPointRef, MultiPolygon, - MultiPolygonAccess, MultiPolygonRef, QueryAttributeSelection, QueryRectangle, - SpatialBounded, SpatialResolution, + AxisAlignedRectangle, BoundingBox2D, Coordinate2D, Line, MultiLineString, + MultiLineStringAccess, MultiLineStringRef, MultiPoint, MultiPointAccess, MultiPointRef, + MultiPolygon, MultiPolygonAccess, MultiPolygonRef, SpatialBounded, SpatialResolution, + }, + raster::{ + BoundedGrid, GeoTransform, GridBoundingBox, GridBounds, GridIdx, GridIdx2D, GridShape, + GridSize, SamplePoints, SpatialGridDefinition, }, spatial_reference::SpatialReference, util::Result, @@ -352,6 +355,17 @@ fn diag_distance(ul_coord: Coordinate2D, lr_coord: Coordinate2D) -> f64 { (proj_ul_lr_vector.x * proj_ul_lr_vector.x + proj_ul_lr_vector.y * proj_ul_lr_vector.y).sqrt() } +pub fn suggest_pixel_size_like_gdal_helper( + bbox: B, + spatial_resolution: SpatialResolution, + source_srs: SpatialReference, + target: SpatialReference, +) -> Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target)?; + + suggest_pixel_size_like_gdal(bbox, spatial_resolution, &projector) +} + /// This method calculates a suggested pixel size for the translation of a raster into a different projection. /// The source raster is described using a `BoundingBox2D` and a pixel size as `SpatialResolution`. /// A suggested pixel size is calculated using the approach used by GDAL: @@ -375,6 +389,150 @@ pub fn suggest_pixel_size_like_gdal Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target_srs)?; + + suggest_output_spatial_grid_like_gdal(spatial_grid, &projector) +} + +pub fn reproject_spatial_grid_bounds( + spatial_grid: &SpatialGridDefinition, + projector: &P, +) -> Result> { + const SAMPLE_POINT_STEPS: usize = 2; + + // First, try to reproject the bounds: + let full_bounds: std::result::Result = spatial_grid + .spatial_partition() + .as_bbox() + .reproject(projector); + + if let Ok(projected_bounds) = full_bounds { + let res = A::from_min_max( + projected_bounds.lower_left(), + projected_bounds.upper_right(), + ); + return Some(res).transpose(); + } + + // Second, create a grid of coordinates project that and use the valid bounds. + // To do this, we generate a `SpatialGridDefinition` ... + let sample_bounds = SpatialGridDefinition::new( + spatial_grid.geo_transform, + GridBoundingBox::new_unchecked( + spatial_grid.grid_bounds.min_index(), + spatial_grid.grid_bounds.max_index() + GridIdx2D::new_y_x(1, 1), + ), + ); + // Then the the obvious way to generate sample points is to use all the pixels in the grid like this: + // let coord_grid = spatial_grid.generate_coord_grid_upper_left_edge(); + // However, this creates a lot of redundant points == work. + // The better way, also employed by GDAL is to use a "Haus vom Nikolaus" strategy which is done below: + let mut coord_grid_sample = sample_bounds.sample_outline(SAMPLE_POINT_STEPS); + coord_grid_sample.append(&mut sample_bounds.sample_diagonals(SAMPLE_POINT_STEPS)); + coord_grid_sample.append(&mut sample_bounds.sample_cross(SAMPLE_POINT_STEPS)); + // Then, we try to reproject the sample coordinates and gather all the valid coordinates. + let proj_outline_coordinates: Vec = + project_coordinates_fail_tolerant(&coord_grid_sample, projector) + .into_iter() + .flatten() + .collect(); + // TODO: we need a way to indicate that the operator might produce no data, e.g. if no points are valid after reprojection. + if proj_outline_coordinates.is_empty() { + return Ok(None); + } + // Then, the maximum bounding box is generated from the valid coordinates. + let out = MultiPoint::new(proj_outline_coordinates)?.spatial_bounds(); + // Finally, the requested bound type is returned. + Some(A::from_min_max(out.lower_left(), out.upper_right())).transpose() +} + +pub fn suggest_output_spatial_grid_like_gdal( + spatial_grid: &SpatialGridDefinition, + projector: &P, +) -> Result { + const ROUND_UP_SIZE: bool = false; + + let in_x_pixels = spatial_grid.grid_bounds().axis_size_x(); + let in_y_pixels = spatial_grid.grid_bounds().axis_size_y(); + + let proj_bbox_option: Option = + reproject_spatial_grid_bounds(spatial_grid, projector)?; + + let Some(proj_bbox) = proj_bbox_option else { + return Err(error::Error::NoIntersectionWithTargetProjection { + srs_in: projector.source_srs(), + srs_out: projector.target_srs(), + bounds: spatial_grid.spatial_partition().as_bbox(), + }); + }; + + let out_x_distance = proj_bbox.size_x(); + let out_y_distance = proj_bbox.size_y(); + + let out_diagonal_dist = + (out_x_distance * out_x_distance + out_y_distance * out_y_distance).sqrt(); + + let pixel_size = + out_diagonal_dist / ((in_x_pixels * in_x_pixels + in_y_pixels * in_y_pixels) as f64).sqrt(); + + let x_pixels_with_frac = out_x_distance / pixel_size; + let y_pixels_with_frac = out_y_distance / pixel_size; + + let (x_pixels, y_pixels) = if ROUND_UP_SIZE { + const EPS_FROM_GDAL: f64 = 1e-5; + ( + (x_pixels_with_frac - EPS_FROM_GDAL).ceil() as usize, + (y_pixels_with_frac - EPS_FROM_GDAL).ceil() as usize, + ) + } else { + ( + (x_pixels_with_frac + 0.5) as usize, + (y_pixels_with_frac + 0.5) as usize, + ) + }; + + // TODO: gdal does some magic to fit to the bounds which might change the pixel size again. + // let x_pixel_size = out_x_distance / x_pixels as f64; + // let y_pixel_size = out_y_distance / y_pixels as f64; + let x_pixel_size = pixel_size; + let y_pixel_size = pixel_size; + + let geo_transform = GeoTransform::new(proj_bbox.upper_left(), x_pixel_size, -y_pixel_size); + let grid_bounds = GridShape::new_2d(y_pixels, x_pixels).bounding_box(); + let out_spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + // if the input grid is anchored at the upper left idx then we don't have to move the origin of the geo transform + if spatial_grid.grid_bounds.min_index() == GridIdx([0, 0]) { + return Ok(SpatialGridDefinition::new(geo_transform, grid_bounds)); + } + + let proj_origin = spatial_grid + .geo_transform() + .origin_coordinate() + .reproject(projector)?; + + let out_spatial_grid_moved_origin = + out_spatial_grid.with_moved_origin_to_nearest_grid_edge(proj_origin); + + Ok(out_spatial_grid_moved_origin.replace_origin(proj_origin)) +} + +pub fn suggest_pixel_size_from_diag_cross_helper( + bbox: B, + spatial_resolution: SpatialResolution, + source_srs: SpatialReference, + target: SpatialReference, +) -> Result { + let projector = CoordinateProjector::from_known_srs(source_srs, target)?; + + suggest_pixel_size_from_diag_cross(bbox, spatial_resolution, &projector) +} + /// This approach uses the GDAL way to suggest the pixel size. However, we check both diagonals and take the smaller one. /// This method fails if the bbox cannot be projected pub fn suggest_pixel_size_from_diag_cross( @@ -387,7 +545,7 @@ pub fn suggest_pixel_size_from_diag_cross = projected_diag_distance(bbox.lower_left(), bbox.upper_right(), projector); let min_dist_r = match (proj_ul_lr_distance, proj_ll_ur_distance) { @@ -456,26 +614,19 @@ pub fn project_coordinates_fail_tolerant( /// this method performs the transformation of a query rectangle in `target` projection /// to a new query rectangle with coordinates in the `source` projection -pub fn reproject_query( - query: QueryRectangle, +pub fn reproject_spatial_query( + spatial_bounds: S, source: SpatialReference, target: SpatialReference, force_clipping: bool, -) -> Result>> { - let (Some(s_bbox), Some(p_bbox)) = - reproject_and_unify_bbox_internal(query.spatial_bounds, target, source, force_clipping)? +) -> Result> { + let (Some(_s_bbox), Some(p_bbox)) = + reproject_and_unify_bbox_internal(spatial_bounds, target, source, force_clipping)? else { return Ok(None); }; - let p_spatial_resolution = - suggest_pixel_size_from_diag_cross_projected(s_bbox, p_bbox, query.spatial_resolution)?; - Ok(Some(QueryRectangle { - spatial_bounds: p_bbox, - spatial_resolution: p_spatial_resolution, - time_interval: query.time_interval, - attributes: query.attributes, - })) + Ok(Some(p_bbox)) } /// Reproject a bounding box to the `target` projection and return the input and output bounding box @@ -534,6 +685,26 @@ fn reproject_and_unify_bbox_internal( } } +/// Reproject the area of use of the `source` projection to the `target` projection and back. Return the back projected bounds and the area of use in the `target` projection. +pub fn reproject_and_unify_proj_bounds( + source: SpatialReference, + target: SpatialReference, +) -> Result<(Option, Option)> { + let proj_from_to = CoordinateProjector::from_known_srs(source, target)?; + let proj_to_from = CoordinateProjector::from_known_srs(target, source)?; + + let target_bbox_clipped = source + .area_of_use_projected::()? + .reproject_clipped(&proj_from_to)?; // TODO: can we intersect areas of use first? + + if let Some(target_b) = target_bbox_clipped { + let source_bbox_clipped = target_b.reproject(&proj_to_from)?; + Ok((Some(source_bbox_clipped), target_bbox_clipped)) + } else { + Ok((None, None)) + } +} + #[cfg(test)] mod tests { diff --git a/datatypes/src/primitives/bounding_box.rs b/datatypes/src/primitives/bounding_box.rs index 4d93a731d..35f9c5991 100644 --- a/datatypes/src/primitives/bounding_box.rs +++ b/datatypes/src/primitives/bounding_box.rs @@ -24,7 +24,7 @@ impl fmt::Display for BoundingBox2D { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "(ll: {}, ur: {})", + "BoundingBox2D: (lower_left: {}, upper_right: {})", self.lower_left_coordinate, self.upper_right_coordinate ) } diff --git a/datatypes/src/primitives/db_types.rs b/datatypes/src/primitives/db_types.rs index 027f2e34b..e4e098224 100644 --- a/datatypes/src/primitives/db_types.rs +++ b/datatypes/src/primitives/db_types.rs @@ -9,7 +9,7 @@ use crate::{ util::NotNanF64, }; use postgres_types::{FromSql, ToSql}; -use std::collections::HashMap; +use std::collections::BTreeMap; #[derive(Debug, ToSql, FromSql)] #[postgres(name = "Measurement")] @@ -77,7 +77,7 @@ impl TryFrom for Measurement { continuous: None, classification: Some(classification), } => { - let mut classes = HashMap::with_capacity(classification.classes.len()); + let mut classes = BTreeMap::new(); for SmallintTextKeyValue { key, value } in classification.classes { classes.insert( u8::try_from(key).map_err(|_| Error::UnexpectedInvalidDbTypeConversion)?, diff --git a/datatypes/src/primitives/measurement.rs b/datatypes/src/primitives/measurement.rs index 2915c9e30..fc4847097 100644 --- a/datatypes/src/primitives/measurement.rs +++ b/datatypes/src/primitives/measurement.rs @@ -1,6 +1,6 @@ use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; use std::fmt; use std::str::FromStr; @@ -18,12 +18,20 @@ impl Measurement { Self::Continuous(ContinuousMeasurement { measurement, unit }) } - pub fn classification(measurement: String, classes: HashMap) -> Self { + pub fn classification(measurement: String, classes: BTreeMap) -> Self { Self::Classification(ClassificationMeasurement { measurement, classes, }) } + + pub fn is_classification(&self) -> bool { + matches!(self, Self::Classification(_)) + } + + pub fn is_continuous(&self) -> bool { + matches!(self, Self::Continuous(_)) + } } #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, FromSql, ToSql)] @@ -39,7 +47,7 @@ pub struct ContinuousMeasurement { )] pub struct ClassificationMeasurement { pub measurement: String, - pub classes: HashMap, + pub classes: BTreeMap, } /// A type that is solely for serde's serializability. @@ -68,13 +76,14 @@ impl TryFrom for ClassificationMeasuremen type Error = ::Err; fn try_from(measurement: SerializableClassificationMeasurement) -> Result { - let mut classes = HashMap::with_capacity(measurement.classes.len()); - for (k, v) in measurement.classes { - classes.insert(k.parse::()?, v); - } + let classes: Result, _> = measurement + .classes + .into_iter() + .map(|(k, v)| k.parse::().map(|x| (x, v))) + .collect(); Ok(Self { measurement: measurement.measurement, - classes, + classes: classes?, }) } } @@ -91,12 +100,12 @@ impl fmt::Display for Measurement { /// # Examples /// ```rust /// use geoengine_datatypes::primitives::Measurement; - /// use std::collections::HashMap; + /// use std::collections::BTreeMap; /// /// assert_eq!(format!("{}", Measurement::Unitless), ""); /// assert_eq!(format!("{}", Measurement::continuous("foo".into(), Some("bar".into()))), "foo in bar"); /// assert_eq!(format!("{}", Measurement::continuous("foo".into(), None)), "foo"); - /// assert_eq!(format!("{}", Measurement::classification("foobar".into(), HashMap::new())), "foobar"); + /// assert_eq!(format!("{}", Measurement::classification("foobar".into(), BTreeMap::new())), "foobar"); /// ``` fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { @@ -138,7 +147,7 @@ mod tests { fn classification_serialization() { let measurement = Measurement::classification( "foo".into(), - HashMap::from([(1_u8, "bar".to_string()), (2, "baz".to_string())]), + BTreeMap::from([(1_u8, "bar".to_string()), (2, "baz".to_string())]), ); let serialized = serde_json::to_string(&measurement).unwrap(); assert_eq!( diff --git a/datatypes/src/primitives/mod.rs b/datatypes/src/primitives/mod.rs index f89da27d0..4646bbd82 100755 --- a/datatypes/src/primitives/mod.rs +++ b/datatypes/src/primitives/mod.rs @@ -16,6 +16,8 @@ mod query_rectangle; mod spatial_partition; mod spatial_resolution; mod spatio_temporal_bounded; +mod time_dimension; +mod time_gap_fill_iter; mod time_instance; mod time_interval; mod time_step; @@ -39,14 +41,21 @@ pub use multi_point::{MultiPoint, MultiPointAccess, MultiPointRef}; pub use multi_polygon::{MultiPolygon, MultiPolygonAccess, MultiPolygonRef}; pub use no_geometry::NoGeometry; pub use query_rectangle::{ - BandSelection, ColumnSelection, PlotQueryRectangle, PlotSeriesSelection, + BandSelection, BandSelectionIter, ColumnSelection, PlotQueryRectangle, PlotSeriesSelection, QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, VectorQueryRectangle, }; pub use spatial_partition::{ AxisAlignedRectangle, SpatialPartition2D, SpatialPartitioned, partitions_extent, }; -pub use spatial_resolution::SpatialResolution; +pub use spatial_resolution::{ + SpatialResolution, find_next_best_overview_level, find_next_best_overview_level_resolution, +}; pub use spatio_temporal_bounded::{SpatialBounded, TemporalBounded}; +pub use time_dimension::{RegularTimeDimension, TimeDimension}; +pub use time_gap_fill_iter::{ + TimeEmptySingleFill, TimeFilledItem, TimeGapFill, TimeGapFillIter, TimeGapFillNextAction, + TimeSingleAppend, TryIrregularTimeFillIterExt, TryRegularTimeFillIterExt, TryTimeGapFillIter, +}; pub use time_instance::TimeInstance; pub use time_interval::{TimeInterval, time_interval_extent}; pub use time_step::{TimeGranularity, TimeStep, TimeStepIter}; diff --git a/datatypes/src/primitives/query_rectangle.rs b/datatypes/src/primitives/query_rectangle.rs index f29c49fb2..64a1a47a0 100644 --- a/datatypes/src/primitives/query_rectangle.rs +++ b/datatypes/src/primitives/query_rectangle.rs @@ -1,7 +1,5 @@ -use super::{ - AxisAlignedRectangle, BoundingBox2D, SpatialPartition2D, SpatialPartitioned, SpatialResolution, - TimeInterval, -}; +use super::{AxisAlignedRectangle, BoundingBox2D, TimeInterval}; +use crate::raster::{GeoTransform, GridBoundingBox2D}; use crate::{ error::{DuplicateBandInQueryBandSelection, QueryBandSelectionMustNotBeEmpty}, util::Result, @@ -12,14 +10,121 @@ use snafu::ensure; /// A spatio-temporal rectangle with a specified resolution and the selected bands #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct QueryRectangle< - SpatialBounds: AxisAlignedRectangle, - AttributeSelection: QueryAttributeSelection, -> { - pub spatial_bounds: SpatialBounds, - pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, - pub attributes: AttributeSelection, +pub struct QueryRectangle { + spatial_bounds: SpatialBounds, + time_interval: TimeInterval, + attributes: AttributeSelection, +} + +impl QueryRectangle { + pub fn time_interval(&self) -> TimeInterval { + self.time_interval + } + + pub fn spatial_bounds(&self) -> SpatialBounds { + self.spatial_bounds + } + + pub fn spatial_bounds_mut(&mut self) -> &mut SpatialBounds { + &mut self.spatial_bounds + } + + pub fn time_interval_mut(&mut self) -> &mut TimeInterval { + &mut self.time_interval + } + + pub fn attributes(&self) -> &A { + &self.attributes + } + + pub fn attributes_mut(&mut self) -> &mut A { + &mut self.attributes + } + + /// Creates a new `QueryRectangle` from a `BoundingBox2D`, and a `TimeInterval` + pub fn new(spatial_bounds: SpatialBounds, time_interval: TimeInterval, attributes: A) -> Self { + Self { + spatial_bounds, + time_interval, + attributes, + } + } + + /// Create a clone of `self` with another `TimeInterval`. + #[must_use] + pub fn select_time_interval(&self, time_interval: TimeInterval) -> Self { + Self { + spatial_bounds: self.spatial_bounds(), + time_interval, + attributes: self.attributes().clone(), + } + } + + /// Create a clone of `self` with other `SpatialBounds`. + #[must_use] + pub fn select_spatial_bounds(&self, spatial_bounds: SpatialBounds) -> Self { + Self { + spatial_bounds, + time_interval: self.time_interval(), + attributes: self.attributes().clone(), + } + } + + /// Create a copy of `self` with other `QueryAttributeSelection`. + /// This method also allow to change the type of `QueryAttributeSelection`. + #[must_use] + pub fn select_attributes( + &self, + attributes: B, + ) -> QueryRectangle { + QueryRectangle { + spatial_bounds: self.spatial_bounds, + time_interval: self.time_interval, + attributes, + } + } +} + +pub type VectorQueryRectangle = QueryRectangle; +pub type RasterQueryRectangle = QueryRectangle; +pub type PlotQueryRectangle = QueryRectangle; + +// Implementation for VectorQueryRectangle and PlotQueryRectangle +impl QueryRectangle +where + A: QueryAttributeSelection, +{ + /// Creates a new `QueryRectangle` with bounds and time from a `RasterQueryRectangle` and supplied attributes. + /// + /// # Panics + /// If the `geo_transform` can't transform the raster bounds into a valid `SpatialPartition` + pub fn from_raster_query_and_geo_transform_replace_attributes( + raster_query: &RasterQueryRectangle, + geo_transform: GeoTransform, + attributes: A, + ) -> QueryRectangle { + let bounds = geo_transform.grid_to_spatial_bounds(&raster_query.spatial_bounds()); + let bounding_box = BoundingBox2D::from_min_max(bounds.lower_left(), bounds.upper_right()) + .expect("Bounds are already valid"); + + QueryRectangle::new(bounding_box, raster_query.time_interval, attributes) + } +} + +impl RasterQueryRectangle { + /// Creates a new `QueryRectangle` that describes the requested grid. + /// The spatial query is derived from a vector query rectangle and a `GeoTransform`. + /// The temporal query is defined by a `TimeInterval`. + /// NOTE: If the distance between the upper left of the spatial partition and the origin coordinate is not at a multiple of the spatial resolution, the grid bounds will be shifted. + pub fn from_bounds_and_geo_transform( + query: &QueryRectangle, + bands: BandSelection, + geo_transform: GeoTransform, + ) -> Self { + let grid_bounds = + geo_transform.bounding_box_2d_to_intersecting_grid_bounds(&query.spatial_bounds()); + Self::new(grid_bounds, query.time_interval, bands) + } } pub trait QueryAttributeSelection: Clone + Send + Sync {} @@ -67,6 +172,10 @@ impl BandSelection { pub fn as_vec(&self) -> Vec { self.0.clone() } + + pub fn is_single(&self) -> bool { + self.count() == 1 + } } impl From for BandSelection { @@ -93,112 +202,57 @@ impl TryFrom<[u32; N]> for BandSelection { impl QueryAttributeSelection for BandSelection {} -#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ColumnSelection {} - -impl ColumnSelection { - pub fn all() -> Self { - Self {} - } -} - -impl QueryAttributeSelection for ColumnSelection {} - -#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct PlotSeriesSelection {} - -impl PlotSeriesSelection { - pub fn all() -> Self { - Self {} - } +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct BandSelectionIter { + band_selection: BandSelection, + next_index: usize, } -impl QueryAttributeSelection for PlotSeriesSelection {} - -pub type VectorQueryRectangle = QueryRectangle; -pub type RasterQueryRectangle = QueryRectangle; -pub type PlotQueryRectangle = QueryRectangle; - -impl RasterQueryRectangle { - pub fn from_qrect_and_bands( - query: &QueryRectangle, - bands: BandSelection, - ) -> Self - where - A: QueryAttributeSelection, - QueryRectangle: SpatialPartitioned, - { - Self { - spatial_bounds: query.spatial_partition(), - time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: bands, - } - } - - #[must_use] - pub fn select_bands(&self, bands: BandSelection) -> Self { +impl BandSelectionIter { + pub fn new(band_selection: BandSelection) -> Self { Self { - spatial_bounds: self.spatial_bounds, - time_interval: self.time_interval, - spatial_resolution: self.spatial_resolution, - attributes: bands, + band_selection, + next_index: 0, } } -} - -impl SpatialPartitioned for QueryRectangle { - fn spatial_partition(&self) -> SpatialPartition2D { - SpatialPartition2D::with_bbox_and_resolution(self.spatial_bounds, self.spatial_resolution) - } -} -impl SpatialPartitioned for QueryRectangle { - fn spatial_partition(&self) -> SpatialPartition2D { - SpatialPartition2D::with_bbox_and_resolution(self.spatial_bounds, self.spatial_resolution) + pub fn reset(&mut self) { + self.next_index = 0; } } -impl SpatialPartitioned for QueryRectangle { - fn spatial_partition(&self) -> SpatialPartition2D { - self.spatial_bounds - } -} +impl Iterator for BandSelectionIter { + type Item = u32; -impl From> - for QueryRectangle -{ - fn from(value: QueryRectangle) -> Self { - Self { - spatial_bounds: value.spatial_bounds, - time_interval: value.time_interval, - spatial_resolution: value.spatial_resolution, - attributes: value.attributes.into(), + fn next(&mut self) -> Option { + if self.next_index >= self.band_selection.0.len() { + return None; } - } -} -impl From> - for QueryRectangle -{ - fn from(value: QueryRectangle) -> Self { - Self { - spatial_bounds: value.spatial_bounds, - time_interval: value.time_interval, - spatial_resolution: value.spatial_resolution, - attributes: value.attributes.into(), - } + let item = self.band_selection.0[self.next_index]; + self.next_index += 1; + Some(item) } } -impl From for PlotSeriesSelection { - fn from(_: ColumnSelection) -> Self { +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ColumnSelection {} + +impl ColumnSelection { + pub fn all() -> Self { Self {} } } -impl From for ColumnSelection { - fn from(_: PlotSeriesSelection) -> Self { +impl QueryAttributeSelection for ColumnSelection {} + +#[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PlotSeriesSelection {} + +impl PlotSeriesSelection { + pub fn all() -> Self { Self {} } } + +impl QueryAttributeSelection for PlotSeriesSelection {} diff --git a/datatypes/src/primitives/spatial_partition.rs b/datatypes/src/primitives/spatial_partition.rs index f368f9f76..bea267484 100644 --- a/datatypes/src/primitives/spatial_partition.rs +++ b/datatypes/src/primitives/spatial_partition.rs @@ -335,22 +335,10 @@ impl From<&SpatialPartition2D> for geo::Rect { } /// Compute the extent of all input partitions. If one partition is None, the output will also be None -pub fn partitions_extent>>( - mut bboxes: I, +pub fn partitions_extent>( + bboxes: I, ) -> Option { - let Some(Some(mut extent)) = bboxes.next() else { - return None; - }; - - for bbox in bboxes { - if let Some(bbox) = bbox { - extent.extend(&bbox); - } else { - return None; - } - } - - Some(extent) + bboxes.reduce(|s, other| s.extended(&other)) } #[cfg(test)] @@ -524,37 +512,16 @@ mod tests { #[test] fn extent() { - assert_eq!(partitions_extent([None].into_iter()), None); assert_eq!( partitions_extent( [ - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()), - Some(SpatialPartition2D::new((0., 70.).into(), (70., 0.).into()).unwrap()) + SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap(), + SpatialPartition2D::new((0., 70.).into(), (70., 0.).into()).unwrap() ] .into_iter() ), Some(SpatialPartition2D::new((-50., 70.).into(), (70., -50.).into()).unwrap()) ); - assert_eq!( - partitions_extent( - [ - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()), - None - ] - .into_iter() - ), - None - ); - assert_eq!( - partitions_extent( - [ - None, - Some(SpatialPartition2D::new((-50., 50.).into(), (50., -50.).into()).unwrap()) - ] - .into_iter() - ), - None - ); } #[test] diff --git a/datatypes/src/primitives/spatial_resolution.rs b/datatypes/src/primitives/spatial_resolution.rs index b892ec046..ffa59e851 100644 --- a/datatypes/src/primitives/spatial_resolution.rs +++ b/datatypes/src/primitives/spatial_resolution.rs @@ -2,6 +2,7 @@ use std::{convert::TryFrom, ops::Add, ops::Div, ops::Mul, ops::Sub}; use crate::primitives::error; use crate::util::Result; +use float_cmp::{ApproxEq, F64Margin, approx_eq}; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -36,6 +37,16 @@ impl SpatialResolution { pub fn one() -> Self { SpatialResolution { x: 1., y: 1. } } + + pub fn with_native_resolution_and_gdal_overview_level( + native_resolution: SpatialResolution, + gdal_overview_level: u32, + ) -> SpatialResolution { + SpatialResolution::new_unchecked( + native_resolution.x * f64::from(gdal_overview_level), + native_resolution.y * f64::from(gdal_overview_level), + ) + } } impl TryFrom<(f64, f64)> for SpatialResolution { @@ -90,6 +101,66 @@ impl Div for SpatialResolution { } } +impl ApproxEq for SpatialResolution { + type Margin = F64Margin; + + fn approx_eq>(self, other: Self, margin: M) -> bool { + let m = margin.into(); + approx_eq!(f64, self.x, other.x, m) && approx_eq!(f64, self.y, other.y, m) + } +} + +#[allow(clippy::float_cmp)] +impl PartialOrd for SpatialResolution { + fn partial_cmp(&self, other: &Self) -> Option { + if self.x < other.x && self.y < other.y { + Some(std::cmp::Ordering::Less) + } else if self.x > other.x && self.y > other.y { + Some(std::cmp::Ordering::Greater) + } else if self.x == other.x && self.y == other.y { + // TODO: use `approx_eq`? + Some(std::cmp::Ordering::Equal) + } else { + None + } + } +} + +/// finds the coarsest overview level that is still at least as fine as the required resolution +// TODO: add option to only use overview levels available for a given gdal dataset. +pub fn find_next_best_overview_level( + native_resolution: SpatialResolution, + target_resolution: SpatialResolution, +) -> u32 { + let mut current_overview_level = 0; + let mut next_overview_level = 2; + + while SpatialResolution::with_native_resolution_and_gdal_overview_level( + native_resolution, + next_overview_level, + ) <= target_resolution + { + current_overview_level = next_overview_level; + next_overview_level *= 2; + } + + current_overview_level +} + +/// Scale up the given resolution to the next best overview level resolution, i.e., a resolution that is a power of 2 and still finer than the target +pub fn find_next_best_overview_level_resolution( + mut current_resolution: SpatialResolution, + target_resolution: SpatialResolution, +) -> SpatialResolution { + debug_assert!(current_resolution <= target_resolution); + + while current_resolution * 2.0 <= target_resolution { + current_resolution = current_resolution * 2.0; + } + + current_resolution +} + #[cfg(test)] mod test { use super::*; @@ -127,4 +198,39 @@ mod test { let res = SpatialResolution { x: 4., y: 8. } / 2.; assert_eq!(res, SpatialResolution { x: 2., y: 4. }); } + + #[test] + fn it_finds_next_best_overview_level() { + assert_eq!( + find_next_best_overview_level( + SpatialResolution::new_unchecked(0.1, 0.1), + SpatialResolution::new_unchecked(0.1, 0.1) + ), + 0 + ); + + assert_eq!( + find_next_best_overview_level( + SpatialResolution::new_unchecked(0.1, 0.1), + SpatialResolution::new_unchecked(0.2, 0.2) + ), + 2 + ); + + assert_eq!( + find_next_best_overview_level( + SpatialResolution::new_unchecked(0.1, 0.1), + SpatialResolution::new_unchecked(0.3, 0.3) + ), + 2 + ); + + assert_eq!( + find_next_best_overview_level( + SpatialResolution::new_unchecked(0.1, 0.1), + SpatialResolution::new_unchecked(0.4, 0.4) + ), + 4 + ); + } } diff --git a/datatypes/src/primitives/time_dimension.rs b/datatypes/src/primitives/time_dimension.rs new file mode 100644 index 000000000..7f097df74 --- /dev/null +++ b/datatypes/src/primitives/time_dimension.rs @@ -0,0 +1,444 @@ +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; + +use crate::{ + delegate_from_to_sql, + error::Error, + primitives::{TimeInstance, TimeInterval, TimeStep, TimeStepIter}, + util::Result, +}; + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RegularTimeDimension { + pub origin: TimeInstance, + pub step: TimeStep, +} + +impl RegularTimeDimension { + pub fn new_with_epoch_origin(step: TimeStep) -> Self { + Self { + origin: TimeInstance::EPOCH_START, + step, + } + } + + pub fn new(origin: TimeInstance, step: TimeStep) -> Self { + Self { origin, step } + } + + /// Checks whether two `RegularTimeDimension`s are compatible, i.e. have the same step and + /// their origins align on the same time steps. + pub fn compatible_with(&self, other: RegularTimeDimension) -> bool { + let step_identical = self.step == other.step; + // TODO: handle special cases like 12 months and 1 year, or 60 seconds and 1 minute + let Ok(snapped_origin_other) = self.snap_prev(other.origin) else { + return false; + }; + let origin_compatible = snapped_origin_other == other.origin; + step_identical && origin_compatible + } + + pub fn merge(&self, other: RegularTimeDimension) -> Option { + if self.compatible_with(other) { + let origin = if self.origin < other.origin { + self.origin + } else { + other.origin + }; + // TODO: handle special cases like 12 months and 1 year, or 60 seconds and 1 minute where we must decide which step to take + Some(RegularTimeDimension { + origin, + step: self.step, + }) + } else { + None + } + } + + pub fn valid_step(&self, time: TimeInstance) -> bool { + self.snap_prev(time) + .map(|snapped| snapped == time) + .unwrap_or(false) + } + + pub fn valid_interval(&self, time: TimeInterval) -> bool { + self.valid_step(time.start()) && self.valid_step(time.end()) + } + + /// Snaps the given `time_instance` to the previous (smaller or equal) time instance + pub fn snap_prev(&self, time_instance: TimeInstance) -> Result { + self.step.snap_relative(self.origin, time_instance) + } + + /// Snaps the given `time_instance` to the next (larger or equal) time instance + pub fn snap_next(&self, time_instance: TimeInstance) -> Result { + let snapped = self.snap_prev(time_instance)?; + if snapped < time_instance { + snapped + self.step + } else { + Ok(snapped) + } + } + + /// Returns the largest `TimeInterval` that is fully contained in the given `time_interval`. + /// If no time steps are contained, `Ok(None)` is returned. + pub fn contained_interval(&self, time_interval: TimeInterval) -> Result> { + let intersected = self.intersected_interval(time_interval)?; + + let start = if time_interval.start() == intersected.start() { + intersected.start() + } else { + (intersected.start() + self.step)? + }; + + let end = if time_interval.end() == intersected.end() { + intersected.end() + } else { + (intersected.end() - self.step)? + }; + + if end <= start { + Ok(None) + } else { + TimeInterval::new(start, end).map(Some) + } + } + + /// Returns the smallest `TimeInterval` that contains the given `time_interval` + pub fn intersected_interval(&self, time_interval: TimeInterval) -> Result { + let start = self.snap_prev(time_interval.start())?; + let end = self.snap_next(time_interval.end())?; + TimeInterval::new(start, end) + } + + /// Returns the number of time steps that are fully contained in the given `time_interval` + /// If no time steps are contained, `0` is returned. + pub fn steps_contained_in(&self, time_interval: TimeInterval) -> Result { + let Some(contained) = self.contained_interval(time_interval)? else { + return Ok(0); + }; + self.step.num_steps_in_interval(contained) + } + + /// Returns the number of time steps that intersect with the given `time_interval` + pub fn steps_intersecting(&self, time_interval: TimeInterval) -> Result { + let intersected = self.intersected_interval(time_interval)?; + self.step.num_steps_in_interval(intersected) + } + + /// Returns an iterator over all time steps that are fully contained in the given `time_interval` + /// If no time steps are contained, `Ok(None)` is returned. + /// + /// # Panics + /// IF `time_interval` is not valid + pub fn contained_intervals( + &self, + time_interval: TimeInterval, + ) -> Result>> { + let Some(contained) = self.contained_interval(time_interval)? else { + return Ok(None); + }; + let iter = TimeStepIter::new_with_interval(contained, self.step)?; + let intervals = iter.map(|start| { + TimeInterval::new_unchecked( + start, + (start + self.step).expect("is included in valid interval"), + ) + }); + Ok(Some(intervals)) + } + + /// Returns an iterator over all time steps that intersect with the given `time_interval` + /// + /// # Panics + /// IF `time_interval` is not valid + pub fn intersecting_intervals( + &self, + time_interval: TimeInterval, + ) -> Result + use<>> { + let intersected = self.intersected_interval(time_interval)?; + let iter = TimeStepIter::new_with_interval(intersected, self.step)?; + let step = self.step; + let intervals = iter.map(move |start| { + TimeInterval::new_unchecked( + start, + (start + step).expect("is included in valid interval"), + ) + }); + Ok(intervals) + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum TimeDimension { + Regular(RegularTimeDimension), + Irregular, +} + +impl TimeDimension { + pub fn is_regular(&self) -> bool { + matches!(self, TimeDimension::Regular(_)) + } + + pub fn origin(&self) -> Option { + match self { + TimeDimension::Regular(r) => Some(r.origin), + TimeDimension::Irregular => None, + } + } + + pub fn new_irregular() -> Self { + TimeDimension::Irregular + } + + pub fn new_regular(origin: TimeInstance, step: TimeStep) -> Self { + TimeDimension::Regular(RegularTimeDimension { origin, step }) + } + + pub fn new_regular_with_epoch(step: TimeStep) -> Self { + TimeDimension::Regular(RegularTimeDimension::new_with_epoch_origin(step)) + } + + pub fn unwrap_regular(self) -> Option { + match self { + TimeDimension::Regular(r) => Some(r), + TimeDimension::Irregular => None, + } + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, ToSql, FromSql)] +#[postgres(name = "TimeDimensionDiscriminator")] +pub enum TimeDimensionDiscriminatorDbType { + Regular, + Irregular, +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq, ToSql, FromSql)] +#[postgres(name = "TimeDimension")] +pub struct TimeDimensionDbType { + pub regular_dimension: Option, + pub discriminant: TimeDimensionDiscriminatorDbType, +} + +impl From<&TimeDimension> for TimeDimensionDbType { + fn from(value: &TimeDimension) -> Self { + match value { + TimeDimension::Regular(r) => Self { + regular_dimension: Some(*r), + discriminant: TimeDimensionDiscriminatorDbType::Regular, + }, + TimeDimension::Irregular => Self { + regular_dimension: None, + discriminant: TimeDimensionDiscriminatorDbType::Irregular, + }, + } + } +} + +impl TryFrom for TimeDimension { + type Error = Error; + + fn try_from(value: TimeDimensionDbType) -> Result { + match value.discriminant { + TimeDimensionDiscriminatorDbType::Regular => Ok(TimeDimension::Regular( + value + .regular_dimension + .ok_or(Error::UnexpectedInvalidDbTypeConversion)?, + )), + TimeDimensionDiscriminatorDbType::Irregular => Ok(TimeDimension::Irregular), + } + } +} + +delegate_from_to_sql!(TimeDimension, TimeDimensionDbType); + +#[cfg(test)] +mod tests { + + use crate::primitives::DateTime; + + use super::*; + + #[test] + fn test_compatible() { + let dim1 = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let dim2 = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + assert!(dim1.compatible_with(dim2)); + assert!(dim2.compatible_with(dim1)); + let dim3 = RegularTimeDimension::new( + DateTime::new_utc(2020, 1, 1, 0, 0, 0).into(), + TimeStep::years(1).unwrap(), + ); + assert!(dim1.compatible_with(dim3)); + assert!(dim3.compatible_with(dim1)); + let dim4 = RegularTimeDimension::new( + DateTime::new_utc(2021, 2, 1, 0, 0, 0).into(), + TimeStep::years(1).unwrap(), + ); + assert!(!dim1.compatible_with(dim4)); + assert!(!dim4.compatible_with(dim1)); + + let dim5 = RegularTimeDimension::new( + DateTime::new_utc(2021, 2, 1, 0, 0, 0).into(), + TimeStep::years(7).unwrap(), + ); + assert!(!dim1.compatible_with(dim5)); + assert!(!dim5.compatible_with(dim1)); + } + + #[test] + fn test_merge() { + let dim1 = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let dim2 = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + assert_eq!(dim1.merge(dim2).unwrap(), dim1); + let dim3 = RegularTimeDimension::new( + DateTime::new_utc(2020, 1, 1, 0, 0, 0).into(), + TimeStep::years(1).unwrap(), + ); + assert_eq!(dim1.merge(dim3).unwrap(), dim1); + let dim4 = RegularTimeDimension::new( + DateTime::new_utc(2021, 1, 1, 0, 0, 0).into(), + TimeStep::years(2).unwrap(), + ); + assert!(dim1.merge(dim4).is_none()); + } + + #[test] + fn test_snap() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let time = DateTime::new_utc(2023, 6, 15, 12, 0, 0).into(); + let snapped_prev = dim.snap_prev(time).unwrap(); + assert_eq!(snapped_prev, DateTime::new_utc(2023, 1, 1, 0, 0, 0).into()); + let snapped_next = dim.snap_next(time).unwrap(); + assert_eq!(snapped_next, DateTime::new_utc(2024, 1, 1, 0, 0, 0).into()); + } + + #[test] + fn test_contained_interval() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let contained = dim.contained_interval(interval).unwrap().unwrap(); + assert_eq!( + contained, + TimeInterval::new( + DateTime::new_utc(2021, 1, 1, 0, 0, 0), + DateTime::new_utc(2023, 1, 1, 0, 0, 0) + ) + .unwrap() + ); + } + + #[test] + fn test_intersected_interval() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let intersected = dim.intersected_interval(interval).unwrap(); + assert_eq!( + intersected, + TimeInterval::new( + DateTime::new_utc(2020, 1, 1, 0, 0, 0), + DateTime::new_utc(2024, 1, 1, 0, 0, 0) + ) + .unwrap() + ); + } + + #[test] + fn test_steps_contained_in() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let steps = dim.steps_contained_in(interval).unwrap(); + assert_eq!(steps, 2); + } + + #[test] + fn test_steps_intersecting() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let steps = dim.steps_intersecting(interval).unwrap(); + assert_eq!(steps, 4); + } + + #[test] + fn test_contained_intervals() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let contained = dim.contained_intervals(interval).unwrap().unwrap(); + let contained: Vec = contained.collect(); + assert_eq!( + contained, + vec![ + TimeInterval::new( + DateTime::new_utc(2021, 1, 1, 0, 0, 0), + DateTime::new_utc(2022, 1, 1, 0, 0, 0) + ) + .unwrap(), + TimeInterval::new( + DateTime::new_utc(2022, 1, 1, 0, 0, 0), + DateTime::new_utc(2023, 1, 1, 0, 0, 0) + ) + .unwrap(), + ] + ); + } + + #[test] + fn test_intersecting_intervals() { + let dim = RegularTimeDimension::new_with_epoch_origin(TimeStep::years(1).unwrap()); + let interval = TimeInterval::new( + DateTime::new_utc(2020, 6, 15, 0, 0, 0), + DateTime::new_utc(2023, 6, 15, 0, 0, 0), + ) + .unwrap(); + let intersecting = dim.intersecting_intervals(interval).unwrap(); + let intersecting: Vec = intersecting.collect(); + assert_eq!( + intersecting, + vec![ + TimeInterval::new( + DateTime::new_utc(2020, 1, 1, 0, 0, 0), + DateTime::new_utc(2021, 1, 1, 0, 0, 0) + ) + .unwrap(), + TimeInterval::new( + DateTime::new_utc(2021, 1, 1, 0, 0, 0), + DateTime::new_utc(2022, 1, 1, 0, 0, 0) + ) + .unwrap(), + TimeInterval::new( + DateTime::new_utc(2022, 1, 1, 0, 0, 0), + DateTime::new_utc(2023, 1, 1, 0, 0, 0) + ) + .unwrap(), + TimeInterval::new( + DateTime::new_utc(2023, 1, 1, 0, 0, 0), + DateTime::new_utc(2024, 1, 1, 0, 0, 0) + ) + .unwrap(), + ] + ); + } +} diff --git a/datatypes/src/primitives/time_gap_fill_iter.rs b/datatypes/src/primitives/time_gap_fill_iter.rs new file mode 100644 index 000000000..53b32f334 --- /dev/null +++ b/datatypes/src/primitives/time_gap_fill_iter.rs @@ -0,0 +1,1474 @@ +use crate::primitives::{RegularTimeDimension, TimeStep}; +use crate::primitives::{TimeInstance, TimeInterval}; +use std::iter::Peekable; + +pub trait TimeFilledItem { + fn create_fill_element(ti: TimeInterval) -> Self; + fn time(&self) -> TimeInterval; +} + +impl TimeFilledItem for TimeInterval { + fn create_fill_element(ti: TimeInterval) -> Self { + ti + } + + fn time(&self) -> TimeInterval { + // TODO: use Result here? Then, we coult impl TimeFilledItem for Option and Result... + *self + } +} + +type TryTimeIrregularRangeFillType = TryTimeGapFillIter< + T, + E, + TryTimeGapFillIter< + T, + E, + TryTimeGapFillIter, TimeGapSingleFill>, + TimeSingleAppend, + >, + TimeEmptySingleFill, +>; + +pub trait TryIrregularTimeFillIterExt: Iterator> +where + Self: Iterator> + Sized, +{ + /// This creates an Iterator where time gaps between items are filled with a single tile inteval + fn try_time_irregular_fill(self) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeGapSingleFill::new()) + } + + /// This creates an Iterator where a single fill item is prepended if the first item from the source starts after the specified start + fn try_time_irregular_prepend( + self, + first_element_start_required: TimeInstance, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeSinglePrepend::new(first_element_start_required)) + } + + /// This creates an Iterator where a single fill item is appended if the last item from the source ends after the specified end + fn try_time_irregular_append( + self, + last_element_end_required: TimeInstance, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeSingleAppend::new(last_element_end_required)) + } + + /// This creates an Iterator where a single fill item is returned if the source is empty + fn try_time_irregular_empty_fill( + self, + range: TimeInterval, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeEmptySingleFill::new(range)) + } + + /// This creates an Iterator where: + /// - IF the source is empty + /// - a single fill item is returned covering the source is empty + /// - ELSE + /// - a single fill item is prepended if the first item from the source starts after the specified start + /// - time gaps between items are filled with a single tile inteval + /// - a single fill item is appended if the last item from the source ends after the specified end + /// + fn try_time_irregular_range_fill( + self, + range_to_cover: TimeInterval, + ) -> TryTimeIrregularRangeFillType { + self.try_time_irregular_prepend(range_to_cover.start()) + .try_time_irregular_fill() + .try_time_irregular_append(range_to_cover.end()) + .try_time_irregular_empty_fill(range_to_cover) + } +} + +impl>> TryIrregularTimeFillIterExt + for I +{ +} + +pub trait TimeGapFill { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction; + fn name(&self) -> &'static str { + std::any::type_name::() + } +} + +pub enum TimeGapFillNextAction { + SourceNext, + CreateFillElement(TimeInterval), + End, +} + +impl TimeGapFillNextAction { + pub fn variant_name(&self) -> &'static str { + match self { + TimeGapFillNextAction::SourceNext => "SourceNext", + TimeGapFillNextAction::CreateFillElement(_) => "CreateFillElement", + TimeGapFillNextAction::End => "End", + } + } +} + +pub struct TimeGapFillIter, S: TimeGapFill> { + source: Peekable, + state: S, +} + +impl, S: TimeGapFill> TimeGapFillIter { + pub fn new(source: I, state: S) -> Self { + let peekable_source = Iterator::peekable(source); + + Self { + source: peekable_source, + state, + } + } +} + +impl, S: TimeGapFill> Iterator + for TimeGapFillIter +{ + type Item = T; + + fn next(&mut self) -> Option { + let peek_time: Option = self.source.peek().map(T::time); + let action = self.state.next_action(peek_time.as_ref()); + + //dbg!( + // self.state.name(), + // action.variant_name(), + // next.as_ref().map(|t| t.time()) + //); + match action { + TimeGapFillNextAction::End => None, + TimeGapFillNextAction::SourceNext => self.source.next(), + TimeGapFillNextAction::CreateFillElement(ti) => Some(T::create_fill_element(ti)), + } + } +} + +pub struct TryTimeGapFillIter< + T: TimeFilledItem, + E, + I: Iterator>, + S: TimeGapFill, +> { + source: Peekable, + state: S, +} + +impl>, S: TimeGapFill> + TryTimeGapFillIter +{ + pub fn new(source: I, state: S) -> Self { + let peekable_source = Iterator::peekable(source); + + Self { + source: peekable_source, + state, + } + } +} + +impl>, S: TimeGapFill> Iterator + for TryTimeGapFillIter +{ + type Item = Result; + + fn next(&mut self) -> Option { + let peek_time = self + .source + .peek() + .and_then(|r| r.as_ref().ok()) + .map(T::time); + match self.state.next_action(peek_time.as_ref()) { + TimeGapFillNextAction::End => None, + TimeGapFillNextAction::SourceNext => self.source.next(), + TimeGapFillNextAction::CreateFillElement(ti) => Some(Ok(T::create_fill_element(ti))), + } + } +} + +pub struct TimeGapSingleFill { + last_time_end: Option, +} + +impl Default for TimeGapSingleFill { + fn default() -> Self { + Self::new() + } +} + +impl TimeGapSingleFill { + pub fn new() -> Self { + Self { + last_time_end: None, + } + } +} + +impl TimeGapFill for TimeGapSingleFill { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + match (self.last_time_end, source_peek) { + // the source produced None in a previous next() call + // the next element in the source is None. Nothing to fill here + (None | Some(_), None) => { + self.last_time_end = None; + TimeGapFillNextAction::End + } + + // the source produces an element that starts later then the prevous element ended + (Some(last_end), Some(peek)) if last_end < peek.start() => { + self.last_time_end = Some(peek.end()); + TimeGapFillNextAction::CreateFillElement(TimeInterval::new_unchecked( + last_end, + peek.start(), + )) + } + // the next element aligns with the previous element + // the source produces its first element + (None | Some(_), Some(peek)) => { + self.last_time_end = Some(peek.end()); + TimeGapFillNextAction::SourceNext + } + } + } +} + +pub struct TimeSinglePrepend { + start: Option, +} + +impl TimeSinglePrepend { + pub fn new(first_element_start_required: TimeInstance) -> Self { + Self { + start: Some(first_element_start_required), + } + } +} + +impl TimeGapFill for TimeSinglePrepend { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + let nn = match (self.start, source_peek) { + // default case where the start has already been checked + (None, _) => TimeGapFillNextAction::SourceNext, + // the source never produced an element, can't prepend an element here + (Some(_start), None) => TimeGapFillNextAction::End, + // the first element starts after the required start so a single fill element is inserted here + (Some(start), Some(peek)) if start < peek.start() => { + TimeGapFillNextAction::CreateFillElement(TimeInterval::new_unchecked( + start, + peek.start(), + )) + } + // the first element starts bevore or equal to the requred first start + (Some(_start), Some(_peek)) => TimeGapFillNextAction::SourceNext, + }; + // we only need to check the first element, so we can set start to None here + self.start = None; + nn + } +} + +pub struct TimeSingleAppend { + last_time_end: Option, + req_end: TimeInstance, +} + +impl TimeSingleAppend { + pub fn new(last_element_end: TimeInstance) -> Self { + Self { + last_time_end: None, + req_end: last_element_end, + } + } +} + +impl TimeGapFill for TimeSingleAppend { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + match (self.last_time_end, source_peek) { + // the source produces an element + (_, Some(peek)) => { + self.last_time_end = Some(peek.end()); + TimeGapFillNextAction::SourceNext + } + + // the next element in the source is None. Check if we need to append a fill element + (Some(last_end), None) if last_end < self.req_end => { + self.last_time_end = None; + TimeGapFillNextAction::CreateFillElement(TimeInterval::new_unchecked( + last_end, + self.req_end, + )) + } + _ => TimeGapFillNextAction::End, + } + } +} + +pub struct TimeEmptySingleFill { + range: TimeInterval, + pristine: bool, +} + +impl TimeEmptySingleFill { + pub fn new(range: TimeInterval) -> Self { + Self { + range, + pristine: true, + } + } +} + +impl TimeGapFill for TimeEmptySingleFill { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + match (self.pristine, source_peek) { + (true, None) => { + self.pristine = false; + TimeGapFillNextAction::CreateFillElement(self.range) + } + (_, Some(_)) => { + self.pristine = false; + TimeGapFillNextAction::SourceNext + } + (false, None) => TimeGapFillNextAction::End, + } + } +} + +pub struct TimeGapRegularFill { + step: TimeStep, + last_time: Option, + finished: bool, +} + +impl TimeGapRegularFill { + pub fn new(step: TimeStep) -> Self { + Self { + step, + last_time: None, + finished: false, + } + } +} + +impl TimeGapFill for TimeGapRegularFill { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + if self.finished { + return TimeGapFillNextAction::End; + } + + match (self.last_time, source_peek) { + // case where no TI is availabe from source + // no new element from source + (Some(_) | None, None) => { + self.finished = true; + TimeGapFillNextAction::End + } + + // the source produces an element that starts later then the previous element ended + (Some(last), Some(peek)) if last < peek.start() => { + let next_start = last; + let next_end = (last + self.step).expect("TimeStep addition overflowed"); // TODO: check range overflow? + self.last_time = Some(next_end); + TimeGapFillNextAction::CreateFillElement(TimeInterval::new_unchecked( + next_start, next_end, + )) + } + + // first element from source + // the next element aligns with the previous element + (None | Some(_), Some(peek)) => { + self.last_time = Some(peek.end()); + TimeGapFillNextAction::SourceNext + } + } + } +} + +pub struct TimeGapRegularPrepend { + step: TimeStep, + next_time: Option, +} + +impl TimeGapRegularPrepend { + pub fn new(step: TimeStep, first_element_start_required: TimeInstance) -> Self { + Self { + step, + next_time: Some(first_element_start_required), + } + } +} + +impl TimeGapFill for TimeGapRegularPrepend { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + match (self.next_time, source_peek) { + // default case where the source produces elements + (None, _) => TimeGapFillNextAction::SourceNext, + // the source never produced an element, can't prepend an element here + (Some(_start), None) => TimeGapFillNextAction::End, + // the first element starts after the required start so a single fill element is inserted here + (Some(next_time), Some(peek)) if next_time < peek.start() => { + let next_element = TimeInterval::new_unchecked( + next_time, + (next_time + self.step).expect("TimeStep addition overflowed"), // TODO: check range overflow? + ); + self.next_time = Some(next_element.end()); // we only need to check the first element, so we + TimeGapFillNextAction::CreateFillElement(next_element) + } + // the first element starts bevore or equal to the requred first start --> no prepend needed + (Some(_start), Some(_peek)) => { + self.next_time = None; + TimeGapFillNextAction::SourceNext + } + } + } +} + +pub struct TimeGapRegularAppend { + step: TimeStep, + last_time: TimeInstance, + last_seen: Option, + finished: bool, +} + +impl TimeGapRegularAppend { + pub fn new(step: TimeStep, last_element_end_required: TimeInstance) -> Self { + Self { + step, + last_time: last_element_end_required, + last_seen: None, + finished: false, + } + } +} + +impl TimeGapFill for TimeGapRegularAppend { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + if self.finished { + return TimeGapFillNextAction::End; + } + + match (self.last_seen, source_peek) { + // the source produces an element + (_, Some(peek)) => { + self.last_seen = Some(peek.end()); + TimeGapFillNextAction::SourceNext + } + + // the next element in the source is None. Check if we need to append a fill element + (Some(last_seen), None) if last_seen < self.last_time => { + let next_start = last_seen; + let next_end = (last_seen + self.step).expect("TimeStep addition overflowed"); // TODO: check range overflow? + + self.last_seen = Some(next_end); + TimeGapFillNextAction::CreateFillElement(TimeInterval::new_unchecked( + next_start, next_end, + )) + } + _ => { + self.finished = true; + TimeGapFillNextAction::End + } + } + } +} + +pub struct TimeEmptyRegularFill { + range: TimeInterval, + step: TimeStep, + last_time: Option, + fuse: bool, +} + +impl TimeEmptyRegularFill { + pub fn new(regular_dimension: RegularTimeDimension, range: TimeInterval) -> Self { + Self { + range, + step: regular_dimension.step, + last_time: None, // TODO: check range overflow? + fuse: false, + } + } +} + +impl TimeGapFill for TimeEmptyRegularFill { + fn next_action(&mut self, source_peek: Option<&TimeInterval>) -> TimeGapFillNextAction { + if self.fuse { + return TimeGapFillNextAction::SourceNext; + } + + match (self.last_time, source_peek) { + (None, None) if !self.fuse => { + let next = TimeInterval::new_unchecked( + self.range.start(), + (self.range.start() + self.step).expect("TimeStep addition overflowed"), // TODO: check range overflow? + ); + self.last_time = Some(next); + TimeGapFillNextAction::CreateFillElement(next) + } + (Some(last), None) if last.end() < self.range.end() => { + let next_start = last.end(); + let next_end = (next_start + self.step).expect("TimeStep addition overflowed"); // TODO: check range overflow? + let next = TimeInterval::new_unchecked(next_start, next_end); + self.last_time = Some(next); + TimeGapFillNextAction::CreateFillElement(next) + } + (Some(_) | None, None) => TimeGapFillNextAction::End, + (_, Some(_)) => { + self.fuse = true; + TimeGapFillNextAction::SourceNext + } + } + } +} + +/// Type alias for the complex nested `TryTimeGapFillIter` used in `try_time_regular_range_fill` +pub type TryTimeRegularRangeFillType = TryTimeGapFillIter< + T, + E, + TryTimeGapFillIter< + T, + E, + TryTimeGapFillIter< + T, + E, + TryTimeGapFillIter, + TimeGapRegularFill, + >, + TimeGapRegularAppend, + >, + TimeEmptyRegularFill, +>; + +pub trait TryRegularTimeFillIterExt: Iterator> +where + Self: Iterator> + Sized, +{ + /// This creates an Iterator where time gaps between items are filled with a single tile inteval + fn try_time_regular_fill( + self, + step: TimeStep, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeGapRegularFill::new(step)) + } + + /// This creates an Iterator where a single fill item is prepended if the first item from the source starts after the specified start + fn try_time_regular_prepend( + self, + step: TimeStep, + first_element_start_required: TimeInstance, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new( + self, + TimeGapRegularPrepend::new(step, first_element_start_required), + ) + } + + /// This creates an Iterator where a single fill item is appended if the last item from the source ends after the specified end + fn try_time_regular_append( + self, + step: TimeStep, + last_element_end_required: TimeInstance, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new( + self, + TimeGapRegularAppend::new(step, last_element_end_required), + ) + } + + /// This creates an Iterator where empty sources are filled with regular timesteps + fn try_time_regular_empty_fill( + self, + regular_dimension: RegularTimeDimension, + range: TimeInterval, + ) -> TryTimeGapFillIter { + TryTimeGapFillIter::new(self, TimeEmptyRegularFill::new(regular_dimension, range)) + } + + /// This creates an Iterator where: + /// - gaps before the first and after the last element are filled with regular step to fill the specified range + /// - gaps are filled with regular step intervals + /// - IF the source is empty, regular steps are inserted anchored at the query start! + fn try_time_regular_range_fill( + self, + regular_dimension: RegularTimeDimension, + range_to_cover: TimeInterval, + ) -> TryTimeRegularRangeFillType { + self.try_time_regular_prepend(regular_dimension.step, range_to_cover.start()) + .try_time_regular_fill(regular_dimension.step) + .try_time_regular_append(regular_dimension.step, range_to_cover.end()) + .try_time_regular_empty_fill(regular_dimension, range_to_cover) + } +} + +impl>> TryRegularTimeFillIterExt for I {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_no_source_intervals() { + let start = TimeInstance::from_millis(0).unwrap(); + let end = TimeInstance::from_millis(100).unwrap(); + let iter = vec![] + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + assert_eq!( + result.unwrap(), + vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(100).unwrap() + )] + ); + } + + #[test] + fn test_source_covers_entire_range() { + let start = TimeInstance::from_millis(0).unwrap(); + let end = TimeInstance::from_millis(100).unwrap(); + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(100).unwrap(), + )]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(100).unwrap() + )] + ); + } + + #[test] + fn test_source_starts_after_hint() { + let start = TimeInstance::from_millis(0).unwrap(); + let end = TimeInstance::from_millis(100).unwrap(); + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(40).unwrap(), + )]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(100).unwrap() + ), + ] + ); + } + + #[test] + fn test_source_starts_before_hint() { + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(50).unwrap(); + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ), + ]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(50).unwrap() + ), + ] + ); + } + + #[test] + fn test_source_with_gaps() { + let start = TimeInstance::from_millis(0).unwrap(); + let end = TimeInstance::from_millis(100).unwrap(); + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(60).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(80).unwrap(), + TimeInstance::from_millis(90).unwrap(), + ), + ]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(60).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(60).unwrap(), + TimeInstance::from_millis(80).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(80).unwrap(), + TimeInstance::from_millis(90).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(90).unwrap(), + TimeInstance::from_millis(100).unwrap() + ), + ] + ); + } + + #[test] + fn test_source_ends_before_hint() { + let start = TimeInstance::from_millis(0).unwrap(); + let end = TimeInstance::from_millis(100).unwrap(); + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ), + ]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(100).unwrap() + ), + ] + ); + } + + #[test] + fn test_source_exactly_matches_hint() { + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(20).unwrap(); + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + )]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + )] + ); + } + + #[test] + fn test_source_overlapping_start() { + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(50).unwrap(); + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(60).unwrap(), + ), + ]; + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(60).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_gap_fill_no_gaps() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_fill(step); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_gap_fill_with_gaps() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_fill(step); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_start_prepend() { + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + )]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(0).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_prepend(step, start); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_start_prepend_start_contained() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(5).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_prepend(step, start); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_start_prepend_no_action_1() { + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + )]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(0).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_prepend(step, start); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + )] + ); + } + + #[test] + fn test_regular_time_start_prepend_no_action_2() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(0).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_prepend(step, start); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ) + ] + ); + } + + #[test] + fn test_regular_time_end_append() { + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap(), + )]; + let step = TimeStep::millis(10).unwrap(); + let end = TimeInstance::from_millis(30).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_append(step, end); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(0).unwrap(), + TimeInstance::from_millis(10).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_end_append_end_contained() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let end = TimeInstance::from_millis(35).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_append(step, end); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ) + ] + ); + } + + #[test] + fn test_regular_time_end_append_no_action_1() { + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + )]; + let step = TimeStep::millis(10).unwrap(); + let end = TimeInstance::from_millis(30).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_append(step, end); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + )] + ); + } + + #[test] + fn test_regular_time_end_append_no_action_2() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let end = TimeInstance::from_millis(40).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_append(step, end); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap(), + ) + ] + ); + } + + #[test] + fn test_regular_time_start_prepend_and_end_append() { + let intervals = vec![TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + )]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(40).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_prepend(step, start) + .try_time_regular_append(step, end); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + ] + ); + } + + #[test] + fn test_regular_time_gap_fill_empty_source() { + let intervals: Vec = vec![]; + let step = TimeStep::millis(10).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_fill(step); + let result: Result, _> = iter.collect::, _>>(); + assert_eq!(result.unwrap(), vec![]); + } + + #[test] + fn time_regular_range_fill_no_gaps() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(30).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_range_fill( + RegularTimeDimension::new(TimeInstance::from_millis(0).unwrap(), step), + TimeInterval::new_unchecked(start, end), + ); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + ] + ); + } + + #[test] + fn time_regular_all_cases() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(50).unwrap(), + TimeInstance::from_millis(60).unwrap(), + ), + ]; + let step = TimeStep::millis(10).unwrap(); + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(70).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_range_fill( + RegularTimeDimension::new(TimeInstance::from_millis(0).unwrap(), step), + TimeInterval::new_unchecked(start, end), + ); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(20).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(20).unwrap(), + TimeInstance::from_millis(30).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(30).unwrap(), + TimeInstance::from_millis(40).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(40).unwrap(), + TimeInstance::from_millis(50).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(50).unwrap(), + TimeInstance::from_millis(60).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(60).unwrap(), + TimeInstance::from_millis(70).unwrap() + ), + ] + ); + } + + #[test] + fn time_irregular_all_caes() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(15).unwrap(), + TimeInstance::from_millis(25).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(45).unwrap(), + TimeInstance::from_millis(55).unwrap(), + ), + ]; + let start = TimeInstance::from_millis(10).unwrap(); + let end = TimeInstance::from_millis(60).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_irregular_range_fill(TimeInterval::new_unchecked(start, end)); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(10).unwrap(), + TimeInstance::from_millis(15).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(15).unwrap(), + TimeInstance::from_millis(25).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(25).unwrap(), + TimeInstance::from_millis(45).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(45).unwrap(), + TimeInstance::from_millis(55).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(55).unwrap(), + TimeInstance::from_millis(60).unwrap() + ), + ] + ); + } + + #[test] + fn regularr_non_zero_origin() { + let intervals = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_735_689_600_000).unwrap(), + TimeInstance::from_millis(1_738_368_000_000).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_738_368_000_000).unwrap(), + TimeInstance::from_millis(1_740_787_200_000).unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_743_465_600_000).unwrap(), + TimeInstance::from_millis(1_746_057_600_000).unwrap(), + ), + ]; + let step = TimeStep::months(1).unwrap(); + let start = TimeInstance::from_millis(1_733_011_200_000).unwrap(); + let end = TimeInstance::from_millis(1_747_267_200_000).unwrap(); + let iter = intervals + .into_iter() + .map(|t| -> Result { Ok(t) }) + .try_time_regular_range_fill( + RegularTimeDimension::new( + TimeInstance::from_millis(1_388_534_400_000).unwrap(), + step, + ), + TimeInterval::new_unchecked(start, end), + ); + let result: Result, _> = iter.collect::, _>>(); + + assert_eq!( + result.unwrap(), + vec![ + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_733_011_200_000).unwrap(), + TimeInstance::from_millis(1_735_689_600_000).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_735_689_600_000).unwrap(), + TimeInstance::from_millis(1_738_368_000_000).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_738_368_000_000).unwrap(), + TimeInstance::from_millis(1_740_787_200_000).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_740_787_200_000).unwrap(), + TimeInstance::from_millis(1_743_465_600_000).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_743_465_600_000).unwrap(), + TimeInstance::from_millis(1_746_057_600_000).unwrap() + ), + TimeInterval::new_unchecked( + TimeInstance::from_millis(1_746_057_600_000).unwrap(), + TimeInstance::from_millis(1_748_736_000_000).unwrap() + ), + ] + ); + } +} diff --git a/datatypes/src/primitives/time_instance.rs b/datatypes/src/primitives/time_instance.rs index 7df872443..125177cb9 100644 --- a/datatypes/src/primitives/time_instance.rs +++ b/datatypes/src/primitives/time_instance.rs @@ -13,7 +13,7 @@ use std::{ str::FromStr, }; -#[derive(Clone, Copy, Serialize, PartialEq, Eq, PartialOrd, Ord, Debug, FromSql, ToSql)] +#[derive(Clone, Copy, Serialize, PartialEq, Eq, PartialOrd, Ord, Debug, FromSql, ToSql, Hash)] #[repr(C)] #[postgres(transparent)] pub struct TimeInstance(i64); diff --git a/datatypes/src/primitives/time_interval.rs b/datatypes/src/primitives/time_interval.rs index 15b48b014..8b02b1cc3 100755 --- a/datatypes/src/primitives/time_interval.rs +++ b/datatypes/src/primitives/time_interval.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use std::{cmp::Ordering, convert::TryInto}; /// Stores time intervals in ms in close-open semantic [start, end) -#[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq, ToSql, FromSql)] +#[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq, ToSql, FromSql, Hash)] #[repr(C)] pub struct TimeInterval { start: TimeInstance, @@ -88,6 +88,23 @@ impl TimeInterval { } ); + ensure!( + start_instant <= end_instant, + error::TimeIntervalEndBeforeStart { + start: start_instant, + end: end_instant + } + ); + ensure!( + start_instant >= TimeInstance::MIN && end_instant <= TimeInstance::MAX, + error::TimeIntervalOutOfBounds { + start: start_instant, + end: end_instant, + min: TimeInstance::MIN, + max: TimeInstance::MAX, + } + ); + Ok(Self { start: start_instant, end: end_instant, @@ -173,6 +190,10 @@ impl TimeInterval { self == other || ((self.start..self.end).contains(&other.start) && (self.end >= other.end)) } + pub fn contains_instance(&self, instance: TimeInstance) -> bool { + self.start <= instance && instance < self.end + } + /// Returns whether the given interval intersects this interval /// /// # Examples @@ -246,10 +267,10 @@ impl TimeInterval { i2: *other, } ); - Ok(Self { - start: TimeInstance::min(self.start, other.start), - end: TimeInstance::max(self.end, other.end), - }) + Self::new( + TimeInstance::min(self.start, other.start), + TimeInstance::max(self.end, other.end), + ) } pub fn start(&self) -> TimeInstance { diff --git a/datatypes/src/primitives/time_step.rs b/datatypes/src/primitives/time_step.rs index dde9fa00e..5a0fd1a98 100644 --- a/datatypes/src/primitives/time_step.rs +++ b/datatypes/src/primitives/time_step.rs @@ -33,6 +33,49 @@ pub struct TimeStep { } impl TimeStep { + /// Creates a new `TimeStep`. + /// # Panics + /// This method panics if `step` is 0. + pub fn new_unchecked(granularity: TimeGranularity, step: u32) -> Self { + // TODO: try and new_unchecked? + assert!(step > 0, "step must be greater than 0"); + Self { granularity, step } + } + + pub fn new(granularity: TimeGranularity, step: u32) -> Result { + ensure!(step > 0, error::TimeStepStepMustBeGreaterThanZero { step }); + + Ok(Self { granularity, step }) + } + + pub fn years(step: u32) -> Result { + Self::new(TimeGranularity::Years, step) + } + + pub fn months(step: u32) -> Result { + Self::new(TimeGranularity::Months, step) + } + + pub fn days(step: u32) -> Result { + Self::new(TimeGranularity::Days, step) + } + + pub fn hours(step: u32) -> Result { + Self::new(TimeGranularity::Hours, step) + } + + pub fn minutes(step: u32) -> Result { + Self::new(TimeGranularity::Minutes, step) + } + + pub fn seconds(step: u32) -> Result { + Self::new(TimeGranularity::Seconds, step) + } + + pub fn millis(step: u32) -> Result { + Self::new(TimeGranularity::Millis, step) + } + /// Resolves how many `TimeStep`-sized intervals fit into a given `TimeInterval`. /// Remember that `TimeInterval` is not inclusive. /// @@ -416,6 +459,26 @@ impl Mul for TimeStep { } } +impl TryFrom for TimeStep { + type Error = Error; + + fn try_from(value: Duration) -> Result { + match [ + value.num_days(), + value.num_hours(), + value.num_minutes(), + value.num_seconds(), + value.num_milliseconds(), + ] { + [d, 0, 0, 0, 0] => TimeStep::days(d as u32), + [_, h, 0, 0, 0] => TimeStep::hours(h as u32), + [_, _, m, 0, 0] => TimeStep::minutes(m as u32), + [_, _, _, s, 0] => TimeStep::seconds(s as u32), + [_, _, _, _, ms] => TimeStep::millis(ms as u32), + } + } +} + /// An `Iterator` to iterate over time in steps #[derive(Debug, Clone)] pub struct TimeStepIter { @@ -498,6 +561,10 @@ impl Iterator for TimeStepIter { Some(next) } + + fn size_hint(&self) -> (usize, Option) { + (self.steps as usize, Some(self.steps as usize)) + } } #[cfg(test)] diff --git a/datatypes/src/raster/db_types.rs b/datatypes/src/raster/db_types.rs new file mode 100644 index 000000000..449cc06f2 --- /dev/null +++ b/datatypes/src/raster/db_types.rs @@ -0,0 +1,39 @@ +use postgres_types::{FromSql, ToSql}; + +use crate::delegate_from_to_sql; + +use super::GridBoundingBox2D; + +#[derive(Debug, PartialEq, ToSql, FromSql)] +#[postgres(name = "GridBoundingBox2D")] +pub struct GridBoundingBox2DDbType { + y_min: i64, + y_max: i64, + x_min: i64, + x_max: i64, +} + +impl From<&GridBoundingBox2D> for GridBoundingBox2DDbType { + fn from(value: &GridBoundingBox2D) -> Self { + Self { + y_min: value.y_min() as i64, + y_max: value.y_max() as i64, + x_min: value.x_min() as i64, + x_max: value.x_max() as i64, + } + } +} + +impl From for GridBoundingBox2D { + fn from(value: GridBoundingBox2DDbType) -> Self { + GridBoundingBox2D::new_min_max( + value.y_min as isize, + value.y_max as isize, + value.x_min as isize, + value.x_max as isize, + ) + .expect("conversion must be correct") + } +} + +delegate_from_to_sql!(GridBoundingBox2D, GridBoundingBox2DDbType); diff --git a/datatypes/src/raster/empty_grid.rs b/datatypes/src/raster/empty_grid.rs index e9842ee61..dd3d43c31 100644 --- a/datatypes/src/raster/empty_grid.rs +++ b/datatypes/src/raster/empty_grid.rs @@ -42,7 +42,7 @@ where impl GridSize for EmptyGrid where - D: GridSize + GridSpaceToLinearSpace, + D: GridSpaceToLinearSpace, { type ShapeArray = D::ShapeArray; @@ -85,23 +85,30 @@ where } } -impl ChangeGridBounds for EmptyGrid +impl ChangeGridBounds for EmptyGrid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Copy, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = EmptyGrid, T>; + type BoundedOutput = EmptyGrid, T>; + type UnboundedOutput = EmptyGrid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { EmptyGrid::new(self.shift_bounding_box(offset)) } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Ok(EmptyGrid::new(bounds)) } + + fn unbounded(self) -> Self::UnboundedOutput { + EmptyGrid::new(self.grid_shape()) + } } impl ByteSize for EmptyGrid {} diff --git a/datatypes/src/raster/geo_transform.rs b/datatypes/src/raster/geo_transform.rs index beba9a1dc..43a4ac353 100644 --- a/datatypes/src/raster/geo_transform.rs +++ b/datatypes/src/raster/geo_transform.rs @@ -1,11 +1,14 @@ +use super::{GeoTransformAccess, GridBoundingBox2D, GridBounds, GridIdx, GridIdx2D}; use crate::{ - primitives::{AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialResolution}, + primitives::{ + AxisAlignedRectangle, BoundingBox2D, Coordinate2D, SpatialPartition2D, SpatialResolution, + }, util::test::TestDefault, }; +use float_cmp::{ApproxEq, F64Margin, approx_eq}; +use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Deserializer, Serialize, de}; -use super::{GridBoundingBox2D, GridIdx, GridIdx2D}; - /// This is a typedef for the `GDAL GeoTransform`. It represents an affine transformation matrix. pub type GdalGeoTransform = [f64; 6]; @@ -13,7 +16,7 @@ pub type GdalGeoTransform = [f64; 6]; /// In Geo Engine x pixel size is always postive and y pixel size is always negative. For raster tiles /// the origin is always the upper left corner. In the global grid for the `TilingStrategy` the origin /// is always located at (0, 0). -#[derive(Copy, Clone, PartialEq, Debug, Serialize, Deserialize)] +#[derive(Copy, Clone, PartialEq, Debug, Serialize, Deserialize, ToSql, FromSql)] #[serde(rename_all = "camelCase")] pub struct GeoTransform { pub origin_coordinate: Coordinate2D, @@ -34,9 +37,8 @@ impl GeoTransform { /// #[inline] pub fn new(origin_coordinate: Coordinate2D, x_pixel_size: f64, y_pixel_size: f64) -> Self { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - + debug_assert!(x_pixel_size != 0.0); + debug_assert!(y_pixel_size != 0.0); Self { origin_coordinate, x_pixel_size, @@ -60,9 +62,8 @@ impl GeoTransform { origin_coordinate_y: f64, y_pixel_size: f64, ) -> Self { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - + debug_assert!(x_pixel_size != 0.0); + debug_assert!(y_pixel_size != 0.0); Self { origin_coordinate: (origin_coordinate_x, origin_coordinate_y).into(), x_pixel_size, @@ -78,6 +79,10 @@ impl GeoTransform { self.y_pixel_size } + pub fn y_axis_is_neg(&self) -> bool { + self.y_pixel_size < 0.0 + } + /// Transforms a grid coordinate (row, column) ~ (y, x) into a SRS coordinate (x,y) /// The resulting coordinate is the upper left coordinate of the pixel /// See GDAL documentation for more details (including the two ignored parameters): @@ -114,6 +119,7 @@ impl GeoTransform { } /// Transforms an SRS coordinate (x,y) into a grid coordinate (row, column) ~ (y, x) + /// This method selects the grid index with the center nearest to the coordinate. /// /// # Examples /// @@ -148,6 +154,7 @@ impl GeoTransform { } pub fn spatial_resolution(&self) -> SpatialResolution { + // TODO: should honor negative y size SpatialResolution { x: self.x_pixel_size.abs(), y: self.y_pixel_size.abs(), @@ -156,13 +163,7 @@ impl GeoTransform { /// compute the index of the upper left pixel that is contained in the `partition` pub fn upper_left_pixel_idx(&self, partition: &SpatialPartition2D) -> GridIdx2D { - // choose the epsilon relative to the pixel size - const EPSILON: f64 = 0.000_001; - let epsilon: Coordinate2D = - (self.x_pixel_size() * EPSILON, self.y_pixel_size() * EPSILON).into(); - - let upper_left_coordinate = partition.upper_left() + epsilon; - + let upper_left_coordinate = partition.upper_left(); self.coordinate_to_grid_idx_2d(upper_left_coordinate) } @@ -190,13 +191,25 @@ impl GeoTransform { self.coordinate_to_grid_idx_2d(lower_right) } + /// Transform a `BoundingBox2D` into a `GridBoundingBox2D`. + #[inline] + pub fn bounding_box_2d_to_intersecting_grid_bounds( + &self, + bounding_box: &BoundingBox2D, + ) -> GridBoundingBox2D { + let upper_left = self.coordinate_to_grid_idx_2d(bounding_box.upper_left()); + let lower_right = self.coordinate_to_grid_idx_2d(bounding_box.lower_right()); + + GridBoundingBox2D::new_unchecked(upper_left, lower_right) + } + /// Transform a `SpatialPartition2D` into a `GridBoundingBox` #[inline] pub fn spatial_to_grid_bounds( &self, spatial_partition: &SpatialPartition2D, ) -> GridBoundingBox2D { - let GridIdx([ul_y, ul_x]) = self.upper_left_pixel_idx(spatial_partition); + let GridIdx([ul_y, ul_x]) = self.coordinate_to_grid_idx_2d(spatial_partition.upper_left()); let GridIdx([lr_y, lr_x]) = self.lower_right_pixel_idx(spatial_partition); // this is the pixel inside the spatial partition debug_assert!(ul_x <= lr_x); @@ -222,6 +235,65 @@ impl GeoTransform { Ok(unchecked) } + + pub fn grid_to_spatial_bounds>( + &self, + grid_bounds: &S, + ) -> SpatialPartition2D { + let ul = self.grid_idx_to_pixel_upper_left_coordinate_2d(grid_bounds.min_index()); + let lr = self.grid_idx_to_pixel_upper_left_coordinate_2d(grid_bounds.max_index() + 1); + + SpatialPartition2D::new_unchecked(ul, lr) + } + + pub fn origin_coordinate(&self) -> Coordinate2D { + self.origin_coordinate + } + + #[must_use] + pub fn shift_by_pixel_offset(&self, offset: GridIdx2D) -> Self { + GeoTransform { + origin_coordinate: self.grid_idx_to_pixel_upper_left_coordinate_2d(offset), + x_pixel_size: self.x_pixel_size, + y_pixel_size: self.y_pixel_size, + } + } + + pub fn nearest_pixel_edge(&self, coordinate: Coordinate2D) -> GridIdx2D { + self.coordinate_to_grid_idx_2d( + coordinate + Coordinate2D::new(self.x_pixel_size * 0.5, self.y_pixel_size * 0.5), // by adding a half pixel, we can find flips between edges + ) + } + + pub fn nearest_pixel_edge_coordinate(&self, coordinate: Coordinate2D) -> Coordinate2D { + self.grid_idx_to_pixel_upper_left_coordinate_2d(self.nearest_pixel_edge(coordinate)) + } + + pub fn distance_to_nearest_pixel_edge(&self, coordinate: Coordinate2D) -> Coordinate2D { + let pixel_edge = self.nearest_pixel_edge_coordinate(coordinate); + let dist = coordinate - pixel_edge; + debug_assert!(dist.x.abs() <= self.x_pixel_size.abs() * 0.5); + debug_assert!(dist.y.abs() <= self.y_pixel_size.abs() * 0.5); + dist + } + + /// `coordinate` is a valid pixel edge if it is approximately equal to the nearest pixel edge, within floating point precision + pub fn is_valid_pixel_edge(&self, coordinate: Coordinate2D) -> bool { + let edge = self.nearest_pixel_edge(coordinate); + let edge_coord = self.grid_idx_to_pixel_upper_left_coordinate_2d(edge); + // compare x and y separately to account for difference in magnitude + approx_eq!(f64, coordinate.x, edge_coord.x) && approx_eq!(f64, coordinate.y, edge_coord.y) + } + + pub fn is_compatible_grid(&self, other: GeoTransform) -> bool { + self.is_valid_pixel_edge(other.origin_coordinate) + && approx_eq!(f64, self.x_pixel_size(), other.x_pixel_size()) + && approx_eq!(f64, self.y_pixel_size(), other.y_pixel_size()) + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.is_compatible_grid(g.geo_transform()) + } } impl TestDefault for GeoTransform { @@ -256,6 +328,18 @@ impl From for GdalGeoTransform { } } +impl ApproxEq for GeoTransform { + type Margin = F64Margin; + + fn approx_eq>(self, other: Self, margin: M) -> bool { + let m: F64Margin = margin.into(); + + self.origin_coordinate.approx_eq(other.origin_coordinate, m) + && self.x_pixel_size.approx_eq(other.x_pixel_size, m) + && self.y_pixel_size.approx_eq(other.y_pixel_size, m) + } +} + #[cfg(test)] mod tests { use super::*; @@ -518,4 +602,60 @@ mod tests { assert!(test.is_err()); } + + #[test] + fn shift_by_pixel_offset() { + let geo_transform = GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0); + let shifted = geo_transform.shift_by_pixel_offset([1, 1].into()); + assert_eq!(shifted.origin_coordinate, (1.0, -1.0).into()); + } + + #[test] + fn coordinate_to_nearest_grid_center_idx_2d() { + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let coord = Coordinate2D::new(0.1, -0.1); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(0.5, -0.5); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(0.9, -0.9); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [0, 0].into()); + + let coord = Coordinate2D::new(1.0, -1.0); + let center_coord_idx = geo_transform.coordinate_to_grid_idx_2d(coord); + assert_eq!(center_coord_idx, [1, 1].into()); + } + + #[test] + fn it_detects_valid_pixel_edge_robustly() { + let geo_transform = GeoTransform::new((-180.0, 90.).into(), 0.2, -0.2); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new(-45.0, 22.4))); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new(-45.0, 22.399_999_999_999_99))); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new(-45.0, 22.4_f64.next_up()))); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new(-45.0, 22.4_f64.next_down()))); + assert!(!geo_transform.is_valid_pixel_edge(Coordinate2D::new(-45.0, 22.41))); + + let geo_transform = GeoTransform::new( + (10_000_000_000_000_000_f64, -10_000_000_000_000_000_f64).into(), + 10_000_000_000_000_000_f64, + -10_000_000_000_000_000_f64, + ); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new( + 10_000_000_000_000_000_f64, + -10_000_000_000_000_000_f64 + ))); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new( + 10_000_000_000_000_001_f64, + -10_000_000_000_000_000_f64 + ))); + assert!(geo_transform.is_valid_pixel_edge(Coordinate2D::new( + 10_000_000_000_000_000_f64.next_up(), + -10_000_000_000_000_000_f64.next_up() + ))); + } } diff --git a/datatypes/src/raster/grid.rs b/datatypes/src/raster/grid.rs index 5538917ca..671396fa3 100644 --- a/datatypes/src/raster/grid.rs +++ b/datatypes/src/raster/grid.rs @@ -1,6 +1,6 @@ use super::{ - GridBoundingBox, GridBounds, GridContains, GridIdx, GridIdx2D, GridIndexAccess, - GridIndexAccessMut, GridSize, GridSpaceToLinearSpace, + GridBoundingBox, GridBounds, GridContains, GridIdx, GridIndexAccess, GridIndexAccessMut, + GridSize, GridSpaceToLinearSpace, grid_traits::{ChangeGridBounds, GridShapeAccess}, }; use crate::util::Result; @@ -45,6 +45,48 @@ pub type GridShape1D = GridShape<[usize; 1]>; pub type GridShape2D = GridShape<[usize; 2]>; pub type GridShape3D = GridShape<[usize; 3]>; +impl GridShape1D { + pub fn new_1d(x_size: usize) -> Self { + Self::new([x_size]) + } + + pub fn x(self) -> usize { + self.shape_array[0] + } +} + +impl GridShape2D { + pub fn new_2d(y_size: usize, x_size: usize) -> Self { + Self::new([y_size, x_size]) + } + + pub fn x(&self) -> usize { + self.shape_array[1] + } + + pub fn y(&self) -> usize { + self.shape_array[0] + } +} + +impl GridShape3D { + pub fn new_3d(z_size: usize, y_size: usize, x_size: usize) -> Self { + Self::new([z_size, y_size, x_size]) + } + + pub fn x(&self) -> usize { + self.shape_array[2] + } + + pub fn y(&self) -> usize { + self.shape_array[1] + } + + pub fn z(&self) -> usize { + self.shape_array[0] + } +} + impl From<[usize; 1]> for GridShape1D { fn from(shape: [usize; 1]) -> Self { GridShape1D { shape_array: shape } @@ -266,17 +308,6 @@ impl GridBounds for GridShape3D { } } -/// Method to generate an `Iterator` over all `GridIdx2D` in `GridBounds` -pub fn grid_idx_iter_2d(bounds: &B) -> impl Iterator + use -where - B: GridBounds, -{ - let GridIdx([y_s, x_s]) = bounds.min_index(); - let GridIdx([y_e, x_e]) = bounds.max_index(); - - (y_s..=y_e).flat_map(move |y| (x_s..=x_e).map(move |x| [y, x].into())) -} - #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Grid { @@ -484,26 +515,36 @@ where } } -impl ChangeGridBounds for Grid +impl ChangeGridBounds for Grid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Clone, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = Grid, T>; + type BoundedOutput = Grid, T>; + type UnboundedOutput = Grid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { Grid { shape: self.shift_bounding_box(offset), data: self.data, } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Grid::new(bounds, self.data) } + + fn unbounded(self) -> Self::UnboundedOutput { + Grid { + shape: self.grid_shape(), + data: self.data, + } + } } impl ByteSize for Grid diff --git a/datatypes/src/raster/grid_bounds.rs b/datatypes/src/raster/grid_bounds.rs index 83389f85f..49eecccee 100644 --- a/datatypes/src/raster/grid_bounds.rs +++ b/datatypes/src/raster/grid_bounds.rs @@ -1,13 +1,14 @@ -use snafu::ensure; - -use crate::{error, util::Result}; - use super::{ BoundedGrid, GridBounds, GridContains, GridIdx, GridIntersection, GridShape, GridShapeAccess, GridSize, GridSpaceToLinearSpace, }; +use crate::{error, raster::GridIdx2D, util::Result}; +use serde::{Deserialize, Serialize}; +use snafu::ensure; +use std::ops::Add; -#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone)] +#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize)] +/// A bounding box for a grid where the min and max values are inclusive pub struct GridBoundingBox where A: AsRef<[isize]>, @@ -49,6 +50,70 @@ where } } +impl GridBoundingBox1D { + #[inline] + pub fn x_min(&self) -> isize { + let [x_min] = self.min; + x_min + } + + #[inline] + pub fn x_max(&self) -> isize { + let [x_max] = self.max; + x_max + } + + #[inline] + pub fn x_bounds(&self) -> [isize; 2] { + [self.x_min(), self.x_max()] + } + + #[inline] + pub fn new_min_max(x_min: isize, x_max: isize) -> Result { + Self::new([x_min], [x_max]) + } +} + +impl GridBoundingBox2D { + #[inline] + pub fn x_min(&self) -> isize { + let [_y_min, x_min] = self.min; + x_min + } + + #[inline] + pub fn x_max(&self) -> isize { + let [_y_max, x_max] = self.max; + x_max + } + + #[inline] + pub fn x_bounds(&self) -> [isize; 2] { + [self.x_min(), self.x_max()] + } + + #[inline] + pub fn y_min(&self) -> isize { + let [y_min, _x_min] = self.min; + y_min + } + + #[inline] + pub fn y_max(&self) -> isize { + let [y_max, _x_max] = self.max; + y_max + } + + #[inline] + pub fn y_bounds(&self) -> [isize; 2] { + [self.y_min(), self.y_max()] + } + + pub fn new_min_max(y_min: isize, y_max: isize, x_min: isize, x_max: isize) -> Result { + Self::new([y_min, x_min], [y_max, x_max]) + } +} + impl GridSize for GridBoundingBox<[isize; 1]> { type ShapeArray = [usize; 1]; @@ -326,6 +391,167 @@ where } } +impl GridContains for GridBoundingBox2D { + fn contains(&self, other: &GridBoundingBox2D) -> bool { + let [self_y_min, self_x_min] = self.min; + let [other_y_min, other_x_min] = other.min; + + let [self_y_max, self_x_max] = self.max; + let [other_y_max, other_x_max] = other.max; + + self_y_min <= other_y_min + && self_x_min <= other_x_min + && self_y_max >= other_y_max + && self_x_max >= other_x_max + } +} + +pub trait GridBoundingBoxExt: GridBounds { + fn extend(&mut self, other: &Self); + + #[must_use] + fn extended(&self, other: &Self) -> Self + where + Self: Sized + Clone, + { + let mut extended = self.clone(); + extended.extend(other); + extended + } + + fn shift_by_offset( + &self, + offset: GridIdx, + ) -> GridBoundingBox + where + GridIdx: Add> + Clone, + { + GridBoundingBox::new_unchecked(self.min_index() + offset.clone(), self.max_index() + offset) + } +} + +impl GridBoundingBoxExt for GridBoundingBox1D { + fn extend(&mut self, other: &Self) { + let [self_x_min] = self.min; + let [other_x_min] = other.min; + + let [self_x_max] = self.max; + let [other_x_max] = other.max; + + self.min = [self_x_min.min(other_x_min)]; + self.max = [self_x_max.max(other_x_max)]; + } +} + +impl GridBoundingBoxExt for GridBoundingBox2D { + fn extend(&mut self, other: &Self) { + let [self_y_min, self_x_min] = self.min; + let [other_y_min, other_x_min] = other.min; + + let [self_y_max, self_x_max] = self.max; + let [other_y_max, other_x_max] = other.max; + + self.min = [self_y_min.min(other_y_min), self_x_min.min(other_x_min)]; + self.max = [self_y_max.max(other_y_max), self_x_max.max(other_x_max)]; + } +} + +impl GridBoundingBoxExt for GridBoundingBox3D { + fn extend(&mut self, other: &Self) { + let [self_z_min, self_y_min, self_x_min] = self.min; + let [other_z_min, other_y_min, other_x_min] = other.min; + + let [self_z_max, self_y_max, self_x_max] = self.max; + let [other_z_max, other_y_max, other_x_max] = other.max; + + self.min = [ + self_z_min.min(other_z_min), + self_y_min.min(other_y_min), + self_x_min.min(other_x_min), + ]; + self.max = [ + self_z_max.max(other_z_max), + self_y_max.max(other_y_max), + self_x_max.max(other_x_max), + ]; + } +} + +impl core::fmt::Display for GridBoundingBox2D { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "GridBoundingBox2D: upper_left [y: {}, x:{}], lower_right [y: {}, x: {}]", + self.y_max(), + self.x_min(), + self.y_min(), + self.x_max() + ) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize)] +pub struct GridIdx2DIter { + pub grid_bounds: GridBoundingBox2D, + pub x_pos: isize, + pub y_pos: isize, +} + +impl GridIdx2DIter { + pub fn new(grid_bounds: &B) -> Self + where + B: GridBounds, + { + let grid_bounds = + GridBoundingBox2D::new_unchecked(grid_bounds.min_index(), grid_bounds.max_index()); + + Self { + grid_bounds, + x_pos: grid_bounds.x_min(), + y_pos: grid_bounds.y_min(), + } + } + + pub fn reset(&mut self) { + self.x_pos = self.grid_bounds.x_min(); + self.y_pos = self.grid_bounds.y_min(); + } +} + +impl Iterator for GridIdx2DIter { + type Item = GridIdx2D; + + fn next(&mut self) -> Option { + if self.x_pos > self.grid_bounds.x_max() { + self.x_pos = self.grid_bounds.x_min(); + self.y_pos += 1; + } + + if self.y_pos > self.grid_bounds.y_max() { + return None; + } + + let current = GridIdx2D::new_y_x(self.y_pos, self.x_pos); + + self.x_pos += 1; + + Some(current) + } + + fn size_hint(&self) -> (usize, Option) { + let size = self.grid_bounds.number_of_elements(); + (size, Some(size)) + } +} + +/// Method to generate an `Iterator` over all `GridIdx2D` in `GridBounds` +pub fn grid_idx_iter_2d(bounds: &B) -> GridIdx2DIter +where + B: GridBounds, +{ + GridIdx2DIter::new(bounds) +} + #[cfg(test)] mod tests { use super::*; @@ -461,4 +687,65 @@ mod tests { assert_eq!(l2, 1 * 42 * 42 + 1 * 42 + 1); assert_eq!(a.grid_idx_unchecked(l2), [2, 2, 2].into()); } + + #[test] + fn grid_bounding_box_2d_contains() { + let a = GridBoundingBox::new([1, 1], [42, 42]).unwrap(); + let b = GridBoundingBox::new([2, 2], [41, 41]).unwrap(); + assert!(a.contains(&b)); + assert!(!b.contains(&a)); + } + + #[test] + fn extend_1d() { + let mut a = GridBoundingBox::new([1], [42]).unwrap(); + let b = GridBoundingBox::new([2], [69]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1], [69]).unwrap()); + } + + #[test] + fn extend_2d() { + let mut a = GridBoundingBox::new([1, 2], [42, 69]).unwrap(); + let b = GridBoundingBox::new([2, 1], [69, 42]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1, 1], [69, 69]).unwrap()); + } + + #[test] + fn extend_3d() { + let mut a = GridBoundingBox::new([1, 3, 2], [42, 69, 666]).unwrap(); + let b = GridBoundingBox::new([3, 2, 1], [69, 666, 42]).unwrap(); + a.extend(&b); + assert_eq!(a, GridBoundingBox::new([1, 2, 1], [69, 666, 666]).unwrap()); + } + + #[test] + fn grid_idx_iter_2d() { + let grid_bounds_2d = GridBoundingBox2D::new_unchecked([-1, 1], [2, 4]); + let grid_idx_iter = GridIdx2DIter::new(&grid_bounds_2d); + let res: Vec<_> = grid_idx_iter.collect(); + + assert_eq!( + res, + vec![ + GridIdx2D::new_y_x(-1, 1), + GridIdx2D::new_y_x(-1, 2), + GridIdx2D::new_y_x(-1, 3), + GridIdx2D::new_y_x(-1, 4), + GridIdx2D::new_y_x(0, 1), + GridIdx2D::new_y_x(0, 2), + GridIdx2D::new_y_x(0, 3), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(1, 1), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(1, 4), + GridIdx2D::new_y_x(2, 1), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(2, 3), + GridIdx2D::new_y_x(2, 4) + ] + ); + } } diff --git a/datatypes/src/raster/grid_index.rs b/datatypes/src/raster/grid_index.rs index 05fe9d125..fc5b1da27 100644 --- a/datatypes/src/raster/grid_index.rs +++ b/datatypes/src/raster/grid_index.rs @@ -1,8 +1,10 @@ -use std::ops::{Add, Div, Mul, Rem, Sub}; +use std::ops::{Add, Div, Mul, Neg, Rem, Sub}; use num_traits::{One, Zero}; use serde::{Deserialize, Serialize}; +use super::GridShape2D; + /// /// The grid index struct. This is a wrapper for arrays with added methods and traits, e.g. Add, Sub... /// @@ -228,6 +230,19 @@ where } } +impl Mul for GridIdx2D { + type Output = Self; + + fn mul(self, rhs: GridShape2D) -> Self::Output { + let GridIdx([a, b]) = self; + let GridShape2D { + shape_array: [shape_a, shape_b], + } = rhs; + + GridIdx([a * shape_a as isize, b * shape_b as isize]) + } +} + impl Mul for GridIdx3D where I: Into, @@ -318,3 +333,74 @@ where GridIdx([a % a_other, b % b_other, c % c_other]) } } + +impl GridIdx1D { + pub fn x(self) -> isize { + let [a] = self.0; + a + } + + pub fn to_2d(self) -> GridIdx2D { + let [a] = self.0; + GridIdx([a, 0]) + } + + pub fn to_3d(self) -> GridIdx3D { + let [a] = self.0; + GridIdx([a, 0, 0]) + } + + pub fn new_x(x: isize) -> Self { + GridIdx([x]) + } +} + +impl GridIdx2D { + pub fn x(&self) -> isize { + let [_, x] = self.0; + x + } + + pub fn y(&self) -> isize { + let [y, _] = self.0; + y + } + + pub fn to_3d(self) -> GridIdx3D { + let [a, b] = self.0; + GridIdx([a, b, 0]) + } + + pub fn new_y_x(y: isize, x: isize) -> Self { + GridIdx([y, x]) + } +} + +impl GridIdx3D { + pub fn x(&self) -> isize { + let [_, _, x] = self.0; + x + } + + pub fn y(&self) -> isize { + let [_, y, _] = self.0; + y + } + + pub fn z(&self) -> isize { + let [z, _, _] = self.0; + z + } + + pub fn new_z_y_x(z: isize, y: isize, x: isize) -> Self { + GridIdx([z, y, x]) + } +} + +impl Neg for GridIdx2D { + type Output = Self; + + fn neg(self) -> Self { + GridIdx::new_y_x(-self.y(), -self.x()) + } +} diff --git a/datatypes/src/raster/grid_or_empty.rs b/datatypes/src/raster/grid_or_empty.rs index 5a22943f3..f4c02623e 100644 --- a/datatypes/src/raster/grid_or_empty.rs +++ b/datatypes/src/raster/grid_or_empty.rs @@ -148,7 +148,7 @@ where impl GridBounds for GridOrEmpty where - D: GridBounds + GridSpaceToLinearSpace, + D: GridBounds, T: Clone, I: AsRef<[isize]> + Into>, { @@ -213,29 +213,39 @@ where } } -impl ChangeGridBounds for GridOrEmpty +impl ChangeGridBounds for GridOrEmpty where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone + GridSpaceToLinearSpace, - T: Copy, - GridBoundingBox: GridSize, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, + T: Copy, { - type Output = GridOrEmpty, T>; + type BoundedOutput = GridOrEmpty, T>; + type UnboundedOutput = GridOrEmpty, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { match self { GridOrEmpty::Grid(g) => GridOrEmpty::Grid(g.shift_by_offset(offset)), GridOrEmpty::Empty(n) => GridOrEmpty::Empty(n.shift_by_offset(offset)), } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { match self { GridOrEmpty::Grid(g) => g.set_grid_bounds(bounds).map(Into::into), GridOrEmpty::Empty(n) => n.set_grid_bounds(bounds).map(Into::into), } } + + fn unbounded(self) -> Self::UnboundedOutput { + match self { + GridOrEmpty::Grid(g) => GridOrEmpty::Grid(g.unbounded()), + GridOrEmpty::Empty(n) => GridOrEmpty::Empty(n.unbounded()), + } + } } #[cfg(test)] @@ -247,7 +257,7 @@ mod tests { #[test] fn grid_bounds_2d_empty_grid() { let dim: GridShape2D = [3, 2].into(); - let raster2d: GridOrEmpty2D = EmptyGrid::new(dim).into(); // FIXME: find out why type is needed + let raster2d: GridOrEmpty2D = EmptyGrid::new(dim).into(); assert_eq!(raster2d.min_index(), GridIdx([0, 0])); assert_eq!(raster2d.max_index(), GridIdx([2, 1])); diff --git a/datatypes/src/raster/grid_spatial.rs b/datatypes/src/raster/grid_spatial.rs new file mode 100644 index 000000000..59e7b8b7c --- /dev/null +++ b/datatypes/src/raster/grid_spatial.rs @@ -0,0 +1,627 @@ +use super::{ + FromIndexFn, GeoTransform, GeoTransformAccess, Grid, GridBoundingBox2D, GridBoundingBoxExt, + GridBounds, GridIdx, GridIdx2D, GridIntersection, TilingSpecification, TilingStrategy, +}; +use crate::{ + operations::reproject::{ + CoordinateProjection, Reproject, ReprojectClipped, suggest_output_spatial_grid_like_gdal, + }, + primitives::{ + AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialPartitioned, + SpatialResolution, + }, + util::Result, +}; +use float_cmp::{ApproxEq, approx_eq}; +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDefinition { + pub geo_transform: GeoTransform, + pub grid_bounds: GridBoundingBox2D, +} + +impl SpatialGridDefinition { + pub fn new(geo_transform: GeoTransform, grid_bounds: GridBoundingBox2D) -> Self { + Self { + geo_transform, + grid_bounds, + } + } + + pub fn new_generic>( + geo_transform: GeoTransform, + grid_bounds: G, + ) -> Self { + let grid_bounds: GridBoundingBox2D = grid_bounds.into(); + Self::new(geo_transform, grid_bounds) + } + + pub fn grid_bounds(&self) -> GridBoundingBox2D { + self.grid_bounds + } + + pub fn geo_transform(&self) -> GeoTransform { + self.geo_transform + } + + pub fn spatial_partition(&self) -> SpatialPartition2D { + self.geo_transform.grid_to_spatial_bounds(&self.grid_bounds) + } + + #[must_use] + /// Moves the origin and bounds using a pixel offset. The spatial location stays the same! + pub fn shift_bounds_relative_by_pixel_offset(&self, offset: GridIdx2D) -> Self { + let grid_bounds = self.grid_bounds.shift_by_offset(offset); + let geo_transform = self.geo_transform.shift_by_pixel_offset(-offset); + Self::new(geo_transform, grid_bounds) + } + + /// Moves the origin to another pixel edge. The spatial location stays the same! + /// Check if you can use `shift_bounds_relative_by_pixel_offset`! + pub fn with_moved_origin_exact_grid(&self, new_origin: Coordinate2D) -> Option { + if self.geo_transform.is_valid_pixel_edge(new_origin) { + Some(self.with_moved_origin_to_nearest_grid_edge(new_origin)) + } else { + None + } + } + + /// This method moves the origin to the coordinate of the grid edge nearest to the supplied new origin reference + #[must_use] + pub fn with_moved_origin_to_nearest_grid_edge( + &self, + new_origin_referece: Coordinate2D, + ) -> Self { + let nearest_to_target = self.geo_transform.nearest_pixel_edge(new_origin_referece); + self.shift_bounds_relative_by_pixel_offset(-nearest_to_target) + } + + /// This method moves the origin to the coordinate of the grid edge nearest to the supplied new origin reference + pub fn with_moved_origin_to_nearest_grid_edge_with_distance( + &self, + new_origin_referece: Coordinate2D, + ) -> (Self, Coordinate2D) { + let distance = self + .geo_transform + .distance_to_nearest_pixel_edge(new_origin_referece); + let new_self = self.with_moved_origin_to_nearest_grid_edge(new_origin_referece); + (new_self, distance) + } + + /// Creates a new spatial grid with the self shape and pixel size but new origin. + #[must_use] + pub fn replace_origin(&self, new_origin: Coordinate2D) -> Self { + Self { + geo_transform: GeoTransform::new( + new_origin, + self.geo_transform.x_pixel_size(), + self.geo_transform.y_pixel_size(), + ), + grid_bounds: self.grid_bounds, + } + } + + /// Merges two spatial grids + /// If the second grid is not compatible with selfit returns None + /// If the second grid has a different `GeoTransform` it is transformed to the `GroTransform` of self + pub fn merge(&self, other: &Self) -> Option { + if !self.is_compatible_grid_generic(other) { + return None; + } + + let other_shift = + other.with_moved_origin_exact_grid(self.geo_transform.origin_coordinate)?; + + let merged_bounds = self.grid_bounds().extended(&other_shift.grid_bounds()); + + Some(Self::new(self.geo_transform, merged_bounds)) + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.geo_transform().is_compatible_grid(g.geo_transform()) + } + + /// Computes the intersection of self and other + /// IF other is incompatible with self, None is returned. + /// IF other has a different `GeoTransform` then self it is transformed to to the `GeoTransform` of self. + pub fn intersection(&self, other: &SpatialGridDefinition) -> Option { + if !self.is_compatible_grid_generic(other) { + return None; + } + + let (other_shift, dist) = other.with_moved_origin_to_nearest_grid_edge_with_distance( + self.geo_transform.origin_coordinate, + ); + + if dist.x.abs() > self.geo_transform().x_pixel_size().abs() * 0.00001 // TODO: maybe use exact_grid and another epsilon? + || dist.y.abs() > self.geo_transform().y_pixel_size().abs() * 0.00001 + { + return None; + } + + let intersection_bounds = self + .grid_bounds() + .intersection(&(other_shift.grid_bounds()))?; + + Some(Self::new(self.geo_transform, intersection_bounds)) + } + + /// Creates a new spatial grid that has the same origin as self. + /// The pixel sizes are changed and the grid bounds are adapted to cover the same spatial area. + /// Note: if the new resolution is not a multiple of the old resolution the new grid might cover a larger spatial area then self. + #[must_use] + pub fn with_changed_resolution(&self, new_res: SpatialResolution) -> Self { + let geo_transform = + GeoTransform::new(self.geo_transform.origin_coordinate, new_res.x, -new_res.y); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&self.spatial_partition()); + SpatialGridDefinition::new(geo_transform, grid_bounds) + } + + pub fn generate_coord_grid_upper_left_edge(&self) -> Grid { + let map_fn = |idx: GridIdx2D| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(idx) + }; + + Grid::from_index_fn(&self.grid_bounds, map_fn) + } + + pub fn generate_coord_grid_pixel_center(&self) -> Grid { + let map_fn = |idx: GridIdx2D| { + self.geo_transform + .grid_idx_to_pixel_center_coordinate_2d(idx) + }; + + Grid::from_index_fn(&self.grid_bounds, map_fn) + } + + #[must_use] + pub fn spatial_bounds_to_compatible_spatial_grid( + &self, + spatial_partition: SpatialPartition2D, + ) -> Self { + let grid_bounds = self + .geo_transform + .spatial_to_grid_bounds(&spatial_partition); + Self::new(self.geo_transform, grid_bounds) + } + + #[must_use] + pub fn flip_axis_y(&self) -> Self { + let geo_transform = GeoTransform::new( + self.geo_transform.origin_coordinate, + self.geo_transform.x_pixel_size(), + -self.geo_transform.y_pixel_size(), + ); + + let y_min = -(self.grid_bounds.y_max() + 1); // since grid bounds are inclusive + let y_max = -(self.grid_bounds.y_min() + 1); + + let grid_bounds = GridBoundingBox2D::new_unchecked( + [y_min, self.grid_bounds.x_min()], + [y_max, self.grid_bounds.x_max()], + ); + + Self::new(geo_transform, grid_bounds) + } + + /// snap the grid bounds to be multiples of `overview_level` within the given dataset bounds s.t. directly reading from an overview is possible + /// i.e. starting from multiple of `overview_level` and having a size that is multiple of `overview_level` (or goes to the end of the dataset) + pub fn snap_to_dataset_overview_level( + &self, + dataset_grid: SpatialGridDefinition, + overview_level: u32, + ) -> Option { + debug_assert!( + self.geo_transform + .approx_eq(dataset_grid.geo_transform, float_cmp::F64Margin::default()) + && self.is_compatible_grid_generic(&dataset_grid) + && self + .grid_bounds + .intersection(&dataset_grid.grid_bounds()) + .is_some() + ); + + let overview_level = overview_level as isize; + + let [current_min_x, current_max_x] = self.grid_bounds.x_bounds(); + let [current_min_y, current_max_y] = self.grid_bounds.y_bounds(); + + let snapped_min_x = (current_min_x / overview_level) * overview_level; + let snapped_min_y = (current_min_y / overview_level) * overview_level; + + let current_size_x = current_max_x - current_min_x + 1; + let current_size_y = current_max_y - current_min_y + 1; + + let snapped_size_y = + ((current_size_y + overview_level - 1) / overview_level) * overview_level; + let snapped_size_x = + ((current_size_x + overview_level - 1) / overview_level) * overview_level; + + // Calculate snapped maximum (inclusive) + let snapped_max_y = snapped_min_y + snapped_size_y - 1; + let snapped_max_x = snapped_min_x + snapped_size_x - 1; + + // Create snapped bounds + let snapped_bounds = GridBoundingBox2D::new( + [snapped_min_y, snapped_min_x], + [snapped_max_y, snapped_max_x], + ) + .ok()?; + + // limit snapped bounds to dataset bounds + let intersection = snapped_bounds.intersection(&dataset_grid.grid_bounds())?; + + // Use the intersection as the final bounds + Some(Self::new(self.geo_transform, intersection)) + } +} + +impl SpatialPartitioned for SpatialGridDefinition { + fn spatial_partition(&self) -> SpatialPartition2D { + self.spatial_partition() + } +} + +impl GridBounds for SpatialGridDefinition { + type IndexArray = [isize; 2]; + + fn min_index(&self) -> GridIdx { + self.grid_bounds.min_index() + } + + fn max_index(&self) -> GridIdx { + self.grid_bounds.max_index() + } +} + +impl GeoTransformAccess for SpatialGridDefinition { + fn geo_transform(&self) -> GeoTransform { + self.geo_transform() + } +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize)] +pub struct TilingSpatialGridDefinition { + // Don't make this public to avoid leaking inner + element_grid_definition: SpatialGridDefinition, + tiling_specification: TilingSpecification, +} + +impl TilingSpatialGridDefinition { + pub fn new( + element_grid_definition: SpatialGridDefinition, + tiling_specification: TilingSpecification, + ) -> Self { + Self { + element_grid_definition, + tiling_specification, + } + } + + pub fn tiling_spatial_grid_definition(&self) -> SpatialGridDefinition { + // TODO: maybe do this in new and store it? + self.element_grid_definition + .with_moved_origin_to_nearest_grid_edge( + self.tiling_specification.tiling_origin_reference(), + ) + } + + pub fn tiling_geo_transform(&self) -> GeoTransform { + self.tiling_spatial_grid_definition().geo_transform() + } + + pub fn tiling_grid_bounds(&self) -> GridBoundingBox2D { + self.tiling_spatial_grid_definition().grid_bounds() + } + + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + // TODO: use tiling_spatial_grid_definition? + self.element_grid_definition.is_compatible_grid_generic(g) + } + + pub fn is_same_tiled_grid(&self, other: &TilingSpatialGridDefinition) -> bool { + // TODO: re-implement when decided how to model struct + let a = self.tiling_spatial_grid_definition(); + let b = other.tiling_spatial_grid_definition(); + approx_eq!(GeoTransform, a.geo_transform(), b.geo_transform()) + } + + /// Returns the data tiling strategy for the given tile size in pixels. + #[must_use] + pub fn generate_data_tiling_strategy(&self) -> TilingStrategy { + TilingStrategy { + geo_transform: self.tiling_geo_transform(), + tile_size_in_pixels: self.tiling_specification.tile_size_in_pixels, + } + } + + #[must_use] + pub fn with_other_bounds(&self, new_bounds: GridBoundingBox2D) -> Self { + let new_grid = SpatialGridDefinition::new(self.tiling_geo_transform(), new_bounds); + Self::new(new_grid, self.tiling_specification) + } +} + +impl SpatialPartitioned for TilingSpatialGridDefinition { + fn spatial_partition(&self) -> SpatialPartition2D { + // TODO: use tiling bounds and geotransform? must be equal!!! + self.element_grid_definition.spatial_partition() + } +} + +impl Reproject

for SpatialGridDefinition { + type Out = Self; + + fn reproject(&self, projector: &P) -> Result { + suggest_output_spatial_grid_like_gdal(self, projector) + } +} + +impl ReprojectClipped

for SpatialGridDefinition { + type Out = Self; + + fn reproject_clipped(&self, projector: &P) -> Result> { + let target_bounds_in_source_srs: Option = projector + .source_srs() + .area_of_use_intersection(&projector.target_srs())?; + if target_bounds_in_source_srs.is_none() { + return Ok(None); + } + let target_bounds_in_source_srs = target_bounds_in_source_srs.expect("case checked above"); + let intersection_grid_bounds = + target_bounds_in_source_srs.intersection(&self.spatial_partition()); + if intersection_grid_bounds.is_none() { + return Ok(None); + } + let intersection_grid_bounds = intersection_grid_bounds.expect("case checked above"); + let intersecting_grid = + self.spatial_bounds_to_compatible_spatial_grid(intersection_grid_bounds); + let compatible_intersecting_grid = if target_bounds_in_source_srs + .contains_coordinate(&self.geo_transform().origin_coordinate()) + { + intersecting_grid + } else { + intersecting_grid.with_moved_origin_to_nearest_grid_edge( + intersecting_grid.spatial_partition().upper_left(), + ) + }; + compatible_intersecting_grid + .reproject(projector) + .map(Option::Some) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + operations::reproject::suggest_output_spatial_grid_like_gdal_helper, + primitives::AxisAlignedRectangle, + raster::{BoundedGrid, GridShape}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, + test_data, + util::gdal::gdal_open_dataset, + }; + + use super::*; + + #[test] + fn shift_bounds_relative_by_pixel_offset() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + let shifted_s = s.shift_bounds_relative_by_pixel_offset(GridIdx2D::new([1, 1])); + assert_eq!( + shifted_s.geo_transform(), + GeoTransform::new_with_coordinate_x_y(-1., 1.0, 1., -1.0) + ); + assert_eq!(shifted_s.grid_bounds().min_index(), GridIdx2D::new([-1, 1])); + assert_eq!(shifted_s.grid_bounds().max_index(), GridIdx2D::new([1, 3])); + + assert_eq!(s.spatial_partition(), shifted_s.spatial_partition()); + } + + #[test] + fn merge() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.0, 1.0, -1.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let merged = s.merge(&s_2).unwrap(); + + assert_eq!(s.geo_transform, merged.geo_transform); + assert_eq!( + GridBoundingBox2D::new_min_max(-2, 1, 0, 3).unwrap(), + merged.grid_bounds + ); + + let s_s2_spatial_partition = s.spatial_partition().extended(&s_2.spatial_partition()); + let merged_partition = merged.spatial_partition(); + + assert!(approx_eq!( + Coordinate2D, + s_s2_spatial_partition.upper_left(), + merged_partition.upper_left() + )); + assert!(approx_eq!( + Coordinate2D, + s_s2_spatial_partition.lower_right(), + merged_partition.lower_right() + )); + } + + #[test] + fn no_merge_origin() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.1, 1.0, -1.1, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + assert!(s.merge(&s_2).is_none()); + } + + #[test] + fn no_merge_pixel_size() { + let s = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0.0, 1.0, 0.0, -1.0), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + let s_2 = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(1.0, 1.1, -1.0, -1.1), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ); + + assert!(s.merge(&s_2).is_none()); + } + + #[test] + fn source_resolution() { + let epsg_4326 = SpatialReference::epsg_4326(); + let epsg_3857 = SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857); + + // use ndvi dataset that was reprojected using gdal as ground truth + let dataset_4326 = gdal_open_dataset(test_data!( + "raster/modis_ndvi/MOD13A2_M_NDVI_2014-04-01.TIFF" + )) + .unwrap(); + let geotransform_4326 = dataset_4326.geo_transform().unwrap(); + let res_4326 = SpatialResolution::new(geotransform_4326[1], -geotransform_4326[5]).unwrap(); + + let dataset_3857 = gdal_open_dataset(test_data!( + "raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01.TIFF" + )) + .unwrap(); + let geotransform_3857 = dataset_3857.geo_transform().unwrap(); + + // ndvi was projected from 4326 to 3857. The calculated source_resolution for getting the raster in 3857 with `res_3857` + // should thus roughly be like the original `res_4326` + + let spatial_grid_3857 = SpatialGridDefinition::new( + geotransform_3857.into(), + GridShape::new_2d(dataset_3857.raster_size().1, dataset_3857.raster_size().0) + .bounding_box(), + ); + + let result_res = + suggest_output_spatial_grid_like_gdal_helper(&spatial_grid_3857, epsg_3857, epsg_4326) + .unwrap(); + + assert!(1. - (result_res.geo_transform().x_pixel_size() / res_4326.x).abs() < 0.02); + assert!(1. - (result_res.geo_transform().y_pixel_size() / res_4326.y).abs() < 0.02); + } + + #[test] + fn flip_axis_y() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(20.0, 20.0), 3., 2.), + GridBoundingBox2D::new_min_max(10, 25, 1, 2).unwrap(), + ); + + let fliped = spatial_grid.flip_axis_y(); + + assert_eq!( + fliped.geo_transform, + GeoTransform::new(Coordinate2D::new(20.0, 20.0), 3., -2.) + ); + + assert_eq!( + fliped.grid_bounds, + GridBoundingBox2D::new_min_max(-26, -11, 1, 2).unwrap() + ); + } + + #[test] + fn intersection_with_floating_point_discrepancies() { + let a = SpatialGridDefinition { + geo_transform: GeoTransform::new((-180.0, 90.).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 0], [899, 1799]).unwrap(), + }; + + let b = SpatialGridDefinition { + geo_transform: GeoTransform::new((-45.0, 22.399_999_999_999_99).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 0], [561, 1124]).unwrap(), + }; + + assert_eq!( + a.intersection(&b), + Some(SpatialGridDefinition { + geo_transform: GeoTransform::new((-180.0, 90.).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([338, 675], [899, 1799]).unwrap(), + }) + ); + + let a = SpatialGridDefinition { + geo_transform: GeoTransform::new((-180.0, 90.).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 0], [899, 1799]).unwrap(), + }; + + let b = SpatialGridDefinition { + geo_transform: GeoTransform::new((-45.0, 90.0).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 0], [561, 1124]).unwrap(), + }; + + assert_eq!( + a.intersection(&b), + Some(SpatialGridDefinition { + geo_transform: GeoTransform::new((-180.0, 90.).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 675], [561, 1799]).unwrap(), + }) + ); + } + + #[test] + fn move_origin_with_floating_point_discrepancies() { + let tile = SpatialGridDefinition { + geo_transform: GeoTransform::new((0.0, 0.0).into(), 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new([0, 512], [511, 1023]).unwrap(), + }; + + assert_eq!( + tile.with_moved_origin_exact_grid((-45.0, 22.399_999_999_999_99).into()), + Some(SpatialGridDefinition { + geo_transform: GeoTransform::new((-45.0, 22.400_000_000_000_002).into(), 0.2, -0.2), // TODO: 22.3999..9 vs 22.4 vs. 22.40000..01 + grid_bounds: GridBoundingBox2D::new([112, 737], [623, 1248]).unwrap(), + }) + ); + } + + #[test] + fn it_snaps_to_overview_levels() { + let tile_dataset_intersection_grid = SpatialGridDefinition::new( + GeoTransform::new((-180.0, 90.0).into(), 0.2, -0.2), + GridBoundingBox2D::new([1, 1], [3, 3]).unwrap(), + ); + + let dataset_grid = SpatialGridDefinition::new( + GeoTransform::new((-180.0, 90.0).into(), 0.2, -0.2), + GridBoundingBox2D::new([0, 0], [8, 8]).unwrap(), + ); + + let snapped = + tile_dataset_intersection_grid.snap_to_dataset_overview_level(dataset_grid, 2); + + assert_eq!( + snapped, + Some(SpatialGridDefinition::new( + GeoTransform::new((-180.0, 90.0).into(), 0.2, -0.2), + GridBoundingBox2D::new([0, 0], [3, 3]).unwrap() + )) + ); + } +} diff --git a/datatypes/src/raster/grid_traits.rs b/datatypes/src/raster/grid_traits.rs index d6b8e3204..aa1439209 100644 --- a/datatypes/src/raster/grid_traits.rs +++ b/datatypes/src/raster/grid_traits.rs @@ -82,6 +82,10 @@ where pub trait GridIntersection { // Returns true if Self intesects Rhs fn intersection(&self, other: &Rhs) -> Option; + + fn intersects(&self, other: &Rhs) -> bool { + self.intersection(other).is_some() + } } /// Provides the methods needed to map an n-dimensional `GridIdx` to linear space. @@ -148,13 +152,17 @@ pub trait GridShapeAccess { } /// Change the bounds of gridded data. -pub trait ChangeGridBounds: BoundedGrid +pub trait ChangeGridBounds: + BoundedGrid + GridShapeAccess where I: AsRef<[isize]> + Into> + Clone, - GridBoundingBox: GridSize, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, GridIdx: Add> + From, { - type Output; + type BoundedOutput; + type UnboundedOutput; fn shift_bounding_box(&self, offset: GridIdx) -> GridBoundingBox { let bounds = self.bounding_box(); @@ -165,10 +173,13 @@ where } /// shift using an offset - fn shift_by_offset(self, offset: GridIdx) -> Self::Output; + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput; /// set new bounds. will fail if the axis sizes do not match. - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result; + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result; + + /// remove the bounds. Keep the shape + fn unbounded(self) -> Self::UnboundedOutput; } pub trait GridStep: GridSpaceToLinearSpace diff --git a/datatypes/src/raster/masked_grid.rs b/datatypes/src/raster/masked_grid.rs index bf6b50a7e..eb2cf9330 100644 --- a/datatypes/src/raster/masked_grid.rs +++ b/datatypes/src/raster/masked_grid.rs @@ -272,29 +272,39 @@ where } } -impl ChangeGridBounds for MaskedGrid +impl ChangeGridBounds for MaskedGrid where - I: AsRef<[isize]> + Clone, - D: GridBounds + Clone, - T: Clone, - GridBoundingBox: GridSize, - GridIdx: Add> + From + Clone, + D: GridBounds + GridSize, + I: AsRef<[isize]> + Into> + Clone, + A: AsRef<[usize]> + Into> + Clone, + GridBoundingBox: GridSize, + GridShape: GridSize, + GridIdx: Add> + From, + T: Copy, { - type Output = MaskedGrid, T>; + type BoundedOutput = MaskedGrid, T>; + type UnboundedOutput = MaskedGrid, T>; - fn shift_by_offset(self, offset: GridIdx) -> Self::Output { + fn shift_by_offset(self, offset: GridIdx) -> Self::BoundedOutput { MaskedGrid { inner_grid: self.inner_grid.shift_by_offset(offset.clone()), validity_mask: self.validity_mask.shift_by_offset(offset), } } - fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { + fn set_grid_bounds(self, bounds: GridBoundingBox) -> Result { Ok(MaskedGrid { inner_grid: self.inner_grid.set_grid_bounds(bounds.clone())?, validity_mask: self.validity_mask.set_grid_bounds(bounds)?, }) } + + fn unbounded(self) -> Self::UnboundedOutput { + MaskedGrid { + inner_grid: self.inner_grid.unbounded(), + validity_mask: self.validity_mask.unbounded(), + } + } } impl ByteSize for MaskedGrid diff --git a/datatypes/src/raster/mod.rs b/datatypes/src/raster/mod.rs index 90e9cbc5e..63b120ebd 100755 --- a/datatypes/src/raster/mod.rs +++ b/datatypes/src/raster/mod.rs @@ -6,10 +6,10 @@ pub use self::empty_grid::{EmptyGrid, EmptyGrid1D, EmptyGrid2D, EmptyGrid3D}; pub use self::geo_transform::{GdalGeoTransform, GeoTransform}; pub use self::grid::{ Grid, Grid1D, Grid2D, Grid3D, GridShape, GridShape1D, GridShape2D, GridShape3D, - grid_idx_iter_2d, }; pub use self::grid_bounds::{ - GridBoundingBox, GridBoundingBox1D, GridBoundingBox2D, GridBoundingBox3D, + GridBoundingBox, GridBoundingBox1D, GridBoundingBox2D, GridBoundingBox3D, GridBoundingBoxExt, + GridIdx2DIter, grid_idx_iter_2d, }; pub use self::grid_index::{GridIdx, GridIdx1D, GridIdx2D, GridIdx3D}; pub use self::grid_or_empty::{GridOrEmpty, GridOrEmpty1D, GridOrEmpty2D, GridOrEmpty3D}; @@ -19,7 +19,7 @@ pub use self::grid_traits::{ }; pub use self::grid_typed::{TypedGrid, TypedGrid2D, TypedGrid3D}; pub use self::operations::{ - blit::Blit, convert_data_type::ConvertDataType, convert_data_type::ConvertDataTypeParallel, + convert_data_type::ConvertDataType, convert_data_type::ConvertDataTypeParallel, grid_blit::GridBlit, interpolation::Bilinear, interpolation::InterpolationAlgorithm, interpolation::NearestNeighbor, }; @@ -27,11 +27,13 @@ pub use self::raster_tile::{ BaseTile, MaterializedRasterTile, MaterializedRasterTile2D, MaterializedRasterTile3D, RasterTile, RasterTile2D, RasterTile3D, TilesEqualIgnoringCacheHint, display_raster_tile_2d, }; -pub use self::tiling::{TileInformation, TilingSpecification, TilingStrategy}; +pub use self::tiling::{TileInformation, TileInformationIter, TilingSpecification, TilingStrategy}; pub use self::typed_raster_conversion::TypedRasterConversion; pub use self::typed_raster_tile::{TypedRasterTile2D, TypedRasterTile3D}; pub use self::{grid_traits::ChangeGridBounds, grid_traits::GridShapeAccess}; pub use arrow_conversion::raster_tile_2d_to_arrow_ipc_file; +pub use db_types::GridBoundingBox2DDbType; +pub use grid_spatial::{SpatialGridDefinition, TilingSpatialGridDefinition}; pub use masked_grid::{MaskedGrid, MaskedGrid1D, MaskedGrid2D, MaskedGrid3D}; pub use no_data_value_grid::{ NoDataValueGrid, NoDataValueGrid1D, NoDataValueGrid2D, NoDataValueGrid3D, @@ -43,6 +45,7 @@ pub use operations::checked_scaling::{ pub use operations::from_index_fn::{FromIndexFn, FromIndexFnParallel}; pub use operations::map_elements::{MapElements, MapElementsParallel}; pub use operations::map_indexed_elements::{MapIndexedElements, MapIndexedElementsParallel}; +pub use operations::sample_points::SamplePoints; pub use operations::update_elements::{UpdateElements, UpdateElementsParallel}; pub use operations::update_indexed_elements::{ UpdateIndexedElements, UpdateIndexedElementsParallel, @@ -55,12 +58,14 @@ pub use raster_traits::{CoordinatePixelAccess, GeoTransformAccess, Raster}; mod arrow_conversion; mod band_names; mod data_type; +mod db_types; mod empty_grid; mod geo_transform; mod grid; mod grid_bounds; mod grid_index; mod grid_or_empty; +mod grid_spatial; mod grid_traits; mod grid_typed; mod macros_raster; diff --git a/datatypes/src/raster/operations/blit.rs b/datatypes/src/raster/operations/blit.rs index 804cac7c2..8b1378917 100644 --- a/datatypes/src/raster/operations/blit.rs +++ b/datatypes/src/raster/operations/blit.rs @@ -1,294 +1 @@ -use crate::error; -use crate::raster::{ - ChangeGridBounds, GeoTransformAccess, GridBlit, GridIdx2D, MaterializedRasterTile2D, Pixel, - RasterTile2D, -}; -use crate::util::Result; -use snafu::ensure; - -pub trait Blit { - fn blit(&mut self, source: R) -> Result<()>; -} - -impl Blit> for MaterializedRasterTile2D { - /// Copy `source` raster pixels into this raster, fails if the rasters do not overlap - #[allow(clippy::float_cmp)] - fn blit(&mut self, source: RasterTile2D) -> Result<()> { - // TODO: same crs - // TODO: allow approximately equal pixel sizes? - // TODO: ensure pixels are aligned - - let into_geo_transform = self.geo_transform(); - let from_geo_transform = source.geo_transform(); - - ensure!( - (self.geo_transform().x_pixel_size() == source.geo_transform().x_pixel_size()) - && (self.geo_transform().y_pixel_size() == source.geo_transform().y_pixel_size()), - error::Blit { - details: "Incompatible pixel size" - } - ); - - let offset = from_geo_transform.origin_coordinate - into_geo_transform.origin_coordinate; - - let offset_x_pixels = (offset.x / into_geo_transform.x_pixel_size()).round() as isize; - let offset_y_pixels = (offset.y / into_geo_transform.y_pixel_size()).round() as isize; - - /* - ensure!( - offset_x_pixels.abs() <= self.grid_array.axis_size_x() as isize - && offset_y_pixels.abs() <= self.grid_array.axis_size_y() as isize, - error::Blit { - details: "No overlapping region", - } - ); - */ - - let origin_offset_pixels = GridIdx2D::new([offset_y_pixels, offset_x_pixels]); - - let tile_offset_pixels = source.tile_information().global_upper_left_pixel_idx() - - self.tile_information().global_upper_left_pixel_idx(); - let global_offset_pixels = origin_offset_pixels + tile_offset_pixels; - - let shifted_source = source.grid_array.shift_by_offset(global_offset_pixels); - - self.grid_array.grid_blit_from(&shifted_source); - - self.cache_hint.merge_with(&source.cache_hint); - - Ok(()) - } -} - -impl Blit> for RasterTile2D { - /// Copy `source` raster pixels into this raster, fails if the rasters do not overlap - #[allow(clippy::float_cmp)] - fn blit(&mut self, source: RasterTile2D) -> Result<()> { - // TODO: same crs - // TODO: allow approximately equal pixel sizes? - // TODO: ensure pixels are aligned - - let into_geo_transform = self.geo_transform(); - let from_geo_transform = source.geo_transform(); - - ensure!( - (self.geo_transform().x_pixel_size() == source.geo_transform().x_pixel_size()) - && (self.geo_transform().y_pixel_size() == source.geo_transform().y_pixel_size()), - error::Blit { - details: "Incompatible pixel size" - } - ); - - let offset = from_geo_transform.origin_coordinate - into_geo_transform.origin_coordinate; - - let offset_x_pixels = (offset.x / into_geo_transform.x_pixel_size()).round() as isize; - let offset_y_pixels = (offset.y / into_geo_transform.y_pixel_size()).round() as isize; - - /* - ensure!( - offset_x_pixels.abs() <= self.grid_array.axis_size_x() as isize - && offset_y_pixels.abs() <= self.grid_array.axis_size_y() as isize, - error::Blit { - details: "No overlapping region", - } - ); - */ - - let origin_offset_pixels = GridIdx2D::new([offset_y_pixels, offset_x_pixels]); - - let tile_offset_pixels = source.tile_information().global_upper_left_pixel_idx() - - self.tile_information().global_upper_left_pixel_idx(); - let global_offset_pixels = origin_offset_pixels + tile_offset_pixels; - - let shifted_source = source.grid_array.shift_by_offset(global_offset_pixels); - - self.grid_array.grid_blit_from(&shifted_source); - - self.cache_hint.merge_with(&source.cache_hint); - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use crate::{ - primitives::{CacheHint, TimeInterval}, - raster::{Blit, GeoTransform, Grid2D, RasterTile2D}, - }; - - #[test] - fn test_blit_ur_materialized() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![0, 0, 8, 9, 0, 0, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn test_blit_ul() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![10, 11, 0, 0, 14, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn test_blit_ll() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert_eq!( - t1.grid_array.inner_grid.data, - vec![0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 0, 6, 7, 0, 0] - ); - } - - #[test] - fn test_blit_ur() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::default(), - ); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let t2 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r2, - CacheHint::default(), - ); - - t1.blit(t2).unwrap(); - - assert!(!t1.is_empty()); - - let masked_grid = match t1.grid_array { - crate::raster::GridOrEmpty::Grid(g) => g, - crate::raster::GridOrEmpty::Empty(_) => panic!("exppected a materialized grid"), - }; - - assert_eq!( - masked_grid.inner_grid.data, - vec![0, 0, 8, 9, 0, 0, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0] - ); - } - - #[test] - fn it_attaches_cache_hint() { - let dim = [4, 4]; - let data = vec![0; 16]; - let geo_transform = GeoTransform::new((0.0, 10.0).into(), 10.0 / 4.0, -10.0 / 4.0); - let temporal_bounds: TimeInterval = TimeInterval::default(); - - let r1 = Grid2D::new(dim.into(), data).unwrap(); - let mut t1 = RasterTile2D::new_without_offset( - temporal_bounds, - geo_transform, - r1, - CacheHint::max_duration(), - ) - .into_materialized_tile(); - - let data = vec![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]; - let geo_transform = GeoTransform::new((-5.0, 15.0).into(), 10.0 / 4.0, -10.0 / 4.0); - - let r2 = Grid2D::new(dim.into(), data).unwrap(); - let cache_hint = CacheHint::seconds(1234); - let t2 = RasterTile2D::new_without_offset(temporal_bounds, geo_transform, r2, cache_hint); - - t1.blit(t2).unwrap(); - - assert_eq!(t1.cache_hint.expires(), cache_hint.expires()); - } -} diff --git a/datatypes/src/raster/operations/grid_blit.rs b/datatypes/src/raster/operations/grid_blit.rs index f9cc8f8dc..e0b661326 100644 --- a/datatypes/src/raster/operations/grid_blit.rs +++ b/datatypes/src/raster/operations/grid_blit.rs @@ -1,7 +1,7 @@ use crate::raster::{ - BoundedGrid, Grid, Grid1D, Grid2D, Grid3D, GridBoundingBox, GridBounds, GridIdx, - GridIndexAccessMut, GridIntersection, GridOrEmpty, GridSize, GridSpaceToLinearSpace, - empty_grid::EmptyGrid, masked_grid::MaskedGrid, + BoundedGrid, Grid, Grid1D, Grid3D, GridBoundingBox, GridBounds, GridIdx, GridIndexAccessMut, + GridIntersection, GridOrEmpty, GridSize, GridSpaceToLinearSpace, empty_grid::EmptyGrid, + masked_grid::MaskedGrid, }; pub trait GridBlit @@ -37,11 +37,14 @@ where } } -impl GridBlit, T> for Grid2D +impl GridBlit, T> for Grid where D: GridSize + GridBounds + GridSpaceToLinearSpace, + D2: GridSize + + GridBounds + + GridSpaceToLinearSpace, T: Copy + Sized, { fn grid_blit_from(&mut self, other: &Grid) { @@ -145,11 +148,14 @@ where } } -impl GridBlit, T> for Grid2D +impl GridBlit, T> for Grid where D: GridSize + GridBounds + GridSpaceToLinearSpace, + D2: GridSize + + GridBounds + + GridSpaceToLinearSpace, T: Copy + Sized, { fn grid_blit_from(&mut self, other: &EmptyGrid) { diff --git a/datatypes/src/raster/operations/interpolation.rs b/datatypes/src/raster/operations/interpolation.rs index 768076995..afc30dfd6 100644 --- a/datatypes/src/raster/operations/interpolation.rs +++ b/datatypes/src/raster/operations/interpolation.rs @@ -1,70 +1,60 @@ use super::from_index_fn::FromIndexFnParallel; -use crate::primitives::{AxisAlignedRectangle, SpatialPartitioned}; use crate::raster::{ - EmptyGrid, GridIdx, GridIdx2D, GridIndexAccess, GridOrEmpty, Pixel, RasterTile2D, - TileInformation, + GeoTransform, GridBounds, GridIdx, GridIdx2D, GridIndexAccess, GridOrEmpty, GridShapeAccess, + GridSize, GridSpaceToLinearSpace, Pixel, }; use crate::util::Result; -pub trait InterpolationAlgorithm: Send + Sync + Clone + 'static { +pub trait InterpolationAlgorithm: Send + Sync + Clone + 'static { /// interpolate the given input tile into the output tile /// the output must be fully contained in the input tile and have an additional row and column in order /// to have all the required neighbor pixels. /// Also the output must have a finer resolution than the input fn interpolate( - input: &RasterTile2D

, - output_tile_info: &TileInformation, - ) -> Result>; + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result>; } #[derive(Clone, Debug)] pub struct NearestNeighbor {} -impl

InterpolationAlgorithm

for NearestNeighbor +impl InterpolationAlgorithm for NearestNeighbor where + D: GridShapeAccess + + Clone + + GridSize + + GridBounds + + PartialEq + + Send + + Sync + + GridSpaceToLinearSpace, P: Pixel, + GridOrEmpty: + GridIndexAccess, GridIdx<::IndexArray>>, { - fn interpolate(input: &RasterTile2D

, info_out: &TileInformation) -> Result> { + fn interpolate( + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result> { if input.is_empty() { - return Ok(RasterTile2D::new_with_tile_info( - input.time, - *info_out, - input.band, - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - input.cache_hint.clone_with_current_datetime(), - )); + return Ok(GridOrEmpty::new_empty_shape(out_bounds)); } - let info_in = input.tile_information(); - let in_upper_left = info_in.spatial_partition().upper_left(); - let in_x_size = info_in.global_geo_transform.x_pixel_size(); - let in_y_size = info_in.global_geo_transform.y_pixel_size(); - - let out_upper_left = info_out.spatial_partition().upper_left(); - let out_x_size = info_out.global_geo_transform.x_pixel_size(); - let out_y_size = info_out.global_geo_transform.y_pixel_size(); - let map_fn = |gidx: GridIdx2D| { - let GridIdx([y, x]) = gidx; - let out_y_coord = out_upper_left.y + y as f64 * out_y_size; - let out_x_coord = out_upper_left.x + x as f64 * out_x_size; - let nearest_in_y_idx = ((out_y_coord - in_upper_left.y) / in_y_size).round() as isize; - let nearest_in_x_idx = ((out_x_coord - in_upper_left.x) / in_x_size).round() as isize; - input.get_at_grid_index_unchecked([nearest_in_y_idx, nearest_in_x_idx]) - }; + let coordinate = out_geo_transform.grid_idx_to_pixel_center_coordinate_2d(gidx); // use center coordinate similar to ArgGIS + let pixel_in_input = in_geo_transform.coordinate_to_grid_idx_2d(coordinate); - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. + input.get_at_grid_index_unchecked(pixel_in_input) + }; - let out_tile = RasterTile2D::new( - input.time, - info_out.global_tile_position, - input.band, - info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), - ); + let out_data = GridOrEmpty::from_index_fn_parallel(&out_bounds, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. - Ok(out_tile) + Ok(out_data) } } @@ -99,57 +89,59 @@ impl Bilinear { } } -impl

InterpolationAlgorithm

for Bilinear +impl InterpolationAlgorithm for Bilinear where + D: GridShapeAccess + + Clone + + GridSize + + GridBounds + + PartialEq + + Send + + Sync + + GridSpaceToLinearSpace, P: Pixel, + GridOrEmpty: + GridIndexAccess, GridIdx<::IndexArray>>, { - fn interpolate(input: &RasterTile2D

, info_out: &TileInformation) -> Result> { + fn interpolate( + in_geo_transform: GeoTransform, + input: &GridOrEmpty, + out_geo_transform: GeoTransform, + out_bounds: D, + ) -> Result> { if input.is_empty() { - return Ok(RasterTile2D::new_with_tile_info( - input.time, - *info_out, - input.band, - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - input.cache_hint.clone_with_current_datetime(), - )); + return Ok(GridOrEmpty::new_empty_shape(out_bounds)); } - let info_in = input.tile_information(); - let in_upper_left = info_in.spatial_partition().upper_left(); - let in_x_size = info_in.global_geo_transform.x_pixel_size(); - let in_y_size = info_in.global_geo_transform.y_pixel_size(); - - let out_upper_left = info_out.spatial_partition().upper_left(); - let out_x_size = info_out.global_geo_transform.x_pixel_size(); - let out_y_size = info_out.global_geo_transform.y_pixel_size(); + let map_fn = |out_g_idx: GridIdx2D| { + let out_coord = out_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(out_g_idx); - let map_fn = |g_idx: GridIdx2D| { - let GridIdx([y_idx, x_idx]) = g_idx; + let in_g_idx = in_geo_transform.coordinate_to_grid_idx_2d(out_coord); - let out_y = out_upper_left.y + y_idx as f64 * out_y_size; - let in_y_idx = ((out_y - in_upper_left.y) / in_y_size).floor() as isize; + let in_a_idx = in_g_idx; + let in_b_idx = in_a_idx + [1, 0]; + let in_c_idx = in_a_idx + [0, 1]; + let in_d_idx = in_a_idx + [1, 1]; - let a_y = in_upper_left.y + in_y_size * in_y_idx as f64; - let b_y = a_y + in_y_size; + let in_a_coord = in_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(in_a_idx); + let a_y = in_a_coord.y; + let b_y = a_y + in_geo_transform.y_pixel_size(); - let out_x = out_upper_left.x + x_idx as f64 * out_x_size; - let in_x_idx = ((out_x - in_upper_left.x) / in_x_size).floor() as isize; + let a_x = in_a_coord.x; + let c_x = a_x + in_geo_transform.x_pixel_size(); - let a_x = in_upper_left.x + in_x_size * in_x_idx as f64; - let c_x = a_x + in_x_size; + let a_v = input.get_at_grid_index(in_a_idx).unwrap_or(None); - let a_v = input.get_at_grid_index_unchecked([in_y_idx, in_x_idx]); + let b_v = input.get_at_grid_index(in_b_idx).unwrap_or(None); - let b_v = input.get_at_grid_index_unchecked([in_y_idx + 1, in_x_idx]); + let c_v = input.get_at_grid_index(in_c_idx).unwrap_or(None); - let c_v = input.get_at_grid_index_unchecked([in_y_idx, in_x_idx + 1]); - - let d_v = input.get_at_grid_index_unchecked([in_y_idx + 1, in_x_idx + 1]); + let d_v = input.get_at_grid_index(in_d_idx).unwrap_or(None); let value = match (a_v, b_v, c_v, d_v) { (Some(a), Some(b), Some(c), Some(d)) => Some(Self::bilinear_interpolation( - out_x, - out_y, + out_coord.x, + out_coord.y, a_x, a_y, a.as_(), @@ -165,18 +157,9 @@ where value.map(|v| P::from_(v)) }; - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); // TODO: this will check for empty tiles. Change to MaskedGrid::from.. to avoid this. + let out_data = GridOrEmpty::from_index_fn_parallel(&out_bounds, map_fn); - let out_tile = RasterTile2D::new( - input.time, - info_out.global_tile_position, - input.band, - info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), - ); - - Ok(out_tile) + Ok(out_data) } } @@ -187,7 +170,10 @@ mod tests { use super::*; use crate::{ primitives::CacheHint, - raster::{GeoTransform, Grid2D, GridOrEmpty, MaskedGrid, RasterTile2D, TileInformation}, + raster::{ + GeoTransform, GeoTransformAccess, Grid2D, GridOrEmpty, MaskedGrid, RasterTile2D, + TileInformation, + }, }; #[test] @@ -206,42 +192,48 @@ mod tests { CacheHint::default(), ); + let input_geo_transform = input.geo_transform(); + let input_grid = input.into_inner_positioned_grid(); + let output_info = TileInformation { global_tile_position: [0, 0].into(), - tile_size_in_pixels: [4, 4].into(), + tile_size_in_pixels: [3, 3].into(), global_geo_transform: GeoTransform::new((0.0, 2.0).into(), 0.5, -0.5), }; + let output_geo_transform = output_info.global_geo_transform; + let output_bounds = output_info.global_pixel_bounds(); + let pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); let output = pool - .install(|| NearestNeighbor::interpolate(&input, &output_info)) + .install(|| { + NearestNeighbor::interpolate( + input_geo_transform, + &input_grid, + output_geo_transform, + output_bounds, + ) + }) .unwrap(); assert!(!output.is_empty()); - let output_data = output.grid_array.as_masked_grid().unwrap(); + let output_data = output.as_masked_grid().unwrap(); assert_eq!( output_data .masked_element_deref_iterator() .collect::>(), vec![ + Some(1), Some(1), Some(2), + Some(1), + Some(1), Some(2), - Some(3), Some(4), - Some(5), - Some(5), - Some(6), Some(4), - Some(5), - Some(5), - Some(6), - Some(7), - Some(8), - Some(8), - Some(9) + Some(5) ] ); } @@ -285,20 +277,33 @@ mod tests { CacheHint::default(), ); + let input_geo_transform = input.geo_transform(); + let input_grid = input.into_inner_positioned_grid(); + let output_info = TileInformation { global_tile_position: [0, 0].into(), tile_size_in_pixels: [4, 4].into(), global_geo_transform: GeoTransform::new((0.0, 2.0).into(), 0.5, -0.5), }; + let output_geo_transform = output_info.global_geo_transform; + let output_bounds = output_info.global_pixel_bounds(); + let pool = ThreadPoolBuilder::new().num_threads(0).build().unwrap(); let output = pool - .install(|| Bilinear::interpolate(&input, &output_info)) + .install(|| { + Bilinear::interpolate( + input_geo_transform, + &input_grid, + output_geo_transform, + output_bounds, + ) + }) .unwrap(); assert!(!output.is_empty()); - let output_data = output.grid_array.as_masked_grid().unwrap(); + let output_data = output.as_masked_grid().unwrap(); assert_eq!( output_data diff --git a/datatypes/src/raster/operations/mod.rs b/datatypes/src/raster/operations/mod.rs index 3e05af7b5..f67130748 100644 --- a/datatypes/src/raster/operations/mod.rs +++ b/datatypes/src/raster/operations/mod.rs @@ -6,5 +6,6 @@ pub mod grid_blit; pub mod interpolation; pub mod map_elements; pub mod map_indexed_elements; +pub mod sample_points; pub mod update_elements; pub mod update_indexed_elements; diff --git a/datatypes/src/raster/operations/sample_points.rs b/datatypes/src/raster/operations/sample_points.rs new file mode 100644 index 000000000..7e35f1eff --- /dev/null +++ b/datatypes/src/raster/operations/sample_points.rs @@ -0,0 +1,256 @@ +use num::range_inclusive; + +use crate::{ + primitives::Coordinate2D, + raster::{GridBoundingBox2D, GridIdx2D, GridSize, SpatialGridDefinition}, +}; + +/// A trait that allows to generate sample points for rectangles (and polygons?). +/// The `Coord` type allows to specify the output type e.g. pixel coordinates or geo coordinates. +/// It is used mainly for the reprojection operation with pixel and geo coordinates. +pub trait SamplePoints { + type Coord; + + fn sample_outline(&self, step: usize) -> Vec; + fn sample_cross(&self, step: usize) -> Vec; + fn sample_diagonals(&self, step: usize) -> Vec; +} + +impl SamplePoints for GridBoundingBox2D { + type Coord = GridIdx2D; + + fn sample_outline(&self, step: usize) -> Vec { + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) * 2 + (self.axis_size_y() / step) * 2; + + let mut collected: Vec = Vec::with_capacity(capacity); + + for x in x_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y_min, x)); + collected.push(GridIdx2D::new_y_x(y_max, x)); + } + + for y in y_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y, x_min)); + collected.push(GridIdx2D::new_y_x(y, x_max)); + } + + collected + } + + fn sample_cross(&self, step: usize) -> Vec { + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + let y_mid = y_min + (self.axis_size_y() / 2) as isize; + let x_mid = x_min + (self.axis_size_x() / 2) as isize; + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) + (self.axis_size_y() / step); + + let mut collected: Vec = Vec::with_capacity(capacity); + + for x in x_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y_mid, x)); + } + + for y in y_range.step_by(step) { + collected.push(GridIdx2D::new_y_x(y, x_mid)); + } + + collected + } + + fn sample_diagonals(&self, step: usize) -> Vec { + enum LongAxis { + X, + Y, + } + + let [y_min, y_max] = self.y_bounds(); + let [x_min, x_max] = self.x_bounds(); + + let x_range = range_inclusive(x_min, x_max); + let y_range = range_inclusive(y_min, y_max); + + let capacity = (self.axis_size_x() / step) * 2 + (self.axis_size_y() / step) * 2; + + let (long_range_id, long_range, b, b_max, m) = if self.axis_size_x() > self.axis_size_y() { + ( + LongAxis::X, + x_range, + y_min, + y_max, + (self.axis_size_y() as f32 / self.axis_size_x() as f32), + ) + } else { + ( + LongAxis::Y, + y_range, + x_min, + x_max, + (self.axis_size_x() as f32 / self.axis_size_y() as f32), + ) + }; + + let mut collected: Vec = Vec::with_capacity(capacity); + + for l in long_range { + let s = (l as f32 * m) as isize + b; + let s_inv = b_max - (l as f32 * m) as isize; + + match long_range_id { + LongAxis::X => { + debug_assert!(l >= x_min); + debug_assert!(l <= x_max); + debug_assert!(s >= y_min); + debug_assert!(s <= y_max); + collected.push(GridIdx2D::new_y_x(s, l)); + collected.push(GridIdx2D::new_y_x(s_inv, l)); + } + LongAxis::Y => { + debug_assert!(s >= x_min); + debug_assert!(s <= x_max); + debug_assert!(l >= y_min); + debug_assert!(l <= y_max); + collected.push(GridIdx2D::new_y_x(l, s)); + collected.push(GridIdx2D::new_y_x(l, s_inv)); + } + } + } + + collected + } +} + +impl SamplePoints for SpatialGridDefinition { + type Coord = Coordinate2D; + + fn sample_outline(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_outline(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } + + fn sample_cross(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_cross(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } + + fn sample_diagonals(&self, step: usize) -> Vec { + let px = self.grid_bounds.sample_diagonals(step); + px.iter() + .map(|gidx| { + self.geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(*gidx) + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_sample_outline() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_outline(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(4, 0), + GridIdx2D::new_y_x(0, 1), + GridIdx2D::new_y_x(4, 1), + GridIdx2D::new_y_x(0, 2), + GridIdx2D::new_y_x(4, 2), + GridIdx2D::new_y_x(0, 3), + GridIdx2D::new_y_x(4, 3), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(4, 4), + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(1, 0), + GridIdx2D::new_y_x(1, 4), + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(3, 0), + GridIdx2D::new_y_x(3, 4), + GridIdx2D::new_y_x(4, 0), + GridIdx2D::new_y_x(4, 4), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_cross() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_cross(1); + let exp = vec![ + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(2, 1), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(2, 3), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(0, 2), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(3, 2), + GridIdx2D::new_y_x(4, 2), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_diagnals() { + let gb = GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(); + let ds = gb.sample_diagonals(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(0, 4), + GridIdx2D::new_y_x(1, 1), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(2, 2), + GridIdx2D::new_y_x(3, 3), + GridIdx2D::new_y_x(3, 1), + GridIdx2D::new_y_x(4, 4), + GridIdx2D::new_y_x(4, 0), + ]; + assert_eq!(ds, exp); + } + + #[test] + fn test_sample_diagnals_non_symetric() { + let gb = GridBoundingBox2D::new_min_max(0, 2, 0, 4).unwrap(); + let ds = gb.sample_diagonals(1); + let exp = vec![ + GridIdx2D::new_y_x(0, 0), + GridIdx2D::new_y_x(2, 0), + GridIdx2D::new_y_x(0, 1), + GridIdx2D::new_y_x(2, 1), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(1, 2), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(1, 3), + GridIdx2D::new_y_x(2, 4), + GridIdx2D::new_y_x(0, 4), + ]; + assert_eq!(ds, exp); + } +} diff --git a/datatypes/src/raster/raster_tile.rs b/datatypes/src/raster/raster_tile.rs index 266a64669..3f021bd4c 100644 --- a/datatypes/src/raster/raster_tile.rs +++ b/datatypes/src/raster/raster_tile.rs @@ -1,10 +1,13 @@ use super::masked_grid::MaskedGrid; +use super::{ + BoundedGrid, ChangeGridBounds, GridBoundingBox2D, GridIndexAccessMut, RasterProperties, + SpatialGridDefinition, +}; use super::{ GeoTransform, GeoTransformAccess, GridBounds, GridIdx2D, GridIndexAccess, GridShape, GridShape2D, GridShape3D, GridShapeAccess, GridSize, Raster, TileInformation, grid_or_empty::GridOrEmpty, }; -use super::{GridIndexAccessMut, RasterProperties}; use crate::primitives::CacheHint; use crate::primitives::{ SpatialBounded, SpatialPartition2D, SpatialPartitioned, SpatialResolution, TemporalBounded, @@ -12,6 +15,7 @@ use crate::primitives::{ }; use crate::raster::Pixel; use crate::util::{ByteSize, Result}; +use float_cmp::approx_eq; use serde::{Deserialize, Serialize}; use std::fmt::Write; @@ -68,6 +72,26 @@ where ) } + pub fn global_pixel_spatial_grid_definition(&self) -> SpatialGridDefinition { + let global_upper_left_idx = self.tile_position + * [ + self.grid_array.axis_size_y() as isize, + self.grid_array.axis_size_x() as isize, + ]; + + SpatialGridDefinition::new( + self.global_geo_transform, + GridBoundingBox2D::new_unchecked( + global_upper_left_idx, + global_upper_left_idx + + [ + self.grid_array.axis_size_y() as isize, + self.grid_array.axis_size_x() as isize, + ], + ), + ) + } + /// Use this geo transform to transform `Coordinate2D` into local grid indices and vice versa. #[inline] pub fn tile_geo_transform(&self) -> GeoTransform { @@ -142,21 +166,35 @@ impl IterableBaseTile for [BaseTile; N] { } } -impl> TilesEqualIgnoringCacheHint for I { +impl> TilesEqualIgnoringCacheHint for I +where + G: GridSize, +{ fn tiles_equal_ignoring_cache_hint(&self, other: &dyn IterableBaseTile) -> bool { let mut iter_self = self.iter_tiles(); let mut iter_other = other.iter_tiles(); - loop { match (iter_self.next(), iter_other.next()) { (Some(a), Some(b)) => { - if a.time != b.time - || a.tile_position != b.tile_position - || a.band != b.band - || a.global_geo_transform != b.global_geo_transform - || a.grid_array != b.grid_array - || a.properties != b.properties - { + if a.time != b.time { + return false; + } + if a.tile_position != b.tile_position { + return false; + } + if a.band != b.band { + return false; + } + if !approx_eq!(GeoTransform, a.global_geo_transform, b.global_geo_transform) { + return false; + } + if a.global_geo_transform != b.global_geo_transform { + return false; + } + if a.properties != b.properties { + return false; + } + if a.grid_array != b.grid_array { return false; } } @@ -340,6 +378,43 @@ where } } +impl RasterTile2D +where + T: Pixel, +{ + /// Converts the tile into a grid with the global pixel bounds of the tile. + /// + /// # Panics + /// Only if the tile was invalid before... + /// + pub fn into_inner_positioned_grid(self) -> GridOrEmpty { + let b = self.bounding_box(); + let g = self.grid_array; + g.set_grid_bounds(b).expect("tile was valid before") + } +} + +impl BoundedGrid for RasterTile2D +where + T: Pixel, +{ + type IndexArray = [isize; 2]; + + fn bounding_box(&self) -> GridBoundingBox2D { + let shape = self.grid_array.shape_ref(); + let offset = + self.tile_position * [shape.axis_size_y() as isize, shape.axis_size_x() as isize]; + GridBoundingBox2D::new_unchecked( + offset, + offset + + [ + shape.axis_size_y() as isize - 1, + shape.axis_size_x() as isize - 1, + ], + ) + } +} + impl TemporalBounded for BaseTile { fn temporal_bounds(&self) -> TimeInterval { self.time diff --git a/datatypes/src/raster/tiling.rs b/datatypes/src/raster/tiling.rs index 71663429d..be66439ae 100644 --- a/datatypes/src/raster/tiling.rs +++ b/datatypes/src/raster/tiling.rs @@ -1,35 +1,30 @@ -use crate::{ - primitives::{AxisAlignedRectangle, Coordinate2D, SpatialPartition2D, SpatialPartitioned}, - util::test::TestDefault, -}; - use super::{ GeoTransform, GridBoundingBox2D, GridIdx, GridIdx2D, GridShape2D, GridShapeAccess, GridSize, + SpatialGridDefinition, +}; +use crate::{ + primitives::{Coordinate2D, SpatialPartition2D, SpatialPartitioned}, + raster::{GridBounds, GridIdx2DIter}, + util::test::TestDefault, }; - use serde::{Deserialize, Serialize}; -/// The static parameters of a `TilingStrategy` +/// The static parameters required to create a `TilingStrategy` #[derive(Debug, Serialize, Deserialize, Clone, Copy)] pub struct TilingSpecification { - pub origin_coordinate: Coordinate2D, pub tile_size_in_pixels: GridShape2D, } impl TilingSpecification { - pub fn new(origin_coordinate: Coordinate2D, tile_size_in_pixels: GridShape2D) -> Self { + pub fn new(tile_size_in_pixels: GridShape2D) -> Self { Self { - origin_coordinate, tile_size_in_pixels, } } - /// create a `TilingStrategy` from self and pixel sizes - pub fn strategy(self, x_pixel_size: f64, y_pixel_size: f64) -> TilingStrategy { - debug_assert!(x_pixel_size > 0.0); - debug_assert!(y_pixel_size < 0.0); - - TilingStrategy::new_with_tiling_spec(self, x_pixel_size, y_pixel_size) + #[allow(clippy::unused_self)] + pub fn tiling_origin_reference(&self) -> Coordinate2D { + Coordinate2D::new(0., 0.) } } @@ -45,10 +40,15 @@ impl GridShapeAccess for TilingSpecification { } } +impl From for GridShape2D { + fn from(val: TilingSpecification) -> Self { + val.tile_size_in_pixels + } +} + impl TestDefault for TilingSpecification { fn test_default() -> Self { Self { - origin_coordinate: Coordinate2D::new(0., 0.), tile_size_in_pixels: GridShape2D::new([512, 512]), } } @@ -69,26 +69,13 @@ impl TilingStrategy { } } - pub fn new_with_tiling_spec( - tiling_specification: TilingSpecification, - x_pixel_size: f64, - y_pixel_size: f64, - ) -> Self { - Self { - tile_size_in_pixels: tiling_specification.tile_size_in_pixels, - geo_transform: GeoTransform::new( - tiling_specification.origin_coordinate, - x_pixel_size, - y_pixel_size, - ), - } - } - pub fn pixel_idx_to_tile_idx(&self, pixel_idx: GridIdx2D) -> GridIdx2D { let GridIdx([y_pixel_idx, x_pixel_idx]) = pixel_idx; let [y_tile_size, x_tile_size] = self.tile_size_in_pixels.into_inner(); - let y_tile_idx = (y_pixel_idx as f64 / y_tile_size as f64).floor() as isize; - let x_tile_idx = (x_pixel_idx as f64 / x_tile_size as f64).floor() as isize; + //let y_tile_idx = (y_pixel_idx as f64 / y_tile_size as f64).floor() as isize; + //let x_tile_idx = (x_pixel_idx as f64 / x_tile_size as f64).floor() as isize; + let y_tile_idx = num::integer::div_floor(y_pixel_idx, y_tile_size as isize); + let x_tile_idx = num::integer::div_floor(x_pixel_idx, x_tile_size as isize); [y_tile_idx, x_tile_idx].into() } @@ -98,45 +85,71 @@ impl TilingStrategy { GridBoundingBox2D::new_unchecked(start, end) } - pub fn num_tiles_intersecting(&self, partition: SpatialPartition2D) -> usize { - let GridIdx([upper_left_tile_y, upper_left_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.upper_left_pixel_idx(&partition)); + pub fn num_tiles_intersecting_partition(&self, partition: SpatialPartition2D) -> usize { + let grid_bounds = self.geo_transform.spatial_to_grid_bounds(&partition); + self.num_tiles_intersecting_grid_bounds(grid_bounds) + } + + pub fn num_tiles_intersecting_grid_bounds(&self, grid_bounds: GridBoundingBox2D) -> usize { + let tile_bounds = self.global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds); - let GridIdx([lower_right_tile_y, lower_right_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.lower_right_pixel_idx(&partition)); + let GridIdx([upper_left_tile_y, upper_left_tile_x]) = tile_bounds.min_index(); + let GridIdx([lower_right_tile_y, lower_right_tile_x]) = tile_bounds.max_index(); (((lower_right_tile_y - upper_left_tile_y) + 1) * ((lower_right_tile_x - upper_left_tile_x) + 1)) as usize } - /// generates the tile idx in \[z,y,x\] order for the tiles intersecting the bounding box - /// the iterator moves once along the x-axis and then increases the y-axis - pub fn tile_idx_iterator( + pub fn global_pixel_grid_bounds_to_tile_grid_bounds( &self, - partition: SpatialPartition2D, - ) -> impl Iterator + use<> { - let GridIdx([upper_left_tile_y, upper_left_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.upper_left_pixel_idx(&partition)); + global_pixel_grid_bounds: GridBoundingBox2D, + ) -> GridBoundingBox2D { + let start = self.pixel_idx_to_tile_idx(global_pixel_grid_bounds.min_index()); + let end = self.pixel_idx_to_tile_idx(global_pixel_grid_bounds.max_index()); + GridBoundingBox2D::new_unchecked(start, end) + } + + /// Transforms a tile position into a global pixel position + pub fn tile_idx_to_global_pixel_idx(&self, tile_idx: GridIdx2D) -> GridIdx2D { + let GridIdx([y_tile_idx, x_tile_idx]) = tile_idx; + GridIdx::new([ + y_tile_idx * self.tile_size_in_pixels.axis_size_y() as isize, + x_tile_idx * self.tile_size_in_pixels.axis_size_x() as isize, + ]) + } - let GridIdx([lower_right_tile_y, lower_right_tile_x]) = - self.pixel_idx_to_tile_idx(self.geo_transform.lower_right_pixel_idx(&partition)); + /// Returns the tile grid bounds for the given `raster_spatial_query`. + /// The query must match the tiling strategy's geo transform for now. + /// + /// # Panics + /// If the query's geo transform does not match the tiling strategy's geo transform. + /// + pub fn raster_spatial_query_to_tiling_grid_box( + &self, + raster_spatial_query: GridBoundingBox2D, + ) -> GridBoundingBox2D { + self.global_pixel_grid_bounds_to_tile_grid_bounds(raster_spatial_query) + } - let y_range = upper_left_tile_y..=lower_right_tile_y; - let x_range = upper_left_tile_x..=lower_right_tile_x; + /// Returns an iterator over all tile indices that intersect with the given `grid_bounds`. + pub fn tile_idx_iterator_from_grid_bounds( + // TODO: indicate that this uses pixel bounds! + &self, + grid_bounds: GridBoundingBox2D, + ) -> GridIdx2DIter { + let tile_bounds = self.global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds); - y_range.flat_map(move |y_tile| x_range.clone().map(move |x_tile| [y_tile, x_tile].into())) + GridIdx2DIter::new(&tile_bounds) } /// generates the tile information for the tiles intersecting the bounding box /// the iterator moves once along the x-axis and then increases the y-axis - pub fn tile_information_iterator( + pub fn tile_information_iterator_from_pixel_bounds( + // TODO: indicate that this uses pixel bounds! &self, - partition: SpatialPartition2D, - ) -> impl Iterator + use<> { - let tile_pixel_size = self.tile_size_in_pixels; - let geo_transform = self.geo_transform; - self.tile_idx_iterator(partition) - .map(move |idx| TileInformation::new(idx, tile_pixel_size, geo_transform)) + grid_bounds: GridBoundingBox2D, + ) -> TileInformationIter { + TileInformationIter::new_with_pixel_bounds(*self, &grid_bounds) } } @@ -161,18 +174,6 @@ impl TileInformation { } } - pub fn with_partition_and_shape(partition: SpatialPartition2D, shape: GridShape2D) -> Self { - Self { - tile_size_in_pixels: shape, - global_tile_position: [0, 0].into(), - global_geo_transform: GeoTransform::new( - partition.upper_left(), - partition.size_x() / shape.axis_size_x() as f64, - -partition.size_y() / shape.axis_size_y() as f64, - ), - } - } - #[allow(clippy::unused_self)] pub fn local_upper_left_pixel_idx(&self) -> GridIdx2D { [0, 0].into() @@ -213,6 +214,13 @@ impl TileInformation { self.global_upper_left_pixel_idx() + self.local_lower_left_pixel_idx() } + pub fn global_pixel_bounds(&self) -> GridBoundingBox2D { + GridBoundingBox2D::new_unchecked( + self.global_upper_left_pixel_idx(), + self.global_lower_right_pixel_idx(), + ) + } + pub fn tile_size_in_pixels(&self) -> GridShape2D { self.tile_size_in_pixels } @@ -232,6 +240,14 @@ impl TileInformation { self.global_geo_transform.y_pixel_size(), ) } + + pub fn spatial_grid_definition(&self) -> SpatialGridDefinition { + SpatialGridDefinition::new(self.global_geo_transform, self.global_pixel_bounds()) + } + + pub fn tiling_strategy(&self) -> TilingStrategy { + TilingStrategy::new(self.tile_size_in_pixels, self.global_geo_transform) + } } impl SpatialPartitioned for TileInformation { @@ -246,96 +262,189 @@ impl SpatialPartitioned for TileInformation { } } +#[derive(Clone, Debug, Serialize)] +pub struct TileInformationIter { + tile_idx_iter: GridIdx2DIter, + tiling_strategy: TilingStrategy, +} + +impl TileInformationIter { + pub fn new_with_pixel_bounds( + tiling_strategy: TilingStrategy, + pixel_bounds: &GridBoundingBox2D, + ) -> Self { + let tile_idx_iter = tiling_strategy.tile_idx_iterator_from_grid_bounds(*pixel_bounds); + + Self { + tile_idx_iter, + tiling_strategy, + } + } + + /// Access the used `TilingStategy`. + pub fn tiling_strategy(&self) -> TilingStrategy { + self.tiling_strategy + } + + pub fn reset(&mut self) { + self.tile_idx_iter.reset(); + } +} + +impl Iterator for TileInformationIter { + type Item = TileInformation; + + fn next(&mut self) -> Option { + self.tile_idx_iter.next().map(|idx| { + TileInformation::new( + idx, + self.tiling_strategy.tile_size_in_pixels, + self.tiling_strategy.geo_transform, + ) + }) + } + + fn size_hint(&self) -> (usize, Option) { + self.tile_idx_iter.size_hint() + } +} + #[cfg(test)] mod tests { use super::*; + use crate::raster::GridIntersection; #[test] fn it_generates_only_intersected_tiles() { + let origin_coordinate = (0., 0.).into(); + + let geo_transform = GeoTransform::new( + origin_coordinate, + 2.095_475_792_884_826_7E-8, + -2.095_475_792_884_826_7E-8, + ); + let strat = TilingStrategy { tile_size_in_pixels: [600, 600].into(), - geo_transform: GeoTransform::new( - (0., 0.).into(), - 2.095_475_792_884_826_7E-8, - -2.095_475_792_884_826_7E-8, - ), + geo_transform, }; - let partition = SpatialPartition2D::new( - (12.477_738_261_222_84, 43.881_293_535_232_544).into(), - (12.477_743_625_640_87, 43.881_288_170_814_514).into(), - ) - .unwrap(); + let ul_idx = strat + .geo_transform + .coordinate_to_grid_idx_2d((12.477_738_261_222_84, 43.881_293_535_232_544).into()); + + let lr_idx = strat + .geo_transform + .coordinate_to_grid_idx_2d((12.477_743_625_640_87, 43.881_288_170_814_514).into()); + + let grid_bounds = GridBoundingBox2D::new_unchecked(ul_idx, lr_idx); let tiles = strat - .tile_information_iterator(partition) + .tile_information_iterator_from_pixel_bounds(grid_bounds) .collect::>(); assert_eq!(tiles.len(), 2); for tile in tiles { - assert!(partition.intersects(&tile.spatial_partition())); + assert!(grid_bounds.intersects(&tile.global_pixel_bounds())); } } #[test] - fn num_tiles_intersecting_insid_esingle() { + fn it_generates_all_interesected_tiles() { let strat = TilingStrategy { tile_size_in_pixels: [512, 512].into(), - geo_transform: GeoTransform::new((0., 0.).into(), 10.0, -10.0), + geo_transform: GeoTransform::new((0., -0.).into(), 10., -10.), }; - let partition = SpatialPartition2D::new( - Coordinate2D { - x: 0. + 70., // add / remove 7 pixels to not be at a tile border - y: 5120. - 70., - }, - Coordinate2D { - x: 5120. - 70., - y: 0. + 70., - }, - ) - .unwrap(); + let bounds = + GridBoundingBox2D::new(GridIdx2D::new([-513, -513]), GridIdx2D::new([512, 512])) + .unwrap(); - assert_eq!(strat.num_tiles_intersecting(partition), 1); + let tiles_idxs = strat + .tile_idx_iterator_from_grid_bounds(bounds) + .collect::>(); + + assert_eq!(tiles_idxs.len(), 4 * 4); + assert_eq!(tiles_idxs[0], [-2, -2].into()); + assert_eq!(tiles_idxs[1], [-2, -1].into()); + assert_eq!(tiles_idxs[14], [1, 0].into()); + assert_eq!(tiles_idxs[15], [1, 1].into()); } #[test] - fn num_tiles_intersecting_border() { - let strat = TilingStrategy { - tile_size_in_pixels: [512, 512].into(), - geo_transform: GeoTransform::new((0., 0.).into(), 10.0, -10.0), - }; - - let partition = SpatialPartition2D::new( - Coordinate2D { x: 0., y: 51200. }, - Coordinate2D { x: 51200., y: 0. }, - ) - .unwrap(); - - assert_eq!(strat.num_tiles_intersecting(partition), 10 * 10); + fn tiling_tile_tile() { + let geo_transform = GeoTransform::new( + (-1_234_567_890., 1_234_567_890.).into(), + 0.000_033_337_4, + -0.000_033_337_4, + ); + + let tile_pixel_size = GridShape2D::new_2d(512, 512); + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + + let tiling_origin_reference = Coordinate2D::new(0., 0.); // This is the _currently_ fixed tiling origin reference. + let nearest_to_tiling_origin = geo_transform.nearest_pixel_edge(tiling_origin_reference); + + let tile_idx = tiling_strat.pixel_idx_to_tile_idx(nearest_to_tiling_origin); + let expected_near_tiling_origin_idx = GridIdx::new([72_329_138_149, 72_329_138_149]); + assert_eq!(tile_idx, expected_near_tiling_origin_idx); + + let pixel_distance_reverse = nearest_to_tiling_origin * -1; + + let origin_pixel_tile = tiling_strat.pixel_idx_to_tile_idx(pixel_distance_reverse); + let origin_pixel_offset = + tiling_strat.tile_idx_to_global_pixel_idx(origin_pixel_tile) - pixel_distance_reverse; + + let expected_origin_in_tiling_based_pixels = + GridIdx::new([-72_329_138_150, -72_329_138_150]); + let expected_tile_offset_from_tiling = GridIdx::new([-85, -85]); + assert_eq!(origin_pixel_tile, expected_origin_in_tiling_based_pixels); + assert_eq!(origin_pixel_offset, expected_tile_offset_from_tiling); } #[test] - fn num_tiles_intersecting_inside() { - let strat = TilingStrategy { - tile_size_in_pixels: [512, 512].into(), - geo_transform: GeoTransform::new((0., 0.).into(), 10.0, -10.0), - }; - - let partition = SpatialPartition2D::new( - Coordinate2D { - x: 0. + 70., // add / remove 7 pixels to not be at a tile border - y: 51200. - 70., - }, - Coordinate2D { - x: 51200. - 70., - y: 0. + 70., - }, - ) - .unwrap(); + fn pixel_idx_to_tile_idx() { + let geo_transform = GeoTransform::new((123., 321.).into(), 1.0, -1.0); + let tile_pixel_size = GridShape2D::new_2d(100, 100); + + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(0, 0)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(1, 1)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(57, 57)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(100, 100)); + assert_eq!(GridIdx2D::new_y_x(1, 1), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(200, 200)); + assert_eq!(GridIdx2D::new_y_x(2, 2), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(1000, 1000)); + assert_eq!(GridIdx2D::new_y_x(10, 10), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(-57, -57)); + assert_eq!(GridIdx2D::new_y_x(-1, -1), pixels); + let pixels = tiling_strat.pixel_idx_to_tile_idx(GridIdx2D::new_y_x(-300, -300)); + assert_eq!(GridIdx2D::new_y_x(-3, -3), pixels); + } - assert_eq!(strat.num_tiles_intersecting(partition), 10 * 10); + #[test] + fn tile_idx_to_pixel_idx() { + let geo_transform = GeoTransform::new((123., 321.).into(), 1.0, -1.0); + let tile_pixel_size = GridShape2D::new_2d(100, 100); + + let tiling_strat = TilingStrategy::new(tile_pixel_size, geo_transform); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(0, 0)); + assert_eq!(GridIdx2D::new_y_x(0, 0), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(1, 1)); + assert_eq!(GridIdx2D::new_y_x(100, 100), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(2, 2)); + assert_eq!(GridIdx2D::new_y_x(200, 200), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(3, 3)); + assert_eq!(GridIdx2D::new_y_x(300, 300), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(10, 10)); + assert_eq!(GridIdx2D::new_y_x(1000, 1000), pixels); + let pixels = tiling_strat.tile_idx_to_global_pixel_idx(GridIdx2D::new_y_x(-3, -3)); + assert_eq!(GridIdx2D::new_y_x(-300, -300), pixels); } } diff --git a/datatypes/src/spatial_reference.rs b/datatypes/src/spatial_reference.rs index d7d541992..6092d1344 100644 --- a/datatypes/src/spatial_reference.rs +++ b/datatypes/src/spatial_reference.rs @@ -260,6 +260,10 @@ impl SpatialReferenceOption { pub fn is_unreferenced(self) -> bool { !self.is_spatial_ref() } + + pub fn as_option(self) -> Option { + self.into() + } } impl ToSql for SpatialReferenceOption { diff --git a/datatypes/src/util/gdal.rs b/datatypes/src/util/gdal.rs index fc5fdb1d4..3066972b6 100644 --- a/datatypes/src/util/gdal.rs +++ b/datatypes/src/util/gdal.rs @@ -1,10 +1,33 @@ +use gdal::{Dataset, DatasetOptions}; use serde::{Deserialize, Serialize}; -use std::fmt::Display; +use snafu::ResultExt; +use std::{fmt::Display, path::Path}; + +use crate::error; +use crate::util::Result; pub fn hide_gdal_errors() { gdal::config::set_error_handler(|_, _, _| {}); } +/// Opens a Gdal Dataset with the given `path`. +/// Other crates should use this method for Gdal Dataset access as a workaround to avoid strange errors. +pub fn gdal_open_dataset(path: &Path) -> Result { + gdal_open_dataset_ex(path, DatasetOptions::default()) +} + +/// Opens a Gdal Dataset with the given `path` and `dataset_options`. +/// Other crates should use this method for Gdal Dataset access as a workaround to avoid strange errors. +pub fn gdal_open_dataset_ex(path: &Path, dataset_options: DatasetOptions) -> Result { + let dataset_options = { + let mut dataset_options = dataset_options; + dataset_options.open_flags |= gdal::GdalOpenFlags::GDAL_OF_VERBOSE_ERROR; + dataset_options + }; + + Dataset::open_ex(path, dataset_options).context(error::Gdal) +} + // TODO: push to `rust-gdal` #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] diff --git a/datatypes/src/util/ranges.rs b/datatypes/src/util/ranges.rs index dbe43cf56..0236c67ec 100644 --- a/datatypes/src/util/ranges.rs +++ b/datatypes/src/util/ranges.rs @@ -4,14 +4,14 @@ pub fn value_in_range(value: T, min: T, max: T) -> bool where T: PartialOrd + Copy, { - (value >= min) && (value < max) + (min..max).contains(&value) } pub fn value_in_range_inclusive(value: T, min: T, max: T) -> bool where T: PartialOrd + Copy, { - (value >= min) && (value <= max) + (min..=max).contains(&value) } pub fn value_in_range_inv(value: T, min: T, max: T) -> bool diff --git a/datatypes/src/util/test.rs b/datatypes/src/util/test.rs index 837a42e03..963fc9619 100644 --- a/datatypes/src/util/test.rs +++ b/datatypes/src/util/test.rs @@ -1,4 +1,9 @@ -use crate::raster::{EmptyGrid, Grid, GridOrEmpty, GridSize, MaskedGrid}; +use float_cmp::approx_eq; + +use crate::raster::{ + EmptyGrid, GeoTransform, Grid, GridIndexAccess, GridOrEmpty, GridSize, MaskedGrid, Pixel, + RasterTile2D, grid_idx_iter_2d, +}; use std::panic; pub trait TestDefault { @@ -77,6 +82,121 @@ pub fn save_test_bytes(bytes: &[u8], filename: &str) { .expect("it should be possible to write this file for testing"); } +/// Method that compares two lists of tiles and panics with a message why there is a difference. +/// +/// # Panics +/// If there is a difference between two tiles or the length of the lists +pub fn assert_eq_two_list_of_tiles_u8( + list_a: &[RasterTile2D], + list_b: &[RasterTile2D], + compare_cache_hint: bool, +) { + assert_eq_two_list_of_tiles::(list_a, list_b, compare_cache_hint); +} + +/// Method that compares two lists of tiles and panics with a message why there is a difference. +/// +/// # Panics +/// If there is a difference between two tiles or the length of the lists +pub fn assert_eq_two_list_of_tiles( + list_a: &[RasterTile2D

], + list_b: &[RasterTile2D

], + compare_cache_hint: bool, +) { + assert_eq!( + list_a.len(), + list_b.len(), + "len() of input_a: {}, len of input_b: {}", + list_a.len(), + list_b.len() + ); + + list_a + .iter() + .zip(list_b) + .enumerate() + .for_each(|(i, (a, b))| { + assert_eq!( + a.time, b.time, + "time of tile {} input_a: {}, input_b: {}", + i, a.time, b.time + ); + assert_eq!( + a.band, b.band, + "band of tile {} input_a: {}, input_b: {}", + i, a.band, b.band + ); + assert_eq!( + a.tile_position, b.tile_position, + "tile position of tile {} input_a: {:?}, input_b: {:?}", + i, a.tile_position, b.tile_position + ); + + let spatial_grid_a = a.global_pixel_spatial_grid_definition(); + let spatial_grid_b = b.global_pixel_spatial_grid_definition(); + assert_eq!( + spatial_grid_a.grid_bounds(), + spatial_grid_b.grid_bounds(), + "grid bounds of tile {} input_a: {:?}, input_b {:?}", + i, + spatial_grid_a.grid_bounds(), + spatial_grid_b.grid_bounds() + ); + assert!( + approx_eq!( + GeoTransform, + spatial_grid_a.geo_transform(), + spatial_grid_b.geo_transform() + ), + "geo transform of tile {} input_a: {:?}, input_b: {:?}", + i, + spatial_grid_a.geo_transform(), + spatial_grid_b.geo_transform() + ); + assert_eq!( + a.grid_array.is_empty(), + b.grid_array.is_empty(), + "grid shape of tile {} input_a is_empty: {:?}, input_b is_empty: {:?}", + i, + a.grid_array.is_empty(), + b.grid_array.is_empty(), + ); + if !a.grid_array.is_empty() { + let mat_a = a.grid_array.clone().into_materialized_masked_grid(); + let mat_b = b.grid_array.clone().into_materialized_masked_grid(); + + assert_eq!( + mat_a.inner_grid.data.len(), + mat_b.inner_grid.data.len(), + "grid data len of tile {} input_a: {:?}, input_b: {:?}", + i, + mat_a.inner_grid.data.len(), + mat_b.inner_grid.data.len(), + ); + + for (pi, idx) in grid_idx_iter_2d(&mat_a).enumerate() { + let a_v = mat_a + .get_at_grid_index(idx) + .expect("tile a must contain idx inside tile bounds"); + let b_v = mat_b + .get_at_grid_index(idx) + .expect("tile b must contain idx inside tile bounds"); + assert_eq!( + a_v, b_v, + "tile {i} pixel {pi} at {idx:?} input_a: {a_v:?}, input_b: {b_v:?}", + ); + } + } + if compare_cache_hint { + assert_eq!( + a.cache_hint, b.cache_hint, + "cache hint of tile {} input_a: {:?}, input_b: {:?}", + i, a.cache_hint, b.cache_hint + ); + } + }); +} + #[cfg(test)] mod tests { use crate::{ diff --git a/openapi.json b/openapi.json index 0a3165050..c7e7a3838 100644 --- a/openapi.json +++ b/openapi.json @@ -11,7 +11,7 @@ "name": "Apache-2.0", "url": "https://github.com/geo-engine/geoengine/blob/main/LICENSE" }, - "version": "0.8.0" + "version": "0.9.0" }, "servers": [ { @@ -805,6 +805,46 @@ ] } }, + "/dataset/{dataset}/tiles": { + "post": { + "tags": [ + "Datasets" + ], + "summary": "Add a tile to a gdal dataset.", + "operationId": "add_dataset_tiles_handler", + "parameters": [ + { + "name": "dataset", + "in": "path", + "description": "Dataset Name", + "required": true, + "schema": { + "$ref": "#/components/schemas/DatasetName" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AutoCreateDataset" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "session_token": [] + } + ] + } + }, "/datasetFromWorkflow/{id}": { "post": { "tags": [ @@ -4152,13 +4192,13 @@ ] } }, - "/wcs/{workflow}?request=DescribeCoverage": { + "/wcs/{workflow}": { "get": { "tags": [ "OGC WCS" ], - "summary": "Get WCS Coverage Description", - "operationId": "wcs_describe_coverage_handler", + "summary": "OGC WCS endpoint", + "operationId": "wcs_handler", "parameters": [ { "name": "workflow", @@ -4170,104 +4210,141 @@ } }, { - "name": "version", + "name": "boundingbox", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WcsVersion" - } + "type": "string" + }, + "example": "-90,-180,90,180,urn:ogc:def:crs:EPSG::4326" }, { - "name": "service", + "name": "format", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WcsService" + "$ref": "#/components/schemas/GetCoverageFormat" } }, { - "name": "request", + "name": "gridbasecrs", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/DescribeCoverageRequest" - } + "type": "string" + }, + "example": "urn:ogc:def:crs:EPSG::4326" + }, + { + "name": "gridoffsets", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "example": "-0.1,0.1" + }, + { + "name": "gridorigin", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "example": "90,-180" + }, + { + "name": "identifier", + "in": "query", + "required": false, + "schema": { + "type": "string" + }, + "example": "" }, { "name": "identifiers", "in": "query", - "required": true, + "required": false, "schema": { "type": "string" }, "example": "" - } - ], - "responses": { - "200": { - "description": "OK", - "content": { - "text/xml": { - "schema": { - "type": "string" - } - } - } - } - }, - "security": [ + }, { - "session_token": [] - } - ] - } - }, - "/wcs/{workflow}?request=GetCapabilities": { - "get": { - "tags": [ - "OGC WCS" - ], - "summary": "Get WCS Capabilities", - "operationId": "wcs_capabilities_handler", - "parameters": [ + "name": "nodatavalue", + "in": "query", + "required": false, + "schema": { + "type": [ + "number", + "null" + ], + "format": "double" + } + }, { - "name": "workflow", - "in": "path", - "description": "Workflow id", + "name": "request", + "in": "query", + "description": "type of WCS request", "required": true, "schema": { - "$ref": "#/components/schemas/WorkflowId" + "$ref": "#/components/schemas/WcsRequest" } }, { - "name": "version", + "name": "resx", "in": "query", "required": false, "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/WcsVersion" - } - ] + "type": [ + "number", + "null" + ], + "format": "double" + } + }, + { + "name": "resy", + "in": "query", + "required": false, + "schema": { + "type": [ + "number", + "null" + ], + "format": "double" } }, { "name": "service", "in": "query", - "required": true, + "required": false, "schema": { "$ref": "#/components/schemas/WcsService" } }, { - "name": "request", + "name": "time", "in": "query", - "required": true, + "required": false, + "schema": { + "type": "string" + } + }, + { + "name": "version", + "in": "query", + "required": false, "schema": { - "$ref": "#/components/schemas/GetCapabilitiesRequest" + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/WcsVersion" + } + ] } } ], @@ -4290,13 +4367,13 @@ ] } }, - "/wcs/{workflow}?request=GetCoverage": { + "/wfs/{workflow}": { "get": { "tags": [ - "OGC WCS" + "OGC WFS" ], - "summary": "Get WCS Coverage", - "operationId": "wcs_get_coverage_handler", + "summary": "OGC WFS endpoint", + "operationId": "wfs_handler", "parameters": [ { "name": "workflow", @@ -4308,272 +4385,110 @@ } }, { - "name": "version", + "name": "bbox", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WcsVersion" - } + "$ref": "#/components/schemas/OgcBoundingBox" + }, + "example": "-90,-180,90,180" }, { - "name": "service", + "name": "count", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WcsService" + "type": [ + "integer", + "null" + ], + "format": "int64", + "minimum": 0 } }, { - "name": "request", + "name": "filter", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/GetCoverageRequest" + "type": [ + "string", + "null" + ] } }, { - "name": "format", + "name": "namespaces", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/GetCoverageFormat" + "type": [ + "string", + "null" + ] } }, { - "name": "identifier", + "name": "propertyName", "in": "query", - "required": true, + "required": false, "schema": { - "type": "string" - }, - "example": "" + "type": [ + "string", + "null" + ] + } }, { - "name": "boundingbox", + "name": "request", "in": "query", + "description": "type of WFS request", "required": true, "schema": { - "type": "string" - }, - "example": "-90,-180,90,180,urn:ogc:def:crs:EPSG::4326" + "$ref": "#/components/schemas/WfsRequest" + } }, { - "name": "gridbasecrs", + "name": "resultType", "in": "query", - "required": true, + "required": false, "schema": { - "type": "string" - }, - "example": "urn:ogc:def:crs:EPSG::4326" + "type": [ + "string", + "null" + ] + } }, { - "name": "gridorigin", + "name": "service", "in": "query", "required": false, "schema": { - "type": "string" - }, - "example": "90,-180" + "$ref": "#/components/schemas/WfsService" + } }, { - "name": "gridoffsets", - "in": "query", - "required": false, - "schema": { - "type": "string" - }, - "example": "-0.1,0.1" - }, - { - "name": "time", - "in": "query", - "required": false, - "schema": { - "type": "string" - } - }, - { - "name": "resx", - "in": "query", - "required": false, - "schema": { - "type": [ - "number", - "null" - ], - "format": "double" - } - }, - { - "name": "resy", + "name": "sortBy", "in": "query", "required": false, "schema": { "type": [ - "number", + "string", "null" - ], - "format": "double" + ] } }, { - "name": "nodatavalue", + "name": "srsName", "in": "query", "required": false, "schema": { "type": [ - "number", + "string", "null" - ], - "format": "double" - } - } - ], - "responses": { - "200": { - "$ref": "#/components/responses/PngResponse" - } - }, - "security": [ - { - "session_token": [] - } - ] - } - }, - "/wfs/{workflow}?request=GetCapabilities": { - "get": { - "tags": [ - "OGC WFS" - ], - "summary": "Get WFS Capabilities", - "operationId": "wfs_capabilities_handler", - "parameters": [ - { - "name": "workflow", - "in": "path", - "description": "Workflow id", - "required": true, - "schema": { - "$ref": "#/components/schemas/WorkflowId" - } - }, - { - "name": "version", - "in": "path", - "required": true, - "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/WfsVersion" - } - ] - } - }, - { - "name": "service", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/WfsService" - } - }, - { - "name": "request", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/GetCapabilitiesRequest" - } - } - ], - "responses": { - "200": { - "description": "OK", - "content": { - "text/xml": { - "schema": { - "type": "string" - } - } - } - } - }, - "security": [ - { - "session_token": [] - } - ] - } - }, - "/wfs/{workflow}?request=GetFeature": { - "get": { - "tags": [ - "OGC WFS" - ], - "summary": "Get WCS Features", - "operationId": "wfs_feature_handler", - "parameters": [ - { - "name": "workflow", - "in": "path", - "description": "Workflow id", - "required": true, - "schema": { - "$ref": "#/components/schemas/WorkflowId" - } - }, - { - "name": "version", - "in": "query", - "required": false, - "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/WfsVersion" - } ] - } - }, - { - "name": "service", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/WfsService" - } - }, - { - "name": "request", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/GetFeatureRequest" - } - }, - { - "name": "typeNames", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/TypeNames" }, - "example": "" - }, - { - "name": "bbox", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/OgcBoundingBox" - }, - "example": "-90,-180,90,180" + "example": "EPSG:4326" }, { "name": "time", @@ -4585,89 +4500,17 @@ "example": "2014-04-01T12:00:00.000Z" }, { - "name": "srsName", + "name": "typeNames", "in": "query", "required": false, "schema": { - "type": [ - "string", - "null" - ] + "$ref": "#/components/schemas/TypeNames" }, - "example": "EPSG:4326" - }, - { - "name": "namespaces", - "in": "query", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "count", - "in": "query", - "required": false, - "schema": { - "type": [ - "integer", - "null" - ], - "format": "int64", - "minimum": 0 - } - }, - { - "name": "sortBy", - "in": "query", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "resultType", - "in": "query", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "filter", - "in": "query", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } - }, - { - "name": "propertyName", - "in": "query", - "required": false, - "schema": { - "type": [ - "string", - "null" - ] - } + "example": "" }, { - "name": "queryResolution", + "name": "version", "in": "query", - "description": "Vendor parameter for specifying a spatial query resolution", "required": false, "schema": { "oneOf": [ @@ -4675,7 +4518,7 @@ "type": "null" }, { - "$ref": "#/components/schemas/WfsResolution" + "$ref": "#/components/schemas/WfsVersion" } ] } @@ -4795,13 +4638,13 @@ ] } }, - "/wms/{workflow}?request=GetCapabilities": { + "/wms/{workflow}": { "get": { "tags": [ "OGC WMS" ], - "summary": "Get WMS Capabilities", - "operationId": "wms_capabilities_handler", + "summary": "OGC WMS endpoint", + "operationId": "wms_handler", "parameters": [ { "name": "workflow", @@ -4813,190 +4656,82 @@ } }, { - "name": "version", - "in": "path", - "required": true, - "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/WmsVersion" - } - ] - } - }, - { - "name": "service", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/WmsService" - } - }, - { - "name": "request", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/GetCapabilitiesRequest" - } - }, - { - "name": "format", - "in": "path", - "required": true, - "schema": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/GetCapabilitiesFormat" - } - ] - } - } - ], - "responses": { - "200": { - "description": "OK", - "content": { - "text/xml": { - "schema": { - "type": "string" - } - } - } - } - }, - "security": [ - { - "session_token": [] - } - ] - } - }, - "/wms/{workflow}?request=GetLegendGraphic": { - "get": { - "tags": [ - "OGC WMS" - ], - "summary": "Get WMS Legend Graphic", - "operationId": "wms_legend_graphic_handler", - "parameters": [ - { - "name": "workflow", - "in": "path", - "description": "Workflow id", - "required": true, - "schema": { - "$ref": "#/components/schemas/WorkflowId" - } - }, - { - "name": "version", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/WmsVersion" - } - }, - { - "name": "service", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/WmsService" - } - }, - { - "name": "request", - "in": "path", - "required": true, - "schema": { - "$ref": "#/components/schemas/GetLegendGraphicRequest" - } - }, - { - "name": "layer", - "in": "path", - "required": true, + "name": "bbox", + "in": "query", + "required": false, "schema": { - "type": "string" + "$ref": "#/components/schemas/OgcBoundingBox" }, - "example": "" - } - ], - "responses": { - "501": { - "description": "Not implemented" - } - }, - "security": [ - { - "session_token": [] - } - ] - } - }, - "/wms/{workflow}?request=GetMap": { - "get": { - "tags": [ - "OGC WMS" - ], - "summary": "Get WMS Map", - "operationId": "wms_map_handler", - "parameters": [ + "example": "-90,-180,90,180" + }, { - "name": "workflow", - "in": "path", - "description": "Workflow id", - "required": true, + "name": "bgcolor", + "in": "query", + "required": false, "schema": { - "$ref": "#/components/schemas/WorkflowId" + "type": [ + "string", + "null" + ] } }, { - "name": "version", + "name": "crs", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WmsVersion" - } + "type": [ + "string", + "null" + ] + }, + "example": "EPSG:4326" }, { - "name": "service", + "name": "elevation", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/WmsService" + "type": [ + "string", + "null" + ] } }, { - "name": "request", + "name": "exceptions", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/GetMapRequest" + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/GetMapExceptionFormat" + } + ] } }, { - "name": "width", + "name": "format", "in": "query", - "required": true, + "required": false, "schema": { - "type": "integer", - "format": "int32", - "minimum": 0 - }, - "example": 512 + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/WmsResponseFormat" + } + ] + } }, { "name": "height", "in": "query", - "required": true, + "required": false, "schema": { "type": "integer", "format": "int32", @@ -5005,74 +4740,71 @@ "example": 256 }, { - "name": "bbox", + "name": "info_format", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/OgcBoundingBox" - }, - "example": "-90,-180,90,180" + "type": [ + "string", + "null" + ] + } }, { - "name": "format", + "name": "layer", "in": "query", - "required": true, + "required": false, "schema": { - "$ref": "#/components/schemas/GetMapFormat" + "type": "string" } }, { "name": "layers", "in": "query", - "required": true, + "required": false, "schema": { "type": "string" }, "example": "" }, { - "name": "crs", + "name": "query_layers", "in": "query", "required": false, "schema": { - "type": [ - "string", - "null" - ] - }, - "example": "EPSG:4326" + "type": "string" + } }, { - "name": "styles", + "name": "request", "in": "query", + "description": "type of WMS request", "required": true, "schema": { - "type": "string" - }, - "example": "custom:{\"type\":\"linearGradient\",\"breakpoints\":[{\"value\":1,\"color\":[0,0,0,255]},{\"value\":255,\"color\":[255,255,255,255]}],\"noDataColor\":[0,0,0,0],\"defaultColor\":[0,0,0,0]}" + "$ref": "#/components/schemas/WmsRequest" + } }, { - "name": "time", + "name": "service", "in": "query", "required": false, "schema": { - "type": "string" - }, - "example": "2014-04-01T12:00:00.000Z" + "$ref": "#/components/schemas/WmsService" + } }, { - "name": "transparent", + "name": "sld", "in": "query", "required": false, "schema": { "type": [ - "boolean", + "string", "null" ] } }, { - "name": "bgcolor", + "name": "sld_body", "in": "query", "required": false, "schema": { @@ -5083,40 +4815,36 @@ } }, { - "name": "sld", + "name": "styles", "in": "query", "required": false, "schema": { - "type": [ - "string", - "null" - ] - } + "type": "string" + }, + "example": "custom:{\"type\":\"linearGradient\",\"breakpoints\":[{\"value\":1,\"color\":[0,0,0,255]},{\"value\":255,\"color\":[255,255,255,255]}],\"noDataColor\":[0,0,0,0],\"defaultColor\":[0,0,0,0]}" }, { - "name": "sld_body", + "name": "time", "in": "query", "required": false, "schema": { - "type": [ - "string", - "null" - ] - } + "type": "string" + }, + "example": "2014-04-01T12:00:00.000Z" }, { - "name": "elevation", + "name": "transparent", "in": "query", "required": false, "schema": { "type": [ - "string", + "boolean", "null" ] } }, { - "name": "exceptions", + "name": "version", "in": "query", "required": false, "schema": { @@ -5125,10 +4853,21 @@ "type": "null" }, { - "$ref": "#/components/schemas/GetMapExceptionFormat" + "$ref": "#/components/schemas/WmsVersion" } ] } + }, + { + "name": "width", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "example": 512 } ], "responses": { @@ -5438,14 +5177,6 @@ "type": "string" } }, - { - "name": "spatialResolution", - "in": "query", - "required": true, - "schema": { - "$ref": "#/components/schemas/SpatialResolution" - } - }, { "name": "attributes", "in": "query", @@ -5535,6 +5266,37 @@ } } }, + "AddDatasetTile": { + "type": "object", + "required": [ + "time", + "spatial_partition", + "band", + "z_index", + "params" + ], + "properties": { + "band": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "params": { + "$ref": "#/components/schemas/GdalDatasetParameters" + }, + "spatial_partition": { + "$ref": "#/components/schemas/SpatialPartition2D" + }, + "time": { + "$ref": "#/components/schemas/TimeInterval" + }, + "z_index": { + "type": "integer", + "format": "int32", + "minimum": 0 + } + } + }, "AddLayer": { "type": "object", "required": [ @@ -6255,6 +6017,16 @@ "sourceOperator" ], "properties": { + "dataPath": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/DataPath" + } + ] + }, "description": { "type": "string" }, @@ -6524,12 +6296,6 @@ } } }, - "DescribeCoverageRequest": { - "type": "string", - "enum": [ - "DescribeCoverage" - ] - }, "EbvPortalDataProviderDefinition": { "type": "object", "required": [ @@ -7076,6 +6842,24 @@ } } }, + "GdalMultiBand": { + "type": "object", + "required": [ + "type", + "resultDescriptor" + ], + "properties": { + "resultDescriptor": { + "$ref": "#/components/schemas/RasterResultDescriptor" + }, + "type": { + "type": "string", + "enum": [ + "GdalMultiBand" + ] + } + } + }, "GdalSourceTimePlaceholder": { "type": "object", "required": [ @@ -7107,59 +6891,38 @@ } } }, - "GetCapabilitiesFormat": { - "type": "string", - "enum": [ - "text/xml" - ] - }, - "GetCapabilitiesRequest": { - "type": "string", - "enum": [ - "GetCapabilities" - ] - }, - "GetCoverageFormat": { - "type": "string", - "enum": [ - "image/tiff" - ] - }, - "GetCoverageRequest": { - "type": "string", - "enum": [ - "GetCoverage" - ] - }, - "GetFeatureRequest": { - "type": "string", - "enum": [ - "GetFeature" - ] - }, - "GetLegendGraphicRequest": { - "type": "string", - "enum": [ - "GetLegendGraphic" - ] - }, - "GetMapExceptionFormat": { - "type": "string", - "enum": [ - "XML", - "JSON" - ] + "GeoTransform": { + "type": "object", + "required": [ + "originCoordinate", + "xPixelSize", + "yPixelSize" + ], + "properties": { + "originCoordinate": { + "$ref": "#/components/schemas/Coordinate2D" + }, + "xPixelSize": { + "type": "number", + "format": "double" + }, + "yPixelSize": { + "type": "number", + "format": "double" + } + } }, - "GetMapFormat": { + "GetCoverageFormat": { "type": "string", "enum": [ - "image/png" + "image/tiff" ] }, - "GetMapRequest": { + "GetMapExceptionFormat": { "type": "string", "enum": [ - "GetMap" + "XML", + "JSON" ] }, "GfbioAbcdDataProviderDefinition": { @@ -7248,6 +7011,36 @@ } } }, + "GridBoundingBox2D": { + "type": "object", + "required": [ + "topLeftIdx", + "bottomRightIdx" + ], + "properties": { + "bottomRightIdx": { + "$ref": "#/components/schemas/GridIdx2D" + }, + "topLeftIdx": { + "$ref": "#/components/schemas/GridIdx2D" + } + } + }, + "GridIdx2D": { + "type": "object", + "required": [ + "yIdx", + "xIdx" + ], + "properties": { + "xIdx": { + "type": "integer" + }, + "yIdx": { + "type": "integer" + } + } + }, "InternalDataId": { "type": "object", "required": [ @@ -7636,6 +7429,9 @@ }, { "$ref": "#/components/schemas/GdalMetaDataList" + }, + { + "$ref": "#/components/schemas/GdalMultiBand" } ], "discriminator": { @@ -7644,6 +7440,7 @@ "GdalMetaDataList": "#/components/schemas/GdalMetaDataList", "GdalMetaDataNetCdfCf": "#/components/schemas/GdalMetadataNetCdfCf", "GdalMetaDataRegular": "#/components/schemas/GdalMetaDataRegular", + "GdalMultiBand": "#/components/schemas/GdalMultiBand", "GdalStatic": "#/components/schemas/GdalMetaDataStatic", "MockMetaData": "#/components/schemas/MockMetaData", "OgrMetaData": "#/components/schemas/OgrMetaData" @@ -8712,26 +8509,6 @@ "ImagePng" ] }, - "PlotQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/BoundingBox2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "PlotResultDescriptor": { "type": "object", "description": "A `ResultDescriptor` for plot queries", @@ -9218,7 +8995,7 @@ ] }, "query": { - "$ref": "#/components/schemas/RasterQueryRectangle" + "$ref": "#/components/schemas/RasterToDatasetQueryRectangle" } }, "example": { @@ -9239,10 +9016,6 @@ "timeInterval": { "start": 1388534400000, "end": 1388534401000 - }, - "spatialResolution": { - "x": 0.1, - "y": 0.1 } } } @@ -9287,73 +9060,31 @@ } } }, - "RasterQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/SpatialPartition2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "RasterResultDescriptor": { "type": "object", "description": "A `ResultDescriptor` for raster queries", "required": [ "dataType", "spatialReference", + "time", + "spatialGrid", "bands" ], "properties": { "bands": { "$ref": "#/components/schemas/RasterBandDescriptors" }, - "bbox": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/SpatialPartition2D" - } - ] - }, "dataType": { "$ref": "#/components/schemas/RasterDataType" }, - "resolution": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/SpatialResolution" - } - ] + "spatialGrid": { + "$ref": "#/components/schemas/SpatialGridDescriptor" }, "spatialReference": { "type": "string" }, "time": { - "oneOf": [ - { - "type": "null" - }, - { - "$ref": "#/components/schemas/TimeInterval" - } - ] + "$ref": "#/components/schemas/TimeDescriptor" } } }, @@ -9387,6 +9118,37 @@ } } }, + "RasterToDatasetQueryRectangle": { + "type": "object", + "description": "A spatio-temporal rectangle with a specified resolution", + "required": [ + "spatialBounds", + "timeInterval" + ], + "properties": { + "spatialBounds": { + "$ref": "#/components/schemas/SpatialPartition2D" + }, + "timeInterval": { + "$ref": "#/components/schemas/TimeInterval" + } + } + }, + "RegularTimeDimension": { + "type": "object", + "required": [ + "origin", + "step" + ], + "properties": { + "origin": { + "$ref": "#/components/schemas/TimeInstance" + }, + "step": { + "$ref": "#/components/schemas/TimeStep" + } + } + }, "Resource": { "oneOf": [ { @@ -9539,20 +9301,12 @@ "name", "id", "description", - "apiUrl", - "bands", - "zones" + "apiUrl" ], "properties": { "apiUrl": { "type": "string" }, - "bands": { - "type": "array", - "items": { - "$ref": "#/components/schemas/StacBand" - } - }, "cacheTtl": { "$ref": "#/components/schemas/CacheTtlSeconds" }, @@ -9587,12 +9341,6 @@ "enum": [ "SentinelS2L2ACogs" ] - }, - "zones": { - "type": "array", - "items": { - "$ref": "#/components/schemas/StacZone" - } } } }, @@ -9647,6 +9395,43 @@ } } }, + "SpatialGridDefinition": { + "type": "object", + "required": [ + "geoTransform", + "gridBounds" + ], + "properties": { + "geoTransform": { + "$ref": "#/components/schemas/GeoTransform" + }, + "gridBounds": { + "$ref": "#/components/schemas/GridBoundingBox2D" + } + } + }, + "SpatialGridDescriptor": { + "type": "object", + "required": [ + "spatialGrid", + "descriptor" + ], + "properties": { + "descriptor": { + "$ref": "#/components/schemas/SpatialGridDescriptorState" + }, + "spatialGrid": { + "$ref": "#/components/schemas/SpatialGridDefinition" + } + } + }, + "SpatialGridDescriptorState": { + "type": "string", + "enum": [ + "source", + "derived" + ] + }, "SpatialPartition2D": { "type": "object", "description": "A partition of space that include the upper left but excludes the lower right coordinate", @@ -9758,28 +9543,6 @@ } } }, - "StacBand": { - "type": "object", - "required": [ - "name", - "dataType" - ], - "properties": { - "dataType": { - "$ref": "#/components/schemas/RasterDataType" - }, - "name": { - "type": "string" - }, - "noDataValue": { - "type": [ - "number", - "null" - ], - "format": "double" - } - } - }, "StacQueryBuffer": { "type": "object", "description": "A struct that represents buffers to apply to stac requests", @@ -9798,23 +9561,6 @@ } } }, - "StacZone": { - "type": "object", - "required": [ - "name", - "epsg" - ], - "properties": { - "epsg": { - "type": "integer", - "format": "int32", - "minimum": 0 - }, - "name": { - "type": "string" - } - } - }, "StaticColor": { "type": "object", "required": [ @@ -10133,6 +9879,66 @@ } } }, + "TimeDescriptor": { + "type": "object", + "required": [ + "dimension" + ], + "properties": { + "bounds": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/TimeInterval" + } + ] + }, + "dimension": { + "$ref": "#/components/schemas/TimeDimension" + } + } + }, + "TimeDimension": { + "oneOf": [ + { + "allOf": [ + { + "$ref": "#/components/schemas/RegularTimeDimension" + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "regular" + ] + } + } + } + ] + }, + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "irregular" + ] + } + } + } + ] + }, "TimeGranularity": { "type": "string", "description": "A time granularity.", @@ -10883,26 +10689,6 @@ "MultiPolygon" ] }, - "VectorQueryRectangle": { - "type": "object", - "description": "A spatio-temporal rectangle with a specified resolution", - "required": [ - "spatialBounds", - "timeInterval", - "spatialResolution" - ], - "properties": { - "spatialBounds": { - "$ref": "#/components/schemas/BoundingBox2D" - }, - "spatialResolution": { - "$ref": "#/components/schemas/SpatialResolution" - }, - "timeInterval": { - "$ref": "#/components/schemas/TimeInterval" - } - } - }, "VectorResultDescriptor": { "type": "object", "required": [ @@ -11003,6 +10789,14 @@ } } }, + "WcsRequest": { + "type": "string", + "enum": [ + "GetCapabilities", + "DescribeCoverage", + "GetCoverage" + ] + }, "WcsService": { "type": "string", "enum": [ @@ -11016,8 +10810,12 @@ "1.1.1" ] }, - "WfsResolution": { - "type": "string" + "WfsRequest": { + "type": "string", + "enum": [ + "GetCapabilities", + "GetFeature" + ] }, "WfsService": { "type": "string", @@ -11087,6 +10885,23 @@ } } }, + "WmsRequest": { + "type": "string", + "enum": [ + "GetCapabilities", + "GetMap", + "GetFeatureInfo", + "GetStyles", + "GetLegendGraphic" + ] + }, + "WmsResponseFormat": { + "type": "string", + "enum": [ + "text/xml", + "image/png" + ] + }, "WmsService": { "type": "string", "enum": [ diff --git a/operators/Cargo.toml b/operators/Cargo.toml index 75659c8bf..42b00b6ab 100644 --- a/operators/Cargo.toml +++ b/operators/Cargo.toml @@ -8,8 +8,6 @@ license-file.workspace = true documentation.workspace = true repository.workspace = true -[features] - [dependencies] arrow = { workspace = true } async-trait = { workspace = true } diff --git a/operators/benches/bands.rs b/operators/benches/bands.rs index 28064dc31..c9e077ae4 100644 --- a/operators/benches/bands.rs +++ b/operators/benches/bands.rs @@ -2,17 +2,14 @@ use futures::{Future, StreamExt}; use geoengine_datatypes::{ - primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - TimeStep, - }, - raster::{RasterDataType, RasterTile2D, RenameBands}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval, TimeStep}, + raster::{GridBoundingBox2D, RasterDataType, RasterTile2D, RenameBands}, util::test::TestDefault, }; use geoengine_operators::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterOperator, - SingleRasterSource, WorkflowOperatorPath, + MockExecutionContext, MultipleRasterSources, RasterOperator, SingleRasterSource, + WorkflowOperatorPath, }, processing::{ Aggregation, RasterStacker, RasterStackerParams, TemporalRasterAggregation, @@ -51,15 +48,15 @@ fn ndvi_source(execution_context: &mut MockExecutionContext) -> Box>().try_into().unwrap(), - }; + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + (0..bands).collect::>().try_into().unwrap(), + ); let mut times = NumberStatistics::default(); @@ -183,25 +178,14 @@ async fn all_bands_at_once(runs: usize, bands: u32, resolution: SpatialResolutio async fn main() { const RUNS: usize = 5; const BANDS: u32 = 8; - const RESOLUTION: f64 = 0.1; println!("one band at a time"); - one_band_at_a_time( - RUNS, - BANDS, - SpatialResolution::new(RESOLUTION, RESOLUTION).unwrap(), - ) - .await; + one_band_at_a_time(RUNS, BANDS).await; println!("all bands at once"); - all_bands_at_once( - RUNS, - BANDS, - SpatialResolution::new(RESOLUTION, RESOLUTION).unwrap(), - ) - .await; + all_bands_at_once(RUNS, BANDS).await; } async fn time_it(f: F) -> (f64, Vec>) diff --git a/operators/benches/cache.rs b/operators/benches/cache.rs index 499ec7227..31db50b07 100644 --- a/operators/benches/cache.rs +++ b/operators/benches/cache.rs @@ -2,17 +2,15 @@ use futures::StreamExt; use geoengine_datatypes::{ - primitives::{ - BandSelection, QueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::TilesEqualIgnoringCacheHint, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::{GridBoundingBox2D, TilesEqualIgnoringCacheHint}, util::test::TestDefault, }; use geoengine_operators::{ cache::{cache_operator::InitializedCacheOperator, shared_cache::SharedCache}, engine::{ - ChunkByteSize, InitializedRasterOperator, MockExecutionContext, MockQueryContext, - QueryProcessor, RasterOperator, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, InitializedRasterOperator, MockExecutionContext, QueryProcessor, + RasterOperator, SingleRasterSource, WorkflowOperatorPath, }, processing::{ AggregateFunctionParams, NeighborhoodAggregate, NeighborhoodAggregateParams, @@ -41,7 +39,7 @@ async fn main() { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), }, @@ -57,7 +55,7 @@ async fn main() { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -68,15 +66,11 @@ async fn main() { let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -92,15 +86,11 @@ async fn main() { let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await diff --git a/operators/benches/cache_concurrent.rs b/operators/benches/cache_concurrent.rs index cb39949e6..00e151c50 100644 --- a/operators/benches/cache_concurrent.rs +++ b/operators/benches/cache_concurrent.rs @@ -1,9 +1,9 @@ #![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] // okay in benchmarks use futures::future::join_all; +use geoengine_datatypes::primitives::DateTime; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::primitives::{DateTime, SpatialPartition2D, SpatialResolution}; -use geoengine_datatypes::raster::RasterProperties; +use geoengine_datatypes::raster::{GridBoundingBox2D, RasterProperties}; use geoengine_datatypes::{ primitives::{RasterQueryRectangle, TimeInterval}, raster::{Grid, RasterTile2D}, @@ -118,12 +118,11 @@ async fn read_cache(tile_cache: &SharedCache, op_no: usize) -> ReadMeasurement { } fn query_rect() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ) } fn op(idx: usize) -> CanonicOperatorName { diff --git a/operators/benches/expression.rs b/operators/benches/expression.rs index f8ed69c40..29843c5f3 100644 --- a/operators/benches/expression.rs +++ b/operators/benches/expression.rs @@ -2,16 +2,15 @@ use futures::{Future, StreamExt}; use geoengine_datatypes::{ - primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{RasterDataType, RasterTile2D, RenameBands}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::RenameBands, + raster::{GridBoundingBox2D, RasterDataType, RasterTile2D}, util::test::TestDefault, }; use geoengine_operators::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterOperator, - SingleRasterSource, WorkflowOperatorPath, + MockExecutionContext, MultipleRasterSources, RasterOperator, SingleRasterSource, + WorkflowOperatorPath, }, processing::{Expression, ExpressionParams, RasterStacker, RasterStackerParams}, source::{GdalSource, GdalSourceParameters}, @@ -55,7 +54,7 @@ fn ndvi_source(execution_context: &mut MockExecutionContext) -> Box (StatisticsWrappingMockExecutionContext, MockQueryContext let workflow = uuid::Uuid::new_v4(); let computation = uuid::Uuid::new_v4(); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), None, Some(QuotaTracking::new( @@ -105,23 +105,17 @@ fn setup_benchmarks(exe_ctx: &mut StatisticsWrappingMockExecutionContext) -> Vec }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: ndvi_id.clone(), - }, + params: GdalSourceParameters::new(ndvi_id.clone()), } .boxed(), }, } .boxed(), - query_rectangle: QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + query_rectangle: RasterQueryRectangle::new( + GridBoundingBox2D::new([-1800, -900], [1799, 899]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), }, Benchmark::Vector { name: "raster_vector_join".to_string(), @@ -144,22 +138,18 @@ fn setup_benchmarks(exe_ctx: &mut StatisticsWrappingMockExecutionContext) -> Vec .boxed(), rasters: vec![ GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), ], }, } .boxed(), - query_rectangle: QueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + query_rectangle: VectorQueryRectangle::new( + BoundingBox2D::new_unchecked([-180., -90.].into(), [180., 90.].into()), + TimeInterval::default(), + ColumnSelection::all(), + ), }, ] } @@ -335,7 +325,7 @@ fn run_benchmark( fn collect_raster_query( runtime: &tokio::runtime::Runtime, query_ctx: &dyn QueryContext, - processor: &Box>, + processor: &BoxRasterQueryProcessor

, query_rectangle: RasterQueryRectangle, ) { let stream = diff --git a/operators/benches/sources.rs b/operators/benches/sources.rs index afde6a93b..881528e4d 100644 --- a/operators/benches/sources.rs +++ b/operators/benches/sources.rs @@ -1,41 +1,42 @@ #![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] // okay in benchmarks -use std::time::Instant; -use std::{hint::black_box, marker::PhantomData}; - use futures::StreamExt; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::raster::{ + BoundedGrid, GridBoundingBox2D, GridShapeAccess, RasterDataType, +}; use geoengine_datatypes::{ - primitives::{RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{RasterQueryRectangle, TimeInterval}, raster::{ GeoTransform, Grid2D, GridOrEmpty2D, GridSize, Pixel, RasterTile2D, TilingSpecification, }, util::test::TestDefault, }; -use geoengine_operators::engine::RasterResultDescriptor; +use geoengine_operators::engine::{MockExecutionContext, RasterResultDescriptor, TimeDescriptor}; use geoengine_operators::{ - engine::{ChunkByteSize, MockQueryContext, QueryContext, RasterQueryProcessor}, + engine::{ChunkByteSize, RasterQueryProcessor}, mock::MockRasterSourceProcessor, source::{GdalMetaDataRegular, GdalSourceProcessor}, util::gdal::create_ndvi_meta_data, }; +use std::time::Instant; +use std::{hint::black_box, marker::PhantomData}; fn setup_gdal_source( meta_data: GdalMetaDataRegular, tiling_specification: TilingSpecification, ) -> GdalSourceProcessor { GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification, + overview_level: 0, meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, } } +#[allow(clippy::too_many_lines)] fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProcessor { let grid: GridOrEmpty2D = Grid2D::new( tiling_spec.tile_size_in_pixels, @@ -44,6 +45,18 @@ fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProces .unwrap() .into(); let geo_transform = GeoTransform::test_default(); + let grid_bounds = grid.grid_shape().bounding_box(); + let grid_bounds = GridBoundingBox2D::new( + [ + grid_bounds.y_min() - grid.axis_size_y() as isize, + grid_bounds.x_min() - grid.axis_size_x() as isize, + ], + [ + grid_bounds.y_min() + 2 * grid.axis_size_y() as isize, + grid_bounds.x_min() + 2 * grid.axis_size_x() as isize, + ], + ) + .unwrap(); let time = TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(); @@ -51,6 +64,9 @@ fn setup_mock_source(tiling_spec: TilingSpecification) -> MockRasterSourceProces result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( RasterDataType::U8, 1, + grid_bounds, + geo_transform, + TimeDescriptor::new_irregular(Some(time)), ), data: vec![ RasterTile2D::new( @@ -135,23 +151,24 @@ fn bench_raster_processor< T: Pixel, F: Fn(TilingSpecification) -> S, S: RasterQueryProcessor, - C: QueryContext, >( bench_id: &'static str, list_of_named_querys: &[(&str, RasterQueryRectangle)], list_of_tiling_specs: &[TilingSpecification], tile_producing_operator_builderr: F, - ctx: &C, run_time: &tokio::runtime::Runtime, ) { for tiling_spec in list_of_tiling_specs { + let exe_ctx = MockExecutionContext::new_with_tiling_spec_and_thread_count(*tiling_spec, 8); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); + let operator = (tile_producing_operator_builderr)(*tiling_spec); for &(qrect_name, ref qrect) in list_of_named_querys { run_time.block_on(async { // query the operator let start_query = Instant::now(); - let query = operator.raster_query(qrect.clone(), ctx).await.unwrap(); + let query = operator.raster_query(qrect.clone(), &ctx).await.unwrap(); let query_elapsed = start_query.elapsed(); let start = Instant::now(); @@ -184,72 +201,55 @@ fn bench_no_data_tiles() { let qrects = vec![ ( "1 tile", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 60.).into(), (60., 0.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-60, 0], [-1, 59]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "2 tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 50.).into(), (60., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-50, 0], [9, 59]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "4 tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-5., 50.).into(), (55., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-55, -5], [9, 54]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "2 tiles, 2 no-data tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((130., 120.).into(), (190., 60.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-120, 130], [59, 189]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "empty tiles", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-5., 50.).into(), (55., -10.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_000_000_000_000, 1_000_000_000_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-50, -5], [-9, 54]).unwrap(), + TimeInterval::new(1_000_000_000_000, 1_000_000_000_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ]; - let tiling_specs = vec![TilingSpecification::new((0., 0.).into(), [600, 600].into())]; + let tiling_specs = vec![TilingSpecification::new([600, 600].into())]; let run_time = tokio::runtime::Runtime::new().unwrap(); - let ctx = MockQueryContext::with_chunk_size_and_thread_count(ChunkByteSize::MAX, 8); bench_raster_processor( "no_data_tiles", &qrects, &tiling_specs, setup_mock_source, - &ctx, &run_time, ); bench_raster_processor( @@ -257,7 +257,6 @@ fn bench_no_data_tiles() { &qrects, &tiling_specs, |ts| setup_gdal_source(create_ndvi_meta_data(), ts), - &ctx, &run_time, ); } @@ -265,30 +264,27 @@ fn bench_no_data_tiles() { fn bench_tile_size() { let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let run_time = tokio::runtime::Runtime::new().unwrap(); - let ctx = MockQueryContext::with_chunk_size_and_thread_count(ChunkByteSize::MAX, 8); let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [600, 600].into()), - TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), + TilingSpecification::new([32, 32].into()), + TilingSpecification::new([64, 64].into()), + TilingSpecification::new([128, 128].into()), + TilingSpecification::new([256, 256].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([600, 600].into()), + TilingSpecification::new([900, 900].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), ]; bench_raster_processor( @@ -296,7 +292,6 @@ fn bench_tile_size() { &qrects, &tiling_specs, |ts| setup_gdal_source(create_ndvi_meta_data(), ts), - &ctx, &run_time, ); } diff --git a/operators/benches/workflows.rs b/operators/benches/workflows.rs index 4b7dad7cf..1b7492d7a 100644 --- a/operators/benches/workflows.rs +++ b/operators/benches/workflows.rs @@ -5,38 +5,40 @@ clippy::missing_panics_doc )] // okay in benchmarks -use std::hint::black_box; -use std::time::{Duration, Instant}; - use futures::TryStreamExt; -use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + +use geoengine_datatypes::primitives::Coordinate2D; +use geoengine_datatypes::primitives::RasterQueryRectangle; use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::primitives::{QueryRectangle, RasterQueryRectangle, SpatialPartitioned}; -use geoengine_datatypes::raster::{Grid2D, RasterDataType, RenameBands}; +use geoengine_datatypes::raster::RenameBands; +use geoengine_datatypes::raster::{ + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, TilingStrategy, +}; use geoengine_datatypes::spatial_reference::SpatialReference; +use geoengine_operators::engine::SingleRasterOrVectorSource; +use geoengine_operators::engine::SpatialGridDescriptor; +use geoengine_operators::engine::TimeDescriptor; +use geoengine_operators::processing::Reprojection; +use geoengine_operators::processing::ReprojectionParams; +use std::hint::black_box; +use std::time::{Duration, Instant}; -use geoengine_datatypes::util::Identifier; use geoengine_datatypes::{ - primitives::{SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::TimeInterval, raster::{GridSize, RasterTile2D, TilingSpecification}, }; use geoengine_operators::call_on_generic_raster_processor; use geoengine_operators::engine::{ - MetaData, MultipleRasterSources, RasterBandDescriptors, RasterResultDescriptor, - SingleRasterOrVectorSource, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, RasterOperator, RasterQueryProcessor, +}; +use geoengine_operators::engine::{ + MultipleRasterSources, RasterBandDescriptors, RasterResultDescriptor, SingleRasterSource, + WorkflowOperatorPath, }; use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; use geoengine_operators::processing::{ - Expression, ExpressionParams, RasterStacker, RasterStackerParams, Reprojection, - ReprojectionParams, + Expression, ExpressionParams, RasterStacker, RasterStackerParams, }; -use geoengine_operators::source::GdalSource; -use geoengine_operators::{ - engine::{ChunkByteSize, MockExecutionContext, RasterOperator, RasterQueryProcessor}, - source::GdalSourceParameters, - util::gdal::create_ndvi_meta_data, -}; - use serde::{Serialize, Serializer}; pub struct BenchmarkCollector { @@ -65,7 +67,7 @@ pub trait BenchmarkRunner { pub struct WorkflowSingleBenchmark { bench_id: &'static str, query_name: &'static str, - query_rect: QueryRectangle, + query_rect: RasterQueryRectangle, tiling_spec: TilingSpecification, chunk_byte_size: ChunkByteSize, num_threads: usize, @@ -76,10 +78,7 @@ pub struct WorkflowSingleBenchmark { impl WorkflowSingleBenchmark where F: Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Fn(TilingSpecification, RasterQueryRectangle) -> Box, { #[inline(never)] pub fn run_bench(&self) -> WorkflowBenchmarkResult { @@ -139,10 +138,7 @@ where impl BenchmarkRunner for WorkflowSingleBenchmark where F: Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Fn(TilingSpecification, RasterQueryRectangle) -> Box, { fn run_all_benchmarks(self, bencher: &mut BenchmarkCollector) { bencher.add_benchmark_result(WorkflowSingleBenchmark::run_bench(&self)); @@ -166,11 +162,7 @@ where T: IntoIterator + Clone, B: IntoIterator + Clone, F: Clone + Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Clone - + Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Clone + Fn(TilingSpecification, RasterQueryRectangle) -> Box, { pub fn new( bench_id: &'static str, @@ -250,11 +242,7 @@ where T: IntoIterator + Clone, B: IntoIterator + Clone, F: Clone + Fn(TilingSpecification, usize) -> MockExecutionContext, - O: Clone - + Fn( - TilingSpecification, - QueryRectangle, - ) -> Box, + O: Clone + Fn(TilingSpecification, RasterQueryRectangle) -> Box, { fn run_all_benchmarks(self, bencher: &mut BenchmarkCollector) { self.into_benchmark_iterator() @@ -289,16 +277,18 @@ where serializer.serialize_u128(duration.as_millis()) } +#[allow(clippy::needless_pass_by_value)] // must match signature fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { - #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_pixel_bounds(query_rect.spatial_bounds()); let mock_data = tile_iter .enumerate() @@ -309,7 +299,7 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval(), tile_info, 0, data.into(), @@ -321,26 +311,27 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + TimeDescriptor::new_irregular(None), + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, } .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }; - let tiling_spec = TilingSpecification::new((0., 0.).into(), [512, 512].into()); + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); + let tiling_spec = TilingSpecification::new([512, 512].into()); let qrects = vec![("World in 36000x18000 pixels", qrect)]; let tiling_specs = vec![tiling_spec]; @@ -359,16 +350,19 @@ fn bench_mock_source_operator(bench_collector: &mut BenchmarkCollector) { .run_all_benchmarks(bench_collector); } +#[allow(clippy::too_many_lines)] fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCollector) { #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_pixel_bounds(query_rect.spatial_bounds()); let mock_data = tile_iter .enumerate() @@ -379,7 +373,7 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval(), tile_info, 0, data.into(), @@ -391,14 +385,16 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + TimeDescriptor::new_irregular(None), + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, }; @@ -427,21 +423,20 @@ fn bench_mock_source_operator_with_expression(bench_collector: &mut BenchmarkCol .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.005, 0.005).unwrap(), - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); let qrects = vec![("World in 72000x36000 pixels", qrect)]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), - TilingSpecification::new((0., 0.).into(), [18000, 18000].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), + TilingSpecification::new([18000, 18000].into()), ]; WorkflowMultiBenchmark::new( @@ -464,10 +459,12 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0.0, 0.), 0.01, -0.01), + ); + let tile_iter = tileing_strategy + .tile_information_iterator_from_pixel_bounds(query_rect.spatial_bounds()); let mock_data = tile_iter .enumerate() .map(|(id, tile_info)| { @@ -477,7 +474,7 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval(), tile_info, 0, data.into(), @@ -489,41 +486,44 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + TimeDescriptor::new_irregular(None), + SpatialGridDescriptor::source_from_parts( + tileing_strategy.geo_transform, + query_rect.spatial_bounds(), + ), + RasterBandDescriptors::new_single_band(), + ), }, }; Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: + geoengine_operators::processing::DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(mock_raster_operator.boxed()), } .boxed() } - let qrect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-18000, -9000], [17999, 8999]).unwrap(), // TODO: should be output bounds? + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ); let qrects = vec![("World in 36000x18000 pixels", qrect)]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [512, 512].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), - TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), - TilingSpecification::new((0., 0.).into(), [18000, 18000].into()), + TilingSpecification::new([512, 512].into()), + TilingSpecification::new([1024, 1024].into()), + TilingSpecification::new([2048, 2048].into()), + TilingSpecification::new([4096, 4096].into()), + TilingSpecification::new([9000, 9000].into()), + TilingSpecification::new([18000, 18000].into()), ]; WorkflowMultiBenchmark::new( @@ -540,18 +540,38 @@ fn bench_mock_source_operator_with_identity_reprojection(bench_collector: &mut B .run_all_benchmarks(bench_collector); } +/* fn bench_mock_source_operator_with_4326_to_3857_reprojection( bench_collector: &mut BenchmarkCollector, ) { + let qrect = SpatialPartition2D::new( + (-20_037_508.342_789_244, 20_048_966.104_014_594).into(), + (20_037_508.342_789_244, -20_048_966.104_014_594).into(), + ) + .unwrap(); + + let qtime = TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(); + let qband = BandSelection::first(); + + let tiling_spec = TilingSpecification::new([512, 512].into()); + + let qrects = vec![("World in 36000x18000 pixels", qrect)]; + let tiling_specs = vec![tiling_spec]; + #[allow(clippy::needless_pass_by_value)] // must match signature fn operator_builder( tiling_spec: TilingSpecification, query_rect: RasterQueryRectangle, ) -> Box { - let query_resolution = query_rect.spatial_resolution; - let query_time = query_rect.time_interval; - let tileing_strategy = tiling_spec.strategy(query_resolution.x, -query_resolution.y); - let tile_iter = tileing_strategy.tile_information_iterator(query_rect.spatial_partition()); + // FIXME: The query origin must match the tiling strategy's origin for now. Also use grid bounds not spatial bounds. + + let tileing_strategy = TilingStrategy::new( + tiling_spec.tile_size_in_pixels, + GeoTransform::new(Coordinate2D::new(0., 0.), 0.01, -0.01), + ); + + let tile_iter = tileing_strategy + .tile_information_iterator_from_grid_bounds(query_rect.grid_bounds()); let mock_data = tile_iter .enumerate() .map(|(id, tile_info)| { @@ -561,7 +581,7 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( ) .unwrap(); RasterTile2D::new_with_tile_info( - query_time, + query_rect.time_interval, tile_info, 0, data.into(), @@ -572,20 +592,21 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( let mock_raster_operator = MockRasterSource { params: MockRasterSourceParams { data: mock_data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: RasterResultDescriptor::new( + RasterDataType::U8, + SpatialReference::epsg_4326().into(), + None, + tileing_strategy.geo_transform, + query_rect.grid_bounds(), + RasterBandDescriptors::new_single_band(), + ), }, }; Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(mock_raster_operator.boxed()), } @@ -622,42 +643,42 @@ fn bench_mock_source_operator_with_4326_to_3857_reprojection( } fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![ ( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ( "World in 72000x36000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.005, 0.005).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.005, 0.005).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ), ]; let tiling_specs = vec![ - TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [32, 32].into()), + TilingSpecification::new(tiling_origin, [64, 64].into()), + TilingSpecification::new(tiling_origin, [128, 128].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -666,7 +687,7 @@ fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(); @@ -688,28 +709,30 @@ fn bench_gdal_source_operator_tile_size(bench_collector: &mut BenchmarkCollector } fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ // TilingSpecification::new((0., 0.).into(), [32, 32].into()), - TilingSpecification::new((0., 0.).into(), [64, 64].into()), - TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [64, 64].into()), + TilingSpecification::new(tiling_origin, [128, 128].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -718,7 +741,7 @@ fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut Be let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let expression_operator = Expression { @@ -760,28 +783,30 @@ fn bench_gdal_source_operator_with_expression_tile_size(bench_collector: &mut Be } fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut BenchmarkCollector) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in 36000x18000 pixels", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(0.01, 0.01).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + SpatialResolution::new(0.01, 0.01).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ // TilingSpecification::new((0., 0.).into(), [32, 32].into()), // TilingSpecification::new((0., 0.).into(), [64, 64].into()), // TilingSpecification::new((0., 0.).into(), [128, 128].into()), - TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [256, 256].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -790,12 +815,13 @@ fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut B let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let projection_operator = Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(gdal_operator.boxed()), } @@ -821,18 +847,21 @@ fn bench_gdal_source_operator_with_identity_reprojection(bench_collector: &mut B fn bench_gdal_source_operator_with_4326_to_3857_reprojection( bench_collector: &mut BenchmarkCollector, ) { + let tiling_origin = Coordinate2D::new(0., 0.); + let qrects = vec![( "World in EPSG:3857 ~ 40000 x 20000 px", - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( + RasterQueryRectangle::with_partition_and_resolution_and_origin( + SpatialPartition2D::new( (-20_037_508.342_789_244, 20_048_966.104_014_594).into(), (20_037_508.342_789_244, -20_048_966.104_014_594).into(), ) .unwrap(), - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), - spatial_resolution: SpatialResolution::new(1050., 2100.).unwrap(), - attributes: BandSelection::first(), - }, + SpatialResolution::new(1050., 2100.).unwrap(), + tiling_origin, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), )]; let tiling_specs = vec![ @@ -840,12 +869,12 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( // TilingSpecification::new((0., 0.).into(), [64, 64].into()), // TilingSpecification::new((0., 0.).into(), [128, 128].into()), // TilingSpecification::new((0., 0.).into(), [256, 256].into()), - TilingSpecification::new((0., 0.).into(), [512, 512].into()), + TilingSpecification::new(tiling_origin, [512, 512].into()), // TilingSpecification::new((0., 0.).into(), [600, 600].into()), // TilingSpecification::new((0., 0.).into(), [900, 900].into()), - TilingSpecification::new((0., 0.).into(), [1024, 1024].into()), - TilingSpecification::new((0., 0.).into(), [2048, 2048].into()), - TilingSpecification::new((0., 0.).into(), [4096, 4096].into()), + TilingSpecification::new(tiling_origin, [1024, 1024].into()), + TilingSpecification::new(tiling_origin, [2048, 2048].into()), + TilingSpecification::new(tiling_origin, [4096, 4096].into()), // TilingSpecification::new((0., 0.).into(), [9000, 9000].into()), ]; @@ -854,7 +883,7 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( let meta_data = create_ndvi_meta_data(); let gdal_operator = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), }; let projection_operator = Reprojection { @@ -863,6 +892,7 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( geoengine_datatypes::spatial_reference::SpatialReferenceAuthority::Epsg, 3857, ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource::from(gdal_operator.boxed()), } @@ -885,15 +915,19 @@ fn bench_gdal_source_operator_with_4326_to_3857_reprojection( .run_all_benchmarks(bench_collector); } +*/ + fn main() { let mut bench_collector = BenchmarkCollector::default(); bench_mock_source_operator(&mut bench_collector); bench_mock_source_operator_with_expression(&mut bench_collector); bench_mock_source_operator_with_identity_reprojection(&mut bench_collector); + /* bench_mock_source_operator_with_4326_to_3857_reprojection(&mut bench_collector); bench_gdal_source_operator_tile_size(&mut bench_collector); bench_gdal_source_operator_with_expression_tile_size(&mut bench_collector); bench_gdal_source_operator_with_identity_reprojection(&mut bench_collector); bench_gdal_source_operator_with_4326_to_3857_reprojection(&mut bench_collector); + */ } diff --git a/operators/src/adapters/feature_collection_merger.rs b/operators/src/adapters/feature_collection_merger.rs index e71b79f5b..8f672fefa 100644 --- a/operators/src/adapters/feature_collection_merger.rs +++ b/operators/src/adapters/feature_collection_merger.rs @@ -140,22 +140,19 @@ mod tests { use super::*; use crate::engine::{ - MockExecutionContext, MockQueryContext, QueryProcessor, TypedVectorQueryProcessor, - VectorOperator, WorkflowOperatorPath, + MockExecutionContext, QueryProcessor, TypedVectorQueryProcessor, VectorOperator, + WorkflowOperatorPath, }; use crate::error::Error; use crate::mock::{MockFeatureCollectionSource, MockPointSource, MockPointSourceParams}; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; + use geoengine_datatypes::collections::{DataCollection, MultiPointCollection}; use geoengine_datatypes::primitives::{ BoundingBox2D, Coordinate2D, MultiPoint, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - collections::{DataCollection, MultiPointCollection}, - primitives::SpatialResolution, - }; #[tokio::test] async fn simple() { @@ -165,9 +162,7 @@ mod tests { .collect(); let source = MockPointSource { - params: MockPointSourceParams { - points: coordinates.clone(), - }, + params: MockPointSourceParams::new(coordinates.clone()), }; let source = source @@ -184,13 +179,13 @@ mod tests { unreachable!(); }; - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, 0.0).into(), (10.0, 10.0).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let cx = MockQueryContext::new((std::mem::size_of::() * 2).into()); + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((0.0, 0.0).into(), (10.0, 10.0).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ); + let ecx = MockExecutionContext::test_default(); + let cx = ecx.mock_query_context((std::mem::size_of::() * 2).into()); let number_of_source_chunks = processor .query(qrect.clone(), &cx) @@ -262,13 +257,13 @@ mod tests { unreachable!(); }; - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, 0.0).into(), (0.0, 0.0).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let cx = MockQueryContext::new((0).into()); + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((0.0, 0.0).into(), (0.0, 0.0).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ); + let ecx = MockExecutionContext::test_default(); + let cx = ecx.mock_query_context((0).into()); let collections = FeatureCollectionChunkMerger::new(processor.query(qrect, &cx).await.unwrap().fuse(), 0) diff --git a/operators/src/adapters/mod.rs b/operators/src/adapters/mod.rs index e9d759011..7918878a5 100644 --- a/operators/src/adapters/mod.rs +++ b/operators/src/adapters/mod.rs @@ -7,23 +7,25 @@ mod raster_time_substream; mod simple_raster_stacker; mod sparse_tiles_fill_adapter; mod stream_statistics_adapter; +mod time_stream_merge; use band_extractor::BandExtractor; pub use feature_collection_merger::FeatureCollectionChunkMerger; pub use raster_stacker::{RasterStackerAdapter, RasterStackerSource}; pub use raster_subquery::{ FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, - TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }; pub use raster_time::{QueryWrapper, Queryable, RasterArrayTimeAdapter, RasterTimeAdapter}; pub use simple_raster_stacker::{ SimpleRasterStackerAdapter, SimpleRasterStackerSource, stack_individual_aligned_raster_bands, }; -pub use sparse_tiles_fill_adapter::{ +pub(crate) use sparse_tiles_fill_adapter::{ FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, SparseTilesFillAdapterError, }; pub use stream_statistics_adapter::StreamStatisticsAdapter; +pub use time_stream_merge::TimeIntervalStreamMerge; use self::raster_time_substream::RasterTimeMultiFold; use crate::util::Result; diff --git a/operators/src/adapters/raster_stacker.rs b/operators/src/adapters/raster_stacker.rs index 41b3bc357..4c40a8d11 100644 --- a/operators/src/adapters/raster_stacker.rs +++ b/operators/src/adapters/raster_stacker.rs @@ -2,11 +2,9 @@ use crate::util::Result; use futures::future::JoinAll; use futures::stream::{Fuse, FusedStream, Stream}; use futures::{Future, StreamExt, ready}; -use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, -}; +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - GridIdx2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, + GridBoundingBox2D, GridIdx2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, }; use pin_project::pin_project; use std::pin::Pin; @@ -66,28 +64,21 @@ impl From<(Q, Vec)> for RasterStackerSource { #[derive(Debug)] pub struct PartialQueryRect { - pub spatial_bounds: SpatialPartition2D, + pub spatial_bounds: GridBoundingBox2D, pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, } impl PartialQueryRect { fn raster_query_rectangle(&self, attributes: BandSelection) -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: self.spatial_bounds, - time_interval: self.time_interval, - spatial_resolution: self.spatial_resolution, - attributes, - } + RasterQueryRectangle::new(self.spatial_bounds, self.time_interval, attributes) } } impl From for PartialQueryRect { fn from(value: RasterQueryRectangle) -> Self { Self { - spatial_bounds: value.spatial_bounds, - time_interval: value.time_interval, - spatial_resolution: value.spatial_resolution, + spatial_bounds: value.spatial_bounds(), + time_interval: value.time_interval(), } } } @@ -127,22 +118,22 @@ where } } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - // TODO: get tiling strategy from stream or execution context instead of creating it here let strat = TilingStrategy { tile_size_in_pixels: tile_info.tile_size_in_pixels, geo_transform: tile_info.global_geo_transform, }; - - strat.tile_grid_box(partition).number_of_elements() + strat + .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) + .number_of_elements() } fn grid_idx_for_nth_tile( tile_info: &TileInformation, - partition: SpatialPartition2D, + pixel_bounds: GridBoundingBox2D, n: usize, ) -> Option { let strat = TilingStrategy { @@ -150,7 +141,9 @@ where geo_transform: tile_info.global_geo_transform, }; - strat.tile_idx_iterator(partition).nth(n) + strat + .tile_idx_iterator_from_grid_bounds(pixel_bounds) + .nth(n) } } @@ -273,9 +266,9 @@ where }); } - *num_spatial_tiles = Some(Self::number_of_tiles_in_partition( + *num_spatial_tiles = Some(Self::number_of_tiles_in_grid_bounds( &ok_tiles[0].tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_bounds, //TODO: use direct mehtod instead of conversion )); *stream_state = StreamState::ProducingTimeSlice { @@ -393,7 +386,6 @@ where state.set(State::Initial); } } - return Poll::Ready(Some(Ok(tile))); } }, @@ -408,8 +400,11 @@ where mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::{CacheHint, Measurement, SpatialResolution, TimeInterval}, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, Measurement, TimeInterval, TimeStep}, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; @@ -417,8 +412,8 @@ mod tests { use crate::{ adapters::QueryWrapper, engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptor, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, WorkflowOperatorPath, }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -428,6 +423,17 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks() { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [0, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -519,14 +525,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -534,14 +533,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -569,7 +561,7 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stacker = RasterStackerAdapter::new( vec![ @@ -591,9 +583,8 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -615,6 +606,17 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_keeps_single_band_input() { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -659,14 +661,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -685,7 +680,7 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stacker = RasterStackerAdapter::new( vec![ @@ -699,9 +694,8 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -714,6 +708,38 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks_stacks() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -887,19 +913,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -907,19 +921,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -947,7 +949,7 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stacker = RasterStackerAdapter::new( vec![ @@ -969,9 +971,8 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -1000,6 +1001,39 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_aligns_temporally_while_stacking_stacks() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(0, 5), @@ -1173,19 +1207,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -1193,19 +1215,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs2 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs2 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -1233,7 +1243,7 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stacker = RasterStackerAdapter::new( vec![ @@ -1255,9 +1265,8 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); @@ -1517,6 +1526,55 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_stacks_more() { + let result_descriptor_1 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs1 band3".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + + let result_descriptor_2 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![RasterBandDescriptor::new( + "mrs2 band2".to_string(), + Measurement::Unitless, + )] + .try_into() + .unwrap(), + }; + + let result_descriptor_3 = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("mrs3 band1".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("mrs3 band2".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }; + // input 1: 3 bands let data: Vec> = vec![ RasterTile2D { @@ -1784,20 +1842,7 @@ mod tests { let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs1 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band2".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs1 band3".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_1.clone(), }, } .boxed(); @@ -1805,19 +1850,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![RasterBandDescriptor::new( - "mrs2 band1".to_string(), - Measurement::Unitless, - )] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_2, }, } .boxed(); @@ -1825,19 +1858,7 @@ mod tests { let mrs3 = MockRasterSource { params: MockRasterSourceParams { data: data3.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: vec![ - RasterBandDescriptor::new("mrs3 band1".to_string(), Measurement::Unitless), - RasterBandDescriptor::new("mrs3 band2".to_string(), Measurement::Unitless), - ] - .try_into() - .unwrap(), - }, + result_descriptor: result_descriptor_3, }, } .boxed(); @@ -1874,7 +1895,7 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stacker = RasterStackerAdapter::new( vec![ @@ -1904,9 +1925,8 @@ mod tests { .into(), ], PartialQueryRect { - spatial_bounds: SpatialPartition2D::new_unchecked([0., 1.].into(), [3., 0.].into()), + spatial_bounds: GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), }, ); diff --git a/operators/src/adapters/raster_subquery/mod.rs b/operators/src/adapters/raster_subquery/mod.rs index 260c950be..41566f6e6 100644 --- a/operators/src/adapters/raster_subquery/mod.rs +++ b/operators/src/adapters/raster_subquery/mod.rs @@ -6,5 +6,5 @@ pub use raster_subquery_adapter::{ }; pub use raster_subquery_reprojection::{ - TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, fold_by_coordinate_lookup_future, }; diff --git a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs index 585925f07..6678eb282 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_adapter.rs @@ -1,38 +1,27 @@ -use crate::adapters::SparseTilesFillAdapter; -use crate::adapters::sparse_tiles_fill_adapter::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, -}; -use crate::engine::{QueryContext, QueryProcessor, RasterQueryProcessor, RasterResultDescriptor}; +use crate::engine::{QueryContext, RasterQueryProcessor}; use crate::error; use crate::util::Result; +use async_trait::async_trait; +use futures::Stream; use futures::future::BoxFuture; use futures::{Future, stream::FusedStream}; use futures::{ FutureExt, TryFuture, TryStreamExt, ready, stream::{BoxStream, TryFold}, }; -use futures::{Stream, StreamExt}; -use geoengine_datatypes::primitives::BandSelection; -use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialPartitioned, -}; -use geoengine_datatypes::raster::TilingSpecification; -use geoengine_datatypes::raster::{GridBoundingBox2D, GridBounds, GridStep}; -use geoengine_datatypes::{ - primitives::TimeInstance, - raster::{Blit, Pixel, RasterTile2D, TileInformation}, -}; +use geoengine_datatypes::primitives::RasterQueryRectangle; +use geoengine_datatypes::primitives::TimeInterval; +use geoengine_datatypes::primitives::{BandSelectionIter, CacheHint}; +use geoengine_datatypes::raster::{GridOrEmpty, TileInformationIter, TilingStrategy}; +use geoengine_datatypes::raster::{Pixel, RasterTile2D, TileInformation}; +use rayon::ThreadPool; +use std::pin::Pin; use pin_project::pin_project; -use rayon::ThreadPool; use std::sync::Arc; use std::task::Poll; -use std::pin::Pin; - -use async_trait::async_trait; - #[async_trait] pub trait FoldTileAccu { type RasterType: Pixel; @@ -41,7 +30,8 @@ pub trait FoldTileAccu { } pub trait FoldTileAccuMut: FoldTileAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D; + fn set_time(&mut self, new_time: TimeInterval); + fn set_cache_hint(&mut self, new_cache_hint: CacheHint); } pub type RasterFold<'a, T, FoldFuture, FoldMethod, FoldTileAccu> = @@ -55,17 +45,96 @@ type IntoTileFuture<'a, T> = BoxFuture<'a, Result>>; /// This is done using a `TileSubQuery`. /// The sub-query is resolved for each produced tile. +#[derive(Clone, Debug)] +struct TileBandCrossProductIter { + tile_iter: TileInformationIter, + band_iter: BandSelectionIter, + current_tile: Option, +} + +impl TileBandCrossProductIter { + fn new(tile_iter: TileInformationIter, band_iter: BandSelectionIter) -> Self { + let mut tile_iter = tile_iter; + let current_tile = tile_iter.next(); + Self { + tile_iter, + band_iter, + current_tile, + } + } + + fn reset(&mut self) { + self.band_iter.reset(); + self.tile_iter.reset(); + self.current_tile = self.tile_iter.next(); + } +} + +impl Iterator for TileBandCrossProductIter { + type Item = (TileInformation, u32); + + fn next(&mut self) -> Option { + let current_t = self.current_tile; + + match (current_t, self.band_iter.next()) { + (None, _) => None, + (Some(t), Some(b)) => Some((t, b)), + (Some(_t), None) => { + self.band_iter.reset(); + self.current_tile = self.tile_iter.next(); + self.current_tile.map(|t| { + ( + t, + self.band_iter + .next() + .expect("There must be at least one band"), + ) + }) + } + } + } +} + #[pin_project(project=StateInnerProjection)] #[derive(Debug, Clone)] enum StateInner { - CreateNextQuery, + ProgressTileBand { + time: TimeInterval, + }, + CreateNextTime, + // PollingTime(#[pin] N), not needed? + CreateNextQuery { + time: TimeInterval, + tile: TileInformation, + band: u32, + }, RunningQuery { #[pin] query_with_accu: A, + time: TimeInterval, + tile: TileInformation, + band: u32, + }, + RunningFold { + #[pin] + fold: B, + time: TimeInterval, + tile: TileInformation, + band: u32, + }, + RunningIntoTile { + #[pin] + into_tile: D, + time: TimeInterval, + tile: TileInformation, + band: u32, + }, + ReturnResult { + result: Option, + time: TimeInterval, + tile: TileInformation, + band: u32, }, - RunningFold(#[pin] B), - RunningIntoTile(#[pin] D), - ReturnResult(Option), Ended, } @@ -81,33 +150,30 @@ type StateInnerType<'a, P, FoldFuture, FoldMethod, TileAccu> = StateInner< /// This is done using a `TileSubQuery`. /// The sub-query is resolved for each produced tile. #[pin_project(project = RasterSubQueryAdapterProjection)] -pub struct RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery> +pub struct RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery, TimeStream> where PixelType: Pixel, RasterProcessorType: RasterQueryProcessor, SubQuery: SubQueryTileAggregator<'a, PixelType>, + TimeStream: Stream>, { + /// The '`TimeStream`' providing the time steps to produce + #[pin] + time_stream: TimeStream, /// The `RasterQueryProcessor` to answer the sub-queries source_processor: &'a RasterProcessorType, /// The `QueryContext` to use for sub-queries query_ctx: &'a dyn QueryContext, /// The `QueryRectangle` the adapter is queried with query_rect_to_answer: RasterQueryRectangle, - /// The `GridBoundingBox2D` that defines the tile grid space of the query. - grid_bounds: GridBoundingBox2D, - // the selected bands from the source - bands: Vec, - // the band being currently processed - current_band_index: u32, /// The `SubQuery` defines what this adapter does. sub_query: SubQuery, /// This `TimeInterval` is the time currently worked on - current_time_start: TimeInstance, - current_time_end: Option, - /// The `GridIdx2D` currently worked on - current_tile_spec: TileInformation, + current_time: TimeInterval, + + band_tile_iter: TileBandCrossProductIter, /// This current state of the adapter #[pin] @@ -120,127 +186,70 @@ where >, } -impl<'a, PixelType, RasterProcessor, SubQuery> - RasterSubQueryAdapter<'a, PixelType, RasterProcessor, SubQuery> +impl<'a, PixelType, RasterProcessor, SubQuery, TimeStream> + RasterSubQueryAdapter<'a, PixelType, RasterProcessor, SubQuery, TimeStream> where PixelType: Pixel, RasterProcessor: RasterQueryProcessor, SubQuery: SubQueryTileAggregator<'a, PixelType>, + TimeStream: Stream>, { pub fn new( source_processor: &'a RasterProcessor, query_rect_to_answer: RasterQueryRectangle, - tiling_spec: TilingSpecification, + tiling_strategy: TilingStrategy, query_ctx: &'a dyn QueryContext, sub_query: SubQuery, + time_stream: TimeStream, ) -> Self { - debug_assert!(query_rect_to_answer.spatial_resolution.y > 0.); - - let tiling_strat = tiling_spec.strategy( - query_rect_to_answer.spatial_resolution.x, - -query_rect_to_answer.spatial_resolution.y, - ); - - let grid_bounds = tiling_strat.tile_grid_box(query_rect_to_answer.spatial_partition()); - - let first_tile_spec = TileInformation { - global_geo_transform: tiling_strat.geo_transform, - global_tile_position: grid_bounds.min_index(), - tile_size_in_pixels: tiling_strat.tile_size_in_pixels, - }; + let tile_iter = tiling_strategy + .tile_information_iterator_from_pixel_bounds(query_rect_to_answer.spatial_bounds()); + let band_iter = BandSelectionIter::new(query_rect_to_answer.attributes().clone()); + let band_tile_iter = TileBandCrossProductIter::new(tile_iter, band_iter); Self { - current_tile_spec: first_tile_spec, - current_time_end: None, - current_time_start: query_rect_to_answer.time_interval.start(), - current_band_index: 0, - grid_bounds, - bands: query_rect_to_answer.attributes.as_vec(), - query_ctx, + current_time: TimeInterval::default(), // This is overwritten in the first poll_next call! query_rect_to_answer, + band_tile_iter, + query_ctx, source_processor, - state: StateInner::CreateNextQuery, + state: StateInner::CreateNextTime, sub_query, + time_stream, } } - /// Wrap the `RasterSubQueryAdapter` with a filter and a `SparseTilesFillAdapter` to produce a `Stream` compatible with `RasterQueryProcessor`. - /// Set the `cache_expiration` to unlimited, if the filler tiles will alway be empty. - pub fn filter_and_fill( - self, - cache_expiration: FillerTileCacheExpirationStrategy, - ) -> BoxStream<'a, Result>> + pub fn box_pin(self) -> BoxStream<'a, Result>> where - Self: Stream>>> + 'a, + SubQuery: Send + 'static, + TimeStream: Send + 'a, { - let grid_bounds = self.grid_bounds.clone(); - let global_geo_transform = self.current_tile_spec.global_geo_transform; - let tile_shape = self.current_tile_spec.tile_size_in_pixels; - let num_bands = self.bands.len() as u32; - let query_time_bounds = self.query_rect_to_answer.time_interval; - - let s = self.filter_map(|x| async move { - match x { - Ok(Some(t)) => Some(Ok(t)), - Ok(None) => None, - Err(e) => Some(Err(e)), - } - }); - - let s_filled = SparseTilesFillAdapter::new( - s, - grid_bounds, - num_bands, - global_geo_transform, - tile_shape, - cache_expiration, - query_time_bounds, - FillerTimeBounds::from(query_time_bounds), // operator should at least fill the query rect. Adapter will handle overflow at start / end gracefully. - ); - s_filled.boxed() - } - - /// Wrap `RasterSubQueryAdapter` to flatten the inner option. - /// - /// SAFETY: This call will cause panics if there is a None result! - pub(crate) fn expect(self, msg: &'static str) -> BoxStream<'a, Result>> - where - Self: Stream>>> + 'a, - { - self.map(|r| r.map(|o| o.expect(msg))).boxed() + Box::pin(self) } } -impl<'a, PixelType, RasterProcessorType, SubQuery> FusedStream - for RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery> +impl<'a, PixelType, RasterProcessorType, SubQuery, TimeStream> FusedStream + for RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery, TimeStream> where PixelType: Pixel, - RasterProcessorType: QueryProcessor< - Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + RasterProcessorType: RasterQueryProcessor, SubQuery: SubQueryTileAggregator<'a, PixelType> + 'static, + TimeStream: Stream> + Send, { fn is_terminated(&self) -> bool { matches!(self.state, StateInner::Ended) } } -impl<'a, PixelType, RasterProcessorType, SubQuery> Stream - for RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery> +impl<'a, PixelType, RasterProcessorType, SubQuery, TimeStream> Stream + for RasterSubQueryAdapter<'a, PixelType, RasterProcessorType, SubQuery, TimeStream> where PixelType: Pixel, - RasterProcessorType: QueryProcessor< - Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + RasterProcessorType: RasterQueryProcessor, SubQuery: SubQueryTileAggregator<'a, PixelType> + 'static, + TimeStream: Stream> + Send, { - type Item = Result>>; + type Item = Result>; /************************************************************************************************************************************** * This method uses the `StateInner` enum to keep track of the current state @@ -268,22 +277,79 @@ where return Poll::Ready(None); } - // first generate a new query - if matches!(*this.state, StateInner::CreateNextQuery) { + // check if we ended in a previous call + if matches!(*this.state, StateInner::ProgressTileBand { .. }) { + // The state is pinned. Project it to get access to the query stored in the context. + let time = if let StateInnerProjection::ProgressTileBand { time } = + this.state.as_mut().project() + { + *time + } else { + // we already checked that the state is `StateInner::RunningQuery` so this case can not happen. + unreachable!() + }; + + if let Some((next_tile, next_band)) = this.band_tile_iter.next() { + this.state.set(StateInner::CreateNextQuery { + time, + band: next_band, + tile: next_tile, + }); + } else { + this.state.set(StateInner::CreateNextTime); + } + } + + if matches!(*this.state, StateInner::CreateNextTime) { + let time_future = ready!(this.time_stream.poll_next(cx)); + match time_future { + None => { + this.state.set(StateInner::Ended); + return Poll::Ready(None); + } + Some(Err(e)) => { + this.state.set(StateInner::Ended); + return Poll::Ready(Some(Err(e))); + } + Some(Ok(time)) => { + *this.current_time = time; + this.band_tile_iter.reset(); + let (tile, band) = this + .band_tile_iter + .next() + .expect("There must be at least one band/tile"); + this.state + .set(StateInner::CreateNextQuery { time, band, tile }); + } + } + } + + // first generate a new query. + if matches!(*this.state, StateInner::CreateNextQuery { .. }) { + let (time, band, tile) = + if let StateInnerProjection::CreateNextQuery { time, band, tile } = + this.state.as_mut().project() + { + (*time, *band, *tile) + } else { + // we already checked that the state is `StateInner::RunningQuery` so this case can not happen. + unreachable!() + }; + match this.sub_query.tile_query_rectangle( - *this.current_tile_spec, + tile, this.query_rect_to_answer.clone(), - *this.current_time_start, - this.bands[*this.current_band_index as usize], + time, + band, ) { - Ok(Some(tile_query_rectangle)) => { + Ok(Some(raster_query_rect)) => { let tile_query_stream_fut = this .source_processor - .raster_query(tile_query_rectangle.clone(), *this.query_ctx); + .raster_query(raster_query_rect.clone(), *this.query_ctx); let tile_folding_accu_fut = this.sub_query.new_fold_accu( - *this.current_tile_spec, - tile_query_rectangle, + tile, + raster_query_rect, this.query_ctx.thread_pool(), ); @@ -293,9 +359,17 @@ where this.state.set(StateInner::RunningQuery { query_with_accu: joined_future, + band, + tile, + time, }); } - Ok(None) => this.state.set(StateInner::ReturnResult(None)), + Ok(None) => this.state.set(StateInner::ReturnResult { + result: None, + band, + tile, + time, + }), Err(e) => { this.state.set(StateInner::Ended); return Poll::Ready(Some(Err(e))); @@ -305,23 +379,32 @@ where // A query was issued, so we check whether it is finished // To work in this scope we first check if the state is the one we expect. We want to set the state in this scope so we can not borrow it here! - if matches!(*this.state, StateInner::RunningQuery { query_with_accu: _ }) { + if matches!(*this.state, StateInner::RunningQuery { .. }) { // The state is pinned. Project it to get access to the query stored in the context. - let rq_res = if let StateInnerProjection::RunningQuery { query_with_accu } = - this.state.as_mut().project() + let (time, band, tile, query_with_accu) = if let StateInnerProjection::RunningQuery { + query_with_accu, + tile, + band, + time, + } = this.state.as_mut().project() { - ready!(query_with_accu.poll(cx)) + (*time, *band, *tile, query_with_accu) } else { // we already checked that the state is `StateInner::RunningQuery` so this case can not happen. unreachable!() }; - match rq_res { + match ready!(query_with_accu.poll(cx)) { Ok((query, tile_folding_accu)) => { let tile_folding_stream = query.try_fold(tile_folding_accu, this.sub_query.fold_method()); - this.state.set(StateInner::RunningFold(tile_folding_stream)); + this.state.set(StateInner::RunningFold { + fold: tile_folding_stream, + time, + tile, + band, + }); } Err(e) => { this.state.set(StateInner::Ended); @@ -332,18 +415,28 @@ where // We are waiting for/expecting the result of the fold. // This block uses the same check and project pattern as above. - if matches!(*this.state, StateInner::RunningFold(_)) { - let rf_res = - if let StateInnerProjection::RunningFold(fold) = this.state.as_mut().project() { - ready!(fold.poll(cx)) - } else { - unreachable!() - }; + if matches!(*this.state, StateInner::RunningFold { .. }) { + let (tile, band, time, fold) = if let StateInnerProjection::RunningFold { + fold, + time, + tile, + band, + } = this.state.as_mut().project() + { + (*tile, *band, *time, fold) + } else { + unreachable!() + }; - match rf_res { + match ready!(fold.poll(cx)) { Ok(tile_accu) => { - let tile = tile_accu.into_tile(); - this.state.set(StateInner::RunningIntoTile(tile)); + let into_tile = tile_accu.into_tile(); + this.state.set(StateInner::RunningIntoTile { + into_tile, + time, + tile, + band, + }); } Err(e) => { this.state.set(StateInner::Ended); @@ -354,20 +447,29 @@ where // We are waiting for/expecting the result of `into_tile` method. // This block uses the same check and project pattern as above. - if matches!(*this.state, StateInner::RunningIntoTile(_)) { - let rf_res = if let StateInnerProjection::RunningIntoTile(fold) = - this.state.as_mut().project() + if matches!(*this.state, StateInner::RunningIntoTile { .. }) { + let (time, band, tile, into_tile) = if let StateInnerProjection::RunningIntoTile { + into_tile, + time, + tile, + band, + } = this.state.as_mut().project() { - ready!(fold.poll(cx)) + (*time, *band, *tile, into_tile) } else { unreachable!() }; - match rf_res { - Ok(mut tile) => { - // set the tile band to the running index, that is because output bands always start at zero and are consecutive, independent of the input bands - tile.band = *this.current_band_index; - this.state.set(StateInner::ReturnResult(Some(tile))); + match ready!(into_tile.poll(cx)) { + Ok(mut result_tile) => { + // set the tile band to the running index, that is because output bands always start at zero and are consecutive, independent of the input bands. + result_tile.band = band; + this.state.set(StateInner::ReturnResult { + result: Some(result_tile), + time, + tile, + band, + }); } Err(e) => { this.state.set(StateInner::Ended); @@ -376,77 +478,37 @@ where } } + debug_assert!( + matches!(*this.state, StateInner::ReturnResult { .. }), + "Must be in 'ReturnResult' state at this point!" + ); // At this stage we are in ReturnResult state. Either from a running fold or because the tile query rect was not valid. // This block uses the check and project pattern as above. - let tile_option = if let StateInnerProjection::ReturnResult(tile_option) = - this.state.as_mut().project() + let (tile, band, time, result) = if let StateInnerProjection::ReturnResult { + result, + time, + tile, + band, + } = this.state.as_mut().project() { - tile_option.take() + (*tile, *band, *time, result.take()) } else { unreachable!() }; // In the next poll we need to produce a new tile (if nothing else happens) - this.state.set(StateInner::CreateNextQuery); - - // If there is a tile, set the current_time_end option. - if let Some(tile) = &tile_option { - debug_assert!(*this.current_time_start >= tile.time.start()); - *this.current_time_end = Some(tile.time.end()); - } - - // now do progress - - let next_tile_pos = if *this.current_band_index + 1 < this.bands.len() as u32 { - // there is still another band to process for the current tile position - *this.current_band_index += 1; - Some(this.current_tile_spec.global_tile_position) - } else { - // all bands for the current tile are processed, we can go to the next tile in space, if there is one - *this.current_band_index = 0; - this.grid_bounds - .inc_idx_unchecked(this.current_tile_spec.global_tile_position, 1) - }; - - // if the grid idx wraps around set the ne query time instance to the end time instance of the last round - match (next_tile_pos, *this.current_time_end) { - (Some(idx), _) => { - // update the spatial index - this.current_tile_spec.global_tile_position = idx; - } - (None, None) => { - // end the stream since we never recieved a tile from any subquery. Should only happen if we end the first grid iteration. - // NOTE: this assumes that the input operator produces no data tiles for queries where time and space are valid but no data is avalable. - debug_assert!(&tile_option.is_none()); - debug_assert!( - *this.current_time_start == this.query_rect_to_answer.time_interval.start() - ); - this.state.set(StateInner::Ended); - } - (None, Some(end_time)) if end_time == *this.current_time_start => { - // Only for time instants: reset the spatial idx to the first tile of the grid AND increase the request time by 1. - this.current_tile_spec.global_tile_position = this.grid_bounds.min_index(); - *this.current_time_start = end_time + 1; - *this.current_time_end = None; - - // check if the next time to request is inside the bounds we are want to answer. - if *this.current_time_start >= this.query_rect_to_answer.time_interval.end() { - this.state.set(StateInner::Ended); - } - } - (None, Some(end_time)) => { - // reset the spatial idx to the first tile of the grid AND move the requested time to the last known time. - this.current_tile_spec.global_tile_position = this.grid_bounds.min_index(); - *this.current_time_start = end_time; - *this.current_time_end = None; - - // check if the next time to request is inside the bounds we are want to answer. - if *this.current_time_start >= this.query_rect_to_answer.time_interval.end() { - this.state.set(StateInner::Ended); - } - } - } + this.state.set(StateInner::ProgressTileBand { time }); + + let result_tile = result.unwrap_or_else(|| { + RasterTile2D::new_with_tile_info( + time, + tile, + band, + GridOrEmpty::new_empty_shape(tile.tile_size_in_pixels), + CacheHint::max_duration(), + ) + }); - Poll::Ready(Some(Ok(tile_option))) + Poll::Ready(Some(Ok(result_tile))) } } @@ -477,285 +539,40 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, + _query_rect: RasterQueryRectangle, + time: TimeInterval, band_idx: u32, - ) -> Result>; + ) -> Result> { + Ok(Some(RasterQueryRectangle::new( + tile_info.global_pixel_bounds(), + time, + band_idx.into(), + ))) + } /// This method generates the method which combines the accumulator and each tile of the sub-query stream in the `TryFold` stream adapter. fn fold_method(&self) -> Self::FoldMethod; - fn into_raster_subquery_adapter( + fn into_raster_subquery_adapter( self, source: &'a S, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, - tiling_specification: TilingSpecification, - ) -> RasterSubQueryAdapter<'a, T, S, Self> + tiling_strategy: TilingStrategy, + time_stream: G, + ) -> RasterSubQueryAdapter<'a, T, S, Self, G> where S: RasterQueryProcessor, + G: Stream>, Self: Sized, { - RasterSubQueryAdapter::<'a, T, S, Self>::new(source, query, tiling_specification, ctx, self) - } -} - -#[derive(Clone, Debug)] -pub struct RasterTileAccu2D { - pub tile: RasterTile2D, - pub pool: Arc, -} - -impl RasterTileAccu2D { - pub fn new(tile: RasterTile2D, pool: Arc) -> Self { - RasterTileAccu2D { tile, pool } - } -} - -#[async_trait] -impl FoldTileAccu for RasterTileAccu2D { - type RasterType = T; - - async fn into_tile(self) -> Result> { - Ok(self.tile) - } - - fn thread_pool(&self) -> &Arc { - &self.pool - } -} - -impl FoldTileAccuMut for RasterTileAccu2D { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.tile - } -} - -pub fn fold_by_blit_impl( - accu: RasterTileAccu2D, - tile: RasterTile2D, -) -> Result> -where - T: Pixel, -{ - let mut accu_tile = accu.tile; - let pool = accu.pool; - let t_union = accu_tile.time.union(&tile.time)?; - - accu_tile.time = t_union; - - if tile.grid_array.is_empty() && accu_tile.grid_array.is_empty() { - // only skip if both tiles are empty. There might be valid data in one otherwise. - return Ok(RasterTileAccu2D::new(accu_tile, pool)); - } - - let mut materialized_tile = accu_tile.into_materialized_tile(); - - materialized_tile.blit(tile)?; - - Ok(RasterTileAccu2D::new(materialized_tile.into(), pool)) -} - -#[allow(dead_code)] -pub fn fold_by_blit_future( - accu: RasterTileAccu2D, - tile: RasterTile2D, -) -> impl Future>> -where - T: Pixel, -{ - crate::util::spawn_blocking(|| fold_by_blit_impl(accu, tile)).then(|x| async move { - match x { - Ok(r) => r, - Err(e) => Err(e.into()), - } - }) -} - -#[cfg(test)] -mod tests { - use std::marker::PhantomData; - - use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{EmptyGrid2D, Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, - spatial_reference::SpatialReference, - util::test::TestDefault, - }; - - use super::*; - use crate::engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, - }; - use crate::mock::{MockRasterSource, MockRasterSourceParams}; - use futures::{StreamExt, TryFutureExt}; - - #[derive(Debug, Clone)] - pub struct TileSubQueryIdentity { - fold_fn: F, - _phantom_pixel_type: PhantomData, - } - - impl<'a, T, FoldM, FoldF> SubQueryTileAggregator<'a, T> for TileSubQueryIdentity - where - T: Pixel, - FoldM: Send + Sync + 'a + Clone + Fn(RasterTileAccu2D, RasterTile2D) -> FoldF, - FoldF: Send + TryFuture, Error = error::Error>, - { - type FoldFuture = FoldF; - - type FoldMethod = FoldM; - - type TileAccu = RasterTileAccu2D; - type TileAccuFuture = BoxFuture<'a, Result>; - - fn new_fold_accu( - &self, - tile_info: TileInformation, - query_rect: RasterQueryRectangle, - pool: &Arc, - ) -> Self::TileAccuFuture { - identity_accu(tile_info, &query_rect, pool.clone()).boxed() - } - - fn tile_query_rectangle( - &self, - tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, - band_idx: u32, - ) -> Result> { - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: query_rect.spatial_resolution, - attributes: band_idx.into(), - })) - } - - fn fold_method(&self) -> Self::FoldMethod { - self.fold_fn.clone() - } - } - - pub fn identity_accu( - tile_info: TileInformation, - query_rect: &RasterQueryRectangle, - pool: Arc, - ) -> impl Future>> + use { - let time_interval = query_rect.time_interval; - crate::util::spawn_blocking(move || { - let output_raster = EmptyGrid2D::new(tile_info.tile_size_in_pixels).into(); - let output_tile = RasterTile2D::new_with_tile_info( - time_interval, - tile_info, - 0, - output_raster, - CacheHint::max_duration(), - ); - RasterTileAccu2D::new(output_tile, pool) - }) - .map_err(From::from) - } - - #[tokio::test] - async fn identity() { - let data: Vec> = vec![ - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(0, 5), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 0].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - RasterTile2D { - time: TimeInterval::new_unchecked(5, 10), - tile_position: [-1, 1].into(), - band: 0, - global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) - .unwrap() - .into(), - properties: Default::default(), - cache_hint: CacheHint::default(), - }, - ]; - - let mrs1 = MockRasterSource { - params: MockRasterSourceParams { - data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(); - - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - shape_array: [2, 2], - }; - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - - let query_ctx = MockQueryContext::test_default(); - let tiling_strat = exe_ctx.tiling_specification; - - let op = mrs1 - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - - let qp = op.query_processor().unwrap().get_u8().unwrap(); - - let a = RasterSubQueryAdapter::new( - &qp, - query_rect, - tiling_strat, - &query_ctx, - TileSubQueryIdentity { - fold_fn: fold_by_blit_future, - _phantom_pixel_type: PhantomData, - }, - ); - let res = a - .map(Result::unwrap) - .map(Option::unwrap) - .collect::>>() - .await; - assert!(data.tiles_equal_ignoring_cache_hint(&res)); + RasterSubQueryAdapter::<'a, T, S, Self, G>::new( + source, + query, + tiling_strategy, + ctx, + self, + time_stream, + ) } } diff --git a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs index 3f0f6a4fd..ac056b6e6 100644 --- a/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs +++ b/operators/src/adapters/raster_subquery/raster_subquery_reprojection.rs @@ -7,39 +7,44 @@ use async_trait::async_trait; use futures::future::BoxFuture; use futures::{Future, FutureExt, TryFuture, TryFutureExt}; use geoengine_datatypes::operations::reproject::Reproject; -use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialPartitioned, + AxisAlignedRectangle, BoundingBox2D, CacheHint, RasterQueryRectangle, SpatialPartition2D, + SpatialPartitioned, }; use geoengine_datatypes::raster::{ - Grid2D, GridIndexAccess, GridSize, UpdateIndexedElementsParallel, + Grid2D, GridIndexAccess, GridIntersection, GridSize, SpatialGridDefinition, + UpdateIndexedElementsParallel, }; use geoengine_datatypes::{ operations::reproject::{CoordinateProjection, CoordinateProjector}, - primitives::{SpatialResolution, TimeInterval}, + primitives::TimeInterval, raster::EmptyGrid, spatial_reference::SpatialReference, }; use geoengine_datatypes::{ - primitives::{Coordinate2D, TimeInstance}, + primitives::Coordinate2D, raster::{CoordinatePixelAccess, GridIdx2D, Pixel, RasterTile2D, TileInformation}, }; use num; use rayon::ThreadPool; use rayon::iter::{IndexedParallelIterator, ParallelIterator}; -use rayon::slice::ParallelSliceMut; +use rayon::slice::{ParallelSlice, ParallelSliceMut}; use tracing::debug; use super::{FoldTileAccu, FoldTileAccuMut, SubQueryTileAggregator}; +#[derive(Debug, Clone, Copy)] +pub struct TileReprojectionSubqueryGridInfo { + pub in_spatial_grid: SpatialGridDefinition, + pub out_spatial_grid: SpatialGridDefinition, +} + #[derive(Debug)] pub struct TileReprojectionSubQuery { pub in_srs: SpatialReference, pub out_srs: SpatialReference, pub fold_fn: F, - pub in_spatial_res: SpatialResolution, - pub valid_bounds_in: SpatialPartition2D, - pub valid_bounds_out: SpatialPartition2D, + pub state: TileReprojectionSubqueryGridInfo, pub _phantom_data: PhantomData, } @@ -66,13 +71,11 @@ where query_rect: RasterQueryRectangle, pool: &Arc, ) -> Self::TileAccuFuture { - // println!("new_fold_accu {:?}", &tile_info.global_tile_position); - build_accu( &query_rect, pool.clone(), tile_info, - self.valid_bounds_out, + self.state, self.out_srs, self.in_srs, ) @@ -83,25 +86,42 @@ where &self, tile_info: TileInformation, query_rect: RasterQueryRectangle, - start_time: TimeInstance, + time: TimeInterval, band_idx: u32, ) -> Result> { - // this is the spatial partition we are interested in - let valid_spatial_bounds = self - .valid_bounds_out - .intersection(&tile_info.spatial_partition()) - .and_then(|vo| vo.intersection(&query_rect.spatial_partition())); + // this are the pixels we are interested in + debug_assert_eq!( + tile_info.global_geo_transform, + self.state.out_spatial_grid.geo_transform() + ); + + let valid_pixel_bounds = self + .state + .out_spatial_grid + .grid_bounds() + .intersection(&tile_info.global_pixel_bounds()) + .and_then(|b| b.intersection(&query_rect.spatial_bounds())); + + let valid_spatial_bounds = valid_pixel_bounds.map(|pb| { + self.state + .out_spatial_grid + .geo_transform() + .grid_to_spatial_bounds(&pb) + }); + if let Some(bounds) = valid_spatial_bounds { let proj = CoordinateProjector::from_known_srs(self.out_srs, self.in_srs)?; let projected_bounds = bounds.reproject(&proj); match projected_bounds { - Ok(pb) => Ok(Some(RasterQueryRectangle { - spatial_bounds: pb, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: self.in_spatial_res, - attributes: band_idx.into(), - })), + Ok(pb) => Ok(Some(RasterQueryRectangle::new( + self.state + .in_spatial_grid + .geo_transform() + .spatial_to_grid_bounds(&pb), + time, + band_idx.into(), + ))), // In some strange cases the reprojection can return an empty box. // We ignore it since it contains no pixels. Err(geoengine_datatypes::error::Error::OutputBboxEmpty { bbox: _ }) => Ok(None), @@ -122,11 +142,11 @@ fn build_accu( query_rect: &RasterQueryRectangle, pool: Arc, tile_info: TileInformation, - valid_bounds_out: SpatialPartition2D, + state: TileReprojectionSubqueryGridInfo, out_srs: SpatialReference, in_srs: SpatialReference, ) -> impl Future>> + use { - let time_interval = query_rect.time_interval; + let time_interval = query_rect.time_interval(); crate::util::spawn_blocking(move || { let output_raster = EmptyGrid::new(tile_info.tile_size_in_pixels); @@ -138,7 +158,7 @@ fn build_accu( tile_info, out_srs, in_srs, - &valid_bounds_out, + &state.out_spatial_grid.spatial_partition(), )?; Ok(TileWithProjectionCoordinates { @@ -181,10 +201,12 @@ fn projected_coordinate_grid_parallel( &tile_info.global_tile_position ); - let mut coord_grid: Grid2D> = + let mut in_coord_grid: Grid2D> = Grid2D::new_filled(tile_info.tile_size_in_pixels, None); - let tile_geo_transform = tile_info.tile_geo_transform(); + let out_coords = tile_info + .spatial_grid_definition() + .generate_coord_grid_pixel_center(); let parallelism = pool.current_num_threads(); let par_chunk_split = @@ -198,65 +220,53 @@ fn projected_coordinate_grid_parallel( par_chunk_size ); - let axis_size_x = tile_info.tile_size_in_pixels.axis_size_x(); - - let res = coord_grid + in_coord_grid .data .par_chunks_mut(par_chunk_size) - .enumerate() - .try_for_each(|(chunk_idx, opt_coord_slice)| { - let chunk_start_y = chunk_idx * par_chunk_split; - let chunk_len = opt_coord_slice.len(); - let chunk_end_y = chunk_start_y + (chunk_len / axis_size_x) - 1; - let out_coords = (0..chunk_len) - .map(|lin_idx| { - let x_idx = lin_idx % axis_size_x; - let y_idx = lin_idx / axis_size_x + chunk_start_y; - let grid_idx = GridIdx2D::from([y_idx as isize, x_idx as isize]); - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(grid_idx) - }) - .collect::>(); - - // the output bounds start at the top left corner of the chunk. - let ul_grid_idx = GridIdx2D::from([chunk_start_y as isize, 0_isize]); - let ul_coord = - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(ul_grid_idx); - - // the output bounds must cover the whole chunk pixels. - let lr_grid_idx = - GridIdx2D::from([chunk_end_y as isize, (axis_size_x - 1) as isize]); - let lr_coord = - tile_geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(lr_grid_idx + 1); - - let chunk_bounds = SpatialPartition2D::new_unchecked(ul_coord, lr_coord); + .zip(out_coords.data.par_chunks(par_chunk_size)) + .try_for_each(|(in_coord_slice, out_coord_slice)| { + debug_assert_eq!( + in_coord_slice.len(), + out_coord_slice.len(), + "slices must be equal" + ); + let chunk_bounds = BoundingBox2D::from_coord_ref_iter(out_coord_slice.iter()); + + if chunk_bounds.is_none() { + debug!("reprojection early exit"); + return Ok(()); + } + + let chunk_bounds = chunk_bounds.expect("checked above"); + let valid_out_area = valid_out_area.as_bbox(); let proj = CoordinateProjector::from_known_srs(out_srs, in_srs)?; - if valid_out_area.contains(&chunk_bounds) { + if valid_out_area.contains_bbox(&chunk_bounds) { debug!("reproject whole tile chunk"); - let in_coords = proj.project_coordinates(&out_coords)?; - opt_coord_slice + let in_coords = proj.project_coordinates(out_coord_slice)?; + in_coord_slice .iter_mut() - .zip(in_coords) - .for_each(|(opt_coord, in_coord)| *opt_coord = Some(in_coord)); - } else if valid_out_area.intersects(&chunk_bounds) { + .zip(in_coords.into_iter()) + .for_each(|(a, b)| *a = Some(b)); + } else if valid_out_area.intersects_bbox(&chunk_bounds) { debug!("reproject part of tile chunk"); - opt_coord_slice.iter_mut().zip(out_coords).for_each( - |(opt_coord, idx_coord)| { - let in_coord = if valid_out_area.contains_coordinate(&idx_coord) { - proj.project_coordinate(idx_coord).ok() + in_coord_slice + .iter_mut() + .zip(out_coord_slice.iter()) + .for_each(|(in_coord, out_coord)| { + *in_coord = if valid_out_area.contains_coordinate(out_coord) { + proj.project_coordinate(*out_coord).ok() } else { None }; - *opt_coord = in_coord; - }, - ); + }); } else { - debug!("reproject empty tile chunk"); + // do nothing. Should be unreachable } Result::<(), crate::error::Error>::Ok(()) - }); - res.map(|()| coord_grid) + })?; + Ok(in_coord_grid) }); debug!( "projected_coordinate_grid_parallel took {} (ns)", @@ -296,8 +306,8 @@ where let mut accu = accu; let t_union = accu.accu_tile.time.union(&tile.time)?; - accu.tile_mut().time = t_union; - accu.tile_mut().cache_hint.merge_with(&tile.cache_hint); + accu.set_time(t_union); + accu.set_cache_hint(accu.accu_tile.cache_hint.merged(&tile.cache_hint)); if tile.grid_array.is_empty() { return Ok(accu); @@ -351,8 +361,12 @@ impl FoldTileAccu for TileWithProjectionCoordinates { } impl FoldTileAccuMut for TileWithProjectionCoordinates { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.accu_tile + fn set_time(&mut self, time: TimeInterval) { + self.accu_tile.time = time; + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.accu_tile.cache_hint = cache_hint; } } @@ -360,22 +374,26 @@ impl FoldTileAccuMut for TileWithProjectionCoordinates { mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::BandSelection, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, - util::test::TestDefault, + primitives::{BandSelection, TimeStep}, + raster::{ + BoundedGrid, GeoTransform, Grid, GridBoundingBox2D, GridShape, GridShape2D, + RasterDataType, SpatialGridDefinition, TilingSpecification, + }, + util::test::{TestDefault, assert_eq_two_list_of_tiles_u8}, }; use crate::{ adapters::RasterSubQueryAdapter, engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, + MockExecutionContext, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, TimeDescriptor, WorkflowOperatorPath, }, mock::{MockRasterSource, MockRasterSourceParams}, }; use super::*; + #[allow(clippy::too_many_lines)] #[tokio::test] async fn identity_projection() { let projection = SpatialReference::epsg_4326(); @@ -386,7 +404,9 @@ mod tests { tile_position: [-1, 0].into(), band: 0, global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), properties: Default::default(), cache_hint: CacheHint::default(), }, @@ -423,60 +443,74 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 2.), 1., -1.), + GridShape::new_2d(2, 4).bounding_box(), + )), + bands: RasterBandDescriptors::new_single_band(), + }; + + let tiling_spec = TilingSpecification::new(GridShape2D::new([2, 2])); + + let tiling_grid = result_descriptor.tiling_grid_definition(tiling_spec); + let tiling_strat = tiling_grid.generate_data_tiling_strategy(); + + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_spec); + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - shape_array: [2, 2], - }; - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); - let query_ctx = MockQueryContext::test_default(); - let tiling_strat = exe_ctx.tiling_specification; + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let op = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); - let qp = op.query_processor().unwrap().get_u8().unwrap(); - - let valid_bounds = projection.area_of_use_projected().unwrap(); + let qp = op.query_processor().unwrap().get_u8(); + let qp = qp.unwrap(); let state_gen = TileReprojectionSubQuery { in_srs: projection, out_srs: projection, fold_fn: fold_by_coordinate_lookup_future, - in_spatial_res: query_rect.spatial_resolution, - valid_bounds_in: valid_bounds, - valid_bounds_out: valid_bounds, + state: TileReprojectionSubqueryGridInfo { + in_spatial_grid: tiling_grid.tiling_spatial_grid_definition(), + out_spatial_grid: tiling_grid.tiling_spatial_grid_definition(), + }, _phantom_data: PhantomData, }; - let a = RasterSubQueryAdapter::new(&qp, query_rect, tiling_strat, &query_ctx, state_gen); - let res = a - .map(Result::unwrap) - .map(Option::unwrap) - .collect::>>() - .await; - assert!(data.tiles_equal_ignoring_cache_hint(&res)); + + let time_stream = qp + .time_query(query_rect.time_interval(), &query_ctx) + .await + .unwrap(); + + let a = RasterSubQueryAdapter::new( + &qp, + query_rect, + tiling_strat, + &query_ctx, + state_gen, + time_stream, + ); + let res = a.map(Result::unwrap).collect::>().await; + + assert_eq_two_list_of_tiles_u8(&data, &res, false); } } diff --git a/operators/src/adapters/raster_time.rs b/operators/src/adapters/raster_time.rs index 0a14ec545..8cfbbc7bf 100644 --- a/operators/src/adapters/raster_time.rs +++ b/operators/src/adapters/raster_time.rs @@ -3,10 +3,11 @@ use crate::util::Result; use crate::util::stream_zip::StreamArrayZip; use futures::future::{self, BoxFuture, Join, JoinAll}; use futures::stream::{BoxStream, FusedStream, Zip}; -use futures::{Future, Stream}; -use futures::{StreamExt, ready}; -use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartition2D, TimeInterval}; -use geoengine_datatypes::raster::{GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy}; +use futures::{Future, Stream, StreamExt, ready}; +use geoengine_datatypes::primitives::{RasterQueryRectangle, TimeInterval}; +use geoengine_datatypes::raster::{ + GridBoundingBox2D, GridSize, Pixel, RasterTile2D, TileInformation, TilingStrategy, +}; use pin_project::pin_project; use std::cmp::min; use std::pin::Pin; @@ -110,17 +111,17 @@ where (tile_a, tile_b) } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - // TODO: get tiling strategy from stream or execution context instead of creating it here let strat = TilingStrategy { tile_size_in_pixels: tile_info.tile_size_in_pixels, geo_transform: tile_info.global_geo_transform, }; - - strat.tile_grid_box(partition).number_of_elements() + strat + .global_pixel_grid_bounds_to_tile_grid_bounds(grid_bounds) + .number_of_elements() } } @@ -212,11 +213,11 @@ where tiles } - fn number_of_tiles_in_partition( + fn number_of_tiles_in_grid_bounds( tile_info: &TileInformation, - partition: SpatialPartition2D, + grid_bounds: GridBoundingBox2D, ) -> usize { - RasterTimeAdapter::::number_of_tiles_in_partition(tile_info, partition) + RasterTimeAdapter::::number_of_tiles_in_grid_bounds(tile_info, grid_bounds) } } @@ -284,9 +285,9 @@ where Some((Ok(tile_a), Ok(tile_b))) => { // TODO: calculate at start when tiling info is available before querying first tile let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_partition( + Self::number_of_tiles_in_grid_bounds( &tile_a.tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_bounds(), // TODO: this should be calculated from the tile grid bounds and not the spatial bounds. ) }); @@ -304,21 +305,22 @@ where // advance current query rectangle let mut new_start = min(tile_a.time.end(), tile_b.time.end()); - if new_start == query_rect.time_interval.start() { + if new_start == query_rect.time_interval().start() { // in the case that the time interval has no length, i.e. start=end, // we have to advance `new_start` to prevent infinite loops. // Otherwise, the new query rectangle would be equal to the previous one. new_start += 1; } - if new_start >= query_rect.time_interval.end() { + if new_start >= query_rect.time_interval().end() { // the query window is exhausted, end the stream state.set(State::Finished); } else { - query_rect.time_interval = TimeInterval::new_unchecked( - new_start, - query_rect.time_interval.end(), - ); + *query_rect.time_interval_mut() = + TimeInterval::new_unchecked( + new_start, + query_rect.time_interval().end(), + ); state.set(State::Initial); } @@ -437,9 +439,9 @@ where // TODO: calculate at start when tiling info is available before querying first tile let num_spatial_tiles = *num_spatial_tiles.get_or_insert_with(|| { - Self::number_of_tiles_in_partition( + Self::number_of_tiles_in_grid_bounds( &tiles[0].tile_information(), - query_rect.spatial_bounds, + query_rect.spatial_bounds(), ) }); @@ -466,20 +468,20 @@ where .min() .expect("N > 0"); - if new_start == query_rect.time_interval.start() { + if new_start == query_rect.time_interval().start() { // in the case that the time interval has no length, i.e. start=end, // we have to advance `new_start` to prevent infinite loops. // Otherwise, the new query rectangle would be equal to the previous one. new_start += 1; } - if new_start >= query_rect.time_interval.end() { + if new_start >= query_rect.time_interval().end() { // the query window is exhausted, end the stream state.set(ArrayState::Finished); } else { - query_rect.time_interval = TimeInterval::new_unchecked( + *query_rect.time_interval_mut() = TimeInterval::new_unchecked( new_start, - query_rect.time_interval.end(), + query_rect.time_interval().end(), ); state.set(ArrayState::Initial); @@ -569,16 +571,18 @@ where mod tests { use super::*; use crate::engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, WorkflowOperatorPath, + MockExecutionContext, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, TimeDescriptor, WorkflowOperatorPath, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use futures::StreamExt; - use geoengine_datatypes::primitives::{BandSelection, CacheHint}; - use geoengine_datatypes::raster::{EmptyGrid, Grid, RasterDataType, RasterProperties}; + use geoengine_datatypes::primitives::{BandSelection, CacheHint, TimeStep}; + use geoengine_datatypes::raster::{ + BoundedGrid, EmptyGrid, GeoTransform, Grid, GridShape2D, RasterDataType, RasterProperties, + SpatialGridDefinition, TilingSpecification, + }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{primitives::SpatialResolution, raster::TilingSpecification}; #[tokio::test] #[allow(clippy::too_many_lines)] @@ -634,9 +638,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + )), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -716,26 +725,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + )), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -866,9 +874,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -948,26 +961,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1098,9 +1110,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1158,26 +1175,28 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1292,9 +1311,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1352,26 +1376,28 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1464,9 +1490,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1520,26 +1551,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1627,9 +1657,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1687,26 +1722,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1811,9 +1845,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -1867,26 +1906,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 8), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 8), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1996,9 +2034,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -2056,27 +2099,25 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 8), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 8), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp1 = mrs1 .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2134,6 +2175,7 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn array_tiles_without_lengths() { + // FIXME: this shuld be illegal, because the tiles have no length let raster_source_a = MockRasterSource { params: MockRasterSourceParams:: { data: vec![ @@ -2185,9 +2227,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -2223,26 +2267,28 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(1, 3), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(1, 3), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let query_processor_a = raster_source_a .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/operators/src/adapters/simple_raster_stacker.rs b/operators/src/adapters/simple_raster_stacker.rs index bd8d3255c..1951e7551 100644 --- a/operators/src/adapters/simple_raster_stacker.rs +++ b/operators/src/adapters/simple_raster_stacker.rs @@ -140,14 +140,14 @@ where Fut: Future>>>>, P: Pixel, { - if query.attributes.count() == 1 { + if query.attributes().count() == 1 { // special case of single band query requires no tile stacking return create_single_bands_stream_fn(query.clone(), ctx).await; } // compute the aggreation for each band separately and stack the streams to get a multi band raster tile stream - let band_streams = join_all(query.attributes.as_slice().iter().map(|band| { - let query = query.select_bands(BandSelection::new_single(*band)); + let band_streams = join_all(query.attributes().as_slice().iter().map(|band| { + let query = query.select_attributes(BandSelection::new_single(*band)); async { Ok(SimpleRasterStackerSource { diff --git a/operators/src/adapters/sparse_tiles_fill_adapter.rs b/operators/src/adapters/sparse_tiles_fill_adapter.rs index 22b9213c2..d1e2747f1 100644 --- a/operators/src/adapters/sparse_tiles_fill_adapter.rs +++ b/operators/src/adapters/sparse_tiles_fill_adapter.rs @@ -1,13 +1,10 @@ use crate::util::Result; use futures::{Stream, ready}; use geoengine_datatypes::{ - primitives::{ - CacheExpiration, CacheHint, RasterQueryRectangle, SpatialPartitioned, TimeInstance, - TimeInterval, - }, + primitives::{CacheExpiration, CacheHint, TimeInstance, TimeInterval}, raster::{ EmptyGrid2D, GeoTransform, GridBoundingBox2D, GridBounds, GridIdx2D, GridShape2D, GridStep, - Pixel, RasterTile2D, TilingSpecification, + Pixel, RasterTile2D, }, }; use pin_project::pin_project; @@ -307,6 +304,11 @@ impl StateContainer { } fn update_current_time(&mut self, new_time: TimeInterval) { + debug_assert!( + !new_time.is_instant(), + "Tile time is the data validity and must not be an instant!" + ); + if let Some(old_time) = self.current_time { if old_time == new_time { return; @@ -359,6 +361,21 @@ impl StateContainer { true } + + fn store_tile(&mut self, tile: RasterTile2D) { + debug_assert!(self.next_tile.is_none()); + let current_time = self + .current_time + .expect("Time must be set when the first tile arrives"); + debug_assert!(current_time.start() <= tile.time.start()); + debug_assert!( + current_time.start() < tile.time.start() + || (self.current_idx.y() < tile.tile_position.y() + || (self.current_idx.y() == tile.tile_position.y() + && self.current_idx.x() < tile.tile_position.x())) + ); + self.next_tile = Some(tile); + } } #[pin_project(project=SparseTilesFillAdapterProjection)] @@ -413,33 +430,6 @@ where } } - pub fn new_like_subquery( - stream: S, - query_rect_to_answer: &RasterQueryRectangle, - tiling_spec: TilingSpecification, - cache_expiration: FillerTileCacheExpirationStrategy, - time_bounds: FillerTimeBounds, - ) -> Self { - debug_assert!(query_rect_to_answer.spatial_resolution.y > 0.); - - let tiling_strat = tiling_spec.strategy( - query_rect_to_answer.spatial_resolution.x, - -query_rect_to_answer.spatial_resolution.y, - ); - - let grid_bounds = tiling_strat.tile_grid_box(query_rect_to_answer.spatial_partition()); - Self::new( - stream, - grid_bounds, - query_rect_to_answer.attributes.count(), - tiling_strat.geo_transform, - tiling_spec.tile_size_in_pixels, - cache_expiration, - query_rect_to_answer.time_interval, - time_bounds, - ) - } - // TODO: return Result with SparseTilesFillAdapterError and map it to Error in the poll_next method if possible #[allow(clippy::too_many_lines, clippy::missing_panics_doc)] pub fn next_step( @@ -479,7 +469,7 @@ where this.sc.state = State::PollingForNextTile; // return the received tile and set state to polling for the next tile tile } else { - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; // save the tile and go to fill mode this.sc.current_no_data_tile() } @@ -585,7 +575,7 @@ where tile } else { // the tile is not the next to produce. Save it and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } @@ -606,13 +596,13 @@ where } else { // save the tile and go to fill mode. this.sc.update_current_time(tile.time); - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } } else { // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } @@ -629,12 +619,12 @@ where .end(), tile.time.start(), )?); - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } else { // the received tile is in a new TimeInterval but we still need to finish the current one. Store tile and go to fill mode. - this.sc.next_tile = Some(tile); + this.sc.store_tile(tile); this.sc.state = State::FillAndProduceNextTile; this.sc.current_no_data_tile() } @@ -786,6 +776,7 @@ where /// It can either be a fixed value for all produced tiles, or it can be derived from the surrounding tiles that are not produced by the adapter. /// In the latter case it will use the cache expiration of the preceeding tile if it is available, otherwise it will use the cache expiration of the following tile. #[derive(Debug, Clone, PartialEq)] +#[allow(dead_code)] pub enum FillerTileCacheExpirationStrategy { NoCache, FixedValue(CacheExpiration), diff --git a/operators/src/adapters/time_stream_merge.rs b/operators/src/adapters/time_stream_merge.rs new file mode 100644 index 000000000..406c4ec35 --- /dev/null +++ b/operators/src/adapters/time_stream_merge.rs @@ -0,0 +1,290 @@ +use futures::stream::{Fuse, Stream}; +use geoengine_datatypes::primitives::TimeInterval; +use pin_project::pin_project; +use std::pin::Pin; +use std::task::{Context, Poll}; + +#[pin_project] +pub struct TimeIntervalStreamMerge +where + S: Stream>, +{ + #[pin] + streams: Vec>, + current: Vec>, + current_time: Option, +} + +impl TimeIntervalStreamMerge +where + S: Stream> + Unpin, +{ + pub fn new(streams: Vec) -> Self { + let len = streams.len(); + Self { + streams: streams.into_iter().map(futures::StreamExt::fuse).collect(), + current: vec![None; len], + current_time: None, + } + } +} + +impl Stream for TimeIntervalStreamMerge +where + S: Stream> + Unpin, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + // Now, fill the current intervals from the streams if they are None. + // This happens either at the start, or after producing an interval. + // TODO: do this concurrently + for (i, stream) in this.streams.iter_mut().enumerate() { + if this.current[i].is_none() && !stream.is_done() { + match Pin::new(stream).poll_next(cx) { + Poll::Ready(Some(Ok(interval))) => this.current[i] = Some(interval), + Poll::Ready(Some(Err(e))) => return Poll::Ready(Some(Err(e))), + Poll::Ready(None) => this.current[i] = None, + Poll::Pending => return Poll::Pending, + } + } + } + + // If all streams are exhausted, we are done + if this.current.iter().all(std::option::Option::is_none) { + return Poll::Ready(None); + } + + // Collect all available intervals + let mut active: Vec<(usize, TimeInterval)> = Vec::new(); + for (i, interval) in this.current.iter().enumerate() { + if let Some(interval) = interval { + active.push((i, *interval)); + } + } + + // If no active intervals, we are done + // This is already covered by the all-none check above, but for safety... + if active.is_empty() { + return Poll::Ready(None); + } + + // Find all intervals that contain the end of the current_time. The one with the minimum end is a candidate for the next merged interval. + let candidate_case_a = this.current_time.and_then(|current_t| { + active + .iter() + .filter(|(_a_i, a)| a.contains_instance(current_t.end())) + .min_by_key(|(_a_i, a)| a.end()) + .map(|(i, interval)| { + ( + *i, + TimeInterval::new_unchecked(current_t.end(), interval.end()), + ) + }) + }); + + let candidate_case_b = this.current_time.and_then(|current_t| { + active + .iter() + .filter(|(_, interval)| interval.start() > current_t.end()) + .min_by_key(|(_, interval)| interval.start()) + .map(|(i, interval)| (*i, *interval)) + }); + + let candidate_case_c = active + .iter() + .min_by_key(|(_, interval)| interval.start()) + .map(|(i, interval)| (*i, *interval)); + + let candidate = candidate_case_a.or(candidate_case_b).or(candidate_case_c); + + let Some((_candidate_i, candidate)) = candidate else { + // This should not happen since we checked active is not empty + return Poll::Ready(None); + }; + + // Now, we have a candidate but we need to check if there are other intervals that start before its end + let candidate_start_as_end = active + .iter() + .filter(|(_a_i, a)| { + candidate.contains_instance(a.start()) && a.start() > candidate.start() + }) + .min_by_key(|(_a_i, a)| a.start()) + .map(|(i, interval)| (*i, *interval)); + + // Produce the merged interval + let merged_interval = if let Some((_, csae)) = candidate_start_as_end { + TimeInterval::new_unchecked(candidate.start(), csae.start()) + } else { + candidate + }; + + for interval in this.current.iter_mut() { + if let Some(iv) = interval + && iv.end() <= merged_interval.end() + { + *interval = None; + } + } + + // Update current_time + *this.current_time = Some(merged_interval); + + // Return the merged interval + Poll::Ready(Some(Ok(merged_interval))) + } +} + +#[cfg(test)] +mod tests { + use futures::StreamExt; + + use super::*; + + #[test] + fn test_non_overlapping_intervals() { + let stream1 = futures::stream::iter(vec![ + Result::::Ok(TimeInterval::new_unchecked(1, 2)), + Ok(TimeInterval::new_unchecked(3, 4)), + Ok(TimeInterval::new_unchecked(9, 10)), + ]); + let stream2 = futures::stream::iter(vec![ + Ok(TimeInterval::new_unchecked(5, 6)), + Ok(TimeInterval::new_unchecked(7, 8)), + Ok(TimeInterval::new_unchecked(11, 12)), + ]); + + let mut merged_stream = TimeIntervalStreamMerge::new(vec![stream1, stream2]); + + let expected = vec![ + TimeInterval::new_unchecked(1, 2), + TimeInterval::new_unchecked(3, 4), + TimeInterval::new_unchecked(5, 6), + TimeInterval::new_unchecked(7, 8), + TimeInterval::new_unchecked(9, 10), + TimeInterval::new_unchecked(11, 12), + ]; + + for expected_interval in expected { + let polled = futures::executor::block_on(merged_stream.next()); + assert_eq!(polled.unwrap().unwrap(), expected_interval); + } + + assert!(futures::executor::block_on(merged_stream.next()).is_none()); + } + + #[test] + fn test_touching_intervals() { + let stream1 = futures::stream::iter(vec![ + Result::::Ok(TimeInterval::new_unchecked(1, 3)), + Ok(TimeInterval::new_unchecked(5, 7)), + Ok(TimeInterval::new_unchecked(9, 11)), + ]); + let stream2 = futures::stream::iter(vec![ + Ok(TimeInterval::new_unchecked(3, 5)), + Ok(TimeInterval::new_unchecked(7, 9)), + Ok(TimeInterval::new_unchecked(11, 13)), + ]); + + let mut merged_stream = TimeIntervalStreamMerge::new(vec![stream1, stream2]); + + let expected = vec![ + TimeInterval::new_unchecked(1, 3), + TimeInterval::new_unchecked(3, 5), + TimeInterval::new_unchecked(5, 7), + TimeInterval::new_unchecked(7, 9), + TimeInterval::new_unchecked(9, 11), + TimeInterval::new_unchecked(11, 13), + ]; + + for expected_interval in expected { + let polled = futures::executor::block_on(merged_stream.next()); + assert_eq!(polled.unwrap().unwrap(), expected_interval); + } + + assert!(futures::executor::block_on(merged_stream.next()).is_none()); + } + + #[test] + fn test_overlapping_intervals() { + let stream1 = futures::stream::iter(vec![ + Result::::Ok(TimeInterval::new_unchecked(1, 4)), + Ok(TimeInterval::new_unchecked(6, 8)), + Ok(TimeInterval::new_unchecked(10, 12)), + ]); + let stream2 = futures::stream::iter(vec![ + Ok(TimeInterval::new_unchecked(2, 5)), + Ok(TimeInterval::new_unchecked(7, 9)), + Ok(TimeInterval::new_unchecked(11, 13)), + ]); + let mut merged_stream = TimeIntervalStreamMerge::new(vec![stream1, stream2]); + let expected = vec![ + TimeInterval::new_unchecked(1, 2), + TimeInterval::new_unchecked(2, 4), + TimeInterval::new_unchecked(4, 5), + TimeInterval::new_unchecked(6, 7), + TimeInterval::new_unchecked(7, 8), + TimeInterval::new_unchecked(8, 9), + TimeInterval::new_unchecked(10, 11), + TimeInterval::new_unchecked(11, 12), + TimeInterval::new_unchecked(12, 13), + ]; + for expected_interval in expected { + let polled = futures::executor::block_on(merged_stream.next()); + assert_eq!(polled.unwrap().unwrap(), expected_interval); + } + assert!(futures::executor::block_on(merged_stream.next()).is_none()); + } + + #[test] + fn test_synchronized_non_overlapping_intervals() { + let stream1 = futures::stream::iter(vec![ + Result::::Ok(TimeInterval::new_unchecked(1, 5)), + Ok(TimeInterval::new_unchecked(6, 10)), + Ok(TimeInterval::new_unchecked(11, 15)), + ]); + let stream2 = futures::stream::iter(vec![ + Ok(TimeInterval::new_unchecked(1, 5)), + Ok(TimeInterval::new_unchecked(6, 10)), + Ok(TimeInterval::new_unchecked(11, 15)), + ]); + let mut merged_stream = TimeIntervalStreamMerge::new(vec![stream1, stream2]); + let expected = vec![ + TimeInterval::new_unchecked(1, 5), + TimeInterval::new_unchecked(6, 10), + TimeInterval::new_unchecked(11, 15), + ]; + for expected_interval in expected { + let polled = futures::executor::block_on(merged_stream.next()); + assert_eq!(polled.unwrap().unwrap(), expected_interval); + } + assert!(futures::executor::block_on(merged_stream.next()).is_none()); + } + + #[test] + fn test_synchronized_touching_intervals() { + let stream1 = futures::stream::iter(vec![ + Result::::Ok(TimeInterval::new_unchecked(1, 3)), + Ok(TimeInterval::new_unchecked(3, 6)), + Ok(TimeInterval::new_unchecked(6, 9)), + ]); + let stream2 = futures::stream::iter(vec![ + Ok(TimeInterval::new_unchecked(1, 3)), + Ok(TimeInterval::new_unchecked(3, 6)), + Ok(TimeInterval::new_unchecked(6, 9)), + ]); + let mut merged_stream = TimeIntervalStreamMerge::new(vec![stream1, stream2]); + let expected = vec![ + TimeInterval::new_unchecked(1, 3), + TimeInterval::new_unchecked(3, 6), + TimeInterval::new_unchecked(6, 9), + ]; + for expected_interval in expected { + let polled = futures::executor::block_on(merged_stream.next()); + assert_eq!(polled.unwrap().unwrap(), expected_interval); + } + assert!(futures::executor::block_on(merged_stream.next()).is_none()); + } +} diff --git a/operators/src/cache/cache_chunks.rs b/operators/src/cache/cache_chunks.rs index 73db18733..c1038be1d 100644 --- a/operators/src/cache/cache_chunks.rs +++ b/operators/src/cache/cache_chunks.rs @@ -179,12 +179,12 @@ where // If the chunk has no time bounds it must be empty so we can skip the temporal check and return true. let temporal_hit = self .time_interval - .is_none_or(|tb| tb.intersects(&query.time_interval)); + .is_none_or(|tb| tb.intersects(&query.time_interval())); // If the chunk has no spatial bounds it is either an empty collection or a no geometry collection. let spatial_hit = self .spatial_bounds - .is_none_or(|sb| sb.intersects_bbox(&query.spatial_bounds)); + .is_none_or(|sb| sb.intersects_bbox(&query.spatial_bounds())); temporal_hit && spatial_hit } @@ -258,7 +258,7 @@ impl CacheElementSpatialBounds for FeatureCollection { let time_filter_bools = self .time_intervals() .iter() - .map(|t| t.intersects(&query_rect.time_interval)) + .map(|t| t.intersects(&query_rect.time_interval())) .collect::>(); self.filter(time_filter_bools) .map_err(|_err| CacheError::CouldNotFilterResults) @@ -283,14 +283,14 @@ macro_rules! impl_cache_result_check { ) -> Result { let geoms_filter_bools = self.geometries().map(|g| { g.bbox() - .map(|bbox| bbox.intersects_bbox(&query_rect.spatial_bounds)) + .map(|bbox| bbox.intersects_bbox(&query_rect.spatial_bounds())) .unwrap_or(false) }); let time_filter_bools = self .time_intervals() .iter() - .map(|t| t.intersects(&query_rect.time_interval)); + .map(|t| t.intersects(&query_rect.time_interval())); let filter_bools = geoms_filter_bools .zip(time_filter_bools) @@ -496,8 +496,8 @@ mod tests { use geoengine_datatypes::{ collections::MultiPointCollection, primitives::{ - BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, SpatialResolution, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, TimeInterval, + VectorQueryRectangle, }, }; use std::{collections::HashMap, sync::Arc}; @@ -561,12 +561,11 @@ mod tests { #[test] fn landing_zone_to_cache_entry() { let cols = create_test_collection(); - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (1., 1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((0., 0.).into(), (1., 1.).into()), + Default::default(), + ColumnSelection::all(), + ); let mut lq = VectorLandingQueryEntry::create_empty::>( query.clone(), ); @@ -584,36 +583,33 @@ mod tests { let cols = create_test_collection(); // elemtes are all fully contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (12., 12.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((0., 0.).into(), (12., 12.).into()), + Default::default(), + ColumnSelection::all(), + ); for c in &cols { assert!(c.intersects_query(&query)); } // first element is not contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(!cols[0].intersects_query(&query)); for c in &cols[1..] { assert!(c.intersects_query(&query)); } // all elements are not contained - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((13., 13.).into(), (26., 26.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((13., 13.).into(), (26., 26.).into()), + Default::default(), + ColumnSelection::all(), + ); for col in &cols { assert!(!col.intersects_query(&query)); } @@ -623,12 +619,11 @@ mod tests { fn cache_entry_matches() { let cols = create_test_collection(); - let cache_entry_bounds = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((1., 1.).into(), (11., 11.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let cache_entry_bounds = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((1., 1.).into(), (11., 11.).into()), + Default::default(), + ColumnSelection::all(), + ); let cache_query_entry = VectorCacheQueryEntry { query: cache_entry_bounds.clone(), @@ -640,21 +635,19 @@ mod tests { assert!(cache_query_entry.query().is_match(&query)); // query is fully contained - let query2 = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query2 = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((2., 2.).into(), (10., 10.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(cache_query_entry.query().is_match(&query2)); // query is exceeds cached bounds - let query3 = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((0., 0.).into(), (8., 8.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query3 = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((0., 0.).into(), (8., 8.).into()), + Default::default(), + ColumnSelection::all(), + ); assert!(!cache_query_entry.query().is_match(&query3)); } } diff --git a/operators/src/cache/cache_operator.rs b/operators/src/cache/cache_operator.rs index adac3acb0..bd46f7b68 100644 --- a/operators/src/cache/cache_operator.rs +++ b/operators/src/cache/cache_operator.rs @@ -5,7 +5,7 @@ use crate::adapters::FeatureCollectionChunkMerger; use crate::cache::shared_cache::{AsyncCache, SharedCache}; use crate::engine::{ CanonicOperatorName, ChunkByteSize, InitializedRasterOperator, InitializedVectorOperator, - QueryContext, QueryProcessor, RasterResultDescriptor, ResultDescriptor, + QueryContext, QueryProcessor, RasterQueryProcessor, RasterResultDescriptor, ResultDescriptor, TypedRasterQueryProcessor, WorkflowOperatorPath, }; use crate::error::Error; @@ -15,9 +15,10 @@ use futures::stream::{BoxStream, FusedStream}; use futures::{Stream, StreamExt, TryStreamExt, ready}; use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Geometry, QueryAttributeSelection, QueryRectangle, VectorQueryRectangle, + BandSelection, Geometry, QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, + VectorQueryRectangle, }; -use geoengine_datatypes::raster::{Pixel, RasterTile2D}; +use geoengine_datatypes::raster::{GridBoundingBox2D, Pixel, RasterTile2D}; use geoengine_datatypes::util::arrow::ArrowTyped; use geoengine_datatypes::util::helpers::ge_report; use pin_project::{pin_project, pinned_drop}; @@ -98,6 +99,14 @@ impl InitializedRasterOperator for InitializedCacheOperator WorkflowOperatorPath { self.source.path() } + + fn optimize( + &self, + resolution: geoengine_datatypes::primitives::SpatialResolution, + ) -> Result, crate::optimization::OptimizationError> + { + self.source.optimize(resolution) + } } impl InitializedVectorOperator for InitializedCacheOperator> { @@ -153,6 +162,14 @@ impl InitializedVectorOperator for InitializedCacheOperator WorkflowOperatorPath { self.source.path() } + + fn optimize( + &self, + resolution: geoengine_datatypes::primitives::SpatialResolution, + ) -> Result, crate::optimization::OptimizationError> + { + self.source.optimize(resolution) + } } /// A cache operator that caches the results of its source operator @@ -182,7 +199,7 @@ where impl QueryProcessor for CacheQueryProcessor where P: QueryProcessor + Sized, - S: AxisAlignedRectangle + Send + Sync + 'static, + S: Clone + Send + Sync + 'static, U: QueryAttributeSelection, E: CacheElement> + Send @@ -280,6 +297,34 @@ where } } +#[async_trait] +impl RasterQueryProcessor + for CacheQueryProcessor< + P, + RasterTile2D, + GridBoundingBox2D, + BandSelection, + RasterResultDescriptor, + > +where + P: RasterQueryProcessor + Sized, + SharedCache: AsyncCache>, + T: Pixel + Send + Sync + 'static, + RasterTile2D: CacheElement + Send + Sync + 'static, + as CacheElement>::ResultStream: + Stream, CacheError>> + Send + Sync + 'static, +{ + type RasterType = T; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.processor.time_query(query, ctx).await // TODO: investigate if we can use caching here? + } +} + #[allow(clippy::large_enum_variant)] // TODO: Box instead? enum SourceStreamEvent { Element(E), @@ -430,8 +475,8 @@ mod tests { use crate::{ engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, MultipleRasterSources, - RasterOperator, SingleRasterSource, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, MultipleRasterSources, RasterOperator, + SingleRasterSource, WorkflowOperatorPath, }, processing::{Expression, ExpressionParams, RasterStacker, RasterStackerParams}, source::{GdalSource, GdalSourceParameters}, @@ -439,8 +484,8 @@ mod tests { }; use futures::StreamExt; use geoengine_datatypes::{ - primitives::{BandSelection, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{RasterDataType, RenameBands, TilesEqualIgnoringCacheHint}, + primitives::{BandSelection, RasterQueryRectangle, TimeInterval}, + raster::{GridBoundingBox2D, RasterDataType, RenameBands, TilesEqualIgnoringCacheHint}, util::test::TestDefault, }; use std::sync::Arc; @@ -452,9 +497,7 @@ mod tests { let ndvi_id = add_ndvi_dataset(&mut exe_ctx); let operator = GdalSource { - params: GdalSourceParameters { - data: ndvi_id.clone(), - }, + params: GdalSourceParameters::new(ndvi_id.clone()), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -467,7 +510,7 @@ mod tests { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -476,15 +519,11 @@ mod tests { let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -501,15 +540,11 @@ mod tests { let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ), &query_ctx, ) .await @@ -539,6 +574,7 @@ mod tests { GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -553,6 +589,7 @@ mod tests { raster: GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -573,7 +610,7 @@ mod tests { let tile_cache = Arc::new(SharedCache::test_default()); - let query_ctx = MockQueryContext::new_with_query_extensions( + let query_ctx = exe_ctx.mock_query_context_with_query_extensions( ChunkByteSize::test_default(), Some(tile_cache), None, @@ -583,15 +620,11 @@ mod tests { // query the first two bands let stream = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::new(vec![0, 1]).unwrap(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new(vec![0, 1]).unwrap(), + ), &query_ctx, ) .await @@ -621,15 +654,11 @@ mod tests { // now query only the second band let stream_from_cache = processor .query( - QueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - [-180., -90.].into(), - [180., 90.].into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::new_single(1), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::default(), + BandSelection::new_single(1), + ), &query_ctx, ) .await diff --git a/operators/src/cache/cache_stream.rs b/operators/src/cache/cache_stream.rs index 14d718159..fc227ee72 100644 --- a/operators/src/cache/cache_stream.rs +++ b/operators/src/cache/cache_stream.rs @@ -162,8 +162,7 @@ mod tests { collections::MultiPointCollection, primitives::{ BandSelection, BoundingBox2D, CacheHint, ColumnSelection, FeatureData, MultiPoint, - RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - VectorQueryRectangle, + RasterQueryRectangle, TimeInterval, VectorQueryRectangle, }, raster::{GeoTransform, Grid2D, GridIdx2D, RasterTile2D}, }; @@ -173,6 +172,7 @@ mod tests { cache_stream::CacheStreamInner, cache_tiles::{CompressedRasterTile2D, CompressedRasterTileExt}, }; + use geoengine_datatypes::raster::GridBoundingBox2D; fn create_test_raster_data() -> Vec> { let mut data = Vec::new(); @@ -226,12 +226,11 @@ mod tests { #[test] fn test_cache_stream_inner_raster() { let data = Arc::new(create_test_raster_data()); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((2., -2.).into(), (8., -8.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([4, 4], [15, 15]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let mut res = Vec::new(); let mut inner = CacheStreamInner::new(data, query); @@ -247,12 +246,11 @@ mod tests { #[test] fn test_cache_stream_inner_vector() { let data = Arc::new(create_test_vecor_data()); - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((2.1, 2.1).into(), (7.9, 7.9).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((2.1, 2.1).into(), (7.9, 7.9).into()), + TimeInterval::new_unchecked(0, 10), + ColumnSelection::all(), + ); let mut res = Vec::new(); let mut inner = CacheStreamInner::new(data, query); diff --git a/operators/src/cache/cache_tiles.rs b/operators/src/cache/cache_tiles.rs index a7c63310b..0dfb02a0c 100644 --- a/operators/src/cache/cache_tiles.rs +++ b/operators/src/cache/cache_tiles.rs @@ -6,10 +6,9 @@ use super::shared_cache::{ RasterLandingQueryEntry, }; use crate::util::Result; -use geoengine_datatypes::primitives::SpatialPartitioned; use geoengine_datatypes::raster::{ - BaseTile, EmptyGrid, Grid, GridOrEmpty, GridShape2D, GridSize, GridSpaceToLinearSpace, - MaskedGrid, RasterTile, + BaseTile, EmptyGrid, Grid, GridBoundingBoxExt, GridIntersection, GridOrEmpty, GridShape2D, + GridSize, GridSpaceToLinearSpace, MaskedGrid, RasterTile, }; use geoengine_datatypes::{ primitives::RasterQueryRectangle, @@ -197,18 +196,23 @@ where } fn update_stored_query(&self, query: &mut Self::Query) -> Result<(), CacheError> { - query.spatial_bounds.extend(&self.spatial_partition()); - query.time_interval = query - .time_interval + let stored_spatial_query_mut = query.spatial_bounds_mut(); + + stored_spatial_query_mut.extend(&self.tile_information().global_pixel_bounds()); + + *query.time_interval_mut() = query + .time_interval() .union(&self.time) .map_err(|_| CacheError::ElementAndQueryDoNotIntersect)?; Ok(()) } fn intersects_query(&self, query: &Self::Query) -> bool { - self.spatial_partition().intersects(&query.spatial_bounds) - && self.time.intersects(&query.time_interval) - && query.attributes.as_slice().contains(&self.band) + self.tile_information() + .global_pixel_bounds() + .intersects(&query.spatial_bounds()) + && self.time.intersects(&query.time_interval()) + && query.attributes().as_slice().contains(&self.band) } } @@ -602,10 +606,8 @@ impl TileCompression for Lz4FlexCompression { mod tests { use std::sync::Arc; - use geoengine_datatypes::{ - primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution}, - raster::GeoTransform, - util::test::TestDefault, + use super::{ + CompressedGridOrEmpty, CompressedMaskedGrid, CompressedRasterTile2D, LandingZoneQueryTiles, }; use crate::cache::{ @@ -615,9 +617,10 @@ mod tests { RasterLandingQueryEntry, }, }; - - use super::{ - CompressedGridOrEmpty, CompressedMaskedGrid, CompressedRasterTile2D, LandingZoneQueryTiles, + use geoengine_datatypes::{ + primitives::{BandSelection, RasterQueryRectangle}, + raster::{GeoTransform, GridBoundingBox2D}, + util::test::TestDefault, }; fn create_test_tile() -> CompressedRasterTile2D { @@ -665,12 +668,11 @@ mod tests { #[test] fn landing_zone_to_cache_entry() { let tile = create_test_tile(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., 1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); let mut lq = RasterLandingQueryEntry::create_empty::>(query.clone()); tile.move_element_into_landing_zone(lq.elements_mut()) @@ -685,47 +687,37 @@ mod tests { let tile = create_test_tile(); // tile is fully contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(tile.intersects_query(&query)); // tile is partially contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0.5, -0.5).into(), - (1.5, -1.5).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, -1], [0, 0]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(tile.intersects_query(&query)); // tile is not contained - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (10., -10.).into(), - (11., -11.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([10, 10], [11, 11]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(!tile.intersects_query(&query)); } #[test] fn cache_entry_matches() { - let cache_entry_bounds = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 0.).into(), (1., -1.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let cache_entry_bounds = RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [10, 10]).unwrap(), + Default::default(), + BandSelection::first(), + ); let cache_query_entry = RasterCacheQueryEntry { query: cache_entry_bounds.clone(), elements: CachedTiles::U8(Arc::new(Vec::new())), @@ -736,27 +728,19 @@ mod tests { assert!(cache_query_entry.query().is_match(&query)); // query is fully contained - let query2 = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0.1, -0.1).into(), - (0.9, -0.9).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query2 = RasterQueryRectangle::new( + GridBoundingBox2D::new([1, 1], [9, 9]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(cache_query_entry.query().is_match(&query2)); // query is exceeds cached bounds - let query3 = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-0.1, 0.1).into(), - (1.1, -1.1).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query3 = RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [11, 11]).unwrap(), + Default::default(), + BandSelection::first(), + ); assert!(!cache_query_entry.query().is_match(&query3)); } } diff --git a/operators/src/cache/shared_cache.rs b/operators/src/cache/shared_cache.rs index a103e0bea..6b3ee2cdc 100644 --- a/operators/src/cache/shared_cache.rs +++ b/operators/src/cache/shared_cache.rs @@ -11,7 +11,7 @@ use futures::Stream; use geoengine_datatypes::{ identifier, primitives::{CacheHint, Geometry, RasterQueryRectangle, VectorQueryRectangle}, - raster::Pixel, + raster::{GridContains, Pixel}, util::{ByteSize, Identifier, arrow::ArrowTyped, test::TestDefault}, }; use lru::LruCache; @@ -853,23 +853,22 @@ pub trait CacheQueryMatch { impl CacheQueryMatch for RasterQueryRectangle { fn is_match(&self, query: &RasterQueryRectangle) -> bool { - self.spatial_bounds.contains(&query.spatial_bounds) - && self.time_interval.contains(&query.time_interval) - && self.spatial_resolution == query.spatial_resolution - && query - .attributes - .as_slice() - .iter() - .all(|b| self.attributes.as_slice().contains(b)) + let cache_spatial_query = self.spatial_bounds(); + let query_spatial_query = query.spatial_bounds(); + + cache_spatial_query.contains(&query_spatial_query) + && self.time_interval().contains(&query.time_interval()) } } impl CacheQueryMatch for VectorQueryRectangle { - // TODO: check if that is what we need fn is_match(&self, query: &VectorQueryRectangle) -> bool { - self.spatial_bounds.contains_bbox(&query.spatial_bounds) - && self.time_interval.contains(&query.time_interval) - && self.spatial_resolution == query.spatial_resolution + let cache_spatial_query = self.spatial_bounds(); + let query_spatial_query = query.spatial_bounds(); + + cache_spatial_query.contains_bbox(&query_spatial_query) + && self.time_interval().contains(&query.time_interval()) + && self.attributes() == query.attributes() } } @@ -1025,10 +1024,8 @@ where #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, DateTime, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{Grid, RasterProperties, RasterTile2D}, + primitives::{BandSelection, CacheHint, DateTime, TimeInterval}, + raster::{Grid, GridBoundingBox2D, RasterProperties, RasterTile2D}, }; use serde_json::json; use std::sync::Arc; @@ -1108,16 +1105,11 @@ mod tests { } fn query_rect() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new( + GridBoundingBox2D::new([-90, -180], [89, 179]).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ) } fn op(idx: usize) -> CanonicOperatorName { diff --git a/operators/src/engine/clonable_operator.rs b/operators/src/engine/clonable_operator.rs index b1d14fbb5..ad89a1985 100644 --- a/operators/src/engine/clonable_operator.rs +++ b/operators/src/engine/clonable_operator.rs @@ -77,3 +77,18 @@ pub trait CloneableInitializedVectorOperator { pub trait CloneableInitializedPlotOperator { fn clone_boxed_plot(&self) -> Box; } + +impl CloneableInitializedRasterOperator for T +where + T: 'static + InitializedRasterOperator + Clone, +{ + fn clone_boxed_raster(&self) -> Box { + Box::new(self.clone()) + } +} + +impl Clone for Box { + fn clone(&self) -> Box { + self.clone_boxed_raster() + } +} diff --git a/operators/src/engine/execution_context.rs b/operators/src/engine/execution_context.rs index e36d699ca..c2821e241 100644 --- a/operators/src/engine/execution_context.rs +++ b/operators/src/engine/execution_context.rs @@ -1,16 +1,20 @@ -use super::query::QueryAbortRegistration; use super::{ CreateSpan, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, MockQueryContext, }; +use crate::cache::shared_cache::SharedCache; use crate::engine::{ ChunkByteSize, RasterResultDescriptor, ResultDescriptor, VectorResultDescriptor, }; use crate::error::Error; use crate::machine_learning::MlModelLoadingInfo; +use crate::meta::quota::{QuotaChecker, QuotaTracking}; use crate::meta::wrapper::InitializedOperatorWrapper; use crate::mock::MockDatasetDataSourceLoadingInfo; -use crate::source::{GdalLoadingInfo, OgrSourceDataset}; +use crate::source::{ + GdalLoadingInfo, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, + OgrSourceDataset, +}; use crate::util::{Result, create_rayon_thread_pool}; use async_trait::async_trait; use geoengine_datatypes::dataset::{DataId, NamedData}; @@ -33,6 +37,11 @@ pub trait ExecutionContext: Send + MetaDataProvider + MetaDataProvider + MetaDataProvider + + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > { fn thread_pool(&self) -> &Arc; fn tiling_specification(&self) -> TilingSpecification; @@ -157,17 +166,40 @@ impl MockExecutionContext { } } + pub fn mock_query_context_test_default(&self) -> MockQueryContext { + MockQueryContext::new(ChunkByteSize::test_default(), self.tiling_specification) + } + pub fn mock_query_context(&self, chunk_byte_size: ChunkByteSize) -> MockQueryContext { - let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); - MockQueryContext { + MockQueryContext::new(chunk_byte_size, self.tiling_specification) + } + + pub fn mock_query_context_with_query_extensions( + &self, + chunk_byte_size: ChunkByteSize, + cache: Option>, + quota_tracking: Option, + quota_checker: Option, + ) -> MockQueryContext { + MockQueryContext::new_with_query_extensions( chunk_byte_size, - thread_pool: self.thread_pool.clone(), - cache: None, - quota_checker: None, - quota_tracking: None, - abort_registration, - abort_trigger: Some(abort_trigger), - } + self.tiling_specification, + cache, + quota_tracking, + quota_checker, + ) + } + + pub fn mock_query_context_with_chunk_size_and_thread_count( + &self, + chunk_byte_size: ChunkByteSize, + num_threads: usize, + ) -> MockQueryContext { + MockQueryContext::with_chunk_size_and_thread_count( + chunk_byte_size, + self.tiling_specification, + num_threads, + ) } } @@ -363,6 +395,23 @@ impl TestDefault for StatisticsWrappingMockExecutionContext { } } +impl StatisticsWrappingMockExecutionContext { + pub fn mock_query_context_with_query_extensions( + &self, + chunk_byte_size: ChunkByteSize, + cache: Option>, + quota_tracking: Option, + quota_checker: Option, + ) -> MockQueryContext { + self.inner.mock_query_context_with_query_extensions( + chunk_byte_size, + cache, + quota_tracking, + quota_checker, + ) + } +} + #[async_trait::async_trait] impl ExecutionContext for StatisticsWrappingMockExecutionContext { fn thread_pool(&self) -> &Arc { diff --git a/operators/src/engine/mod.rs b/operators/src/engine/mod.rs index 5b4fe0bf8..d4741aab0 100644 --- a/operators/src/engine/mod.rs +++ b/operators/src/engine/mod.rs @@ -7,6 +7,11 @@ pub use execution_context::{ ExecutionContext, MetaData, MetaDataProvider, MockExecutionContext, StaticMetaData, StatisticsWrappingMockExecutionContext, }; +pub use initialized_sources::{ + InitializedMultiRasterOrVectorOperator, InitializedMultiRasterOrVectorSource, + InitializedSingleRasterOrVectorOperator, InitializedSingleRasterOrVectorSource, + InitializedSingleRasterSource, InitializedSingleVectorSource, InitializedSources, +}; pub use operator::{ CanonicOperatorName, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, OperatorData, OperatorName, PlotOperator, RasterOperator, @@ -27,18 +32,11 @@ pub use query_processor::{ }; pub use result_descriptor::{ PlotResultDescriptor, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, - ResultDescriptor, TypedResultDescriptor, VectorColumnInfo, VectorResultDescriptor, + ResultDescriptor, SpatialGridDescriptor, SpatialGridDescriptorState, TimeDescriptor, + TypedResultDescriptor, VectorColumnInfo, VectorResultDescriptor, }; use tracing::Span; - pub use workflow_path::WorkflowOperatorPath; - -pub use initialized_sources::{ - InitializedMultiRasterOrVectorOperator, InitializedMultiRasterOrVectorSource, - InitializedSingleRasterOrVectorOperator, InitializedSingleRasterOrVectorSource, - InitializedSingleRasterSource, InitializedSingleVectorSource, InitializedSources, -}; - mod clonable_operator; mod execution_context; mod initialized_sources; diff --git a/operators/src/engine/operator.rs b/operators/src/engine/operator.rs index 7da4b9a05..753d27581 100644 --- a/operators/src/engine/operator.rs +++ b/operators/src/engine/operator.rs @@ -1,10 +1,11 @@ use serde::{Deserialize, Serialize}; +use snafu::ResultExt; use tracing::debug; -use crate::error; use crate::util::Result; +use crate::{error, optimization::OptimizationError}; use async_trait::async_trait; -use geoengine_datatypes::{dataset::NamedData, util::ByteSize}; +use geoengine_datatypes::{dataset::NamedData, primitives::SpatialResolution, util::ByteSize}; use super::{ CloneablePlotOperator, CloneableRasterOperator, CloneableVectorOperator, CreateSpan, @@ -136,6 +137,7 @@ pub trait PlotOperator: } // TODO: implement a derive macro for common fields of operators: name, path, data, result_descriptor and automatically implement common trait functions +#[async_trait] pub trait InitializedRasterOperator: Send + Sync { /// Get the result descriptor of the `Operator` fn result_descriptor(&self) -> &RasterResultDescriptor; @@ -164,6 +166,26 @@ pub trait InitializedRasterOperator: Send + Sync { fn data(&self) -> Option { None } + + /// Optimize the operator graph for a given resolution + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError>; + + /// optimize the operator graph and reinitialize it + async fn optimize_and_reinitialize( + &self, + resolution: SpatialResolution, + exe_ctx: &dyn ExecutionContext, + ) -> Result> { + let optimized = self + .optimize(resolution) + .context(crate::error::Optimization)?; + optimized + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await + } } pub trait InitializedVectorOperator: Send + Sync { @@ -195,6 +217,12 @@ pub trait InitializedVectorOperator: Send + Sync { fn data(&self) -> Option { None } + + /// Optimize the operator graph for a given resolution + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError>; } /// A canonic name for an operator and its sources @@ -256,6 +284,12 @@ pub trait InitializedPlotOperator: Send + Sync { /// Get a canonic representation of the operator and its sources fn canonic_name(&self) -> CanonicOperatorName; + + /// Optimize the operator graph for a given resolution + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError>; } impl InitializedRasterOperator for Box { @@ -282,6 +316,13 @@ impl InitializedRasterOperator for Box { fn data(&self) -> Option { self.as_ref().data() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.as_ref().optimize(resolution) + } } impl InitializedVectorOperator for Box { @@ -308,6 +349,13 @@ impl InitializedVectorOperator for Box { fn data(&self) -> Option { self.as_ref().data() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.as_ref().optimize(resolution) + } } impl InitializedPlotOperator for Box { @@ -322,6 +370,13 @@ impl InitializedPlotOperator for Box { fn canonic_name(&self) -> CanonicOperatorName { self.as_ref().canonic_name() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.as_ref().optimize(resolution) + } } /// An enum to differentiate between `Operator` variants diff --git a/operators/src/engine/query.rs b/operators/src/engine/query.rs index f0ab9d9cf..66cfa0e89 100644 --- a/operators/src/engine/query.rs +++ b/operators/src/engine/query.rs @@ -9,7 +9,7 @@ use crate::{ }; use crate::{meta::quota::QuotaTracking, util::Result}; use futures::Stream; -use geoengine_datatypes::util::test::TestDefault; +use geoengine_datatypes::{raster::TilingSpecification, util::test::TestDefault}; use pin_project::pin_project; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; @@ -52,6 +52,7 @@ impl TestDefault for ChunkByteSize { pub trait QueryContext: Send + Sync { fn chunk_byte_size(&self) -> ChunkByteSize; + fn tiling_specification(&self) -> TilingSpecification; fn thread_pool(&self) -> &Arc; fn quota_tracking(&self) -> Option<&QuotaTracking>; @@ -117,6 +118,7 @@ impl QueryAbortTrigger { pub struct MockQueryContext { pub chunk_byte_size: ChunkByteSize, + pub tiling_specification: TilingSpecification, pub thread_pool: Arc, pub cache: Option>, @@ -127,26 +129,15 @@ pub struct MockQueryContext { pub abort_trigger: Option, } -impl TestDefault for MockQueryContext { - fn test_default() -> Self { - let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); - Self { - chunk_byte_size: ChunkByteSize::test_default(), - thread_pool: create_rayon_thread_pool(0), - cache: None, - quota_checker: None, - quota_tracking: None, - abort_registration, - abort_trigger: Some(abort_trigger), - } - } -} - impl MockQueryContext { - pub fn new(chunk_byte_size: ChunkByteSize) -> Self { + pub(super) fn new( + chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, + ) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(0), cache: None, quota_checker: None, @@ -156,8 +147,9 @@ impl MockQueryContext { } } - pub fn new_with_query_extensions( + pub(super) fn new_with_query_extensions( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, cache: Option>, quota_tracking: Option, quota_checker: Option, @@ -165,6 +157,7 @@ impl MockQueryContext { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(0), cache, quota_checker, @@ -174,13 +167,15 @@ impl MockQueryContext { } } - pub fn with_chunk_size_and_thread_count( + pub(super) fn with_chunk_size_and_thread_count( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, num_threads: usize, ) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); Self { chunk_byte_size, + tiling_specification, thread_pool: create_rayon_thread_pool(num_threads), cache: None, quota_checker: None, @@ -210,6 +205,10 @@ impl QueryContext for MockQueryContext { .ok_or(error::Error::AbortTriggerAlreadyUsed) } + fn tiling_specification(&self) -> TilingSpecification { + self.tiling_specification + } + fn quota_tracking(&self) -> Option<&QuotaTracking> { self.quota_tracking.as_ref() } diff --git a/operators/src/engine/query_processor.rs b/operators/src/engine/query_processor.rs index 52a926186..2475e9afd 100644 --- a/operators/src/engine/query_processor.rs +++ b/operators/src/engine/query_processor.rs @@ -14,24 +14,24 @@ use geoengine_datatypes::collections::{ use geoengine_datatypes::plots::{PlotData, PlotOutputFormat}; use geoengine_datatypes::primitives::{ AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, - QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, SpatialPartition2D, + QueryAttributeSelection, QueryRectangle, RasterQueryRectangle, TimeInterval, VectorQueryRectangle, }; -use geoengine_datatypes::raster::{DynamicRasterDataType, Pixel}; +use geoengine_datatypes::raster::{DynamicRasterDataType, GridBoundingBox2D, Pixel}; use geoengine_datatypes::{collections::MultiPointCollection, raster::RasterTile2D}; use ouroboros::self_referencing; +use tracing::debug; /// An instantiation of an operator that produces a stream of results for a query #[async_trait] pub trait QueryProcessor: Send + Sync { type Output; - type SpatialBounds: AxisAlignedRectangle + Send + Sync; + type SpatialBounds: Send + Sync; type Selection: QueryAttributeSelection; type ResultDescription: ResultDescriptor< QueryRectangleSpatialBounds = Self::SpatialBounds, QueryRectangleAttributeSelection = Self::Selection, >; - /// inner logic of the processor async fn _query<'a>( &'a self, @@ -84,53 +84,88 @@ impl QueryProcessorExt for Q where Q: QueryProcessor {} /// An instantiation of a raster operator that produces a stream of raster results for a query #[async_trait] -pub trait RasterQueryProcessor: Sync + Send { +pub trait RasterQueryProcessor: + QueryProcessor< + Output = RasterTile2D, + SpatialBounds = GridBoundingBox2D, + Selection = BandSelection, + ResultDescription = RasterResultDescriptor, + > + Sync + + Send +{ type RasterType: Pixel; async fn raster_query<'a>( &'a self, query: RasterQueryRectangle, // TODO: query by reference ctx: &'a dyn QueryContext, - ) -> Result>>>; + ) -> Result>>> { + self.query(query, ctx).await + } - fn boxed(self) -> Box> + fn boxed( + self, + ) -> Box< + dyn RasterQueryProcessor< + RasterType = Self::RasterType, + Output = RasterTile2D, + SpatialBounds = GridBoundingBox2D, + Selection = BandSelection, + ResultDescription = RasterResultDescriptor, + >, + > where Self: Sized + 'static, { Box::new(self) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor; -} - -pub type BoxRasterQueryProcessor

= Box>; + fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + self.result_descriptor() + } -#[async_trait] -impl RasterQueryProcessor for S -where - S: QueryProcessor< - Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - > + Sync - + Send, - T: Pixel, -{ - type RasterType = T; - async fn raster_query<'a>( + async fn _time_query<'a>( &'a self, - query: RasterQueryRectangle, // TODO: query by reference + query: TimeInterval, ctx: &'a dyn QueryContext, - ) -> Result>>> { - self.query(query, ctx).await - } + ) -> Result>>; - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { - self.result_descriptor() + async fn time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + let rdt = self.raster_result_descriptor().time; + if let Some(regular_time) = rdt.dimension.unwrap_regular() { + debug!( + "Using time query shortcut for regular time dimension: {regular_time:?} and query: {query:?}" + ); + let iter = regular_time + .intersecting_intervals(query)? + .inspect(|t| debug!("time_query regular shortcut yielded {t:?}")) + .map(Result::Ok); + return Ok(futures::StreamExt::boxed(futures::stream::iter(iter))); + } + + debug!( + "Delegating time query to the {} query processors implementation", + std::any::type_name::() + ); + #[allow(clippy::used_underscore_items)] // TODO: maybe rename? + self._time_query(query, ctx).await } } +pub type BoxRasterQueryProcessor

= Box< + dyn RasterQueryProcessor< + RasterType = P, + Output = RasterTile2D

, + SpatialBounds = GridBoundingBox2D, + Selection = BandSelection, + ResultDescription = RasterResultDescriptor, + >, +>; + /// An instantiation of a vector operator that produces a stream of vector results for a query #[async_trait] pub trait VectorQueryProcessor: Sync + Send { @@ -225,12 +260,12 @@ where } #[async_trait] -impl QueryProcessor for Box> +impl QueryProcessor for BoxRasterQueryProcessor where T: Pixel, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -247,6 +282,19 @@ where } } +#[async_trait] +impl RasterQueryProcessor for BoxRasterQueryProcessor { + type RasterType = T; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.as_ref().time_query(query, ctx).await + } +} + #[async_trait] impl QueryProcessor for Box> where @@ -272,16 +320,16 @@ where /// An enum to differentiate between outputs of raster processors pub enum TypedRasterQueryProcessor { - U8(Box>), - U16(Box>), - U32(Box>), - U64(Box>), - I8(Box>), - I16(Box>), - I32(Box>), - I64(Box>), - F32(Box>), - F64(Box>), + U8(BoxRasterQueryProcessor), + U16(BoxRasterQueryProcessor), + U32(BoxRasterQueryProcessor), + U64(BoxRasterQueryProcessor), + I8(BoxRasterQueryProcessor), + I16(BoxRasterQueryProcessor), + I32(BoxRasterQueryProcessor), + I64(BoxRasterQueryProcessor), + F32(BoxRasterQueryProcessor), + F64(BoxRasterQueryProcessor), } impl DynamicRasterDataType for TypedRasterQueryProcessor { @@ -320,61 +368,61 @@ impl std::fmt::Debug for TypedRasterQueryProcessor { } impl TypedRasterQueryProcessor { - pub fn get_u8(self) -> Option>> { + pub fn get_u8(self) -> Option> { match self { Self::U8(r) => Some(r), _ => None, } } - pub fn get_u16(self) -> Option>> { + pub fn get_u16(self) -> Option> { match self { Self::U16(r) => Some(r), _ => None, } } - pub fn get_u32(self) -> Option>> { + pub fn get_u32(self) -> Option> { match self { Self::U32(r) => Some(r), _ => None, } } - pub fn get_u64(self) -> Option>> { + pub fn get_u64(self) -> Option> { match self { Self::U64(r) => Some(r), _ => None, } } - pub fn get_i8(self) -> Option>> { + pub fn get_i8(self) -> Option> { match self { Self::I8(r) => Some(r), _ => None, } } - pub fn get_i16(self) -> Option>> { + pub fn get_i16(self) -> Option> { match self { Self::I16(r) => Some(r), _ => None, } } - pub fn get_i32(self) -> Option>> { + pub fn get_i32(self) -> Option> { match self { Self::I32(r) => Some(r), _ => None, } } - pub fn get_i64(self) -> Option>> { + pub fn get_i64(self) -> Option> { match self { Self::I64(r) => Some(r), _ => None, } } - pub fn get_f32(self) -> Option>> { + pub fn get_f32(self) -> Option> { match self { Self::F32(r) => Some(r), _ => None, } } - pub fn get_f64(self) -> Option>> { + pub fn get_f64(self) -> Option> { match self { Self::F64(r) => Some(r), _ => None, @@ -530,61 +578,76 @@ impl TypedRasterQueryProcessor { Self::F64(r) => r, } } + + pub fn result_descriptor(&self) -> &RasterResultDescriptor { + match self { + Self::U8(r) => r.raster_result_descriptor(), + Self::U16(r) => r.raster_result_descriptor(), + Self::U32(r) => r.raster_result_descriptor(), + Self::U64(r) => r.raster_result_descriptor(), + Self::I8(r) => r.raster_result_descriptor(), + Self::I16(r) => r.raster_result_descriptor(), + Self::I32(r) => r.raster_result_descriptor(), + Self::I64(r) => r.raster_result_descriptor(), + Self::F32(r) => r.raster_result_descriptor(), + Self::F64(r) => r.raster_result_descriptor(), + } + } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::U8(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::I8(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::U16(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::I16(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::U32(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::I32(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::U64(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::I64(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::F32(value) } } -impl From>> for TypedRasterQueryProcessor { - fn from(value: Box>) -> Self { +impl From> for TypedRasterQueryProcessor { + fn from(value: BoxRasterQueryProcessor) -> Self { TypedRasterQueryProcessor::F64(value) } } diff --git a/operators/src/engine/result_descriptor.rs b/operators/src/engine/result_descriptor.rs index ddd10e80b..ceafad909 100644 --- a/operators/src/engine/result_descriptor.rs +++ b/operators/src/engine/result_descriptor.rs @@ -1,7 +1,17 @@ +use crate::error::{ + Error, RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, +}; +use crate::util::Result; +use geoengine_datatypes::operations::reproject::{CoordinateProjection, ReprojectClipped}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, FeatureDataType, - Measurement, PlotSeriesSelection, QueryAttributeSelection, QueryRectangle, SpatialPartition2D, - SpatialResolution, TimeInterval, + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, Coordinate2D, + FeatureDataType, Measurement, PlotSeriesSelection, QueryAttributeSelection, QueryRectangle, + RegularTimeDimension, SpatialPartition2D, SpatialResolution, TimeDimension, TimeInstance, + TimeInterval, TimeStep, +}; +use geoengine_datatypes::raster::{ + GeoTransform, GeoTransformAccess, Grid, GridBoundingBox2D, GridShape2D, GridShapeAccess, + SpatialGridDefinition, TilingSpatialGridDefinition, TilingSpecification, }; use geoengine_datatypes::util::ByteSize; use geoengine_datatypes::{ @@ -13,16 +23,11 @@ use snafu::ensure; use std::collections::{HashMap, HashSet}; use std::ops::Index; -use crate::error::{ - Error, RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, -}; -use crate::util::Result; - /// A descriptor that contains information about the query result, for instance, the data type /// and spatial reference. pub trait ResultDescriptor: Clone + Serialize { type DataType; - type QueryRectangleSpatialBounds: AxisAlignedRectangle; + type QueryRectangleSpatialBounds; type QueryRectangleAttributeSelection: QueryAttributeSelection; // Check the `query` against the `ResultDescriptor` and return `true` if the query is valid @@ -69,197 +74,320 @@ pub trait ResultDescriptor: Clone + Serialize { F: Fn(&Option) -> Option; } -/// A `ResultDescriptor` for raster queries -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSql, FromSql)] +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct RasterResultDescriptor { - pub data_type: RasterDataType, - pub spatial_reference: SpatialReferenceOption, - pub time: Option, - pub bbox: Option, - pub resolution: Option, - pub bands: RasterBandDescriptors, +pub enum SpatialGridDescriptorState { + /// The spatial grid represents a native dataset + Source, + /// The spatial grid was created by merging two non equal spatial grids + Merged, } -impl RasterResultDescriptor { - pub fn with_datatype_and_num_bands(data_type: RasterDataType, num_bands: u32) -> Self { - Self { - data_type, - spatial_reference: SpatialReferenceOption::Unreferenced, - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_multiple_bands(num_bands), +impl SpatialGridDescriptorState { + #[must_use] + pub fn merge(self, other: Self) -> Self { + match (self, other) { + (SpatialGridDescriptorState::Source, SpatialGridDescriptorState::Source) => { + SpatialGridDescriptorState::Source + } + _ => SpatialGridDescriptorState::Merged, } } + + pub fn is_source(self) -> bool { + self == SpatialGridDescriptorState::Source + } + + pub fn is_derived(self) -> bool { + !self.is_source() + } } -#[derive(Debug, Clone, PartialEq, Serialize)] -pub struct RasterBandDescriptors(Vec); +/// A `ResultDescriptor` for raster queries +#[derive(Debug, Copy, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition, + state: SpatialGridDescriptorState, +} -impl RasterBandDescriptors { - pub fn new(bands: Vec) -> Result { - let mut names = HashSet::new(); - for value in &bands { - ensure!(!value.name.is_empty(), RasterBandNameMustNotBeEmpty); - ensure!(value.name.byte_size() <= 256, RasterBandNameTooLong); - ensure!( - names.insert(&value.name), - RasterBandNamesMustBeUnique { - duplicate_key: value.name.clone() - } - ); +impl SpatialGridDescriptor { + pub fn new_source(spatial_grid_def: SpatialGridDefinition) -> Self { + Self { + spatial_grid: spatial_grid_def, + state: SpatialGridDescriptorState::Source, } + } - Ok(Self(bands)) + pub fn source_from_parts(geo_transform: GeoTransform, grid_bounds: GridBoundingBox2D) -> Self { + Self::new_source(SpatialGridDefinition::new(geo_transform, grid_bounds)) } - /// Convenience method to crate a single band result descriptor with no specific name and a unitless measurement for single band rasters - pub fn new_single_band() -> Self { - Self(vec![RasterBandDescriptor { - name: "band".into(), - measurement: Measurement::Unitless, - }]) + #[must_use] + pub fn as_derived(self) -> Self { + Self { + state: SpatialGridDescriptorState::Merged, + ..self + } } - /// Convenience method to crate multipe band result descriptors with no specific name and a unitless measurement - pub fn new_multiple_bands(num_bands: u32) -> Self { - Self( - (0..num_bands) - .map(RasterBandDescriptor::new_unitless_with_idx) - .collect(), - ) + pub fn merge(&self, other: &SpatialGridDescriptor) -> Option { + // TODO: merge directly to tiling origin? + let merged_grid = self.spatial_grid.merge(&other.spatial_grid)?; + let state = if self.spatial_grid.grid_bounds == merged_grid.grid_bounds + && other.spatial_grid.grid_bounds == merged_grid.grid_bounds + { + self.state.merge(other.state) + } else { + SpatialGridDescriptorState::Merged + }; + + Some(Self { + spatial_grid: merged_grid, + state, + }) } - pub fn bands(&self) -> &[RasterBandDescriptor] { - &self.0 + #[must_use] + pub fn map SpatialGridDefinition>(&self, map_fn: F) -> Self { + Self { + spatial_grid: map_fn(&self.spatial_grid), + ..*self + } } - pub fn len(&self) -> usize { - self.0.len() + pub fn try_map Result>( + &self, + map_fn: F, + ) -> Result { + Ok(Self { + spatial_grid: map_fn(&self.spatial_grid)?, + ..*self + }) } - pub fn count(&self) -> u32 { - self.0.len() as u32 + pub fn is_compatible_grid_generic(&self, g: &G) -> bool { + self.spatial_grid.is_compatible_grid_generic(g) } - pub fn is_empty(&self) -> bool { - self.len() == 0 + pub fn is_compatible_grid(&self, other: &Self) -> bool { + self.is_compatible_grid_generic(&other.spatial_grid) } - pub fn iter(&self) -> impl Iterator { - self.0.iter() + pub fn tiling_grid_definition( + &self, + tiling_specification: TilingSpecification, + ) -> TilingSpatialGridDefinition { + // TODO: we could also store the tiling_origin_reference and then use that directly? + TilingSpatialGridDefinition::new(self.spatial_grid, tiling_specification) } - pub fn into_vec(self) -> Vec { - self.0 + pub fn is_source(&self) -> bool { + self.state == SpatialGridDescriptorState::Source } - // Merge the bands of two descriptors into a new one, fails if there are duplicate names - pub fn merge(&self, other: &Self) -> Result { - let mut bands = self.0.clone(); - bands.extend(other.0.clone()); - Self::new(bands) + pub fn source_spatial_grid_definition(&self) -> Option { + match self.state { + SpatialGridDescriptorState::Source => Some(self.spatial_grid), + SpatialGridDescriptorState::Merged => None, + } } -} -impl TryFrom> for RasterBandDescriptors { - type Error = Error; + pub fn derived_spatial_grid_definition(&self) -> Option { + match self.state { + SpatialGridDescriptorState::Merged => Some(self.spatial_grid), + SpatialGridDescriptorState::Source => None, + } + } - fn try_from(value: Vec) -> Result { - RasterBandDescriptors::new(value) + pub fn spatial_partition(&self) -> SpatialPartition2D { + self.spatial_grid.spatial_partition() } -} -impl From<&RasterBandDescriptors> for BandSelection { - fn from(value: &RasterBandDescriptors) -> Self { - Self::new_unchecked((0..value.len() as u32).collect()) + pub fn geo_transform(&self) -> GeoTransform { + self.spatial_grid.geo_transform } -} -impl Index for RasterBandDescriptors { - type Output = RasterBandDescriptor; + pub fn spatial_resolution(&self) -> SpatialResolution { + self.spatial_grid.geo_transform.spatial_resolution() + } - fn index(&self, index: usize) -> &Self::Output { - &self.0[index] + pub fn grid_shape(&self) -> GridShape2D { + self.spatial_grid.grid_bounds().grid_shape() } -} -impl<'de> Deserialize<'de> for RasterBandDescriptors { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let vec = Vec::deserialize(deserializer)?; - RasterBandDescriptors::new(vec).map_err(serde::de::Error::custom) + #[must_use] + pub fn with_changed_resolution(&self, new_res: SpatialResolution) -> Self { + self.map(|x| x.with_changed_resolution(new_res)) } -} -impl<'a> FromSql<'a> for RasterBandDescriptors { - fn from_sql( - ty: &Type, - raw: &'a [u8], - ) -> Result> { - let vec = Vec::::from_sql(ty, raw)?; - Ok(RasterBandDescriptors(vec)) + #[must_use] + pub fn replace_origin(&self, new_origin: Coordinate2D) -> Self { + self.map(|x| x.replace_origin(new_origin)) } - fn accepts(ty: &Type) -> bool { - as FromSql>::accepts(ty) + #[must_use] + pub fn with_moved_origin_to_nearest_grid_edge( + &self, + new_origin_referece: Coordinate2D, + ) -> Self { + self.map(|x| x.with_moved_origin_to_nearest_grid_edge(new_origin_referece)) } -} -impl ToSql for RasterBandDescriptors { - fn to_sql( + pub fn reproject_clipped( &self, - ty: &Type, - w: &mut bytes::BytesMut, - ) -> Result> { - ToSql::to_sql(&self.0, ty, w) + projector: &P, + ) -> Result> { + let projected = self.spatial_grid.reproject_clipped(projector)?; + match projected { + Some(p) => Ok(Some(Self { + spatial_grid: p, + state: SpatialGridDescriptorState::Merged, + })), + None => Ok(None), + } } - fn accepts(ty: &Type) -> bool { - as FromSql>::accepts(ty) + pub fn generate_coord_grid_pixel_center(&self) -> Grid { + self.spatial_grid.generate_coord_grid_pixel_center() } - fn to_sql_checked( + #[must_use] + pub fn spatial_bounds_to_compatible_spatial_grid( &self, - ty: &Type, - w: &mut bytes::BytesMut, - ) -> Result> { - ToSql::to_sql_checked(&self.0, ty, w) + spatial_partition: SpatialPartition2D, + ) -> Self { + self.map(|x| x.spatial_bounds_to_compatible_spatial_grid(spatial_partition)) + } + + pub fn intersection_with_tiling_grid( + &self, + tiling_grid: &TilingSpatialGridDefinition, + ) -> Option { + let tiling_spatial_grid = tiling_grid.tiling_spatial_grid_definition(); + let intersection = self.spatial_grid.intersection(&tiling_spatial_grid)?; + + let descriptor = if self.spatial_grid.grid_bounds == intersection.grid_bounds { + self.state + } else { + SpatialGridDescriptorState::Merged + }; + + Some(Self { + spatial_grid: intersection, + state: descriptor, + }) + } + + pub fn as_parts(&self) -> (SpatialGridDescriptorState, SpatialGridDefinition) { + let SpatialGridDescriptor { + spatial_grid, + state, + } = *self; + (state, spatial_grid) } } -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql)] -pub struct RasterBandDescriptor { - pub name: String, - pub measurement: Measurement, +#[derive(Debug, Copy, Clone, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct TimeDescriptor { + pub bounds: Option, + pub dimension: TimeDimension, } -impl RasterBandDescriptor { - pub fn new(name: String, measurement: Measurement) -> Self { - Self { name, measurement } +impl TimeDescriptor { + /// Create a new `TimeDescriptor` + pub fn new(bounds: Option, dimension: TimeDimension) -> Self { + Self { bounds, dimension } } - pub fn new_unitless(name: String) -> Self { + /// Create a new `TimeDescriptor` with an irregular time dimension + pub fn new_irregular(bounds: Option) -> Self { Self { - name, - measurement: Measurement::Unitless, + bounds, + dimension: TimeDimension::Irregular, } } - pub fn new_unitless_with_idx(idx: u32) -> Self { + /// Create a new `TimeDescriptor` with a regular time dimension + pub fn new_regular(bounds: Option, origin: TimeInstance, step: TimeStep) -> Self { Self { - name: format!("band {idx}"), - measurement: Measurement::Unitless, + bounds, + dimension: TimeDimension::Regular(RegularTimeDimension { origin, step }), + } + } + + /// Create a new `TimeDescriptor` with a regular time dimension and an epoch origin + pub fn new_regular_with_epoch(bounds: Option, step: TimeStep) -> Self { + Self { + bounds, + dimension: TimeDimension::Regular(RegularTimeDimension::new_with_epoch_origin(step)), + } + } + + /// Create a new `TimeDescriptor` with a regular time dimension and an origin at the start of the bounds + pub fn new_regular_with_origin_at_start(bounds: TimeInterval, step: TimeStep) -> Self { + Self { + bounds: Some(bounds), + dimension: TimeDimension::Regular(RegularTimeDimension { + origin: bounds.start(), + step, + }), + } + } + + #[must_use] + pub fn merge(&self, other: Self) -> Self { + let bounds = match (self.bounds, other.bounds) { + (Some(a), Some(b)) => Some(a.extend(&b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + }; + + let dimension = match (&self.dimension, other.dimension) { + (TimeDimension::Irregular, _) | (_, TimeDimension::Irregular) => { + TimeDimension::Irregular + } + (TimeDimension::Regular(a), TimeDimension::Regular(b)) => match a.merge(b) { + Some(merged) => TimeDimension::Regular(merged), + None => TimeDimension::Irregular, + }, + }; + + Self { bounds, dimension } + } + + /// Is the time dimension regular? + pub fn is_regular(&self) -> bool { + self.dimension.is_regular() + } + + fn map_time_bounds(&self, f: F) -> Self + where + F: Fn(&Option) -> Option, + { + Self { + bounds: f(&self.bounds), + ..*self } } } +/// A `ResultDescriptor` for raster queries +#[derive(Debug, Clone, Serialize, Deserialize, ToSql, FromSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct RasterResultDescriptor { + pub data_type: RasterDataType, + pub spatial_reference: SpatialReferenceOption, + pub time: TimeDescriptor, + pub spatial_grid: SpatialGridDescriptor, + pub bands: RasterBandDescriptors, +} + impl ResultDescriptor for RasterResultDescriptor { type DataType = RasterDataType; - type QueryRectangleSpatialBounds = SpatialPartition2D; + type QueryRectangleSpatialBounds = GridBoundingBox2D; type QueryRectangleAttributeSelection = BandSelection; fn data_type(&self) -> Self::DataType { @@ -297,7 +425,7 @@ impl ResultDescriptor for RasterResultDescriptor { F: Fn(&Option) -> Option, { Self { - time: f(&self.time), + time: self.time.map_time_bounds(f), // FIXME: remove this extra function and use TimeDescriptor::map_time_bounds bands: self.bands.clone(), ..*self } @@ -310,7 +438,7 @@ impl ResultDescriptor for RasterResultDescriptor { Self::QueryRectangleAttributeSelection, >, ) -> Result<()> { - for band in query.attributes.as_slice() { + for band in query.attributes().as_slice() { if *band as usize >= self.bands.len() { return Err(Error::BandDoesNotExist { band_idx: *band }); } @@ -320,7 +448,64 @@ impl ResultDescriptor for RasterResultDescriptor { } } -impl RasterResultDescriptor {} +impl RasterResultDescriptor { + /// create a new `RasterResultDescriptor` + pub fn new( + data_type: RasterDataType, + spatial_reference: SpatialReferenceOption, + time: TimeDescriptor, + spatial_grid: SpatialGridDescriptor, + bands: RasterBandDescriptors, + ) -> Self { + Self { + data_type, + spatial_reference, + time, + spatial_grid, + bands, + } + } + + pub fn spatial_grid_descriptor(&self) -> &SpatialGridDescriptor { + &self.spatial_grid + } + + /// Returns tiling grid definition of the data. + pub fn tiling_grid_definition( + &self, + tiling_specification: TilingSpecification, + ) -> TilingSpatialGridDefinition { + self.spatial_grid + .tiling_grid_definition(tiling_specification) + } + + pub fn spatial_bounds(&self) -> SpatialPartition2D { + self.spatial_grid.spatial_partition() + } + + pub fn with_datatype_and_num_bands( + data_type: RasterDataType, + num_bands: u32, + pixel_bounds: GridBoundingBox2D, + geo_transform: GeoTransform, + time: TimeDescriptor, + ) -> Self { + Self { + data_type, + spatial_reference: SpatialReferenceOption::Unreferenced, + time, + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + geo_transform, + pixel_bounds, + )), + bands: RasterBandDescriptors::new_multiple_bands(num_bands), + } + } + + pub fn replace_resolution(&mut self, resolution: SpatialResolution) { + self.spatial_grid = self.spatial_grid.with_changed_resolution(resolution); + } +} /// A `ResultDescriptor` for vector queries #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -496,11 +681,9 @@ impl From for PlotResultDescriptor { fn from(descriptor: RasterResultDescriptor) -> Self { Self { spatial_reference: descriptor.spatial_reference, - time: descriptor.time, + time: descriptor.time.bounds, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: descriptor - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(descriptor.spatial_bounds().as_bbox()), } } } @@ -531,6 +714,173 @@ impl From for TypedResultDescriptor { } } +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct RasterBandDescriptors(Vec); + +impl RasterBandDescriptors { + pub fn new(bands: Vec) -> Result { + let mut names = HashSet::new(); + for value in &bands { + ensure!(!value.name.is_empty(), RasterBandNameMustNotBeEmpty); + ensure!(value.name.byte_size() <= 256, RasterBandNameTooLong); + ensure!( + names.insert(&value.name), + RasterBandNamesMustBeUnique { + duplicate_key: value.name.clone() + } + ); + } + + Ok(Self(bands)) + } + + /// Convenience method to crate a single band result descriptor with no specific name and a unitless measurement for single band rasters + pub fn new_single_band() -> Self { + Self(vec![RasterBandDescriptor { + name: "band".into(), + measurement: Measurement::Unitless, + }]) + } + + /// Convenience method to crate multipe band result descriptors with no specific name and a unitless measurement + pub fn new_multiple_bands(num_bands: u32) -> Self { + Self( + (0..num_bands) + .map(RasterBandDescriptor::new_unitless_with_idx) + .collect(), + ) + } + + pub fn bands(&self) -> &[RasterBandDescriptor] { + &self.0 + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn count(&self) -> u32 { + self.0.len() as u32 + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn into_vec(self) -> Vec { + self.0 + } + + // Merge the bands of two descriptors into a new one, fails if there are duplicate names + pub fn merge(&self, other: &Self) -> Result { + let mut bands = self.0.clone(); + bands.extend(other.0.clone()); + Self::new(bands) + } + + pub fn is_single(&self) -> bool { + self.len() == 1 + } +} + +impl TryFrom> for RasterBandDescriptors { + type Error = Error; + + fn try_from(value: Vec) -> Result { + RasterBandDescriptors::new(value) + } +} + +impl From<&RasterBandDescriptors> for BandSelection { + fn from(value: &RasterBandDescriptors) -> Self { + Self::new_unchecked((0..value.len() as u32).collect()) + } +} + +impl Index for RasterBandDescriptors { + type Output = RasterBandDescriptor; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} + +impl<'de> Deserialize<'de> for RasterBandDescriptors { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let vec = Vec::deserialize(deserializer)?; + RasterBandDescriptors::new(vec).map_err(serde::de::Error::custom) + } +} + +impl<'a> FromSql<'a> for RasterBandDescriptors { + fn from_sql( + ty: &Type, + raw: &'a [u8], + ) -> Result> { + let vec = Vec::::from_sql(ty, raw)?; + Ok(RasterBandDescriptors(vec)) + } + + fn accepts(ty: &Type) -> bool { + as FromSql>::accepts(ty) + } +} + +impl ToSql for RasterBandDescriptors { + fn to_sql( + &self, + ty: &Type, + w: &mut bytes::BytesMut, + ) -> Result> { + ToSql::to_sql(&self.0, ty, w) + } + + fn accepts(ty: &Type) -> bool { + as FromSql>::accepts(ty) + } + + fn to_sql_checked( + &self, + ty: &Type, + w: &mut bytes::BytesMut, + ) -> Result> { + ToSql::to_sql_checked(&self.0, ty, w) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql)] +pub struct RasterBandDescriptor { + pub name: String, + pub measurement: Measurement, +} + +impl RasterBandDescriptor { + pub fn new(name: String, measurement: Measurement) -> Self { + Self { name, measurement } + } + + pub fn new_unitless(name: String) -> Self { + Self { + name, + measurement: Measurement::Unitless, + } + } + + pub fn new_unitless_with_idx(idx: u32) -> Self { + Self { + name: format!("band {idx}"), + measurement: Measurement::Unitless, + } + } +} + mod db_types { use super::*; use crate::error::Error; @@ -658,6 +1008,59 @@ mod db_types { } } + #[derive(Debug, ToSql, FromSql)] + #[postgres(name = "SpatialGridDescriptorState")] + pub enum SpatialGridDescriptorStateDbType { + /// The spatial grid represents the original data + Source, + /// The spatial grid was created by merging two non equal spatial grids + Merged, + } + + impl From<&SpatialGridDescriptorState> for SpatialGridDescriptorStateDbType { + fn from(value: &SpatialGridDescriptorState) -> Self { + match value { + SpatialGridDescriptorState::Source => SpatialGridDescriptorStateDbType::Source, + SpatialGridDescriptorState::Merged => SpatialGridDescriptorStateDbType::Merged, + } + } + } + + impl From for SpatialGridDescriptorState { + fn from(value: SpatialGridDescriptorStateDbType) -> Self { + match value { + SpatialGridDescriptorStateDbType::Source => SpatialGridDescriptorState::Source, + SpatialGridDescriptorStateDbType::Merged => SpatialGridDescriptorState::Merged, + } + } + } + + #[derive(Debug, ToSql, FromSql)] + #[postgres(name = "SpatialGridDescriptor")] + pub struct SpatialGridDescriptorDbType { + state: SpatialGridDescriptorStateDbType, + spatial_grid: SpatialGridDefinition, + } + + impl From<&SpatialGridDescriptor> for SpatialGridDescriptorDbType { + fn from(value: &SpatialGridDescriptor) -> Self { + Self { + spatial_grid: value.spatial_grid, + state: SpatialGridDescriptorStateDbType::from(&value.state), + } + } + } + + impl From for SpatialGridDescriptor { + fn from(value: SpatialGridDescriptorDbType) -> Self { + Self { + spatial_grid: value.spatial_grid, + state: SpatialGridDescriptorState::from(value.state), + } + } + } + + delegate_from_to_sql!(SpatialGridDescriptor, SpatialGridDescriptorDbType); delegate_from_to_sql!(VectorResultDescriptor, VectorResultDescriptorDbType); delegate_from_to_sql!(TypedResultDescriptor, TypedResultDescriptorDbType); } diff --git a/operators/src/error.rs b/operators/src/error.rs index 692402f74..df5ca69d8 100644 --- a/operators/src/error.rs +++ b/operators/src/error.rs @@ -1,3 +1,5 @@ +use crate::engine::SpatialGridDescriptor; +use crate::optimization::OptimizationError; use crate::processing::BandNeighborhoodAggregateError; use crate::util::statistics::StatisticsError; use bb8_postgres::bb8; @@ -359,6 +361,10 @@ pub enum Error { InterpolationOperator { source: crate::processing::InterpolationError, }, + #[snafu(context(false))] + DownsampleOperator { + source: crate::processing::DownsamplingError, + }, #[snafu(display("TimeShift error: {source}"), context(false))] TimeShift { source: crate::processing::TimeShiftError, @@ -378,6 +384,11 @@ pub enum Error { source: crate::source::GdalSourceError, }, + #[snafu(display("MultiBandGdalSource error: {source}"), context(false))] + MultiBandGdalSource { + source: crate::source::MultiBandGdalSourceError, + }, + QueryCanceled, AbortTriggerAlreadyUsed, @@ -429,6 +440,12 @@ pub enum Error { source: Box, }, + #[snafu(display("RasterResults are incompatible error: {a:?} vs {b:?}"))] + CantMergeSpatialGridDescriptor { + a: SpatialGridDescriptor, + b: SpatialGridDescriptor, + }, + #[snafu(display( "Input stream {stream_index} is not temporally aligned. Expected {expected:?}, found {found:?}." ))] @@ -498,6 +515,8 @@ pub enum Error { message: String, }, + ReprojectionFailed, + #[snafu(display("PostgresError: {}", source))] Postgres { source: tokio_postgres::Error, @@ -511,6 +530,10 @@ pub enum Error { BandNeighborhoodAggregate { source: BandNeighborhoodAggregateError, }, + #[snafu(display("Error during workflow optimization: {source}"))] + Optimization { + source: OptimizationError, + }, } impl From for Error { diff --git a/operators/src/lib.rs b/operators/src/lib.rs index 8d5382ed3..35f516455 100755 --- a/operators/src/lib.rs +++ b/operators/src/lib.rs @@ -6,6 +6,7 @@ pub mod error; pub mod machine_learning; pub mod meta; pub mod mock; +pub mod optimization; pub mod plot; pub mod processing; pub mod source; diff --git a/operators/src/machine_learning/onnx.rs b/operators/src/machine_learning/onnx.rs index 85df425e6..34f639dff 100644 --- a/operators/src/machine_learning/onnx.rs +++ b/operators/src/machine_learning/onnx.rs @@ -1,9 +1,9 @@ use crate::{ engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, - Operator, OperatorName, QueryContext, RasterBandDescriptor, RasterOperator, - RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, - TypedRasterQueryProcessor, WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, + RasterBandDescriptor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, + SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }, error, machine_learning::{ @@ -13,18 +13,21 @@ use crate::{ check_model_input_features, check_model_shape, load_onnx_model_from_loading_info, }, }, + optimization::OptimizationError, util::Result, }; use async_trait::async_trait; use float_cmp::approx_eq; use futures::StreamExt; use futures::stream::BoxStream; -use geoengine_datatypes::machine_learning::MlModelName; -use geoengine_datatypes::primitives::{Measurement, RasterQueryRectangle}; +use geoengine_datatypes::primitives::{ + BandSelection, Measurement, RasterQueryRectangle, SpatialResolution, TimeInterval, +}; use geoengine_datatypes::raster::{ EmptyGrid2D, Grid2D, GridIdx2D, GridIndexAccess, GridShape2D, GridShapeAccess, GridSize, MaskedGrid, Pixel, RasterTile2D, UpdateIndexedElements, }; +use geoengine_datatypes::{machine_learning::MlModelName, raster::GridBoundingBox2D}; use ndarray::{Array2, Array4}; use ort::{ tensor::{IntoTensorElementType, PrimitiveTensorElementType}, @@ -99,8 +102,7 @@ impl RasterOperator for Onnx { data_type: model_loading_info.metadata.output_type, spatial_reference: in_descriptor.spatial_reference, time: in_descriptor.time, - bbox: in_descriptor.bbox, - resolution: in_descriptor.resolution, + spatial_grid: in_descriptor.spatial_grid, bands: vec![RasterBandDescriptor::new( "prediction".to_string(), // TODO: parameter of the operator? Measurement::Unitless, // TODO: get output measurement from model metadata @@ -111,6 +113,7 @@ impl RasterOperator for Onnx { Ok(Box::new(InitializedOnnx { name, path, + model_name: self.params.model, result_descriptor: out_descriptor, source, model_loading_info, @@ -124,6 +127,7 @@ impl RasterOperator for Onnx { pub struct InitializedOnnx { name: CanonicOperatorName, path: WorkflowOperatorPath, + model_name: MlModelName, result_descriptor: RasterResultDescriptor, source: Box, model_loading_info: MlModelLoadingInfo, @@ -165,10 +169,25 @@ impl InitializedRasterOperator for InitializedOnnx { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Onnx { + params: OnnxParams { + model: self.model_name.clone(), + }, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub(crate) struct OnnxProcessor { - source: Box>, // as most ml algorithms work on f32 we use this as input type + source: BoxRasterQueryProcessor, // as most ml algorithms work on f32 we use this as input type result_descriptor: RasterResultDescriptor, model_loading_info: MlModelLoadingInfo, phantom: std::marker::PhantomData, @@ -177,7 +196,7 @@ pub(crate) struct OnnxProcessor { impl OnnxProcessor { pub fn new( - source: Box>, + source: BoxRasterQueryProcessor, result_descriptor: RasterResultDescriptor, model_loading_info: MlModelLoadingInfo, tile_shape: GridShape2D, @@ -193,15 +212,18 @@ impl OnnxProcessor { } #[async_trait] -impl RasterQueryProcessor for OnnxProcessor +impl QueryProcessor for OnnxProcessor where TIn: Pixel + NoDataValueIn + IntoTensorElementType + PrimitiveTensorElementType, TOut: Pixel + NoDataValueOut + IntoTensorElementType + PrimitiveTensorElementType, { - type RasterType = TOut; + type Output = RasterTile2D; + type SpatialBounds = GridBoundingBox2D; + type Selection = BandSelection; + type ResultDescription = RasterResultDescriptor; #[allow(clippy::too_many_lines)] - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, @@ -223,8 +245,8 @@ where .map(|v| TOut::from_(v)) .or(TOut::NO_DATA_OUT_FALLBACK); // Int types return Some or None while float types fallback to Some(NaN) - let mut source_query = query.clone(); - source_query.attributes = (0..num_bands as u32).collect::>().try_into()?; + let source_query = + query.select_attributes((0..num_bands as u32).collect::>().try_into()?); // TODO: re-use session accross queries? // TODO: use another method: https://github.com/pykeio/ort/issues/402#issuecomment-2949993914 @@ -397,11 +419,28 @@ where Ok(stream.boxed()) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &Self::ResultDescription { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for OnnxProcessor +where + TIn: Pixel + NoDataValueIn + IntoTensorElementType + PrimitiveTensorElementType, + TOut: Pixel + NoDataValueOut + IntoTensorElementType + PrimitiveTensorElementType, +{ + type RasterType = TOut; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.source.time_query(query, ctx).await + } +} + // Trait to handle mapping masked pixels to model inputs for all datatypes. trait NoDataValueIn { const NO_DATA_IN_FALLBACK: Self; @@ -473,27 +512,40 @@ impl_no_data_value_none!(i8, u8, i16, u16, i32, u32, i64, u64); #[cfg(test)] mod tests { - use super::*; + use crate::engine::TimeDescriptor; + use crate::machine_learning::MlModelInputNoDataHandling; + use crate::machine_learning::MlModelLoadingInfo; + use crate::machine_learning::MlModelMetadata; + use crate::machine_learning::MlModelOutputNoDataHandling; + use crate::machine_learning::onnx::Onnx; + use crate::machine_learning::onnx::OnnxParams; use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SingleRasterSource, SpatialGridDescriptor, + WorkflowOperatorPath, }, - machine_learning::{MlModelMetadata, MlModelOutputNoDataHandling}, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, + util::Result, }; use approx::assert_abs_diff_eq; + use futures::StreamExt; + use geoengine_datatypes::primitives::TimeStep; + use geoengine_datatypes::raster::GridBoundingBox2D; + use geoengine_datatypes::raster::RasterTile2D; + use geoengine_datatypes::raster::SpatialGridDefinition; + use geoengine_datatypes::raster::TilesEqualIgnoringCacheHint; use geoengine_datatypes::{ - machine_learning::MlTensorShape3D, - primitives::{CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{ - Grid, GridOrEmpty, GridShape, RasterDataType, RenameBands, TilesEqualIgnoringCacheHint, - }, + machine_learning::{MlModelName, MlTensorShape3D}, + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, + raster::{Grid, GridOrEmpty, GridShape, RasterDataType, RenameBands}, spatial_reference::SpatialReference, test_data, util::test::TestDefault, }; use ndarray::{Array1, Array2, arr2, array}; + use ort::value::TensorRef; #[test] fn ort() { @@ -654,9 +706,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -669,9 +726,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -718,14 +780,13 @@ mod tests { }; exe_ctx.ml_models.insert(model_name, ml_model_loading_info); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = onnx .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -854,9 +915,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -869,9 +935,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -884,9 +955,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + None, + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -933,14 +1009,13 @@ mod tests { }; exe_ctx.ml_models.insert(model_name, ml_model_loading_info); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = onnx .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1027,17 +1102,21 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::F32, + spatial_reference: SpatialReference::epsg_4326().into(), + spatial_grid: SpatialGridDescriptor::new_source(SpatialGridDefinition::new( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-512, -1, 0, 1023).unwrap(), + )), + time: TimeDescriptor::new_regular_with_epoch(None, TimeStep::millis(5).unwrap()), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::F32, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -1045,14 +1124,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::F32, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1097,17 +1169,13 @@ mod tests { }; exe_ctx.ml_models.insert(model_name, ml_model_loading_info); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 511.).into(), - (1023., 0.).into(), - ), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-512, -1, 0, 1023).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = onnx .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/operators/src/meta/wrapper.rs b/operators/src/meta/wrapper.rs index 61561143a..ab4681ac7 100644 --- a/operators/src/meta/wrapper.rs +++ b/operators/src/meta/wrapper.rs @@ -1,17 +1,19 @@ use crate::adapters::StreamStatisticsAdapter; use crate::engine::{ CanonicOperatorName, CreateSpan, InitializedRasterOperator, InitializedVectorOperator, - QueryContext, QueryProcessor, RasterResultDescriptor, ResultDescriptor, - TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorResultDescriptor, - WorkflowOperatorPath, + QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, + ResultDescriptor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorOperator, + VectorResultDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, QueryAttributeSelection, QueryRectangle, + QueryAttributeSelection, QueryRectangle, SpatialResolution, TimeInterval, }; +use geoengine_datatypes::raster::RasterTile2D; use std::sync::atomic::{AtomicUsize, Ordering}; use tracing::{Level, span}; @@ -157,6 +159,13 @@ impl InitializedRasterOperator for InitializedOperatorWrapper WorkflowOperatorPath { self.source.path() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.source.optimize(resolution) + } } impl InitializedVectorOperator for InitializedOperatorWrapper> { @@ -196,6 +205,13 @@ impl InitializedVectorOperator for InitializedOperatorWrapper WorkflowOperatorPath { self.source.path() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.source.optimize(resolution) + } } // A wrapper around a query processor that adds statistics and quota tracking @@ -241,7 +257,7 @@ where impl QueryProcessor for QueryProcessorWrapper where Q: QueryProcessor, - S: AxisAlignedRectangle + Send + Sync + 'static, + S: std::fmt::Display + Send + Sync + 'static + Clone + Copy, A: QueryAttributeSelection + 'static, R: ResultDescriptor + 'static, @@ -289,18 +305,15 @@ where let _enter = span.enter(); + let spbox = query.spatial_bounds(); + let time = query.time_interval(); tracing::trace!( event = %"query_start", path = %self.path, - bbox = %format!("[{},{},{},{}]", - query.spatial_bounds.lower_left().x, - query.spatial_bounds.lower_left().y, - query.spatial_bounds.upper_right().x, - query.spatial_bounds.upper_right().y - ), + bbox = %format!("{}", spbox), time = %format!("[{},{}]", - query.time_interval.start().inner(), - query.time_interval.end().inner() + time.start().inner(), + time.end().inner() ) ); @@ -331,3 +344,19 @@ where self.processor.result_descriptor() } } + +#[async_trait] +impl RasterQueryProcessor for QueryProcessorWrapper> +where + Q: RasterQueryProcessor + QueryProcessor>, +{ + type RasterType = Q::RasterType; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.processor.time_query(query, ctx).await + } +} diff --git a/operators/src/mock/mock_dataset_data_source.rs b/operators/src/mock/mock_dataset_data_source.rs index 653960204..decd7c95f 100644 --- a/operators/src/mock/mock_dataset_data_source.rs +++ b/operators/src/mock/mock_dataset_data_source.rs @@ -3,6 +3,7 @@ use crate::engine::{ OperatorName, QueryContext, ResultDescriptor, SourceOperator, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::StreamExt; @@ -10,7 +11,7 @@ use futures::stream; use futures::stream::BoxStream; use geoengine_datatypes::collections::{MultiPointCollection, VectorDataType}; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::CacheHint; +use geoengine_datatypes::primitives::{CacheHint, SpatialResolution}; use geoengine_datatypes::primitives::{Coordinate2D, TimeInterval, VectorQueryRectangle}; use geoengine_datatypes::spatial_reference::SpatialReferenceOption; use postgres_types::{FromSql, ToSql}; @@ -129,6 +130,7 @@ impl VectorOperator for MockDatasetDataSource { Ok(InitializedMockDatasetDataSource { name: CanonicOperatorName::from(&self), path, + data: self.params.data.clone(), result_descriptor: loading_info.result_descriptor().await?, loading_info, } @@ -147,6 +149,7 @@ impl OperatorData for MockDatasetDataSource { struct InitializedMockDatasetDataSource { name: CanonicOperatorName, path: WorkflowOperatorPath, + data: NamedData, result_descriptor: R, loading_info: Box>, } @@ -179,17 +182,29 @@ impl InitializedVectorOperator fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + _target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(MockDatasetDataSource { + params: MockDatasetDataSourceParams { + data: self.data.clone(), + }, + } + .boxed()) + } } #[cfg(test)] mod tests { use super::*; + use crate::engine::MockExecutionContext; use crate::engine::QueryProcessor; - use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; + use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection}; use geoengine_datatypes::util::Identifier; use geoengine_datatypes::util::test::TestDefault; @@ -222,13 +237,13 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = + execution_context.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_feature_collection_source.rs b/operators/src/mock/mock_feature_collection_source.rs index 6022fc44e..893fe0d25 100644 --- a/operators/src/mock/mock_feature_collection_source.rs +++ b/operators/src/mock/mock_feature_collection_source.rs @@ -6,6 +6,7 @@ use crate::engine::{ SourceOperator, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::{self, BoxStream, StreamExt}; @@ -13,6 +14,7 @@ use geoengine_datatypes::collections::{ FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, }; use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::primitives::{ Geometry, Measurement, MultiLineString, MultiPoint, MultiPolygon, NoGeometry, TimeInterval, VectorQueryRectangle, @@ -49,7 +51,7 @@ where let stream = stream::iter( self.collections .iter() - .map(move |c| filter_time_intervals(c, query.time_interval)), + .map(move |c| filter_time_intervals(c, query.time_interval())), ); Ok(stream.boxed()) @@ -139,9 +141,10 @@ where } } -pub struct InitializedMockFeatureCollectionSource { +pub struct InitializedMockFeatureCollectionSource { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: MockFeatureCollectionSourceParams, result_descriptor: R, collections: Vec>, } @@ -204,6 +207,7 @@ macro_rules! impl_mock_feature_collection_source { Ok(InitializedMockFeatureCollectionSource { name: CanonicOperatorName::from(&self), path, + params: self.params.clone(), result_descriptor, collections: self.params.collections, } @@ -237,6 +241,16 @@ macro_rules! impl_mock_feature_collection_source { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + _target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(MockFeatureCollectionSource { + params: self.params.clone(), + } + .boxed()) + } } }; } @@ -249,14 +263,15 @@ impl_mock_feature_collection_source!(MultiPolygon, MultiPolygon); #[cfg(test)] mod tests { use super::*; + use crate::engine::MockExecutionContext; use crate::engine::QueryProcessor; - use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; - use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; - use geoengine_datatypes::primitives::{BoundingBox2D, Coordinate2D, FeatureData, TimeInterval}; - use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; + use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, ColumnSelection, Coordinate2D, FeatureData, TimeInterval, + }; + use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{collections::MultiPointCollection, primitives::SpatialResolution}; #[test] #[allow(clippy::too_many_lines)] @@ -413,13 +428,13 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context((2 * std::mem::size_of::()).into()); let stream = processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_point_source.rs b/operators/src/mock/mock_point_source.rs index 67029bfea..47443535f 100644 --- a/operators/src/mock/mock_point_source.rs +++ b/operators/src/mock/mock_point_source.rs @@ -1,4 +1,5 @@ use crate::engine::{CanonicOperatorName, OperatorData, QueryContext}; +use crate::optimization::OptimizationError; use crate::{ engine::{ ExecutionContext, InitializedVectorOperator, OperatorName, SourceOperator, @@ -11,8 +12,9 @@ use async_trait::async_trait; use futures::stream::{self, BoxStream, StreamExt}; use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::CacheHint; -use geoengine_datatypes::primitives::VectorQueryRectangle; +use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, SpatialResolution, VectorQueryRectangle, +}; use geoengine_datatypes::{ collections::MultiPointCollection, primitives::{Coordinate2D, TimeInterval}, @@ -35,10 +37,10 @@ impl VectorQueryProcessor for MockPointSourceProcessor { ctx: &'a dyn QueryContext, ) -> Result>> { let chunk_size = usize::from(ctx.chunk_byte_size()) / std::mem::size_of::(); - let bounding_box = query.spatial_bounds; + let spatial_query = query.spatial_bounds(); Ok(stream::iter(&self.points) - .filter(move |&coord| std::future::ready(bounding_box.contains_coordinate(coord))) + .filter(move |&coord| std::future::ready(spatial_query.contains_coordinate(coord))) .chunks(chunk_size) .map(move |chunk| { Ok(MultiPointCollection::from_data( @@ -57,8 +59,37 @@ impl VectorQueryProcessor for MockPointSourceProcessor { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", tag = "type")] +#[derive(Default)] +pub enum SpatialBoundsDerive { + Derive, + Bounds(BoundingBox2D), + #[default] + None, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct MockPointSourceParams { pub points: Vec, + #[serde(default = "SpatialBoundsDerive::default")] + pub spatial_bounds: SpatialBoundsDerive, +} + +impl MockPointSourceParams { + pub fn new(points: Vec) -> Self { + MockPointSourceParams { + points, + spatial_bounds: SpatialBoundsDerive::default(), + } + } + + pub fn new_with_bounds(points: Vec, spatial_bounds: SpatialBoundsDerive) -> Self { + MockPointSourceParams { + points, + spatial_bounds, + } + } } pub type MockPointSource = SourceOperator; @@ -79,15 +110,24 @@ impl VectorOperator for MockPointSource { path: WorkflowOperatorPath, _context: &dyn ExecutionContext, ) -> Result> { + let bounds = match self.params.spatial_bounds { + SpatialBoundsDerive::None => None, + SpatialBoundsDerive::Bounds(b) => Some(b), + SpatialBoundsDerive::Derive => { + BoundingBox2D::from_coord_ref_iter(self.params.points.iter()) + } + }; + Ok(InitializedMockPointSource { name: CanonicOperatorName::from(&self), path, + spatial_bounds: self.params.spatial_bounds.clone(), result_descriptor: VectorResultDescriptor { data_type: VectorDataType::MultiPoint, spatial_reference: SpatialReference::epsg_4326().into(), columns: Default::default(), time: None, - bbox: None, + bbox: bounds, }, points: self.params.points, } @@ -100,6 +140,7 @@ impl VectorOperator for MockPointSource { pub struct InitializedMockPointSource { name: CanonicOperatorName, path: WorkflowOperatorPath, + spatial_bounds: SpatialBoundsDerive, result_descriptor: VectorResultDescriptor, points: Vec, } @@ -130,16 +171,29 @@ impl InitializedVectorOperator for InitializedMockPointSource { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + _target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(MockPointSource { + params: MockPointSourceParams { + points: self.points.clone(), + spatial_bounds: self.spatial_bounds.clone(), + }, + } + .boxed()) + } } #[cfg(test)] mod tests { use super::*; + use crate::engine::MockExecutionContext; use crate::engine::QueryProcessor; - use crate::engine::{MockExecutionContext, MockQueryContext}; use futures::executor::block_on_stream; use geoengine_datatypes::collections::FeatureCollectionInfos; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; + use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection}; use geoengine_datatypes::util::test::TestDefault; #[test] @@ -147,11 +201,11 @@ mod tests { let points = vec![Coordinate2D::new(1., 2.); 3]; let mps = MockPointSource { - params: MockPointSourceParams { points }, + params: MockPointSourceParams::new(points), } .boxed(); let serialized = serde_json::to_string(&mps).unwrap(); - let expect = "{\"type\":\"MockPointSource\",\"params\":{\"points\":[{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0}]}}"; + let expect = "{\"type\":\"MockPointSource\",\"params\":{\"points\":[{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0},{\"x\":1.0,\"y\":2.0}],\"spatialBounds\":{\"type\":\"none\"}}}"; assert_eq!(serialized, expect); let _operator: Box = serde_json::from_str(&serialized).unwrap(); @@ -163,7 +217,7 @@ mod tests { let points = vec![Coordinate2D::new(1., 2.); 3]; let mps = MockPointSource { - params: MockPointSourceParams { points }, + params: MockPointSourceParams::new(points), } .boxed(); let initialized = mps @@ -176,13 +230,13 @@ mod tests { panic!() }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = + execution_context.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/mock/mock_raster_source.rs b/operators/src/mock/mock_raster_source.rs index 3048f550c..345591fb9 100644 --- a/operators/src/mock/mock_raster_source.rs +++ b/operators/src/mock/mock_raster_source.rs @@ -2,19 +2,27 @@ use crate::adapters::{ FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, }; use crate::engine::{ - CanonicOperatorName, InitializedRasterOperator, OperatorData, OperatorName, RasterOperator, - RasterQueryProcessor, RasterResultDescriptor, SourceOperator, TypedRasterQueryProcessor, - WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, InitializedRasterOperator, OperatorData, + OperatorName, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, + SingleRasterSource, SourceOperator, TypedRasterQueryProcessor, WorkflowOperatorPath, +}; +use crate::optimization::{OptimizableOperator, OptimizationError}; +use crate::processing::{ + Downsampling, DownsamplingMethod, DownsamplingParams, DownsamplingResolution, }; use crate::util::Result; use async_trait::async_trait; use futures::{stream, stream::StreamExt}; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{CacheExpiration, TimeInstance}; -use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartitioned}; +use geoengine_datatypes::primitives::{ + BandSelection, CacheExpiration, RasterQueryRectangle, SpatialResolution, TimeFilledItem, + TimeInstance, TryIrregularTimeFillIterExt, TryRegularTimeFillIterExt, +}; use geoengine_datatypes::raster::{ - GridShape2D, GridShapeAccess, GridSize, Pixel, RasterTile2D, TilingSpecification, + GridBoundingBox2D, GridIntersection, GridShape2D, GridShapeAccess, GridSize, Pixel, + RasterTile2D, TilingSpecification, }; +use itertools::Itertools; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -52,11 +60,9 @@ where data: Vec>, tiling_specification: TilingSpecification, ) -> Self { - Self { - result_descriptor, - data, - tiling_specification, - } + // use expect here since the mock source should not be used in production and this provides valuable debug information + Self::_new(result_descriptor, data, tiling_specification) + .expect("can initialize from inputs") } fn _new( @@ -100,48 +106,59 @@ where } #[async_trait] -impl RasterQueryProcessor for MockRasterSourceProcessor +impl QueryProcessor for MockRasterSourceProcessor where T: Pixel, { - type RasterType = T; - async fn raster_query<'a>( + type Output = RasterTile2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; + + async fn _query<'a>( &'a self, query: RasterQueryRectangle, _ctx: &'a dyn crate::engine::QueryContext, - ) -> Result>>> - { + ) -> Result>> { let mut known_time_start: Option = None; let mut known_time_end: Option = None; + let qt = query.time_interval(); + let qg = query.spatial_bounds(); let parts: Vec> = self .data .iter() .inspect(|m| { let time_interval = m.time; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + if time_interval.contains(&qt) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; + } + + if time_interval.end() <= qt.start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= qt.start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.start() >= qt.end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= qt.end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }) .filter(move |t| { - t.time.intersects(&query.time_interval) + t.time.intersects(&qt) && t.tile_information() - .spatial_partition() - .intersects(&query.spatial_bounds) + .global_pixel_bounds() + .intersects(&query.spatial_bounds()) }) .cloned() .collect(); @@ -152,40 +169,66 @@ where let inner_stream = stream::iter(parts.into_iter().map(Result::Ok)); - // TODO: evaluate if there are GeoTransforms with positive y-axis - // The "Pixel-space" starts at the top-left corner of a `GeoTransform`. - // Therefore, the pixel size on the x-axis is always increasing - let spatial_resolution = query.spatial_resolution; + let tiling_grid_spec = self + .result_descriptor + .tiling_grid_definition(self.tiling_specification); - let pixel_size_x = spatial_resolution.x; - debug_assert!(pixel_size_x.is_sign_positive()); - // and the pixel size on the y-axis is always decreasing - let pixel_size_y = -spatial_resolution.y; - debug_assert!(pixel_size_y.is_sign_negative()); - - let tiling_strategy = self - .tiling_specification - .strategy(pixel_size_x, pixel_size_y); + let tiling_strategy = tiling_grid_spec.generate_data_tiling_strategy(); // use SparseTilesFillAdapter to fill all the gaps Ok(SparseTilesFillAdapter::new( inner_stream, - tiling_strategy.tile_grid_box(query.spatial_partition()), + tiling_strategy.global_pixel_grid_bounds_to_tile_grid_bounds(qg), self.result_descriptor.bands.count(), tiling_strategy.geo_transform, tiling_strategy.tile_size_in_pixels, FillerTileCacheExpirationStrategy::FixedValue(CacheExpiration::max()), // cache forever because we know all mock data - query.time_interval, + qt, FillerTimeBounds::new(known_time_before, known_time_after), ) .boxed()) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &Self::ResultDescription { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for MockRasterSourceProcessor +where + T: Pixel, +{ + type RasterType = T; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + _ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { + let unique_times = self + .data + .iter() + .map(|tile| tile.time) + .unique_by(|t| *t) + .filter(move |t| t.intersects(&query)) + .map(Ok); + + let times = match self.result_descriptor.time.dimension { + geoengine_datatypes::primitives::TimeDimension::Irregular => { + let times = unique_times.try_time_irregular_range_fill(query.time()); + stream::iter(times).boxed() + } + geoengine_datatypes::primitives::TimeDimension::Regular(regular_dim) => { + let times = unique_times.try_time_regular_range_fill(regular_dim, query.time()); + stream::iter(times).boxed() + } + }; + + Ok(times) + } +} + #[derive(Debug, Serialize, Deserialize, PartialEq, Clone)] #[serde(rename_all = "camelCase")] pub struct MockRasterSourceParams { @@ -303,7 +346,8 @@ pub struct InitializedMockRasterSource { impl InitializedRasterOperator for InitializedMockRasterSource where - TypedRasterQueryProcessor: From>>, + TypedRasterQueryProcessor: From>, + SourceOperator>: RasterOperator, { fn query_processor(&self) -> Result { let processor = TypedRasterQueryProcessor::from( @@ -333,23 +377,51 @@ where fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let source = MockRasterSource { + params: MockRasterSourceParams { + data: self.data.clone(), + result_descriptor: self.result_descriptor.clone(), + }, + } + .boxed(); + + if target_resolution > self.result_descriptor.spatial_grid.spatial_resolution() { + return Ok(Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution(target_resolution), + output_origin_reference: None, + }, + sources: SingleRasterSource { raster: source }, + } + .boxed()); + } + + Ok(source) + } } #[cfg(test)] mod tests { use super::*; use crate::engine::{ - MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptors, + MockExecutionContext, QueryProcessor, RasterBandDescriptors, SpatialGridDescriptor, + TimeDescriptor, }; - use geoengine_datatypes::primitives::{BandSelection, CacheHint}; - use geoengine_datatypes::primitives::{SpatialPartition2D, SpatialResolution}; - use geoengine_datatypes::raster::{Grid, MaskedGrid, RasterDataType, RasterProperties}; - use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - primitives::TimeInterval, - raster::{Grid2D, TileInformation}, - spatial_reference::SpatialReference, + use geoengine_datatypes::primitives::{BandSelection, CacheHint, TimeInterval, TimeStep}; + use geoengine_datatypes::raster::{ + BoundedGrid, GeoTransform, Grid, Grid2D, GridBoundingBox2D, MaskedGrid, RasterDataType, + RasterProperties, TileInformation, }; + use geoengine_datatypes::spatial_reference::SpatialReference; + use geoengine_datatypes::util::test::TestDefault; #[tokio::test] #[allow(clippy::too_many_lines)] @@ -376,9 +448,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -431,9 +505,24 @@ mod tests { "resultDescriptor": { "dataType": "U8", "spatialReference": "EPSG:4326", - "time": null, - "bbox": null, - "resolution": null, + "time": {"bounds": {"start": -8_334_601_228_800_000_i64, "end": 8_210_266_876_799_999_i64}, "dimension": "irregular"} , + "spatialGrid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": 0.0, + "y": 0.0 + }, + "xPixelSize": 1.0, + "yPixelSize": -1.0 + }, + "gridBounds": { + "max": [2, 1], + "min": [0, 0] + } + }, + "state": "source", + }, "bands": [ { "name": "band", @@ -451,7 +540,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -522,18 +610,22 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(1, 3)), + TimeStep::millis(1).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., -3.).into(), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let query_processor = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -544,16 +636,15 @@ mod tests { .get_u8() .unwrap(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); // QUERY 1 - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(-3, 7), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 4), + BandSelection::first(), + ); let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); @@ -575,12 +666,11 @@ mod tests { // QUERY 2 - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(2, 4), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(2, 4), + BandSelection::first(), + ); let result_stream = query_processor.query(query_rect, &query_ctx).await.unwrap(); diff --git a/operators/src/optimization/mod.rs b/operators/src/optimization/mod.rs new file mode 100644 index 000000000..817689690 --- /dev/null +++ b/operators/src/optimization/mod.rs @@ -0,0 +1,1796 @@ +use geoengine_datatypes::{error::ErrorSource, primitives::SpatialResolution}; +use snafu::{Snafu, ensure}; + +use crate::engine::InitializedRasterOperator; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +#[snafu(context(suffix(false)))] // disables default `Snafu` suffix +pub enum OptimizationError { + TargetResolutionMustNotBeSmallerThanSourceResolution { + source_resolution: SpatialResolution, + target_resolution: SpatialResolution, + }, + TargetResolutionMustBeDivisibleBySourceResolution { + source_resolution: SpatialResolution, + target_resolution: SpatialResolution, + }, + SourcesMustNotUseOverviews { + data: String, + oveview_level: u32, + }, + OptimizationNotYetImplementedForOperator { + operator: String, + }, + ProjectionOptimizationFailed { + source: Box, + }, +} + +// TODO: move `optimize` method from `RasterOperator` to `OptimizableOperator` +pub trait OptimizableOperator: InitializedRasterOperator { + // TODO: automatically call before `optimize` or make it part of the `optimize` method + fn ensure_resolution_is_compatible_for_optimization( + &self, + target_resolution: SpatialResolution, + ) -> Result<(), OptimizationError> { + let original_resolution = self.result_descriptor().spatial_grid.spatial_resolution(); + ensure!( + target_resolution >= original_resolution, + TargetResolutionMustNotBeSmallerThanSourceResolution { + source_resolution: original_resolution, + target_resolution, + } + ); + + ensure!( + target_resolution.x % original_resolution.x == 0. + && target_resolution.y % original_resolution.y == 0., + TargetResolutionMustBeDivisibleBySourceResolution { + source_resolution: original_resolution, + target_resolution, + } + ); + + Ok(()) + } +} + +impl OptimizableOperator for T where T: InitializedRasterOperator {} + +#[cfg(test)] +mod tests { + use geoengine_datatypes::{ + collections::VectorDataType, + dataset::{DataId, DatasetId, NamedData}, + primitives::{BoundingBox2D, CacheTtlSeconds, VectorQueryRectangle}, + raster::{RasterDataType, RenameBands}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, + test_data, + util::{Identifier, test::TestDefault}, + }; + + use crate::{ + engine::{ + MockExecutionContext, MultipleRasterSources, PlotOperator, RasterOperator, + SingleRasterOrVectorSource, SingleRasterSource, SingleVectorMultipleRasterSources, + SingleVectorSource, StaticMetaData, VectorOperator, VectorResultDescriptor, + WorkflowOperatorPath, + }, + plot::{Histogram, HistogramBounds, HistogramBuckets, HistogramParams}, + processing::{ + ColumnNames, ColumnRangeFilter, ColumnRangeFilterParams, DeriveOutRasterSpecsSource, + Downsampling, DownsamplingMethod, DownsamplingParams, DownsamplingResolution, + Expression, ExpressionParams, FeatureAggregationMethod, Interpolation, + InterpolationMethod, InterpolationParams, InterpolationResolution, RasterStacker, + RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams, + RasterVectorJoin, RasterVectorJoinParams, Rasterization, RasterizationParams, + Reprojection, ReprojectionParams, TemporalAggregationMethod, + }, + source::{ + GdalSource, GdalSourceParameters, OgrSource, OgrSourceDataset, + OgrSourceDatasetTimeType, OgrSourceErrorSpec, OgrSourceParameters, + }, + util::{ + gdal::{add_ndvi_dataset, add_ndvi_downscaled_3x_dataset}, + input::RasterOrVectorOperator, + }, + }; + + use super::*; + + #[tokio::test] + async fn it_optimizes_gdal_source() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + + let gdal = GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let gdal_initialized = gdal + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.1, 0.1).unwrap() + ); + + let gdal_optimized = gdal_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&gdal_optimized).unwrap(); + + // TODO: add test where downsampling is injected because there are is no overview available with required level + + // assert_eq!( + // json, + // serde_json::json!({ + // "params": { + // "outputOriginReference": null, + // "outputResolution": { + // "type": "resolution", + // "x":0.8, + // "y": 0.8 + // }, + // "samplingMethod": "nearestNeighbor" + // }, + // "sources": { + // "raster": { + // "params": { + // "data": "ndvi", + // "overviewLevel": 3 + // }, + // "type": "GdalSource" + // } + // }, + // "type": "Downsampling" + // }) + // ); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "data": "ndvi", + "overviewLevel": 8 + }, + "type": "GdalSource" + } + ) + ); + + let gdal_optimized_initialized = gdal_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.8, 0.8).unwrap() + ); + } + + #[tokio::test] + async fn it_optimizes_expression() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + + let gdal = GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let expression = Expression { + params: ExpressionParams { + expression: "A + 1".to_string(), + output_type: RasterDataType::U8, + output_band: None, + map_no_data: false, + }, + sources: SingleRasterSource { raster: gdal }, + } + .boxed(); + + let expression_initialized = expression + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + expression_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.1, 0.1).unwrap() + ); + + let expression_optimized = expression_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&expression_optimized).unwrap(); + + // assert_eq!( + // json, + // serde_json::json!({ + // "params": { + // "expression": "A + 1", + // "mapNoData": false, + // "outputBand": { + // "measurement": { + // "type": "unitless" + // }, + // "name": "expression" + // }, + // "outputType": "U8" + // }, + // "sources": { + // "raster": { + // "params": { + // "outputOriginReference": null, + // "outputResolution": { + // "type": "resolution", + // "x": 0.8, + // "y": 0.8 + // }, + // "samplingMethod": "nearestNeighbor" + // }, + // "sources": { + // "raster": { + // "params": { + // "data": "ndvi", + // "overviewLevel": 3 + // }, + // "type": "GdalSource" + // } + // }, + // "type": "Downsampling" + // } + // }, + // "type": "Expression" + // }) + // ); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "expression": "A + 1", + "mapNoData": false, + "outputBand": { + "measurement": { + "type": "unitless" + }, + "name": "expression" + }, + "outputType": "U8" + }, + "sources": { + "raster": { + "params": { + "data": "ndvi", + "overviewLevel": 8 + }, + "type": "GdalSource" + } + }, + "type": "Expression" + }) + ); + + let expression_optimized_initialized = expression_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + expression_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.8, 0.8).unwrap() + ); + } + + #[tokio::test] + async fn it_removes_upsampling() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let workflow = Interpolation { + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::new_unchecked(0.15, 0.15), + ), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed(); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.15, 0.15).unwrap() + ); + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(0.3, 0.3).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 0 + }, + "type": "GdalSource" + } + ) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.3, 0.3).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_reduces_interpolation_resolution() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let workflow = RasterStacker { + params: RasterStackerParams { + rename_bands: RenameBands::Default, + }, + sources: MultipleRasterSources { + rasters: vec![ + GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(), + Interpolation { + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::new_unchecked(0.1, 0.1), + ), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed(), + ], + }, + } + .boxed(); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.1, 0.1).unwrap() + ); + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(0.2, 0.2).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "renameBands": { + "type": "default" + } + }, + "sources": { + "rasters": [ + { + "params": { + "data": "ndvi", + "overviewLevel": 2 + }, + "type": "GdalSource" + }, + { + "params": { + "interpolation": "nearestNeighbor", + "outputOriginReference": null, + "outputResolution": { + "type": "resolution", + "x": 0.2, + "y": 0.2 + } + }, + "sources": { + "raster": { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 0 + }, + "type": "GdalSource" + } + }, + "type": "Interpolation" + } + ] + }, + "type": "RasterStacker" + }) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.2, 0.2).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_replaces_upsampling_with_downsampling() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let workflow = RasterStacker { + params: RasterStackerParams { + rename_bands: RenameBands::Default, + }, + sources: MultipleRasterSources { + rasters: vec![ + GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(), + Interpolation { + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::new_unchecked(0.1, 0.1), + ), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed(), + ], + }, + } + .boxed(); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.1, 0.1).unwrap() + ); + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "renameBands": { + "type": "default" + } + }, + "sources": { + "rasters": [ + { + "params": { + "data": "ndvi", + "overviewLevel": 8 + }, + "type": "GdalSource" + }, + { + "params": { + "outputOriginReference": null, + "outputResolution": { + "type": "resolution", + "x": 0.8, + "y": 0.8 + }, + "samplingMethod": "nearestNeighbor" + }, + "sources": { + "raster": { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 2 + }, + "type": "GdalSource" + } + }, + "type": "Downsampling" + } + ] + }, + "type": "RasterStacker" + }) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.8, 0.8).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_downsampling() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let workflow = RasterStacker { + params: RasterStackerParams { + rename_bands: RenameBands::Default, + }, + sources: MultipleRasterSources { + rasters: vec![ + GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(), + Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution( + SpatialResolution::new_unchecked(0.3, 0.3), + ), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed(), + ], + }, + } + .boxed(); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.3, 0.3).unwrap() + ); + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(0.6, 0.6).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "renameBands": { + "type": "default" + } + }, + "sources": { + "rasters": [ + { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 2 + }, + "type": "GdalSource" + }, + { + "params": { + "outputOriginReference": null, + "outputResolution": { + "type": "resolution", + "x": 0.6, + "y": 0.6 + }, + "samplingMethod": "nearestNeighbor" + }, + "sources": { + "raster": { + "params": { + "data": "ndvi", + "overviewLevel": 4 + }, + "type": "GdalSource" + } + }, + "type": "Downsampling" + } + ] + }, + "type": "RasterStacker" + }) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.6, 0.6).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_reprojection() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + + let workflow: Box = Box::new(Reprojection { + params: ReprojectionParams { + target_spatial_reference: SpatialReference::new( + SpatialReferenceAuthority::Epsg, + 3857, + ), + derive_out_spec: DeriveOutRasterSpecsSource::default(), + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster( + GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(), + ), + }, + }); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let expected_resolution = 14_255.015_508_816_849; + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(expected_resolution, expected_resolution).unwrap() + ); + + let optimize_resolution = expected_resolution * 2.0; + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + // require an interpolation in addition to the reprojection + // TODO: should we instead load the source in higher resolution and downsample? + // or: we could make an exception that the reprojection may not have to produce the exact resolution, but how would we choose the correct resolution for the top level operator then? + assert_eq!( + json, + serde_json::json!({ + "params": { + "interpolation": "nearestNeighbor", + "outputOriginReference": null, + "outputResolution": { + "type": "resolution", + "x": 28_510.031_017_633_697, + "y": 28_510.031_017_633_697 + } + }, + "sources": { + "raster": { + "params": { + "deriveOutSpec": "projectionBounds", + "targetSpatialReference": "EPSG:3857" + }, + "sources": { + "source": { + "params": { + "data": "ndvi", + "overviewLevel": 2 + }, + "type": "GdalSource" + } + }, + "type": "Reprojection" + } + }, + "type": "Interpolation" + }) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap() + ); + + // check that the reprojection was necessary + let workflow: Box = Box::new(Reprojection { + params: ReprojectionParams { + target_spatial_reference: SpatialReference::new( + SpatialReferenceAuthority::Epsg, + 3857, + ), + derive_out_spec: DeriveOutRasterSpecsSource::default(), + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster( + GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: Some(2), + }, + } + .boxed(), + ), + }, + }); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution() + > SpatialResolution::new(optimize_resolution, optimize_resolution).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_vector_reprojection() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ne_10m_ports"); + + exe_ctx.add_meta_data::( + id.clone(), + name.clone(), + Box::new(StaticMetaData { + loading_info: OgrSourceDataset { + file_name: test_data!( + "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg" + ) + .into(), + layer_name: "ne_10m_ports".to_string(), + data_type: Some(VectorDataType::MultiPoint), + time: OgrSourceDatasetTimeType::None, + default_geometry: None, + columns: None, + force_ogr_time_filter: false, + force_ogr_spatial_filter: false, + on_error: OgrSourceErrorSpec::Ignore, + sql_query: None, + attribute_query: None, + cache_ttl: CacheTtlSeconds::default(), + }, + result_descriptor: VectorResultDescriptor { + data_type: VectorDataType::MultiPoint, + spatial_reference: SpatialReference::epsg_4326().into(), + columns: Default::default(), + time: None, + bbox: Some(BoundingBox2D::new_unchecked( + [-171.757_95, -54.809_444].into(), + [179.309_364, 78.226_111].into(), + )), + }, + phantom: Default::default(), + }), + ); + + let ogr_source = OgrSource { + params: OgrSourceParameters { + data: name, + attribute_projection: None, + attribute_filters: None, + }, + } + .boxed(); + + let gdal_source = GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let gdal_source2 = GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let raster_vector_join = RasterVectorJoin { + params: RasterVectorJoinParams { + names: ColumnNames::Default, + feature_aggregation: FeatureAggregationMethod::First, + feature_aggregation_ignore_no_data: false, + temporal_aggregation: TemporalAggregationMethod::None, + temporal_aggregation_ignore_no_data: false, + }, + sources: SingleVectorMultipleRasterSources { + vector: ogr_source, + rasters: vec![gdal_source, gdal_source2], + }, + } + .boxed(); + + let vector_reprojection = VectorOperator::boxed(Reprojection { + params: ReprojectionParams { + target_spatial_reference: SpatialReference::new( + SpatialReferenceAuthority::Epsg, + 3857, + ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, + }, + sources: SingleRasterOrVectorSource { + source: raster_vector_join.into(), + }, + }); + + let vector_reprojection_initialized = vector_reprojection + .clone() + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let vector_reprojection_optimized = vector_reprojection_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&vector_reprojection_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "deriveOutSpec": "projectionBounds", + "targetSpatialReference": "EPSG:3857" + }, + "sources": { + "source": { + "params": { + "featureAggregation": "first", + "featureAggregationIgnoreNoData": false, + "names": { "type": "default" }, + "temporalAggregation": "none", + "temporalAggregationIgnoreNoData": false + }, + "sources": { + "rasters": [ + { + "params": { "data": "ndvi", "overviewLevel": 8 }, + "type": "GdalSource" + }, + { + "params": { "data": "ndvi_downscaled_3x", "overviewLevel": 2 }, + "type": "GdalSource" + } + ], + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "RasterVectorJoin" + } + }, + "type": "Reprojection" + }) + ); + + assert!( + vector_reprojection_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .is_ok() + ); + + // check case where output is finer than input + let vector_reprojection_optimized = vector_reprojection_initialized + .optimize(SpatialResolution::new(0.05, 0.05).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&vector_reprojection_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "deriveOutSpec": "projectionBounds", + "targetSpatialReference": "EPSG:3857" + }, + "sources": { + "source": { + "params": { + "featureAggregation": "first", + "featureAggregationIgnoreNoData": false, + "names": { "type": "default" }, + "temporalAggregation": "none", + "temporalAggregationIgnoreNoData": false + }, + "sources": { + "rasters": [ + { + "params": { "data": "ndvi", "overviewLevel": 0 }, + "type": "GdalSource" + }, + { + "params": { "data": "ndvi_downscaled_3x", "overviewLevel": 0 }, + "type": "GdalSource" + } + ], + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "RasterVectorJoin" + } + }, + "type": "Reprojection" + }) + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_rasterization() { + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ne_10m_ports"); + let mut exe_ctx = MockExecutionContext::test_default(); + exe_ctx.add_meta_data::( + id.clone(), + name.clone(), + Box::new(StaticMetaData { + loading_info: OgrSourceDataset { + file_name: test_data!( + "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg" + ) + .into(), + layer_name: "ne_10m_ports".to_string(), + data_type: Some(VectorDataType::MultiPoint), + time: OgrSourceDatasetTimeType::None, + default_geometry: None, + columns: None, + force_ogr_time_filter: false, + force_ogr_spatial_filter: false, + on_error: OgrSourceErrorSpec::Ignore, + sql_query: None, + attribute_query: None, + cache_ttl: CacheTtlSeconds::default(), + }, + result_descriptor: VectorResultDescriptor { + data_type: VectorDataType::MultiPoint, + spatial_reference: SpatialReference::epsg_4326().into(), + columns: Default::default(), + time: None, + bbox: Some(BoundingBox2D::new_unchecked( + [-171.757_95, -54.809_444].into(), + [179.309_364, 78.226_111].into(), + )), + }, + phantom: Default::default(), + }), + ); + + let source = OgrSource { + params: OgrSourceParameters { + data: name, + attribute_projection: None, + attribute_filters: None, + }, + } + .boxed(); + + let rasterization = Rasterization { + params: RasterizationParams { + spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1), + origin_coordinate: (0., 0.).into(), + density_params: None, + }, + sources: SingleVectorSource { vector: source }, + } + .boxed(); + + let rasterization_initialized = rasterization + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + rasterization_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.1, 0.1).unwrap() + ); + + let rasterization_optimized = rasterization_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&rasterization_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "densityParams": null, + "originCoordinate": { + "x": 0.0, + "y": 0.0 + }, + "spatialResolution": { + "x": 0.8, + "y": 0.8 + } + }, + "sources": { + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "Rasterization" + }) + ); + + let expression_optimized_initialized = rasterization_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + expression_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.8, 0.8).unwrap() + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_raster_vector_join() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ne_10m_ports"); + + exe_ctx.add_meta_data::( + id.clone(), + name.clone(), + Box::new(StaticMetaData { + loading_info: OgrSourceDataset { + file_name: test_data!( + "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg" + ) + .into(), + layer_name: "ne_10m_ports".to_string(), + data_type: Some(VectorDataType::MultiPoint), + time: OgrSourceDatasetTimeType::None, + default_geometry: None, + columns: None, + force_ogr_time_filter: false, + force_ogr_spatial_filter: false, + on_error: OgrSourceErrorSpec::Ignore, + sql_query: None, + attribute_query: None, + cache_ttl: CacheTtlSeconds::default(), + }, + result_descriptor: VectorResultDescriptor { + data_type: VectorDataType::MultiPoint, + spatial_reference: SpatialReference::epsg_4326().into(), + columns: Default::default(), + time: None, + bbox: Some(BoundingBox2D::new_unchecked( + [-171.757_95, -54.809_444].into(), + [179.309_364, 78.226_111].into(), + )), + }, + phantom: Default::default(), + }), + ); + + let ogr_source = OgrSource { + params: OgrSourceParameters { + data: name, + attribute_projection: None, + attribute_filters: None, + }, + } + .boxed(); + + let gdal_source = GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let gdal_source2 = GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let raster_vector_join = RasterVectorJoin { + params: RasterVectorJoinParams { + names: ColumnNames::Default, + feature_aggregation: FeatureAggregationMethod::First, + feature_aggregation_ignore_no_data: false, + temporal_aggregation: TemporalAggregationMethod::None, + temporal_aggregation_ignore_no_data: false, + }, + sources: SingleVectorMultipleRasterSources { + vector: ogr_source, + rasters: vec![gdal_source, gdal_source2], + }, + } + .boxed(); + + let raster_vector_join_initialized = raster_vector_join + .clone() + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let raster_vector_join_optimized = raster_vector_join_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&raster_vector_join_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "featureAggregation": "first", + "featureAggregationIgnoreNoData": false, + "names": { + "type": "default" + }, + "temporalAggregation": "none", + "temporalAggregationIgnoreNoData": false + }, + "sources": { + "rasters": [ + { + "params": { + "data": "ndvi", + "overviewLevel": 8 + }, + "type": "GdalSource" + }, + { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 2 + }, + "type": "GdalSource" + } + ], + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "RasterVectorJoin" + }) + ); + + assert!( + raster_vector_join_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .is_ok() + ); + + // check case where output is finer than input + let raster_vector_join_optimized = raster_vector_join_initialized + .optimize(SpatialResolution::new(0.05, 0.05).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&raster_vector_join_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "featureAggregation": "first", + "featureAggregationIgnoreNoData": false, + "names": { + "type": "default" + }, + "temporalAggregation": "none", + "temporalAggregationIgnoreNoData": false + }, + "sources": { + "rasters": [ + { + "params": { + "data": "ndvi", + "overviewLevel": 0 + }, + "type": "GdalSource" + }, + { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 0 + }, + "type": "GdalSource" + } + ], + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "RasterVectorJoin" + }) + ); + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_optimizes_complex_workflow() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + let ndvi_downscaled_3x_id = add_ndvi_downscaled_3x_dataset(&mut exe_ctx); + + let id: DataId = DatasetId::new().into(); + let ports_name = NamedData::with_system_name("ne_10m_ports"); + + exe_ctx.add_meta_data::( + id.clone(), + ports_name.clone(), + Box::new(StaticMetaData { + loading_info: OgrSourceDataset { + file_name: test_data!( + "vector/data/ne_10m_ports/with_spatial_index/ne_10m_ports.gpkg" + ) + .into(), + layer_name: "ne_10m_ports".to_string(), + data_type: Some(VectorDataType::MultiPoint), + time: OgrSourceDatasetTimeType::None, + default_geometry: None, + columns: None, + force_ogr_time_filter: false, + force_ogr_spatial_filter: false, + on_error: OgrSourceErrorSpec::Ignore, + sql_query: None, + attribute_query: None, + cache_ttl: CacheTtlSeconds::default(), + }, + result_descriptor: VectorResultDescriptor { + data_type: VectorDataType::MultiPoint, + spatial_reference: SpatialReference::epsg_4326().into(), + columns: Default::default(), + time: None, + bbox: Some(BoundingBox2D::new_unchecked( + [-171.757_95, -54.809_444].into(), + [179.309_364, 78.226_111].into(), + )), + }, + phantom: Default::default(), + }), + ); + + let workflow = Expression { + params: ExpressionParams { + expression: "A + B".to_string(), + output_type: RasterDataType::F64, + output_band: None, + map_no_data: false, + }, + sources: SingleRasterSource { + raster: RasterStacker { + params: RasterStackerParams { + rename_bands: RenameBands::Default, + }, + sources: MultipleRasterSources { + rasters: vec![ + RasterTypeConversion { + params: RasterTypeConversionParams { + output_data_type: RasterDataType::F64, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_downscaled_3x_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed(), + Rasterization { + params: RasterizationParams { + spatial_resolution: SpatialResolution::new_unchecked(0.3, 0.3), + origin_coordinate: (0., 0.).into(), + density_params: None, + }, + sources: SingleVectorSource { + vector: ColumnRangeFilter { + params: ColumnRangeFilterParams { + column: "natlscale".to_string(), + ranges: vec![(1..=2).into()], + keep_nulls: false, + }, + sources: SingleVectorSource { + vector: RasterVectorJoin { + params: RasterVectorJoinParams { + names: ColumnNames::Default, + feature_aggregation: + FeatureAggregationMethod::First, + feature_aggregation_ignore_no_data: false, + temporal_aggregation: + TemporalAggregationMethod::None, + temporal_aggregation_ignore_no_data: false, + }, + sources: SingleVectorMultipleRasterSources { + rasters: vec![Downsampling { + params: DownsamplingParams { + sampling_method: + DownsamplingMethod::NearestNeighbor, + output_resolution: + DownsamplingResolution::Resolution( + SpatialResolution::new_unchecked( + 0.3, 0.3, + ), + ), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(), + }, + } + .boxed()], + vector: OgrSource { + params: OgrSourceParameters { + data: ports_name, + attribute_projection: None, + attribute_filters: None, + }, + } + .boxed(), + }, + } + .boxed(), + }, + } + .boxed(), + }, + } + .boxed(), + ], + }, + } + .boxed(), + }, + } + .boxed(); + + let workflow_initialized = workflow + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + workflow_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.3, 0.3).unwrap() + ); + + let workflow_optimized = workflow_initialized + .optimize(SpatialResolution::new(0.6, 0.6).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&workflow_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "expression": "A + B", + "mapNoData": false, + "outputBand": { + "measurement": { + "type": "unitless" + }, + "name": "expression" + }, + "outputType": "F64" + }, + "sources": { + "raster": { + "params": { + "renameBands": { + "type": "default" + } + }, + "sources": { + "rasters": [ + { + "params": { + "outputDataType": "F64" + }, + "sources": { + "raster": { + "params": { + "data": "ndvi_downscaled_3x", + "overviewLevel": 2 + }, + "type": "GdalSource" + } + }, + "type": "RasterTypeConversion" + }, + { + "params": { + "densityParams": null, + "originCoordinate": { + "x": 0.0, + "y": 0.0 + }, + "spatialResolution": { + "x": 0.6, + "y": 0.6 + } + }, + "sources": { + "vector": { + "params": { + "column": "natlscale", + "keepNulls": false, + "ranges": [ + [ + 1, + 2 + ] + ] + }, + "sources": { + "vector": { + "params": { + "featureAggregation": "first", + "featureAggregationIgnoreNoData": false, + "names": { + "type": "default" + }, + "temporalAggregation": "none", + "temporalAggregationIgnoreNoData": false + }, + "sources": { + "rasters": [ + { + "params": { + "outputOriginReference": null, + "outputResolution": { + "type": "resolution", + "x": 0.6, + "y": 0.6 + }, + "samplingMethod": "nearestNeighbor" + }, + "sources": { + "raster": { + "params": { + "data": "ndvi", + "overviewLevel": 4 + }, + "type": "GdalSource" + } + }, + "type": "Downsampling" + } + ], + "vector": { + "params": { + "attributeFilters": null, + "attributeProjection": null, + "data": "ne_10m_ports" + }, + "type": "OgrSource" + } + }, + "type": "RasterVectorJoin" + } + }, + "type": "ColumnRangeFilter" + } + }, + "type": "Rasterization" + } + ] + }, + "type": "RasterStacker" + } + }, + "type": "Expression" + }) + ); + + let gdal_optimized_initialized = workflow_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + assert_eq!( + gdal_optimized_initialized + .result_descriptor() + .spatial_grid + .spatial_resolution(), + SpatialResolution::new(0.6, 0.6).unwrap() + ); + } + + #[tokio::test] + async fn it_optimizes_plot() { + let mut exe_ctx = MockExecutionContext::test_default(); + + let ndvi_id = add_ndvi_dataset(&mut exe_ctx); + + let gdal = GdalSource { + params: GdalSourceParameters { + data: ndvi_id.clone(), + overview_level: None, + }, + } + .boxed(); + + let expression = Histogram { + params: HistogramParams { + attribute_name: "ndvi".to_string(), + bounds: HistogramBounds::Values { + min: 0.0, + max: 255.0, + }, + buckets: HistogramBuckets::Number { value: 8 }, + interactive: false, + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(gdal), + }, + } + .boxed(); + + let expression_initialized = expression + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let expression_optimized = expression_initialized + .optimize(SpatialResolution::new(0.8, 0.8).unwrap()) + .unwrap(); + + let json = serde_json::to_value(&expression_optimized).unwrap(); + + assert_eq!( + json, + serde_json::json!({ + "params": { + "attributeName": "ndvi", + "bounds": { + "max": 255.0, + "min": 0.0 + }, + "buckets": { + "type": "number", + "value": 8 + }, + "interactive": false + }, + "sources": { + "source": { + "params": { + "data": "ndvi", + "overviewLevel": 8 + }, + "type": "GdalSource" + } + }, + "type": "Histogram" + }) + ); + + expression_optimized + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + } +} diff --git a/operators/src/plot/box_plot.rs b/operators/src/plot/box_plot.rs index 1dc4f264f..8ea6cbfe3 100644 --- a/operators/src/plot/box_plot.rs +++ b/operators/src/plot/box_plot.rs @@ -1,16 +1,3 @@ -use async_trait::async_trait; -use futures::StreamExt; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, PlotQueryRectangle, RasterQueryRectangle, - partitions_extent, time_interval_extent, -}; -use num_traits::AsPrimitive; -use serde::{Deserialize, Serialize}; - -use geoengine_datatypes::collections::FeatureCollectionInfos; -use geoengine_datatypes::plots::{BoxPlotAttribute, Plot, PlotData}; -use geoengine_datatypes::raster::GridOrEmpty; - use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, MultipleRasterOrSingleVectorSource, Operator, OperatorName, @@ -19,9 +6,21 @@ use crate::engine::{ WorkflowOperatorPath, }; use crate::error::{self, Error}; +use crate::optimization::OptimizationError; use crate::util::Result; use crate::util::input::MultiRasterOrVectorOperator; use crate::util::statistics::PSquareQuantileEstimator; +use async_trait::async_trait; +use futures::StreamExt; +use geoengine_datatypes::collections::FeatureCollectionInfos; +use geoengine_datatypes::plots::{BoxPlotAttribute, Plot, PlotData}; +use geoengine_datatypes::primitives::{ + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, + RasterQueryRectangle, SpatialResolution, partitions_extent, time_interval_extent, +}; +use geoengine_datatypes::raster::GridOrEmpty; +use num_traits::AsPrimitive; +use serde::{Deserialize, Serialize}; use snafu::ensure; pub const BOXPLOT_OPERATOR_NAME: &str = "BoxPlot"; @@ -112,8 +111,8 @@ impl PlotOperator for BoxPlot { .map(InitializedRasterOperator::result_descriptor) .collect::>(); - let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); + let time = time_interval_extent(in_descriptors.iter().map(|d| d.time.bounds)); + let bbox = partitions_extent(in_descriptors.iter().map(|d| d.spatial_bounds())); Ok(InitializedBoxPlot::new( name, @@ -221,6 +220,23 @@ impl InitializedPlotOperator for InitializedBoxPlot CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(BoxPlot { + params: BoxPlotParams { + column_names: self.names.clone(), + }, + sources: MultipleRasterOrSingleVectorSource { + source: MultiRasterOrVectorOperator::Vector( + self.source.optimize(target_resolution)?, + ), + }, + } + .boxed()) + } } impl InitializedPlotOperator for InitializedBoxPlot>> { @@ -245,6 +261,26 @@ impl InitializedPlotOperator for InitializedBoxPlot CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(BoxPlot { + params: BoxPlotParams { + column_names: self.names.clone(), + }, + sources: MultipleRasterOrSingleVectorSource { + source: MultiRasterOrVectorOperator::Raster( + self.source + .iter() + .map(|s| s.optimize(target_resolution)) + .collect::, OptimizationError>>()?, + ), + }, + } + .boxed()) + } } /// A query processor that calculates the boxplots about its vector input. @@ -272,8 +308,12 @@ impl PlotQueryProcessor for BoxPlotVectorQueryProcessor { .map(|name| BoxPlotAccum::new(name.clone())) .collect(); + let query = query.select_attributes( + ColumnSelection::all(), // TODO: use columns names? + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -311,10 +351,19 @@ impl BoxPlotRasterQueryProcessor { query: PlotQueryRectangle, ctx: &dyn QueryContext, ) -> Result> { - call_on_generic_raster_processor!(input, processor => { + let result_descrpitor = input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::first(), + result_descrpitor + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); + call_on_generic_raster_processor!(input, processor => { - let mut stream = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), ctx).await?; + let mut stream = processor.query(raster_query_rect, ctx).await?; let mut accum = BoxPlotAccum::new(name); while let Some(tile) = stream.next().await { @@ -487,23 +536,24 @@ impl BoxPlotAccum { #[cfg(test)] mod tests { - use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; + + use geoengine_datatypes::primitives::{CacheHint, Coordinate2D, PlotSeriesSelection}; use serde_json::json; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, SpatialResolution, TimeInterval, + BoundingBox2D, DateTime, FeatureData, NoGeometry, TimeInterval, }; use geoengine_datatypes::raster::{ - EmptyGrid2D, Grid2D, MaskedGrid2D, RasterDataType, RasterTile2D, TileInformation, - TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, Grid2D, GridShape2D, MaskedGrid2D, RasterDataType, + RasterTile2D, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{collections::DataCollection, primitives::MultiPoint}; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, VectorOperator, + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, VectorOperator, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; @@ -621,14 +671,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -688,14 +736,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -799,14 +845,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -852,14 +896,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -907,14 +949,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -938,11 +978,21 @@ mod tests { #[tokio::test] async fn no_data_raster_exclude_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = + geoengine_datatypes::raster::TilingSpecification::new(tile_size_in_pixels); + let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -960,14 +1010,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -988,14 +1031,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1007,11 +1048,19 @@ mod tests { #[tokio::test] async fn no_data_raster_include_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -1031,14 +1080,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1059,13 +1101,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1080,11 +1121,19 @@ mod tests { #[tokio::test] async fn empty_tile_raster_exclude_no_data() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let box_plot = BoxPlot { params: BoxPlotParams { column_names: vec![], @@ -1102,14 +1151,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1130,14 +1172,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1149,11 +1189,19 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { params: BoxPlotParams { @@ -1172,14 +1220,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1198,17 +1239,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::test_default(), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context_test_default(), ) .await .unwrap(); @@ -1223,11 +1259,19 @@ mod tests { #[tokio::test] async fn raster_with_no_data_exclude_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { @@ -1256,14 +1300,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1282,16 +1319,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::test_default(), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context_test_default(), ) .await .unwrap(); @@ -1306,11 +1339,19 @@ mod tests { #[tokio::test] async fn raster_with_no_data_include_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = BoxPlot { @@ -1332,14 +1373,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1358,16 +1392,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::test_default(), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -4.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context_test_default(), ) .await .unwrap(); @@ -1382,11 +1412,19 @@ mod tests { #[tokio::test] async fn multiple_rasters_with_no_data_exclude_no_data() { - let tile_size_in_pixels = [4, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(4, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let src = MockRasterSource { @@ -1411,14 +1449,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, }; @@ -1446,17 +1477,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::test_default(), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context_test_default(), ) .await .unwrap(); diff --git a/operators/src/plot/class_histogram.rs b/operators/src/plot/class_histogram.rs index 5446de4e0..af1eff7c9 100644 --- a/operators/src/plot/class_histogram.rs +++ b/operators/src/plot/class_histogram.rs @@ -7,6 +7,7 @@ use crate::engine::{ use crate::engine::{QueryProcessor, WorkflowOperatorPath}; use crate::error; use crate::error::Error; +use crate::optimization::OptimizationError; use crate::util::Result; use crate::util::input::RasterOrVectorOperator; use async_trait::async_trait; @@ -14,8 +15,8 @@ use futures::StreamExt; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::plots::{BarChart, Plot, PlotData}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, ClassificationMeasurement, FeatureDataType, - Measurement, PlotQueryRectangle, RasterQueryRectangle, + AxisAlignedRectangle, BandSelection, ClassificationMeasurement, ColumnSelection, + FeatureDataType, Measurement, PlotQueryRectangle, RasterQueryRectangle, SpatialResolution, }; use num_traits::AsPrimitive; use serde::{Deserialize, Serialize}; @@ -88,11 +89,9 @@ impl PlotOperator for ClassHistogram { name, PlotResultDescriptor { spatial_reference: in_desc.spatial_reference, - time: in_desc.time, + time: in_desc.time.bounds, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: in_desc - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(in_desc.spatial_bounds().as_bbox()), }, self.params.column_name, source_measurement, @@ -208,6 +207,21 @@ impl InitializedPlotOperator for InitializedClassHistogram CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(ClassHistogram { + params: ClassHistogramParams { + column_name: self.column_name.clone(), + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(self.source.optimize(target_resolution)?), + }, + } + .boxed()) + } } impl InitializedPlotOperator for InitializedClassHistogram> { @@ -228,6 +242,21 @@ impl InitializedPlotOperator for InitializedClassHistogram CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(ClassHistogram { + params: ClassHistogramParams { + column_name: self.column_name.clone(), + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Vector(self.source.optimize(target_resolution)?), + }, + } + .boxed()) + } } /// A query processor that calculates the Histogram about its raster inputs. @@ -290,8 +319,17 @@ impl ClassHistogramRasterQueryProcessor { .map(|key| (*key, 0)) .collect(); + let rd = self.input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::first(), + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); + call_on_generic_raster_processor!(&self.input, processor => { - let mut query = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), ctx).await?; + let mut query = processor.query(raster_query_rect, ctx).await?; while let Some(tile) = query.next().await { match tile?.grid_array { @@ -346,8 +384,12 @@ impl ClassHistogramVectorQueryProcessor { .map(|key| (*key, 0)) .collect(); + let query = query.select_attributes( + ColumnSelection::all(), // TODO: figure out why this is a vector query? + ); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -398,9 +440,9 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptor, - RasterBandDescriptors, RasterOperator, RasterResultDescriptor, StaticMetaData, - VectorColumnInfo, VectorOperator, VectorResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, StaticMetaData, + TimeDescriptor, VectorColumnInfo, VectorOperator, VectorResultDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{ @@ -409,12 +451,13 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, Coordinate2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridShape2D, RasterDataType, RasterTile2D, + TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; @@ -498,9 +541,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "bands".into(), Measurement::classification( @@ -528,7 +573,6 @@ mod tests { async fn simple_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -550,13 +594,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -635,14 +678,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -722,14 +763,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -871,17 +910,30 @@ mod tests { #[tokio::test] async fn no_data_raster() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + + let bands = RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + Measurement::Classification(ClassificationMeasurement { + measurement: "foo".to_string(), + classes: [(1, "A".to_string())].into_iter().collect(), + }), + )]) + .unwrap(); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands, }; - let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); - let measurement = Measurement::classification( - "foo".to_string(), - [(1, "A".to_string())].into_iter().collect(), - ); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = ClassHistogram { params: ClassHistogramParams { column_name: None }, @@ -900,18 +952,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - measurement, - )]) - .unwrap(), - }, + result_descriptor, }, } .boxed() @@ -930,13 +971,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -994,14 +1034,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1059,14 +1097,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1085,18 +1121,30 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + + let bands = RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + Measurement::Classification(ClassificationMeasurement { + measurement: "foo".to_string(), + classes: [(4, "D".to_string())].into_iter().collect(), + }), + )]) + .unwrap(); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands, }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); - let measurement = Measurement::classification( - "foo".to_string(), - [(4, "D".to_string())].into_iter().collect(), - ); - let histogram = ClassHistogram { params: ClassHistogramParams { column_name: None }, sources: MockRasterSource { @@ -1112,18 +1160,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - measurement, - )]) - .unwrap(), - }, + result_descriptor, }, } .boxed() @@ -1142,16 +1179,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/histogram.rs b/operators/src/plot/histogram.rs index c72ed319c..4db88cd48 100644 --- a/operators/src/plot/histogram.rs +++ b/operators/src/plot/histogram.rs @@ -1,12 +1,13 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, - PlotResultDescriptor, QueryContext, SingleRasterOrVectorSource, TypedPlotQueryProcessor, - TypedRasterQueryProcessor, TypedVectorQueryProcessor, + PlotResultDescriptor, QueryContext, QueryProcessor, SingleRasterOrVectorSource, + TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, + WorkflowOperatorPath, }; -use crate::engine::{QueryProcessor, WorkflowOperatorPath}; use crate::error; use crate::error::Error; +use crate::optimization::OptimizationError; use crate::string_token; use crate::util::Result; use crate::util::input::RasterOrVectorOperator; @@ -16,8 +17,8 @@ use futures::stream::BoxStream; use futures::{StreamExt, TryFutureExt}; use geoengine_datatypes::plots::{Plot, PlotData}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, DataRef, FeatureDataRef, FeatureDataType, - Geometry, Measurement, PlotQueryRectangle, RasterQueryRectangle, + AxisAlignedRectangle, BandSelection, ColumnSelection, DataRef, FeatureDataRef, FeatureDataType, + Geometry, Measurement, PlotQueryRectangle, RasterQueryRectangle, SpatialResolution, }; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use geoengine_datatypes::{ @@ -115,11 +116,9 @@ impl PlotOperator for Histogram { name, PlotResultDescriptor { spatial_reference: in_desc.spatial_reference, - time: in_desc.time, + time: in_desc.time.bounds, // converting `SpatialPartition2D` to `BoundingBox2D` is ok here, because is makes the covered area only larger - bbox: in_desc - .bbox - .and_then(|p| BoundingBox2D::new(p.lower_left(), p.upper_right()).ok()), + bbox: Some(in_desc.spatial_bounds().as_bbox()), }, self.params, raster_source, @@ -170,6 +169,7 @@ impl PlotOperator for Histogram { /// The initialization of `Histogram` pub struct InitializedHistogram { name: CanonicOperatorName, + params: HistogramParams, result_descriptor: PlotResultDescriptor, metadata: HistogramMetadataOptions, source: Op, @@ -201,6 +201,7 @@ impl InitializedHistogram { Self { name, + params: params.clone(), result_descriptor, metadata: HistogramMetadataOptions { number_of_buckets, @@ -242,6 +243,19 @@ impl InitializedPlotOperator for InitializedHistogram CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Histogram { + params: self.params.clone(), + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(self.source.optimize(target_resolution)?), + }, + } + .boxed()) + } } impl InitializedPlotOperator for InitializedHistogram> { @@ -269,6 +283,19 @@ impl InitializedPlotOperator for InitializedHistogram CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Histogram { + params: self.params.clone(), + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Vector(self.source.optimize(target_resolution)?), + }, + } + .boxed()) + } } /// A query processor that calculates the Histogram about its raster inputs. @@ -371,10 +398,18 @@ impl HistogramRasterQueryProcessor { return Ok(metadata); } + let rd = self.input.result_descriptor(); + // TODO: compute only number of buckets if possible + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::new_single(self.band_idx), + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); call_on_generic_raster_processor!(&self.input, processor => { - process_metadata(processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::new_single(self.band_idx)), ctx).await?, self.metadata).await + process_metadata(processor.query(raster_query_rect, ctx).await?, self.metadata).await }) } @@ -393,8 +428,17 @@ impl HistogramRasterQueryProcessor { .build() .map_err(Error::from)?; + let rd = self.input.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::new_single(self.band_idx), + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); + call_on_generic_raster_processor!(&self.input, processor => { - let mut query = processor.query(RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::new_single(self.band_idx)), ctx).await?; + let mut query = processor.query(raster_query_rect, ctx).await?; while let Some(tile) = query.next().await { @@ -458,8 +502,10 @@ impl HistogramVectorQueryProcessor { // TODO: compute only number of buckets if possible + let query = query.select_attributes(ColumnSelection::all()); + call_on_generic_vector_processor!(&self.input, processor => { - process_metadata(processor.query(query.into(), ctx).await?, &self.column_name, self.metadata).await + process_metadata(processor.query(query, ctx).await?, &self.column_name, self.metadata).await }) } @@ -478,8 +524,10 @@ impl HistogramVectorQueryProcessor { .build() .map_err(Error::from)?; + let query = query.select_attributes(ColumnSelection::all()); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -654,9 +702,9 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, StaticMetaData, VectorColumnInfo, VectorOperator, - VectorResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, StaticMetaData, TimeDescriptor, + VectorColumnInfo, VectorOperator, VectorResultDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{ @@ -665,12 +713,13 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, Coordinate2D, DateTime, FeatureData, NoGeometry, PlotSeriesSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; use geoengine_datatypes::raster::{ - EmptyGrid2D, Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, Grid2D, GridShape2D, RasterDataType, RasterTile2D, + TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; @@ -816,9 +865,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -830,7 +881,6 @@ mod tests { async fn simple_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -857,13 +907,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -883,7 +932,6 @@ mod tests { async fn simple_raster_without_spec() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -912,13 +960,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -976,14 +1023,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1047,14 +1092,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1194,11 +1237,18 @@ mod tests { #[tokio::test] async fn no_data_raster() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = Histogram { params: HistogramParams { @@ -1222,14 +1272,7 @@ mod tests { EmptyGrid2D::::new(tile_size_in_pixels).into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1248,13 +1291,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1307,14 +1349,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1375,14 +1415,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1405,11 +1443,19 @@ mod tests { #[tokio::test] async fn single_value_raster_stream() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); let histogram = Histogram { params: HistogramParams { @@ -1433,14 +1479,7 @@ mod tests { Grid2D::new(tile_size_in_pixels, vec![4; 6]).unwrap().into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -1459,16 +1498,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2013, 12, 1, 12, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((0., -3.).into(), (2., 0.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2013, 12, 1, 12, 0, 0)).unwrap(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/pie_chart.rs b/operators/src/plot/pie_chart.rs index 8423d5b3e..7ce10e244 100644 --- a/operators/src/plot/pie_chart.rs +++ b/operators/src/plot/pie_chart.rs @@ -6,15 +6,19 @@ use crate::engine::{ }; use crate::engine::{QueryProcessor, SingleVectorSource}; use crate::error::Error; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::StreamExt; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::plots::{Plot, PlotData}; -use geoengine_datatypes::primitives::{FeatureDataRef, Measurement, PlotQueryRectangle}; +use geoengine_datatypes::primitives::{ + ColumnSelection, FeatureDataRef, Measurement, PlotQueryRectangle, SpatialResolution, + VectorQueryRectangle, +}; use serde::{Deserialize, Serialize}; use snafu::Snafu; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; pub const PIE_CHART_OPERATOR_NAME: &str = "PieChart"; @@ -108,7 +112,7 @@ pub struct InitializedCountPieChart { result_descriptor: PlotResultDescriptor, column_name: String, column_label: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, } @@ -119,7 +123,7 @@ impl InitializedCountPieChart { result_descriptor: PlotResultDescriptor, column_name: String, column_label: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, ) -> Self { Self { @@ -154,6 +158,22 @@ impl InitializedPlotOperator for InitializedCountPieChart CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(PieChart { + params: PieChartParams::Count { + column_name: self.column_name.clone(), + donut: self.donut, + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } /// A query processor that calculates the Histogram about its vector inputs. @@ -161,7 +181,7 @@ pub struct CountPieChartVectorQueryProcessor { input: TypedVectorQueryProcessor, column_label: String, column_name: String, - class_mapping: Option>, + class_mapping: Option>, donut: bool, } @@ -186,7 +206,7 @@ impl PlotQueryProcessor for CountPieChartVectorQueryProcessor { /// Null-values are empty strings. pub fn feature_data_strings_iter<'f>( feature_data: &'f FeatureDataRef, - class_mapping: Option<&'f HashMap>, + class_mapping: Option<&'f BTreeMap>, ) -> Box + 'f> { match (feature_data, class_mapping) { (FeatureDataRef::Category(feature_data_ref), Some(class_mapping)) => { @@ -227,9 +247,12 @@ impl CountPieChartVectorQueryProcessor { let mut slices: HashMap = HashMap::new(); // TODO: parallelize + let query: VectorQueryRectangle = query.select_attributes(ColumnSelection::all()); call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + + + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -285,8 +308,8 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, StaticMetaData, VectorColumnInfo, - VectorOperator, VectorResultDescriptor, + ChunkByteSize, MockExecutionContext, StaticMetaData, VectorColumnInfo, VectorOperator, + VectorResultDescriptor, }; use crate::mock::MockFeatureCollectionSource; use crate::source::{ @@ -296,8 +319,8 @@ mod tests { use crate::test_data; use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, FeatureDataType, NoGeometry, PlotSeriesSelection, - SpatialResolution, TimeInterval, + BoundingBox2D, FeatureData, FeatureDataType, NoGeometry, PlotQueryRectangle, + PlotSeriesSelection, TimeInterval, }; use geoengine_datatypes::primitives::{CacheTtlSeconds, VectorQueryRectangle}; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -399,14 +422,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -478,14 +499,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -625,14 +644,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap_err(); @@ -674,14 +691,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -757,14 +772,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -824,14 +837,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/scatter_plot.rs b/operators/src/plot/scatter_plot.rs index c44c4b1ad..cb794fbb5 100644 --- a/operators/src/plot/scatter_plot.rs +++ b/operators/src/plot/scatter_plot.rs @@ -12,8 +12,11 @@ use crate::engine::{ TypedPlotQueryProcessor, TypedVectorQueryProcessor, WorkflowOperatorPath, }; use crate::error::Error; +use crate::optimization::OptimizationError; use crate::util::Result; -use geoengine_datatypes::primitives::{Coordinate2D, PlotQueryRectangle}; +use geoengine_datatypes::primitives::{ + ColumnSelection, Coordinate2D, PlotQueryRectangle, SpatialResolution, +}; pub const SCATTERPLOT_OPERATOR_NAME: &str = "ScatterPlot"; @@ -129,6 +132,22 @@ impl InitializedPlotOperator for InitializedScatterPlot CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(ScatterPlot { + params: ScatterPlotParams { + column_x: self.column_x.clone(), + column_y: self.column_y.clone(), + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } /// A query processor that calculates the scatter plot about its vector input. @@ -154,8 +173,10 @@ impl PlotQueryProcessor for ScatterPlotQueryProcessor { let mut collector = CollectorKind::Values(Collector::new(self.column_x.clone(), self.column_y.clone())); + let query = query.select_attributes(ColumnSelection::all()); + call_on_generic_vector_processor!(&self.input, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -281,17 +302,15 @@ impl CollectorKind { #[cfg(test)] mod tests { - use geoengine_datatypes::util::test::TestDefault; - use serde_json::json; - + use crate::engine::{ChunkByteSize, MockExecutionContext, VectorOperator}; + use crate::mock::MockFeatureCollectionSource; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, NoGeometry, PlotSeriesSelection, SpatialResolution, + BoundingBox2D, FeatureData, NoGeometry, PlotQueryRectangle, PlotSeriesSelection, TimeInterval, }; + use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{collections::DataCollection, primitives::MultiPoint}; - - use crate::engine::{ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator}; - use crate::mock::MockFeatureCollectionSource; + use serde_json::json; use super::*; @@ -377,14 +396,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -456,14 +473,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -643,14 +658,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -698,14 +711,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -755,14 +766,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -837,14 +846,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/statistics.rs b/operators/src/plot/statistics.rs index e03fbd605..8c89feb5b 100644 --- a/operators/src/plot/statistics.rs +++ b/operators/src/plot/statistics.rs @@ -7,6 +7,7 @@ use crate::engine::{ }; use crate::error; use crate::error::Error; +use crate::optimization::OptimizationError; use crate::util::Result; use crate::util::input::MultiRasterOrVectorOperator; use crate::util::number_statistics::NumberStatistics; @@ -16,8 +17,8 @@ use futures::stream::select_all; use futures::{FutureExt, StreamExt, TryFutureExt, TryStreamExt}; use geoengine_datatypes::collections::FeatureCollectionInfos; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, BoundingBox2D, PlotQueryRectangle, RasterQueryRectangle, - partitions_extent, time_interval_extent, + AxisAlignedRectangle, BandSelection, BoundingBox2D, ColumnSelection, PlotQueryRectangle, + RasterQueryRectangle, SpatialResolution, partitions_extent, time_interval_extent, }; use geoengine_datatypes::raster::ConvertDataTypeParallel; use geoengine_datatypes::raster::{GridOrEmpty, GridSize}; @@ -114,8 +115,8 @@ impl PlotOperator for Statistics { ); } - let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); + let time = time_interval_extent(in_descriptors.iter().map(|d| d.time.bounds)); + let bbox = partitions_extent(in_descriptors.iter().map(|d| d.spatial_bounds())); let initialized_operator = InitializedStatistics::new( name, @@ -243,6 +244,29 @@ impl InitializedPlotOperator for InitializedStatistics CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Statistics { + params: StatisticsParams { + column_names: self.column_names.clone(), + percentiles: self + .percentiles + .iter() + .copied() + .map(NotNan::::new) + .collect::,_>>().expect("percentiles should be not nan because they are NotNan during initialization"), + }, + sources: MultipleRasterOrSingleVectorSource { + source: MultiRasterOrVectorOperator::Vector( + self.source.optimize(target_resolution)?, + ), + }, + } + .boxed()) + } } impl InitializedPlotOperator for InitializedStatistics>> { @@ -268,6 +292,32 @@ impl InitializedPlotOperator for InitializedStatistics CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Statistics { + params: StatisticsParams { + column_names: self.column_names.clone(), + percentiles: self + .percentiles + .iter() + .copied() + .map(NotNan::::new) + .collect::,_>>().expect("percentiles should be not nan because they are NotNan during initialization"), + }, + sources: MultipleRasterOrSingleVectorSource { + source: MultiRasterOrVectorOperator::Raster( + self.source + .iter() + .map(|s| s.optimize(target_resolution)) + .collect::, OptimizationError>>()?, + ), + }, + } + .boxed()) + } } /// A query processor that calculates the statistics about its vector input. @@ -301,8 +351,10 @@ impl PlotQueryProcessor for StatisticsVectorQueryProcessor { }) .collect(); + let query = query.select_attributes(ColumnSelection::all()); + call_on_generic_vector_processor!(&self.vector, processor => { - let mut query = processor.query(query.into(), ctx).await?; + let mut query = processor.query(query, ctx).await?; while let Some(collection) = query.next().await { let collection = collection?; @@ -353,12 +405,19 @@ impl PlotQueryProcessor for StatisticsRasterQueryProcessor { ctx: &'a dyn QueryContext, ) -> Result { let mut queries = Vec::with_capacity(self.rasters.len()); - let q: RasterQueryRectangle = - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()); for (i, raster_processor) in self.rasters.iter().enumerate() { + let rd = raster_processor.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::first(), + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); + queries.push( call_on_generic_raster_processor!(raster_processor, processor => { - processor.query(q.clone(), ctx).await? + processor.query(raster_query_rect.clone(), ctx).await? // TODO: avoid cloning query? .and_then(move |tile| crate::util::spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || (i, tile.convert_data_type_parallel()) ).map_err(Into::into)) .boxed() }), @@ -540,23 +599,22 @@ impl From<&StatisticsAggregator> for StatisticsOutput { #[cfg(test)] mod tests { use geoengine_datatypes::collections::DataCollection; - use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; + use geoengine_datatypes::primitives::{CacheHint, Coordinate2D, PlotSeriesSelection}; use geoengine_datatypes::util::test::TestDefault; use serde_json::json; use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterOperator, - RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, TimeDescriptor, }; use crate::engine::{RasterBandDescriptors, VectorOperator}; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::util::input::MultiRasterOrVectorOperator::Raster; - use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, NoGeometry, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{BoundingBox2D, FeatureData, NoGeometry, TimeInterval}; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridShape2D, RasterDataType, + RasterTile2D, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -588,9 +646,8 @@ mod tests { #[tokio::test] async fn empty_raster_input() { - let tile_size_in_pixels = [3, 2].into(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -614,14 +671,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -631,11 +686,18 @@ mod tests { #[tokio::test] async fn single_raster_implicit_name() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = MockRasterSource { params: MockRasterSourceParams { @@ -652,14 +714,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -684,14 +739,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -716,11 +769,18 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn two_rasters_implicit_names() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -738,14 +798,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -764,14 +817,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -797,14 +843,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -837,11 +881,18 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn two_rasters_explicit_names() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -859,14 +910,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -885,14 +929,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -918,14 +955,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -957,11 +992,18 @@ mod tests { #[tokio::test] async fn two_rasters_explicit_names_incomplete() { - let tile_size_in_pixels = [3, 2].into(); - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels, + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); let raster_source = vec![ MockRasterSource { @@ -979,14 +1021,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(), @@ -1005,14 +1040,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(), @@ -1042,7 +1070,6 @@ mod tests { async fn vector_no_column() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1101,14 +1128,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1142,7 +1167,6 @@ mod tests { async fn vector_single_column() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1201,14 +1225,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1234,7 +1256,6 @@ mod tests { async fn vector_two_columns() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1293,14 +1314,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1334,10 +1353,20 @@ mod tests { async fn raster_percentile() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -1353,14 +1382,7 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1385,14 +1407,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -1421,7 +1441,6 @@ mod tests { async fn vector_percentiles() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1480,14 +1499,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/temporal_raster_mean_plot.rs b/operators/src/plot/temporal_raster_mean_plot.rs index 664b49261..7509a04f6 100644 --- a/operators/src/plot/temporal_raster_mean_plot.rs +++ b/operators/src/plot/temporal_raster_mean_plot.rs @@ -1,9 +1,10 @@ use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, - InitializedSources, Operator, OperatorName, PlotOperator, PlotQueryProcessor, - PlotResultDescriptor, QueryContext, QueryProcessor, RasterQueryProcessor, SingleRasterSource, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedPlotOperator, + InitializedRasterOperator, InitializedSources, Operator, OperatorName, PlotOperator, + PlotQueryProcessor, PlotResultDescriptor, QueryContext, QueryProcessor, SingleRasterSource, TypedPlotQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use crate::util::math::average_floor; use async_trait::async_trait; @@ -11,8 +12,8 @@ use futures::StreamExt; use futures::stream::BoxStream; use geoengine_datatypes::plots::{AreaLineChart, Plot, PlotData}; use geoengine_datatypes::primitives::{ - BandSelection, Measurement, PlotQueryRectangle, RasterQueryRectangle, TimeInstance, - TimeInterval, + BandSelection, Measurement, PlotQueryRectangle, RasterQueryRectangle, SpatialResolution, + TimeInstance, TimeInterval, }; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use serde::{Deserialize, Serialize}; @@ -122,11 +123,24 @@ impl InitializedPlotOperator for InitializedMeanRasterPixelValuesOverTime { fn canonic_name(&self) -> CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(MeanRasterPixelValuesOverTime { + params: self.state.clone(), + sources: SingleRasterSource { + raster: self.raster.optimize(target_resolution)?, + }, + } + .boxed()) + } } /// A query processor that calculates the `TemporalRasterMeanPlot` about its input. pub struct MeanRasterPixelValuesOverTimeQueryProcessor { - raster: Box>, + raster: BoxRasterQueryProcessor

, time_position: MeanRasterPixelValuesOverTimePosition, measurement: Measurement, draw_area: bool, @@ -145,13 +159,17 @@ impl PlotQueryProcessor for MeanRasterPixelValuesOverTimeQueryProcesso query: PlotQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result { + let rd = self.raster.result_descriptor(); + + let raster_query_rect = RasterQueryRectangle::from_bounds_and_geo_transform( + &query, + BandSelection::first(), + rd.tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + ); + let means = Self::calculate_means( - self.raster - .query( - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), - ctx, - ) - .await?, + self.raster.query(raster_query_rect, ctx).await?, self.time_position, ) .await?; @@ -262,8 +280,8 @@ mod tests { use crate::{ engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, }, source::GdalSource, }; @@ -272,9 +290,15 @@ mod tests { source::GdalSourceParameters, }; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheHint, Measurement, PlotSeriesSelection, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, Coordinate2D, Measurement, PlotSeriesSelection, TimeInterval, + }; + use geoengine_datatypes::raster::GeoTransform; + use geoengine_datatypes::{ + dataset::NamedData, + plots::PlotMetaData, + primitives::DateTime, + raster::{BoundedGrid, GridShape2D}, }; - use geoengine_datatypes::{dataset::NamedData, plots::PlotMetaData, primitives::DateTime}; use geoengine_datatypes::{raster::TilingSpecification, spatial_reference::SpatialReference}; use geoengine_datatypes::{ raster::{Grid2D, RasterDataType, TileInformation}, @@ -291,9 +315,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("test")), } .boxed(), }, @@ -329,7 +351,6 @@ mod tests { async fn single_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -367,14 +388,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -437,17 +456,28 @@ mod tests { )); } + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + // TODO: find out if this can handle regular time as well + time: TimeDescriptor::new_irregular(Some( + TimeInterval::new( + tiles.first().unwrap().time.start(), + tiles.last().unwrap().time.end(), + ) + .unwrap(), + )), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new_2d(3, 2).bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + MockRasterSource { params: MockRasterSourceParams { data: tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -457,7 +487,6 @@ mod tests { async fn raster_series() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -510,14 +539,12 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/plot/temporal_vector_line_plot.rs b/operators/src/plot/temporal_vector_line_plot.rs index f6066ff6c..cd4aad149 100644 --- a/operators/src/plot/temporal_vector_line_plot.rs +++ b/operators/src/plot/temporal_vector_line_plot.rs @@ -1,25 +1,21 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedSources, InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, - PlotResultDescriptor, QueryContext, SingleVectorSource, TypedPlotQueryProcessor, - VectorQueryProcessor, WorkflowOperatorPath, + PlotResultDescriptor, QueryContext, QueryProcessor, SingleVectorSource, + TypedPlotQueryProcessor, VectorColumnInfo, VectorQueryProcessor, WorkflowOperatorPath, }; -use crate::engine::{QueryProcessor, VectorColumnInfo}; use crate::error; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::TryStreamExt; -use geoengine_datatypes::primitives::{FeatureDataType, PlotQueryRectangle}; +use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::{ - collections::FeatureCollection, - plots::{Plot, PlotData}, -}; -use geoengine_datatypes::{ - collections::FeatureCollectionInfos, - plots::{DataPoint, MultiLineChart}, -}; -use geoengine_datatypes::{ - primitives::{Geometry, Measurement, TimeInterval}, + collections::{FeatureCollection, FeatureCollectionInfos}, + plots::{DataPoint, MultiLineChart, Plot, PlotData}, + primitives::{ + ColumnSelection, FeatureDataType, Geometry, Measurement, PlotQueryRectangle, TimeInterval, + }, util::arrow::ArrowTyped, }; use serde::{Deserialize, Serialize}; @@ -143,6 +139,19 @@ impl InitializedPlotOperator for InitializedFeatureAttributeValuesOverTime { fn canonic_name(&self) -> CanonicOperatorName { self.name.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(FeatureAttributeValuesOverTime { + params: self.state.clone(), + sources: SingleVectorSource { + vector: self.vector_source.optimize(target_resolution)?, + }, + } + .boxed()) + } } /// A query processor that calculates the `TemporalVectorLinePlot` on its input. @@ -172,9 +181,11 @@ where ) -> Result { let values = FeatureAttributeValues::::default(); + let query = query.select_attributes(ColumnSelection::all()); + let values = self .features - .query(query.into(), ctx) + .query(query, ctx) .await? .try_fold(values, |mut acc, features| async move { let ids = features.data(&self.params.id_column)?; @@ -275,22 +286,20 @@ impl FeatureAttributeValues { #[cfg(test)] mod tests { use super::*; + use crate::{ + engine::{ChunkByteSize, MockExecutionContext, VectorOperator}, + mock::MockFeatureCollectionSource, + }; + use geoengine_datatypes::primitives::PlotQueryRectangle; use geoengine_datatypes::primitives::{CacheHint, PlotSeriesSelection}; use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{ collections::MultiPointCollection, plots::PlotMetaData, - primitives::{ - BoundingBox2D, DateTime, FeatureData, MultiPoint, SpatialResolution, TimeInterval, - }, + primitives::{BoundingBox2D, DateTime, FeatureData, MultiPoint, TimeInterval}, }; use serde_json::{Value, json}; - use crate::{ - engine::{ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator}, - mock::MockFeatureCollectionSource, - }; - #[tokio::test] #[allow(clippy::too_many_lines)] async fn plot() { @@ -352,14 +361,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -501,14 +508,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -618,7 +623,7 @@ mod tests { ) .boxed(); - let exe_ctc = MockExecutionContext::test_default(); + let exe_ctx = MockExecutionContext::test_default(); let operator = FeatureAttributeValuesOverTime { params: FeatureAttributeValuesOverTimeParams { @@ -630,7 +635,7 @@ mod tests { let operator = operator .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctc) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); @@ -638,14 +643,12 @@ mod tests { let result = query_processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: PlotSeriesSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + PlotQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + PlotSeriesSelection::all(), + ), + &exe_ctx.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); diff --git a/operators/src/processing/band_neighborhood_aggregate/mod.rs b/operators/src/processing/band_neighborhood_aggregate/mod.rs index bb4e5a7d1..d70c053cc 100644 --- a/operators/src/processing/band_neighborhood_aggregate/mod.rs +++ b/operators/src/processing/band_neighborhood_aggregate/mod.rs @@ -4,18 +4,21 @@ use std::task::{Context, Poll}; use crate::adapters::RasterStreamExt; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - ResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, ResultDescriptor, SingleRasterSource, + TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; use futures::{FutureExt, Stream}; -use geoengine_datatypes::primitives::RasterQueryRectangle; +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; use geoengine_datatypes::raster::{ - GridIdx2D, GridIndexAccess, MapElements, MapIndexedElements, RasterDataType, RasterTile2D, + GridBoundingBox2D, GridIdx2D, GridIndexAccess, MapElements, MapIndexedElements, RasterDataType, + RasterTile2D, }; use pin_project::pin_project; use serde::{Deserialize, Serialize}; @@ -186,17 +189,32 @@ impl InitializedRasterOperator for InitializedBandNeighborhoodAggregate { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(BandNeighborhoodAggregate { + params: BandNeighborhoodAggregateParams { + aggregate: self.aggregate.clone(), + }, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub(crate) struct BandNeighborhoodAggregateProcessor { - source: Box>, + source: BoxRasterQueryProcessor, result_descriptor: RasterResultDescriptor, aggregate: NeighborhoodAggregate, } impl BandNeighborhoodAggregateProcessor { pub fn new( - source: Box>, + source: BoxRasterQueryProcessor, result_descriptor: RasterResultDescriptor, aggregate: NeighborhoodAggregate, ) -> Self { @@ -209,10 +227,13 @@ impl BandNeighborhoodAggregateProcessor { } #[async_trait] -impl RasterQueryProcessor for BandNeighborhoodAggregateProcessor { - type RasterType = f64; +impl QueryProcessor for BandNeighborhoodAggregateProcessor { + type Output = RasterTile2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, @@ -220,11 +241,14 @@ impl RasterQueryProcessor for BandNeighborhoodAggregateProcessor { // query the source with all bands, to compute the aggregate // then, select only the queried bands // TODO: avoid computing the aggregate for bands that are not queried - let mut source_query = query.clone(); let source_result_descriptor = self.source.raster_result_descriptor(); - source_query.attributes = (&source_result_descriptor.bands).into(); + let source_query = RasterQueryRectangle::new( + query.spatial_bounds(), + query.time_interval(), + (&source_result_descriptor.bands).into(), + ); - let must_extract_bands = query.attributes != source_query.attributes; + let must_extract_bands = query.attributes() != source_query.attributes(); let aggregate = match &self.aggregate { NeighborhoodAggregate::FirstDerivative { band_distance } => { @@ -256,7 +280,7 @@ impl RasterQueryProcessor for BandNeighborhoodAggregateProcessor { if must_extract_bands { Ok(Box::pin(aggregate.extract_bands( - query.attributes.as_vec(), + query.attributes().as_vec(), source_result_descriptor.bands.count(), ))) } else { @@ -264,11 +288,24 @@ impl RasterQueryProcessor for BandNeighborhoodAggregateProcessor { } } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &Self::ResultDescription { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for BandNeighborhoodAggregateProcessor { + type RasterType = f64; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.source.time_query(query, ctx).await + } +} + #[pin_project(project = BandNeighborhoodAggregateStreamProjection)] struct BandNeighborhoodAggregateStream { #[pin] @@ -742,16 +779,16 @@ impl Accu for MovingAverageAccu { mod tests { use futures::StreamExt; use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{BandSelection, CacheHint, TimeInterval, TimeStep}, + raster::{Grid, GridBoundingBox2D, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::{MockExecutionContext, MockQueryContext, RasterBandDescriptors}, + engine::{ + MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, TimeDescriptor, + }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -1177,9 +1214,20 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + data.first().unwrap().time.start(), + data.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_multiple_bands(3), }, }, @@ -1201,14 +1249,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new_unchecked(vec![0, 1, 2]), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + BandSelection::new_unchecked(vec![0, 1, 2]), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = band_neighborhood_aggregate .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1319,9 +1366,20 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + data.first().unwrap().time.start(), + data.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), bands: RasterBandDescriptors::new_multiple_bands(3), }, }, @@ -1343,14 +1401,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new_unchecked(vec![0]), // only get first band - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + BandSelection::new_unchecked(vec![0]), // only get first band + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = band_neighborhood_aggregate .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/operators/src/processing/bandwise_expression/mod.rs b/operators/src/processing/bandwise_expression/mod.rs index 883200e5b..cc4308615 100644 --- a/operators/src/processing/bandwise_expression/mod.rs +++ b/operators/src/processing/bandwise_expression/mod.rs @@ -1,18 +1,20 @@ use std::sync::Arc; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - ResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, ResultDescriptor, SingleRasterSource, + TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; -use geoengine_datatypes::primitives::RasterQueryRectangle; +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; use geoengine_datatypes::raster::{ - GridOrEmpty2D, MapElementsParallel, Pixel, RasterDataType, RasterTile2D, + GridBoundingBox2D, GridOrEmpty2D, MapElementsParallel, Pixel, RasterDataType, RasterTile2D, }; use geoengine_expression::{ DataType, ExpressionAst, ExpressionParser, LinkedExpression, Parameter, @@ -73,6 +75,7 @@ impl RasterOperator for BandwiseExpression { Ok(Box::new(InitializedBandwiseExpression { name, path, + params: self.params.clone(), result_descriptor, source, expression, @@ -85,6 +88,7 @@ impl RasterOperator for BandwiseExpression { pub struct InitializedBandwiseExpression { name: CanonicOperatorName, + params: BandwiseExpressionParams, path: WorkflowOperatorPath, result_descriptor: RasterResultDescriptor, source: Box, @@ -136,10 +140,23 @@ impl InitializedRasterOperator for InitializedBandwiseExpression { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(BandwiseExpression { + params: self.params.clone(), + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub(crate) struct BandwiseExpressionProcessor { - source: Box>, + source: BoxRasterQueryProcessor, result_descriptor: RasterResultDescriptor, expression: Arc, map_no_data: bool, @@ -151,7 +168,7 @@ where TO: Pixel, { pub fn new( - source: Box>, + source: BoxRasterQueryProcessor, result_descriptor: RasterResultDescriptor, expression: LinkedExpression, map_no_data: bool, @@ -196,13 +213,16 @@ where } #[async_trait] -impl RasterQueryProcessor for BandwiseExpressionProcessor +impl QueryProcessor for BandwiseExpressionProcessor where TO: Pixel, { - type RasterType = TO; + type Output = RasterTile2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, @@ -240,23 +260,43 @@ where Ok(stream.boxed()) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &Self::ResultDescription { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for BandwiseExpressionProcessor +where + TO: Pixel, +{ + type RasterType = TO; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::{Grid, GridShape, MapElements, RenameBands, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, TimeInterval, TimeStep}, + raster::{ + Grid, GridBoundingBox2D, GridShape, MapElements, RenameBands, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, + SpatialGridDescriptor, TimeDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -355,17 +395,30 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + data.first().unwrap().time.start(), + data.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -373,14 +426,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -410,14 +456,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = expression .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/operators/src/processing/circle_merging_quadtree/operator.rs b/operators/src/processing/circle_merging_quadtree/operator.rs index 02acc6ea8..dc6abe6a6 100644 --- a/operators/src/processing/circle_merging_quadtree/operator.rs +++ b/operators/src/processing/circle_merging_quadtree/operator.rs @@ -8,7 +8,7 @@ use geoengine_datatypes::collections::{ }; use geoengine_datatypes::primitives::{ BoundingBox2D, Circle, FeatureDataType, FeatureDataValue, Measurement, MultiPoint, - MultiPointAccess, VectorQueryRectangle, + MultiPointAccess, SpatialBounded, SpatialResolution, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use serde::{Deserialize, Serialize}; @@ -22,6 +22,7 @@ use crate::engine::{ }; use crate::engine::{InitializedSources, OperatorName, WorkflowOperatorPath}; use crate::error::{self, Error}; +use crate::optimization::OptimizationError; use crate::processing::circle_merging_quadtree::aggregates::MeanAggregator; use crate::processing::circle_merging_quadtree::circle_of_points::CircleOfPoints; use crate::processing::circle_merging_quadtree::circle_radius_model::LogScaledRadius; @@ -37,6 +38,7 @@ use super::quadtree::CircleMergingQuadtree; pub struct VisualPointClusteringParams { pub min_radius_px: f64, pub delta_px: f64, + pub resolution: f64, radius_column: String, count_column: String, column_aggregates: HashMap, @@ -70,14 +72,21 @@ impl VectorOperator for VisualPointClustering { self.params.min_radius_px > 0.0, error::InputMustBeGreaterThanZero { scope: "VisualPointClustering", - name: "min_radius_px" + name: "minRadius" } ); ensure!( self.params.delta_px >= 0.0, error::InputMustBeZeroOrPositive { scope: "VisualPointClustering", - name: "delta_px" + name: "deltaPx" + } + ); + ensure!( + self.params.resolution >= 0.0, + error::InputMustBeZeroOrPositive { + scope: "VisualPointClustering", + name: "resolution" } ); ensure!(!self.params.radius_column.is_empty(), error::EmptyInput); @@ -176,6 +185,7 @@ impl VectorOperator for VisualPointClustering { Ok(InitializedVisualPointClustering { name, path, + params: self.params.clone(), result_descriptor: VectorResultDescriptor { data_type: VectorDataType::MultiPoint, spatial_reference: in_desc.spatial_reference, @@ -188,6 +198,7 @@ impl VectorOperator for VisualPointClustering { radius_column: self.params.radius_column, count_column: self.params.count_column, attribute_mapping: self.params.column_aggregates, + resolution: self.params.resolution, } .boxed()) } @@ -198,12 +209,14 @@ impl VectorOperator for VisualPointClustering { pub struct InitializedVisualPointClustering { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: VisualPointClusteringParams, result_descriptor: VectorResultDescriptor, vector_source: Box, radius_model: LogScaledRadius, radius_column: String, count_column: String, attribute_mapping: HashMap, + resolution: f64, } impl InitializedVectorOperator for InitializedVisualPointClustering { @@ -218,6 +231,7 @@ impl InitializedVectorOperator for InitializedVisualPointClustering { self.count_column.clone(), self.result_descriptor.clone(), self.attribute_mapping.clone(), + self.resolution, ) .boxed(), )) @@ -252,6 +266,19 @@ impl InitializedVectorOperator for InitializedVisualPointClustering { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(VisualPointClustering { + params: self.params.clone(), + sources: SingleVectorSource { + vector: self.vector_source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct VisualPointClusteringProcessor { @@ -261,6 +288,7 @@ pub struct VisualPointClusteringProcessor { count_column: String, result_descriptor: VectorResultDescriptor, attribute_mapping: HashMap, + resolution: f64, } impl VisualPointClusteringProcessor { @@ -271,6 +299,7 @@ impl VisualPointClusteringProcessor { count_column: String, result_descriptor: VectorResultDescriptor, attribute_mapping: HashMap, + resolution: f64, ) -> Self { Self { source, @@ -279,6 +308,7 @@ impl VisualPointClusteringProcessor { count_column, result_descriptor, attribute_mapping, + resolution, } } @@ -286,9 +316,9 @@ impl VisualPointClusteringProcessor { circles_of_points: impl Iterator, radius_column: &str, count_column: &str, - resolution: f64, columns: &HashMap, cache_hint: CacheHint, + resolution: f64, ) -> Result { let mut builder = MultiPointCollection::builder(); @@ -396,12 +426,10 @@ impl QueryProcessor for VisualPointClusteringProcessor { .iter() .map(|(name, column_info)| (name.clone(), column_info.data_type)) .collect(); - - let joint_resolution = f64::max(query.spatial_resolution.x, query.spatial_resolution.y); - let scaled_radius_model = self.radius_model.with_scaled_radii(joint_resolution)?; + let scaled_radius_model = self.radius_model.with_scaled_radii(self.resolution)?; let initial_grid_fold_state = Result::::Ok(GridFoldState { - grid: Grid::new(query.spatial_bounds, scaled_radius_model), + grid: Grid::new(query.spatial_bounds().spatial_bounds(), scaled_radius_model), column_mapping: self.attribute_mapping.clone(), cache_hint: CacheHint::max_duration(), }); @@ -474,7 +502,11 @@ impl QueryProcessor for VisualPointClusteringProcessor { cache_hint, } = grid?; - let mut cmq = CircleMergingQuadtree::new(query.spatial_bounds, *grid.radius_model(), 1); + let mut cmq = CircleMergingQuadtree::new( + query.spatial_bounds().spatial_bounds(), + *grid.radius_model(), + 1, + ); // TODO: worker thread for circle_of_points in grid.drain() { @@ -485,9 +517,9 @@ impl QueryProcessor for VisualPointClusteringProcessor { cmq.into_iter(), &self.radius_column, &self.count_column, - joint_resolution, &column_schema, cache_hint, + self.resolution, ) }); @@ -502,16 +534,13 @@ impl QueryProcessor for VisualPointClusteringProcessor { #[cfg(test)] mod tests { use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; + use geoengine_datatypes::primitives::BoundingBox2D; use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::FeatureData; - use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::primitives::TimeInterval; use geoengine_datatypes::util::test::TestDefault; - use crate::{ - engine::{MockExecutionContext, MockQueryContext}, - mock::MockFeatureCollectionSource, - }; + use crate::{engine::MockExecutionContext, mock::MockFeatureCollectionSource}; use super::*; @@ -532,6 +561,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + resolution: 0.1, radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: Default::default(), @@ -555,14 +585,13 @@ mod tests { .multi_point() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -606,6 +635,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + resolution: 0.1, radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -639,14 +669,13 @@ mod tests { .multi_point() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -691,6 +720,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + resolution: 0.1, radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -724,14 +754,13 @@ mod tests { .multi_point() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -784,6 +813,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + resolution: 0.1, radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -817,14 +847,13 @@ mod tests { .multi_point() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); @@ -892,6 +921,7 @@ mod tests { params: VisualPointClusteringParams { min_radius_px: 8., delta_px: 1., + resolution: 0.1, radius_column: "radius".to_string(), count_column: "count".to_string(), column_aggregates: [( @@ -925,14 +955,13 @@ mod tests { .multi_point() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); - let qrect = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }; + let qrect = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query = query_processor.query(qrect, &query_context).await.unwrap(); diff --git a/operators/src/processing/column_range_filter.rs b/operators/src/processing/column_range_filter.rs index 136afa6dc..9df3affac 100644 --- a/operators/src/processing/column_range_filter.rs +++ b/operators/src/processing/column_range_filter.rs @@ -4,6 +4,7 @@ use crate::engine::{ VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; use crate::error; +use crate::optimization::OptimizationError; use crate::util::Result; use crate::util::input::StringOrNumberRange; use crate::{adapters::FeatureCollectionChunkMerger, engine::SingleVectorSource}; @@ -14,7 +15,7 @@ use geoengine_datatypes::collections::{ FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, }; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, FeatureDataType, FeatureDataValue, Geometry, + BoundingBox2D, ColumnSelection, FeatureDataType, FeatureDataValue, Geometry, SpatialResolution, VectorQueryRectangle, }; use geoengine_datatypes::util::arrow::ArrowTyped; @@ -96,6 +97,19 @@ impl InitializedVectorOperator for InitializedColumnRangeFilter { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(ColumnRangeFilter { + params: self.state.clone(), + sources: SingleVectorSource { + vector: self.vector_source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct ColumnRangeFilterProcessor { @@ -199,14 +213,13 @@ where #[cfg(test)] mod tests { use super::*; - use crate::engine::{MockExecutionContext, MockQueryContext}; + use crate::engine::MockExecutionContext; use crate::mock::MockFeatureCollectionSource; use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, FeatureCollectionModifications, MultiPointCollection, }; - use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, Coordinate2D, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::util::test::TestDefault; @@ -283,11 +296,10 @@ mod tests { } .boxed(); + let exe_ctx = MockExecutionContext::test_default(); + let initialized = filter - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); @@ -297,14 +309,13 @@ mod tests { panic!(); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx = MockQueryContext::new((2 * std::mem::size_of::()).into()); + let ctx = exe_ctx.mock_query_context((2 * std::mem::size_of::()).into()); let stream = point_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/processing/downsample/mod.rs b/operators/src/processing/downsample/mod.rs new file mode 100644 index 000000000..acdfa285c --- /dev/null +++ b/operators/src/processing/downsample/mod.rs @@ -0,0 +1,1023 @@ +use crate::adapters::{ + FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, +}; +use crate::engine::{ + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, + OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, + RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, +}; +use crate::optimization::{OptimizableOperator, OptimizationError}; +use crate::util::Result; +use async_trait::async_trait; +use futures::future::BoxFuture; +use futures::stream::BoxStream; +use futures::{Future, FutureExt, TryFuture, TryFutureExt}; +use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, Coordinate2D, find_next_best_overview_level_resolution, +}; +use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialResolution, TimeInterval}; +use geoengine_datatypes::raster::{ + ChangeGridBounds, GeoTransform, GridBoundingBox2D, GridContains, GridIdx2D, GridIndexAccess, + GridOrEmpty, Pixel, RasterTile2D, TileInformation, TilingSpecification, + UpdateIndexedElementsParallel, +}; +use rayon::ThreadPool; +use serde::{Deserialize, Serialize}; +use snafu::{Snafu, ensure}; +use std::marker::PhantomData; +use std::sync::Arc; + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct DownsamplingParams { + pub sampling_method: DownsamplingMethod, + pub output_resolution: DownsamplingResolution, + pub output_origin_reference: Option, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum DownsamplingResolution { + Resolution(SpatialResolution), + Fraction { x: f64, y: f64 }, +} + +#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub enum DownsamplingMethod { + NearestNeighbor, + // Mean, +} + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)), context(suffix(false)), module(error))] +pub enum DownsamplingError { + #[snafu(display("The fraction used to downsample must be >= 1, was {f}."))] + FractionMustBeOneOrLarger { f: f64 }, + #[snafu(display("The output resolution must be higher than the input resolution."))] + OutputMustBeLowerResolutionThanInput { + input: SpatialResolution, + output: SpatialResolution, + }, +} + +pub type Downsampling = Operator; + +impl OperatorName for Downsampling { + const TYPE_NAME: &'static str = "Downsampling"; +} + +#[typetag::serde] +#[async_trait] +impl RasterOperator for Downsampling { + async fn _initialize( + self: Box, + path: WorkflowOperatorPath, + context: &dyn ExecutionContext, + ) -> Result> { + let name = CanonicOperatorName::from(&self); + let initialized_source = self + .sources + .initialize_sources(path.clone(), context) + .await?; + InitializedDownsampling::new_with_source_and_params( + name, + path, + initialized_source.raster, + self.params, + context.tiling_specification(), + ) + .map(InitializedRasterOperator::boxed) + } + + span_fn!(Downsampling); +} + +pub struct InitializedDownsampling { + name: CanonicOperatorName, + path: WorkflowOperatorPath, + output_result_descriptor: RasterResultDescriptor, + raster_source: O, + sampling_method: DownsamplingMethod, + tiling_specification: TilingSpecification, +} + +impl InitializedDownsampling { + pub fn new_with_source_and_params( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + raster_source: O, + params: DownsamplingParams, + tiling_specification: TilingSpecification, + ) -> Result { + let in_descriptor = raster_source.result_descriptor(); + + let in_spatial_grid = in_descriptor.spatial_grid_descriptor(); + + let output_resolution = match params.output_resolution { + DownsamplingResolution::Resolution(res) => { + ensure!( + res.x.abs() >= in_spatial_grid.spatial_resolution().x.abs(), + error::OutputMustBeLowerResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + ensure!( + res.y.abs() >= in_spatial_grid.spatial_resolution().y.abs(), // TODO: allow neg y size in SpatialResolution + error::OutputMustBeLowerResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + res + } + + DownsamplingResolution::Fraction { x, y } => { + ensure!(x >= 1.0, error::FractionMustBeOneOrLarger { f: x }); + ensure!(y >= 1.0, error::FractionMustBeOneOrLarger { f: y }); + + SpatialResolution::new_unchecked( + in_spatial_grid.spatial_resolution().x * x, + in_spatial_grid.spatial_resolution().y.abs() * y, // TODO: allow negative size + ) + } + }; + + let output_gspatial_grid = if let Some(oc) = params.output_origin_reference { + in_spatial_grid + .with_moved_origin_to_nearest_grid_edge(oc) + .replace_origin(oc) + .with_changed_resolution(output_resolution) + } else { + in_spatial_grid.with_changed_resolution(output_resolution) + }; + + let out_descriptor = RasterResultDescriptor { + spatial_reference: in_descriptor.spatial_reference, + data_type: in_descriptor.data_type, // TODO: datatype depends on resample method! + time: in_descriptor.time, + spatial_grid: output_gspatial_grid, + bands: in_descriptor.bands.clone(), + }; + + Ok(InitializedDownsampling { + name, + path, + output_result_descriptor: out_descriptor, + raster_source, + sampling_method: params.sampling_method, + tiling_specification, + }) + } +} + +impl InitializedRasterOperator for InitializedDownsampling { + fn query_processor(&self) -> Result { + let source_processor = self.raster_source.query_processor()?; + + let res = call_on_generic_raster_processor!( + source_processor, p => match self.sampling_method { + DownsamplingMethod::NearestNeighbor => DownsampleProcessor::<_,_>::new( + p, + self.output_result_descriptor.clone(), + self.tiling_specification, + ).boxed() + .into(), + } + ); + + Ok(res) + } + + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.output_result_descriptor + } + + fn canonic_name(&self) -> CanonicOperatorName { + self.name.clone() + } + + fn name(&self) -> &'static str { + Downsampling::TYPE_NAME + } + + fn path(&self) -> WorkflowOperatorPath { + self.path.clone() + } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let out_descriptor = self.result_descriptor(); + let in_descriptor = self.raster_source.result_descriptor(); + + let input_resolution = in_descriptor.spatial_grid.spatial_resolution(); + + let new_origin = if in_descriptor.spatial_grid.geo_transform().origin_coordinate + == out_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate + { + None + } else { + Some( + out_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate, + ) + }; + + if input_resolution == target_resolution { + // special case where downsampling becomes redundant, unless it also regrids + + // TODO: source does not need to be optimized, but we need it as an `RasterOperator` and not `InitializedRasterOperator` + let optimzed_source = self.raster_source.optimize(target_resolution)?; + + if new_origin.is_some() { + return Ok(Downsampling { + params: DownsamplingParams { + sampling_method: self.sampling_method, + output_resolution: DownsamplingResolution::Resolution(target_resolution), + output_origin_reference: new_origin, + }, + sources: SingleRasterSource { + raster: optimzed_source, + }, + } + .boxed()); + } + return Ok(optimzed_source); + } + + // target resolution must be coarser than input + debug_assert!(input_resolution < target_resolution); + + let snapped_input_resolution = + find_next_best_overview_level_resolution(input_resolution, target_resolution); + + let optimzed_source = self.raster_source.optimize(snapped_input_resolution)?; + + Ok(Downsampling { + params: DownsamplingParams { + sampling_method: self.sampling_method, + output_resolution: DownsamplingResolution::Resolution(target_resolution), + output_origin_reference: new_origin, + }, + sources: SingleRasterSource { + raster: optimzed_source, + }, + } + .boxed()) + } +} + +pub struct DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Copy, +{ + source: Q, + out_result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, +} + +impl DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Copy, +{ + pub fn new( + source: Q, + out_result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + ) -> Self { + Self { + source, + out_result_descriptor, + tiling_specification, + } + } +} + +#[async_trait] +impl RasterQueryProcessor for DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Pixel, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.source.time_query(query, ctx).await + } +} + +#[async_trait] +impl QueryProcessor for DownsampleProcessor +where + Q: RasterQueryProcessor, + P: Pixel, +{ + type Output = RasterTile2D

; + type SpatialBounds = GridBoundingBox2D; + type Selection = BandSelection; + type ResultDescription = RasterResultDescriptor; + + async fn _query<'a>( + &'a self, + query: RasterQueryRectangle, + ctx: &'a dyn QueryContext, + ) -> Result>> { + // do not interpolate if the source resolution is already fine enough + + let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); + let out_spatial_grid = self.result_descriptor().spatial_grid_descriptor(); + + // if the output resolution is the same as the input resolution, we can just forward the query // TODO: except the origin changes? + if in_spatial_grid == out_spatial_grid { + return self.source.query(query, ctx).await; + } + + let tiling_grid_definition = + out_spatial_grid.tiling_grid_definition(ctx.tiling_specification()); + // This is the tiling strategy we want to fill + let tiling_strategy: geoengine_datatypes::raster::TilingStrategy = + tiling_grid_definition.generate_data_tiling_strategy(); + + let sub_query = DownsampleSubQuery::<_, P> { + input_geo_transform: in_spatial_grid + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(), + output_geo_transform: tiling_grid_definition.tiling_geo_transform(), + fold_fn: fold_future, + tiling_specification: self.tiling_specification, + _phantom_pixel_type: PhantomData, + }; + + let time_stream = self.time_query(query.time_interval(), ctx).await?; + + Ok(Box::pin(RasterSubQueryAdapter::<'a, P, _, _, _>::new( + &self.source, + query, + tiling_strategy, + ctx, + sub_query, + time_stream, + ))) + } + + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.out_result_descriptor + } +} + +#[derive(Debug, Clone)] +pub struct DownsampleSubQuery { + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, + fold_fn: F, + tiling_specification: TilingSpecification, + _phantom_pixel_type: PhantomData, +} + +impl<'a, T, FoldM, FoldF> SubQueryTileAggregator<'a, T> for DownsampleSubQuery +where + T: Pixel, + FoldM: Send + Sync + 'a + Clone + Fn(DownsampleAccu, RasterTile2D) -> FoldF, + FoldF: Send + TryFuture, Error = crate::error::Error>, +{ + type FoldFuture = FoldF; + + type FoldMethod = FoldM; + + type TileAccu = DownsampleAccu; + type TileAccuFuture = BoxFuture<'a, Result>; + + fn new_fold_accu( + &self, + tile_info: TileInformation, + query_rect: RasterQueryRectangle, + pool: &Arc, + ) -> Self::TileAccuFuture { + create_accu( + self.input_geo_transform, + self.output_geo_transform, + tile_info, + &query_rect, + pool.clone(), + self.tiling_specification, + ) + .boxed() + } + + fn tile_query_rectangle( + &self, + tile_info: TileInformation, + _query_rect: RasterQueryRectangle, + time: TimeInterval, + band_idx: u32, + ) -> Result> { + let out_tile_pixel_bounds = tile_info.global_pixel_bounds(); + //.intersection(&query_rect.spatial_query.grid_bounds()); + //let out_tile_pixel_bounds = out_tile_pixel_bounds; + let out_tile_spatial_bounds = self + .output_geo_transform + .grid_to_spatial_bounds(&out_tile_pixel_bounds); + let input_pixel_bounds = self + .input_geo_transform + .spatial_to_grid_bounds(&out_tile_spatial_bounds); + + Ok(Some(RasterQueryRectangle::new( + input_pixel_bounds, + time, + BandSelection::new_single(band_idx), + ))) + } + + fn fold_method(&self) -> Self::FoldMethod { + self.fold_fn.clone() + } +} + +#[derive(Clone, Debug)] +pub struct DownsampleAccu { + pub output_tile_info: TileInformation, + pub output_grid: GridOrEmpty, + pub input_global_geo_transform: GeoTransform, + + pub time: Option, + pub cache_hint: CacheHint, + pub pool: Arc, +} + +impl DownsampleAccu { + pub fn new( + output_tile_info: TileInformation, + input_global_geo_transform: GeoTransform, + time: Option, + cache_hint: CacheHint, + pool: Arc, + ) -> Self { + DownsampleAccu { + output_tile_info, + output_grid: GridOrEmpty::new_empty_shape(output_tile_info.global_pixel_bounds()), + input_global_geo_transform, + time, + cache_hint, + pool, + } + } +} + +#[async_trait] +impl FoldTileAccu for DownsampleAccu { + type RasterType = T; + + async fn into_tile(self) -> Result> { + // TODO: later do conversation of accu into tile here + + let output_tile = RasterTile2D::new_with_tile_info( + self.time.expect("there is at least one input"), + self.output_tile_info, + 0, // TODO: need band? + self.output_grid.unbounded(), + self.cache_hint, + ); + + Ok(output_tile) + } + + fn thread_pool(&self) -> &Arc { + &self.pool + } +} + +impl FoldTileAccuMut for DownsampleAccu { + fn set_time(&mut self, time: TimeInterval) { + self.time = Some(time); + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.cache_hint = cache_hint; + } +} + +pub fn create_accu( + input_geo_transform: GeoTransform, + _output_geo_transform: GeoTransform, + tile_info: TileInformation, + _query_rect: &RasterQueryRectangle, + pool: Arc, + _tiling_specification: TilingSpecification, +) -> impl Future>> + use { + crate::util::spawn_blocking(move || { + DownsampleAccu::new( + tile_info, + input_geo_transform, + None, + CacheHint::max_duration(), + pool.clone(), + ) + }) + .map_err(From::from) +} + +pub fn fold_future( + accu: DownsampleAccu, + tile: RasterTile2D, +) -> impl Future>> +where + T: Pixel, +{ + crate::util::spawn_blocking_with_thread_pool(accu.pool.clone(), || fold_impl(accu, tile)).then( + |x| async move { + match x { + Ok(r) => Ok(r), + Err(e) => Err(e.into()), + } + }, + ) +} + +pub fn fold_impl(mut accu: DownsampleAccu, tile: RasterTile2D) -> DownsampleAccu +where + T: Pixel, +{ + // get the time now because it is not known when the accu was created + accu.set_time(tile.time); + accu.cache_hint.merge_with(&tile.cache_hint); + + // TODO: add a skip if both tiles are empty? + if tile.is_empty() { + // TODO: and ignore no-data. + return accu; + } + + // copy all input tiles into the accu to have all data for interpolation + let mut accu_tile = accu.output_grid.into_materialized_masked_grid(); + let in_tile_grid = tile.into_inner_positioned_grid(); + let accu_geo_transform = accu.output_tile_info.global_geo_transform; + let in_geo_transform = accu.input_global_geo_transform; + + let map_fn = |grid_idx: GridIdx2D, current_value: Option| -> Option { + let accu_pixel_coord = accu_geo_transform.grid_idx_to_pixel_center_coordinate_2d(grid_idx); // use center coordinate similar to ArcGIS + let source_pixel_idx = in_geo_transform.coordinate_to_grid_idx_2d(accu_pixel_coord); + + if in_tile_grid.contains(&source_pixel_idx) { + in_tile_grid.get_at_grid_index_unchecked(source_pixel_idx) + } else { + current_value + } + }; + + accu_tile.update_indexed_elements_parallel(map_fn); + + accu.output_grid = accu_tile.into(); + + accu +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, + TimeDescriptor, + }; + use crate::mock::{MockRasterSource, MockRasterSourceParams}; + use futures::StreamExt; + use geoengine_datatypes::primitives::TimeStep; + use geoengine_datatypes::raster::{Grid, GridShape2D, RasterDataType}; + use geoengine_datatypes::spatial_reference::SpatialReference; + use geoengine_datatypes::util::test::TestDefault; + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn nearest_neighbor_4() { + // In this test, 2x2 tiles with 4x4 pixels are downsampled using nearest neighbor to one tile with 4x4 pixels. The resolution is now 1/2 of the original resolution. + // The test uses the following input: + // + // _1, _2, _3, _4 | 21, 22, 23, 24 + // _5, _6, _7, _8 | 25, 26, 27, 28 + // _9, 10, 11, 12 | 29, 30, 31, 32 + // 13, 14, 15, 16 | 33, 34, 35, 36 + // ---------------+--------------- + // 41, 42, 43, 44 | 61, 62, 63, 64 + // 45, 46, 47, 48 | 65, 66, 67, 68 + // 49, 50, 51, 52 | 69, 70, 71, 72 + // 53, 54, 55, 56 | 73, 74, 75, 76 + // + // The input is downsampled to: + // + // _6. _8, 26, 28 + // 14, 16, 33, 36 + // 46, 48, 66, 68 + // 54, 56, 74, 76 + // + // The center of each pixel is mapped to a coordinate. Then, for this coordinate the nearest pixel center is selected. + // In this case, we have the special case that the pixel center of the target pixel hits the edge between two original pixels. + // The pixel which "owns" the edge is selected because pixels are defiend from the upper left edge. + // E.g. _6 is selected for the first pixel since it owns the edge between _1, _2, _5_ and _6. + + let in_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 1.0, -1.0); + let out_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 2.0, -2.0); + let tile_size_in_pixels = GridShape2D { + shape_array: [4, 4], + }; + + let exe_ctx = MockExecutionContext::new_with_tiling_spec_and_thread_count( + TilingSpecification::new(tile_size_in_pixels), + 8, + ); + + let data: Vec> = vec![ + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![ + 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, + ], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + ]; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + data.first().unwrap().time.start(), + data.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + in_geo_transform, + GridBoundingBox2D::new_min_max(0, 7, 0, 7).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let mrs1 = MockRasterSource { + params: MockRasterSourceParams { + data: data.clone(), + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + + let downsampler = Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_origin_reference: None, + output_resolution: DownsamplingResolution::Resolution(SpatialResolution { + x: out_geo_transform.x_pixel_size(), + y: out_geo_transform.y_pixel_size().abs(), + }), + }, + sources: SingleRasterSource { raster: mrs1 }, + } + .boxed(); + + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(0, 3, 0, 3).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); + + let query_ctx = exe_ctx.mock_query_context(ChunkByteSize::test_default()); + + let op = downsampler + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let qp = op.query_processor().unwrap().get_u8().unwrap(); + + let result = qp + .raster_query(query_rect, &query_ctx) + .await + .unwrap() + .collect::>() + .await; + + assert_eq!(result.len(), 1); + + let tile = result[0].as_ref().unwrap(); + let grid = tile.grid_array.clone().into_materialized_masked_grid(); + // _6. _8, 26, 28 + // 14, 16, 33, 36 + // 46, 48, 66, 68 + // 54, 56, 74, 76 + assert_eq!( + grid.inner_grid.data, + &[6, 8, 26, 28, 14, 16, 34, 36, 46, 48, 66, 68, 54, 56, 74, 76] + ); + } + + #[allow(clippy::too_many_lines)] + #[tokio::test] + async fn nearest_neighbor_3() { + // In this test, 3x3 tiles with 3x3 pixels are downsampled using nearest neighbor to one tile with 3x3 pixels. The resolution is now 1/3 of the original resolution. + // The test uses the following input: + // + // _0, _1, _2 | 10, 11, 12 | 20, 21, 22 + // _3, _4, _5 | 13, 14, 15 | 23, 24, 25 + // _6, _7, _8 | 16, 17, 18 | 26, 27, 28 + // -----------+------------+----------- + // 30, 31, 32 | 40, 41, 42 | 50, 51, 52 + // 33, 34, 35 | 43, 44, 45 | 53, 54, 55 + // 36, 37, 38 | 46, 47, 48 | 56, 57, 58 + // -----------+------------+----------- + // 60, 61, 62 | 70, 71, 72 | 80, 81, 82 + // 63, 64, 65 | 73, 74, 75 | 83, 84, 85 + // 66, 67, 68 | 76, 77, 78 | 86, 87, 88 + // + // The input is downsampled to: + // + // _4. 14, 24 + // 34, 44, 54 + // 64, 74, 84 + // + // The center of each pixel is mapped to a coordinate. Then, for this coordinate the nearest pixel center is selected. + // In this case, each pixel corresponds to the center of the corresponding tile. + // E.g. _4 is selected for the first pixel since it is in the center of the tile. + + let in_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 1.0, -1.0); + let out_geo_transform = GeoTransform::new(Coordinate2D::new(0.0, 0.0), 3.0, -3.0); + let tile_size_in_pixels = GridShape2D { + shape_array: [3, 3], + }; + + let exe_ctx = MockExecutionContext::new_with_tiling_spec_and_thread_count( + TilingSpecification::new(tile_size_in_pixels), + 8, + ); + + let data: Vec> = vec![ + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new(tile_size_in_pixels, vec![0, 1, 2, 3, 4, 5, 6, 7, 8]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![10, 11, 12, 13, 14, 15, 16, 17, 18], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![20, 21, 22, 23, 24, 25, 26, 27, 28], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![30, 31, 32, 33, 34, 35, 36, 37, 38], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![40, 41, 42, 43, 44, 45, 46, 47, 48], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [1, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![50, 51, 52, 53, 54, 55, 56, 57, 58], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 0].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![60, 61, 62, 63, 64, 65, 66, 67, 68], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 1].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![70, 71, 72, 73, 74, 75, 76, 77, 78], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [2, 2].into(), + band: 0, + global_geo_transform: in_geo_transform, + grid_array: Grid::new( + tile_size_in_pixels, + vec![80, 81, 82, 83, 84, 85, 86, 87, 88], + ) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + ]; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + data.first().unwrap().time.start(), + data.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + in_geo_transform, + GridBoundingBox2D::new_min_max(0, 8, 0, 8).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let mrs1 = MockRasterSource { + params: MockRasterSourceParams { + data: data.clone(), + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + + let downsampler = Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_origin_reference: None, + output_resolution: DownsamplingResolution::Resolution(SpatialResolution { + x: out_geo_transform.x_pixel_size(), + y: out_geo_transform.y_pixel_size().abs(), + }), + }, + sources: SingleRasterSource { raster: mrs1 }, + } + .boxed(); + + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(0, 2, 0, 2).unwrap(), + TimeInterval::new_unchecked(0, 5), + [0].try_into().unwrap(), + ); + + let query_ctx = exe_ctx.mock_query_context(ChunkByteSize::test_default()); + + let op = downsampler + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap(); + + let qp = op.query_processor().unwrap().get_u8().unwrap(); + + let result = qp + .raster_query(query_rect, &query_ctx) + .await + .unwrap() + .collect::>() + .await; + + assert_eq!(result.len(), 1); + + let tile = result[0].as_ref().unwrap(); + let grid = tile.grid_array.clone().into_materialized_masked_grid(); + // _4. 14, 24 + // 34, 44, 54 + // 64, 74, 84 + + assert_eq!(grid.inner_grid.data, &[4, 14, 24, 34, 44, 54, 64, 74, 84]); + } +} diff --git a/operators/src/processing/expression/raster_operator.rs b/operators/src/processing/expression/raster_operator.rs index cf9eaaa36..fd033802d 100644 --- a/operators/src/processing/expression/raster_operator.rs +++ b/operators/src/processing/expression/raster_operator.rs @@ -10,11 +10,12 @@ use crate::{ WorkflowOperatorPath, }, error::InvalidNumberOfExpressionInputBands, + optimization::{OptimizableOperator, OptimizationError}, processing::expression::canonicalize_name, util::Result, }; use async_trait::async_trait; -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::{primitives::SpatialResolution, raster::RasterDataType}; use geoengine_expression::{ DataType, ExpressionAst, ExpressionParser, LinkedExpression, Parameter, }; @@ -108,8 +109,7 @@ impl RasterOperator for Expression { data_type: self.params.output_type, spatial_reference: in_descriptor.spatial_reference, time: in_descriptor.time, - bbox: in_descriptor.bbox, - resolution: in_descriptor.resolution, + spatial_grid: in_descriptor.spatial_grid, bands: RasterBandDescriptors::new(vec![ self.params .output_band @@ -122,6 +122,7 @@ impl RasterOperator for Expression { path, result_descriptor, source, + expression_string: self.params.expression, expression, map_no_data: self.params.map_no_data, }; @@ -136,11 +137,13 @@ impl OperatorName for Expression { const TYPE_NAME: &'static str = "Expression"; } +#[derive(Clone)] pub struct InitializedExpression { name: CanonicOperatorName, path: WorkflowOperatorPath, result_descriptor: RasterResultDescriptor, source: Box, + expression_string: String, expression: ExpressionAst, map_no_data: bool, } @@ -217,24 +220,43 @@ impl InitializedRasterOperator for InitializedExpression { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(resolution)?; + + Ok(Expression { + params: ExpressionParams { + expression: self.expression_string.clone(), + output_type: self.result_descriptor.data_type, + output_band: Some(self.result_descriptor.bands[0].clone()), + map_no_data: self.map_no_data, + }, + sources: SingleRasterSource { + raster: self.source.optimize(resolution)?, + }, + } + .boxed()) + } } #[cfg(test)] mod tests { use super::*; use crate::engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, QueryProcessor, + MockExecutionContext, MultipleRasterSources, QueryProcessor, SpatialGridDescriptor, + TimeDescriptor, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::processing::{RasterStacker, RasterStackerParams}; use futures::StreamExt; use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds, Measurement}; - use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - Grid2D, GridOrEmpty, MapElements, MaskedGrid2D, RasterTile2D, RenameBands, TileInformation, - TilingSpecification, + Grid2D, GridBoundingBox2D, GridOrEmpty, MapElements, MaskedGrid2D, RasterTile2D, + RenameBands, TileInformation, TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; @@ -291,7 +313,6 @@ mod tests { async fn basic_unary() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -315,18 +336,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -352,7 +369,6 @@ mod tests { async fn unary_map_no_data() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -376,18 +392,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -413,7 +425,6 @@ mod tests { async fn basic_binary() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -448,18 +459,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -481,7 +488,6 @@ mod tests { async fn basic_coalesce() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -523,18 +529,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -563,7 +565,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -599,18 +600,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -644,7 +641,6 @@ mod tests { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -688,18 +684,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -721,7 +713,6 @@ mod tests { async fn it_classifies() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -765,18 +756,14 @@ mod tests { let processor = operator.query_processor().unwrap().get_u8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ctx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -805,7 +792,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -839,18 +825,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -904,9 +886,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::I8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, }, @@ -919,7 +903,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -954,18 +937,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -987,7 +966,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1025,18 +1003,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await @@ -1057,7 +1031,6 @@ mod tests { let no_data_value = 0; let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -1098,18 +1071,14 @@ mod tests { let processor = o.query_processor().unwrap().get_i8().unwrap(); - let ctx = MockQueryContext::new(1.into()); + let ctx = ectx.mock_query_context(1.into()); let result_stream = processor .query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (2., 0.).into(), - ), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ), &ctx, ) .await diff --git a/operators/src/processing/expression/raster_query_processor.rs b/operators/src/processing/expression/raster_query_processor.rs index 71769e446..29be82122 100644 --- a/operators/src/processing/expression/raster_query_processor.rs +++ b/operators/src/processing/expression/raster_query_processor.rs @@ -1,18 +1,19 @@ use super::RasterExpressionError; use crate::{ - engine::{BoxRasterQueryProcessor, QueryContext, QueryProcessor, RasterResultDescriptor}, + engine::{ + BoxRasterQueryProcessor, QueryContext, QueryProcessor, RasterQueryProcessor, + RasterResultDescriptor, + }, util::Result, }; use async_trait::async_trait; use futures::{StreamExt, TryStreamExt, stream::BoxStream}; use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, RasterQueryRectangle, SpatialPartition2D, TimeInterval, - }, + primitives::{BandSelection, CacheHint, RasterQueryRectangle, TimeInterval}, raster::{ - ConvertDataType, FromIndexFnParallel, GeoTransform, GridIdx2D, GridIndexAccess, - GridOrEmpty, GridOrEmpty2D, GridShape2D, GridShapeAccess, MapElementsParallel, Pixel, - RasterTile2D, + ConvertDataType, FromIndexFnParallel, GeoTransform, GridBoundingBox2D, GridIdx2D, + GridIndexAccess, GridOrEmpty, GridOrEmpty2D, GridShape2D, GridShapeAccess, + MapElementsParallel, Pixel, RasterTile2D, }, }; use geoengine_expression::LinkedExpression; @@ -62,7 +63,7 @@ where Tuple: ExpressionTupleProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -72,12 +73,7 @@ where ctx: &'b dyn QueryContext, ) -> Result>> { // rewrite query to request all input bands from the source. They are all combined in the single output band by means of the expression. - let source_query = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: BandSelection::first_n(Tuple::num_bands()), - }; + let source_query = query.select_attributes(BandSelection::first_n(Tuple::num_bands())); let stream = self.sources @@ -97,7 +93,7 @@ where ) = Tuple::metadata(&rasters); let program = self.program.clone(); - let map_no_data = self.map_no_data; + let map_no_data: bool = self.map_no_data; let out = crate::util::spawn_blocking_with_thread_pool( ctx.thread_pool().clone(), @@ -123,6 +119,23 @@ where } } +#[async_trait] +impl RasterQueryProcessor for ExpressionQueryProcessor +where + TO: Pixel, + Tuple: ExpressionTupleProcessor, +{ + type RasterType = TO; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.sources.time_query(query, ctx).await + } +} + #[async_trait] trait ExpressionTupleProcessor: Send + Sync { type Tuple: Send + 'static; @@ -154,6 +167,12 @@ trait ExpressionTupleProcessor: Send + Sync { ) -> Result>; fn num_bands() -> u32; + + async fn time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>>; } #[async_trait] @@ -236,6 +255,14 @@ where fn num_bands() -> u32 { 1 } + + async fn time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.raster.time_query(query, ctx).await + } } // TODO: implement this via macro for 2-8 sources @@ -262,7 +289,7 @@ where } // if there is no error, the source did not produce all bands, which likely means a bug in an operator return Err(crate::error::Error::MustNotHappen { - message: "source did not produce all bands".to_string(), + message: "source did not produce all bands (N: 2)".to_string(), }); } @@ -356,6 +383,14 @@ where fn num_bands() -> u32 { 2 } + + async fn time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.raster.time_query(query, ctx).await + } } type Function3 = fn(Option, Option, Option) -> Option; @@ -426,7 +461,7 @@ macro_rules! impl_expression_tuple_processor { } // if there is no error, the source did not produce all bands, which likely means a bug in an operator return Err(crate::error::Error::MustNotHappen { - message: "source did not produce all bands".to_string(), + message: format!("source did not produce all bands, (N: {})", $N), }); } @@ -510,7 +545,17 @@ macro_rules! impl_expression_tuple_processor { fn num_bands() -> u32 { $N } + + async fn time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.raster.time_query(query, ctx).await + } } + + }; // For any input, generate `f64, bool` diff --git a/operators/src/processing/expression/vector_operator.rs b/operators/src/processing/expression/vector_operator.rs index f8f35c25a..a781ed58c 100644 --- a/operators/src/processing/expression/vector_operator.rs +++ b/operators/src/processing/expression/vector_operator.rs @@ -9,6 +9,7 @@ use crate::{ VectorColumnInfo, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }, + optimization::OptimizationError, util::Result, }; use async_trait::async_trait; @@ -16,7 +17,7 @@ use futures::StreamExt; use futures::stream::BoxStream; use geoengine_datatypes::primitives::{ FeatureData, FeatureDataRef, FeatureDataType, FloatOptionsParIter, Geometry, Measurement, - MultiLineString, MultiPoint, MultiPolygon, VectorQueryRectangle, + MultiLineString, MultiPoint, MultiPolygon, SpatialResolution, VectorQueryRectangle, }; use geoengine_datatypes::util::arrow::ArrowTyped; use geoengine_datatypes::{ @@ -107,6 +108,7 @@ impl From for OutputColumn { struct InitializedVectorExpression { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: VectorExpressionParams, result_descriptor: VectorResultDescriptor, features: Box, expression: Arc, @@ -157,7 +159,7 @@ impl VectorOperator for VectorExpression { insert_new_column( &mut result_descriptor.columns, output_column_name.clone(), - self.params.output_measurement, + self.params.output_measurement.clone(), )?; DataType::Number } @@ -197,6 +199,7 @@ impl VectorOperator for VectorExpression { let initialized_operator = InitializedVectorExpression { name, path, + params: self.params.clone(), result_descriptor, features: initialized_source.vector, expression, @@ -460,6 +463,19 @@ impl InitializedVectorOperator for InitializedVectorExpression { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(VectorExpression { + params: self.params.clone(), + sources: SingleVectorSource { + vector: self.features.optimize(target_resolution)?, + }, + } + .boxed()) + } } /// A processor that evaluates an expression on the columns of a `FeatureCollection`. @@ -711,7 +727,7 @@ where mod tests { use super::*; use crate::{ - engine::{ChunkByteSize, MockExecutionContext, MockQueryContext, QueryProcessor}, + engine::{ChunkByteSize, MockExecutionContext, QueryProcessor}, mock::MockFeatureCollectionSource, }; use geoengine_datatypes::{ @@ -719,10 +735,7 @@ mod tests { ChunksEqualIgnoringCacheHint, IntoGeometryIterator, MultiPointCollection, MultiPolygonCollection, }, - primitives::{ - BoundingBox2D, ColumnSelection, MultiPoint, MultiPolygon, SpatialResolution, - TimeInterval, - }, + primitives::{BoundingBox2D, ColumnSelection, MultiPoint, MultiPolygon, TimeInterval}, util::test::TestDefault, }; @@ -786,6 +799,8 @@ mod tests { let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx = MockExecutionContext::test_default(); + let operator = VectorExpression { params: VectorExpressionParams { input_columns: vec!["foo".into()], @@ -797,22 +812,18 @@ mod tests { sources: point_source.into(), } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -876,6 +887,7 @@ mod tests { .unwrap(); let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx = MockExecutionContext::test_default(); let operator = VectorExpression { params: VectorExpressionParams { @@ -888,22 +900,18 @@ mod tests { sources: point_source.into(), } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await .unwrap(); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -988,12 +996,11 @@ mod tests { }, sources: polygons.into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1071,12 +1078,11 @@ mod tests { }, sources: polygons.into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1168,16 +1174,15 @@ mod tests { .boxed() .into(), }, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + VectorQueryRectangle::new( + BoundingBox2D::new( (0., 0.).into(), (NUMBER_OF_ROWS as f64, NUMBER_OF_ROWS as f64).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), ) .await; @@ -1215,7 +1220,8 @@ mod tests { let query_processor: Box> = operator.query_processor().unwrap().try_into().unwrap(); - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); diff --git a/operators/src/processing/interpolation/mod.rs b/operators/src/processing/interpolation/mod.rs index e897c7337..a22d0405b 100644 --- a/operators/src/processing/interpolation/mod.rs +++ b/operators/src/processing/interpolation/mod.rs @@ -9,19 +9,21 @@ use crate::engine::{ OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::{OptimizableOperator, OptimizationError}; +use crate::processing::{ + Downsampling, DownsamplingMethod, DownsamplingParams, DownsamplingResolution, +}; use crate::util::Result; use async_trait::async_trait; use futures::future::BoxFuture; use futures::stream::BoxStream; use futures::{Future, FutureExt, TryFuture, TryFutureExt}; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, - SpatialPartitioned, SpatialResolution, TimeInstance, TimeInterval, -}; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; +use geoengine_datatypes::primitives::{BandSelection, CacheHint, Coordinate2D}; +use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialResolution, TimeInterval}; use geoengine_datatypes::raster::{ - Bilinear, Blit, EmptyGrid2D, GeoTransform, GridOrEmpty, GridSize, InterpolationAlgorithm, - NearestNeighbor, Pixel, RasterTile2D, TileInformation, TilingSpecification, + Bilinear, ChangeGridBounds, GeoTransform, GridBlit, GridBoundingBox2D, GridOrEmpty, + InterpolationAlgorithm, NearestNeighbor, Pixel, RasterTile2D, TileInformation, + TilingSpecification, }; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; @@ -31,14 +33,15 @@ use snafu::{Snafu, ensure}; #[serde(rename_all = "camelCase")] pub struct InterpolationParams { pub interpolation: InterpolationMethod, - pub input_resolution: InputResolution, + pub output_resolution: InterpolationResolution, + pub output_origin_reference: Option, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] #[serde(rename_all = "camelCase", tag = "type")] -pub enum InputResolution { - Value(SpatialResolution), - Source, +pub enum InterpolationResolution { + Resolution(SpatialResolution), + Fraction { x: f64, y: f64 }, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] @@ -51,10 +54,13 @@ pub enum InterpolationMethod { #[derive(Debug, Snafu)] #[snafu(visibility(pub(crate)), context(suffix(false)), module(error))] pub enum InterpolationError { - #[snafu(display( - "The input resolution was defined as `source` but the source resolution is unknown.", - ))] - UnknownInputResolution, + #[snafu(display("The fraction used to interpolate must be >= 1, was {f}."))] + FractionMustBeOneOrLarger { f: f64 }, + #[snafu(display("The output resolution must be higher than the input resolution."))] + OutputMustBeHigherResolutionThanInput { + input: SpatialResolution, + output: SpatialResolution, + }, } pub type Interpolation = Operator; @@ -78,56 +84,100 @@ impl RasterOperator for Interpolation { .initialize_sources(path.clone(), context) .await?; let raster_source = initialized_sources.raster; + InitializedInterpolation::new_with_source_and_params( + name, + path, + raster_source, + &self.params, + context.tiling_specification(), + ) + .map(InitializedRasterOperator::boxed) + } + + span_fn!(Interpolation); +} + +pub struct InitializedInterpolation { + name: CanonicOperatorName, + output_result_descriptor: RasterResultDescriptor, + raster_source: O, + path: WorkflowOperatorPath, + interpolation_method: InterpolationMethod, + tiling_specification: TilingSpecification, +} + +impl InitializedInterpolation { + pub fn new_with_source_and_params( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + raster_source: O, + params: &InterpolationParams, + tiling_specification: TilingSpecification, + ) -> Result { let in_descriptor = raster_source.result_descriptor(); + let in_spatial_grid = in_descriptor.spatial_grid_descriptor(); + + let output_resolution = match params.output_resolution { + InterpolationResolution::Resolution(res) => { + ensure!( + res.x.abs() <= in_spatial_grid.spatial_resolution().x.abs(), + error::OutputMustBeHigherResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + ensure!( + res.y.abs() <= in_spatial_grid.spatial_resolution().y.abs(), + error::OutputMustBeHigherResolutionThanInput { + input: in_spatial_grid.spatial_resolution(), + output: res + } + ); + res + } - ensure!( - matches!(self.params.input_resolution, InputResolution::Value(_)) - || in_descriptor.resolution.is_some(), - error::UnknownInputResolution - ); + InterpolationResolution::Fraction { x, y } => { + ensure!(x >= 1.0, error::FractionMustBeOneOrLarger { f: x }); + ensure!(y >= 1.0, error::FractionMustBeOneOrLarger { f: y }); - let input_resolution = if let InputResolution::Value(res) = self.params.input_resolution { - res + SpatialResolution::new_unchecked( + in_spatial_grid.spatial_resolution().x / x, + in_spatial_grid.spatial_resolution().y.abs() / y, + ) + } + }; + + let out_spatial_grid = if let Some(oc) = params.output_origin_reference { + in_spatial_grid + .with_changed_resolution(output_resolution) + .with_moved_origin_to_nearest_grid_edge(oc) + .replace_origin(oc) } else { - in_descriptor.resolution.expect("checked in ensure") + in_spatial_grid.with_changed_resolution(output_resolution) }; let out_descriptor = RasterResultDescriptor { spatial_reference: in_descriptor.spatial_reference, data_type: in_descriptor.data_type, - bbox: in_descriptor.bbox, time: in_descriptor.time, - resolution: None, // after interpolation the resolution is uncapped + spatial_grid: out_spatial_grid, bands: in_descriptor.bands.clone(), }; let initialized_operator = InitializedInterpolation { name, path, - result_descriptor: out_descriptor, + output_result_descriptor: out_descriptor, raster_source, - interpolation_method: self.params.interpolation, - input_resolution, - tiling_specification: context.tiling_specification(), + interpolation_method: params.interpolation, + tiling_specification, }; - Ok(initialized_operator.boxed()) + Ok(initialized_operator) } - - span_fn!(Interpolation); -} - -pub struct InitializedInterpolation { - name: CanonicOperatorName, - path: WorkflowOperatorPath, - result_descriptor: RasterResultDescriptor, - raster_source: Box, - interpolation_method: InterpolationMethod, - input_resolution: SpatialResolution, - tiling_specification: TilingSpecification, } -impl InitializedRasterOperator for InitializedInterpolation { +impl InitializedRasterOperator for InitializedInterpolation { fn query_processor(&self) -> Result { let source_processor = self.raster_source.query_processor()?; @@ -135,15 +185,13 @@ impl InitializedRasterOperator for InitializedInterpolation { source_processor, p => match self.interpolation_method { InterpolationMethod::NearestNeighbor => InterploationProcessor::<_,_, NearestNeighbor>::new( p, - self.result_descriptor.clone(), - self.input_resolution, + self.output_result_descriptor.clone(), self.tiling_specification, ).boxed() .into(), InterpolationMethod::BiLinear =>InterploationProcessor::<_,_, Bilinear>::new( p, - self.result_descriptor.clone(), - self.input_resolution, + self.output_result_descriptor.clone(), self.tiling_specification, ).boxed() .into(), @@ -154,7 +202,7 @@ impl InitializedRasterOperator for InitializedInterpolation { } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.output_result_descriptor } fn canonic_name(&self) -> CanonicOperatorName { @@ -168,17 +216,105 @@ impl InitializedRasterOperator for InitializedInterpolation { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let out_descriptor = self.result_descriptor(); + let in_descriptor = self.raster_source.result_descriptor(); + + let input_resolution = in_descriptor.spatial_grid.spatial_resolution(); + + let new_origin = if in_descriptor.spatial_grid.geo_transform().origin_coordinate + == out_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate + { + None + } else { + Some( + out_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate, + ) + }; + + if input_resolution == target_resolution { + // special case where interpolation becomes redundant, unless it also regrids + + // TODO: source does not need to be optimized, but we need it as an `RasterOperator` and not `InitializedRasterOperator` + let optimzed_source = self.raster_source.optimize(target_resolution)?; + + if new_origin.is_some() { + return Ok(Interpolation { + params: InterpolationParams { + interpolation: self.interpolation_method, + output_resolution: InterpolationResolution::Resolution(target_resolution), + output_origin_reference: new_origin, + }, + sources: SingleRasterSource { + raster: optimzed_source, + }, + } + .boxed()); + } + return Ok(optimzed_source); + } + + // snap the input resolution to an overview level + let mut snapped_input_resolution = input_resolution; + + while snapped_input_resolution * 2.0 < target_resolution { + snapped_input_resolution = snapped_input_resolution * 2.0; + } + + let optimzed_source = self.raster_source.optimize(snapped_input_resolution)?; + + if snapped_input_resolution < target_resolution { + // result must be coarser than the source, so we need to convert to Downsampling + return Ok(Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution(target_resolution), + output_origin_reference: new_origin, + }, + sources: SingleRasterSource { + raster: optimzed_source, + }, + } + .boxed()); + } + + // target resolution is still finer than what the source produces + debug_assert!(snapped_input_resolution > target_resolution); + + Ok(Interpolation { + params: InterpolationParams { + interpolation: self.interpolation_method, + output_resolution: InterpolationResolution::Resolution(target_resolution), + output_origin_reference: new_origin, + }, + sources: SingleRasterSource { + raster: optimzed_source, + }, + } + .boxed()) + } } pub struct InterploationProcessor where Q: RasterQueryProcessor, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { source: Q, - result_descriptor: RasterResultDescriptor, - input_resolution: SpatialResolution, + out_result_descriptor: RasterResultDescriptor, tiling_specification: TilingSpecification, interpolation: PhantomData, } @@ -187,18 +323,16 @@ impl InterploationProcessor where Q: RasterQueryProcessor, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { pub fn new( source: Q, - result_descriptor: RasterResultDescriptor, - input_resolution: SpatialResolution, + out_result_descriptor: RasterResultDescriptor, tiling_specification: TilingSpecification, ) -> Self { Self { source, - result_descriptor, - input_resolution, + out_result_descriptor, tiling_specification, interpolation: PhantomData, } @@ -208,17 +342,12 @@ where #[async_trait] impl QueryProcessor for InterploationProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor + Send + Sync, P: Pixel, - I: InterpolationAlgorithm

, + I: InterpolationAlgorithm, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -228,41 +357,77 @@ where ctx: &'a dyn QueryContext, ) -> Result>> { // do not interpolate if the source resolution is already fine enough - if query.spatial_resolution.x >= self.input_resolution.x - && query.spatial_resolution.y >= self.input_resolution.y - { - // TODO: should we use the query or the input resolution here? + + let in_spatial_grid = self.source.result_descriptor().spatial_grid_descriptor(); + let out_spatial_grid = self.result_descriptor().spatial_grid_descriptor(); + + // if the output resolution is the same as the input resolution, we can just forward the query // TODO: except the origin changes? + if in_spatial_grid == out_spatial_grid { return self.source.query(query, ctx).await; } + let tiling_grid_definition = + out_spatial_grid.tiling_grid_definition(ctx.tiling_specification()); + + // This is the tiling strategy we want to fill + let tiling_strategy: geoengine_datatypes::raster::TilingStrategy = + tiling_grid_definition.generate_data_tiling_strategy(); + + let input_geo_transform = in_spatial_grid + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform(); + + let output_geo_transform = tiling_grid_definition.tiling_geo_transform(); + let sub_query = InterpolationSubQuery::<_, P, I> { - input_resolution: self.input_resolution, + input_geo_transform, + output_geo_transform, fold_fn: fold_future, tiling_specification: self.tiling_specification, phantom: PhantomData, _phantom_pixel_type: PhantomData, }; - Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( + let time_stream = self.time_query(query.time_interval(), ctx).await?; + + Ok(Box::pin(RasterSubQueryAdapter::<'a, P, _, _, _>::new( &self.source, query, - self.tiling_specification, + tiling_strategy, ctx, sub_query, - ) - .filter_and_fill( - crate::adapters::FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - )) + time_stream, + ))) } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.out_result_descriptor + } +} + +#[async_trait] +impl RasterQueryProcessor for InterploationProcessor +where + P: Pixel, + Q: RasterQueryProcessor + Send + Sync, + I: InterpolationAlgorithm, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await } } #[derive(Debug, Clone)] pub struct InterpolationSubQuery { - input_resolution: SpatialResolution, + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, // TODO remove because in adapter? fold_fn: F, tiling_specification: TilingSpecification, phantom: PhantomData, @@ -274,7 +439,7 @@ where T: Pixel, FoldM: Send + Sync + 'a + Clone + Fn(InterpolationAccu, RasterTile2D) -> FoldF, FoldF: Send + TryFuture, Error = crate::error::Error>, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { type FoldFuture = FoldF; @@ -290,6 +455,8 @@ where pool: &Arc, ) -> Self::TileAccuFuture { create_accu( + self.input_geo_transform, + self.output_geo_transform, tile_info, &query_rect, pool.clone(), @@ -302,23 +469,35 @@ where &self, tile_info: TileInformation, _query_rect: RasterQueryRectangle, - start_time: TimeInstance, + time: TimeInterval, band_idx: u32, ) -> Result> { // enlarge the spatial bounds in order to have the neighbor pixels for the interpolation - let spatial_bounds = tile_info.spatial_partition(); - let enlarge: Coordinate2D = (self.input_resolution.x, -self.input_resolution.y).into(); - let spatial_bounds = SpatialPartition2D::new( - spatial_bounds.upper_left(), - spatial_bounds.lower_right() + enlarge, - )?; - - Ok(Some(RasterQueryRectangle { - spatial_bounds, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: self.input_resolution, - attributes: band_idx.into(), - })) + + let tile_pixel_bounds = tile_info.global_pixel_bounds(); + let tile_spatial_bounds = self + .output_geo_transform + .grid_to_spatial_bounds(&tile_pixel_bounds); + let input_pixel_bounds = self + .input_geo_transform + .spatial_to_grid_bounds(&tile_spatial_bounds); + let enlarged_input_pixel_bounds = GridBoundingBox2D::new( + [ + input_pixel_bounds.y_min() - 1, + input_pixel_bounds.x_min() - 1, + ], + [ + input_pixel_bounds.y_max() + 1, + input_pixel_bounds.x_max() + 1, + ], + ) + .expect("max bounds must be larger then min bounds already"); + + Ok(Some(RasterQueryRectangle::new( + enlarged_input_pixel_bounds, + time, + BandSelection::new_single(band_idx), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -327,21 +506,30 @@ where } #[derive(Clone, Debug)] -pub struct InterpolationAccu> { +pub struct InterpolationAccu> { pub output_info: TileInformation, - pub input_tile: RasterTile2D, + pub input_tile: GridOrEmpty, + pub input_geo_transform: GeoTransform, + pub time: TimeInterval, + pub cache_hint: CacheHint, pub pool: Arc, phantom: PhantomData, } -impl> InterpolationAccu { +impl> InterpolationAccu { pub fn new( - input_tile: RasterTile2D, + input_tile: GridOrEmpty, + input_geo_transform: GeoTransform, + time: TimeInterval, + cache_hint: CacheHint, output_info: TileInformation, pool: Arc, ) -> Self { InterpolationAccu { input_tile, + input_geo_transform, + time, + cache_hint, output_info, pool, phantom: Default::default(), @@ -350,17 +538,32 @@ impl> InterpolationAccu { } #[async_trait] -impl> FoldTileAccu for InterpolationAccu { +impl> FoldTileAccu + for InterpolationAccu +{ type RasterType = T; async fn into_tile(self) -> Result> { // now that we collected all the input tile pixels we perform the actual interpolation let output_tile = crate::util::spawn_blocking_with_thread_pool(self.pool, move || { - I::interpolate(&self.input_tile, &self.output_info) + I::interpolate( + self.input_geo_transform, + &self.input_tile, + self.output_info.global_geo_transform, + self.output_info.global_pixel_bounds(), + ) }) .await??; + let output_tile = RasterTile2D::new_with_tile_info( + self.time, + self.output_info, + 0, + output_tile.unbounded(), + self.cache_hint, + ); + Ok(output_tile) } @@ -369,59 +572,58 @@ impl> FoldTileAccu for InterpolationAccu< } } -impl> FoldTileAccuMut for InterpolationAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.input_tile +impl> FoldTileAccuMut + for InterpolationAccu +{ + fn set_time(&mut self, time: TimeInterval) { + self.time = time; + } + + fn set_cache_hint(&mut self, cache_hint: CacheHint) { + self.cache_hint = cache_hint; } } -pub fn create_accu>( +pub fn create_accu>( + input_geo_transform: GeoTransform, + output_geo_transform: GeoTransform, tile_info: TileInformation, query_rect: &RasterQueryRectangle, pool: Arc, - tiling_specification: TilingSpecification, + _tiling_specification: TilingSpecification, ) -> impl Future>> + use { + let query_rect = query_rect.clone(); + // create an accumulator as a single tile that fits all the input tiles - let spatial_bounds = query_rect.spatial_bounds; - let spatial_resolution = query_rect.spatial_resolution; - let time_interval = query_rect.time_interval; + let time_interval = query_rect.time_interval(); crate::util::spawn_blocking(move || { - let tiling = tiling_specification.strategy(spatial_resolution.x, -spatial_resolution.y); - - let origin_coordinate = tiling - .tile_information_iterator(spatial_bounds) - .next() - .expect("a query contains at least one tile") - .spatial_partition() - .upper_left(); - - let geo_transform = GeoTransform::new( - origin_coordinate, - spatial_resolution.x, - -spatial_resolution.y, - ); - - let bbox = tiling.tile_grid_box(spatial_bounds); - - let shape = [ - bbox.axis_size_y() * tiling.tile_size_in_pixels.axis_size_y(), - bbox.axis_size_x() * tiling.tile_size_in_pixels.axis_size_x(), - ]; + let tile_pixel_bounds = tile_info.global_pixel_bounds(); + let tile_spatial_bounds = output_geo_transform.grid_to_spatial_bounds(&tile_pixel_bounds); + let input_pixel_bounds = input_geo_transform.spatial_to_grid_bounds(&tile_spatial_bounds); + let enlarged_input_pixel_bounds = GridBoundingBox2D::new( + [ + input_pixel_bounds.y_min() - 1, + input_pixel_bounds.x_min() - 1, + ], + [ + input_pixel_bounds.y_max() + 1, + input_pixel_bounds.x_max() + 1, + ], + ) + .expect("max bounds must be larger then min bounds already"); // create a non-aligned (w.r.t. the tiling specification) grid by setting the origin to the top-left of the tile and the tile-index to [0, 0] - let grid = EmptyGrid2D::new(shape.into()); + let grid = GridOrEmpty::new_empty_shape(enlarged_input_pixel_bounds); - let input_tile = RasterTile2D::new( + InterpolationAccu::new( + grid, + input_geo_transform, time_interval, - [0, 0].into(), - 0, - geo_transform, - GridOrEmpty::from(grid), CacheHint::max_duration(), - ); - - InterpolationAccu::new(input_tile, tile_info, pool) + tile_info, + pool, + ) }) .map_err(From::from) } @@ -432,11 +634,11 @@ pub fn fold_future( ) -> impl Future>> where T: Pixel, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { crate::util::spawn_blocking(|| fold_impl(accu, tile)).then(|x| async move { match x { - Ok(r) => r, + Ok(r) => Ok(r), Err(e) => Err(e.into()), } }) @@ -445,25 +647,23 @@ where pub fn fold_impl( mut accu: InterpolationAccu, tile: RasterTile2D, -) -> Result> +) -> InterpolationAccu where T: Pixel, - I: InterpolationAlgorithm, + I: InterpolationAlgorithm, { // get the time now because it is not known when the accu was created - accu.input_tile.time = tile.time; + accu.set_time(tile.time); + accu.cache_hint.merge_with(&tile.cache_hint); // TODO: add a skip if both tiles are empty? // copy all input tiles into the accu to have all data for interpolation - let mut accu_input_tile = accu.input_tile.into_materialized_tile(); - accu_input_tile.blit(tile)?; - - Ok(InterpolationAccu::new( - accu_input_tile.into(), - accu.output_info, - accu.pool, - )) + let in_tile = &tile.into_inner_positioned_grid(); + + accu.input_tile.grid_blit_from(in_tile); + + accu } #[cfg(test)] @@ -471,7 +671,9 @@ mod tests { use super::*; use futures::StreamExt; use geoengine_datatypes::{ - primitives::{RasterQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{ + Coordinate2D, RasterQueryRectangle, SpatialResolution, TimeInterval, TimeStep, + }, raster::{ Grid2D, GridOrEmpty, RasterDataType, RasterTile2D, RenameBands, TileInformation, TilingSpecification, @@ -482,8 +684,8 @@ mod tests { use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -491,17 +693,34 @@ mod tests { #[tokio::test] async fn nearest_neighbor_operator() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); + + // test raster: + // [0, 10) + // || 1 | 2 || 3 | 4 || + // || 5 | 6 || 7 | 8 || + // + // [10, 20) + // || 8 | 7 || 6 | 5 || + // || 4 | 3 || 2 | 1 || + + // exptected raster: + // [0, 10) + // ||1 | 1 || 2 | 2 || + // ||1 | 1 || 2 | 2 || + // ||5 | 5 || 6 | 6 || + // ||5 | 5 || 6 | 6 || let raster = make_raster(CacheHint::max_duration()); let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster }, } @@ -511,13 +730,12 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-4, 0], [-1, 7]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -528,48 +746,48 @@ mod tests { times.append(&mut vec![TimeInterval::new_unchecked(10, 20); 8]); let data = vec![ - vec![1, 2, 5, 6], - vec![2, 3, 6, 7], - vec![3, 4, 7, 8], - vec![4, 0, 8, 0], - vec![5, 6, 0, 0], - vec![6, 7, 0, 0], - vec![7, 8, 0, 0], - vec![8, 0, 0, 0], - vec![8, 7, 4, 3], - vec![7, 6, 3, 2], - vec![6, 5, 2, 1], - vec![5, 0, 1, 0], - vec![4, 3, 0, 0], - vec![3, 2, 0, 0], - vec![2, 1, 0, 0], - vec![1, 0, 0, 0], + vec![1; 4], + vec![2; 4], + vec![3; 4], + vec![4; 4], + vec![5; 4], + vec![6; 4], + vec![7; 4], + vec![8; 4], + vec![8; 4], + vec![7; 4], + vec![6; 4], + vec![5; 4], + vec![4; 4], + vec![3; 4], + vec![2; 4], + vec![1; 4], ]; let valid = vec![ vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], ]; for (i, tile) in result.into_iter().enumerate() { let tile = tile.into_materialized_tile(); assert_eq!(tile.time, times[i]); - assert_eq!(tile.grid_array.inner_grid.data, data[i]); assert_eq!(tile.grid_array.validity_mask.data, valid[i]); + assert_eq!(tile.grid_array.inner_grid.data, data[i]); } Ok(()) @@ -631,17 +849,30 @@ mod tests { ), ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::I8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some( + TimeInterval::new( + raster_tiles.first().unwrap().time.start(), + raster_tiles.last().unwrap().time.end(), + ) + .unwrap(), + ), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1.0, -1.0), + GridBoundingBox2D::new_min_max(-2, -1, 0, 3).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::I8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -649,10 +880,8 @@ mod tests { #[tokio::test] async fn it_attaches_cache_hint() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let cache_hint = CacheHint::seconds(1234); let raster = make_raster(cache_hint); @@ -660,7 +889,10 @@ mod tests { let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster }, } @@ -670,13 +902,12 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -694,15 +925,15 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_interpolates_multiple_bands() -> Result<()> { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [2, 2].into(), - )); - + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let operator = Interpolation { params: InterpolationParams { interpolation: InterpolationMethod::NearestNeighbor, - input_resolution: InputResolution::Value(SpatialResolution::one()), + output_resolution: InterpolationResolution::Resolution( + SpatialResolution::zero_point_five(), + ), + output_origin_reference: None, }, sources: SingleRasterSource { raster: RasterStacker { @@ -725,13 +956,12 @@ mod tests { let processor = operator.query_processor()?.get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 2.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::zero_point_five(), - attributes: [0, 1].try_into().unwrap(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-4, 0], [-1, 7]).unwrap(), + TimeInterval::new_unchecked(0, 20), + [0, 1].try_into().unwrap(), + ); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let result_stream = processor.query(query_rect, &query_ctx).await?; @@ -749,48 +979,50 @@ mod tests { .collect::>(); let data = vec![ - vec![1, 2, 5, 6], - vec![2, 3, 6, 7], - vec![3, 4, 7, 8], - vec![4, 0, 8, 0], - vec![5, 6, 0, 0], - vec![6, 7, 0, 0], - vec![7, 8, 0, 0], - vec![8, 0, 0, 0], - vec![8, 7, 4, 3], - vec![7, 6, 3, 2], - vec![6, 5, 2, 1], - vec![5, 0, 1, 0], - vec![4, 3, 0, 0], - vec![3, 2, 0, 0], - vec![2, 1, 0, 0], - vec![1, 0, 0, 0], + vec![1; 4], + vec![2; 4], + vec![3; 4], + vec![4; 4], + vec![5; 4], + vec![6; 4], + vec![7; 4], + vec![8; 4], + vec![8; 4], + vec![7; 4], + vec![6; 4], + vec![5; 4], + vec![4; 4], + vec![3; 4], + vec![2; 4], + vec![1; 4], ]; - let data = data - .clone() - .into_iter() - .zip(data) - .flat_map(|(a, b)| vec![a, b]) - .collect::>(); let valid = vec![ vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], vec![true; 4], vec![true; 4], vec![true; 4], - vec![true, false, true, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, true, false, false], - vec![true, false, false, false], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], + vec![true; 4], ]; + + let data = data + .clone() + .into_iter() + .zip(data) + .flat_map(|(a, b)| vec![a, b]) + .collect::>(); + let valid = valid .clone() .into_iter() diff --git a/operators/src/processing/line_simplification.rs b/operators/src/processing/line_simplification.rs index 32dd9a76b..fe1011f0e 100644 --- a/operators/src/processing/line_simplification.rs +++ b/operators/src/processing/line_simplification.rs @@ -5,6 +5,7 @@ use crate::{ TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }, + optimization::OptimizationError, util::Result, }; use async_trait::async_trait; @@ -43,11 +44,7 @@ impl VectorOperator for LineSimplification { path: WorkflowOperatorPath, context: &dyn ExecutionContext, ) -> Result> { - if self - .params - .epsilon - .map_or(false, |e| !e.is_finite() || e <= 0.0) - { + if self.params.epsilon <= 0.0 || !self.params.epsilon.is_finite() { return Err(LineSimplificationError::InvalidEpsilon.into()); } @@ -85,8 +82,7 @@ impl VectorOperator for LineSimplification { pub struct LineSimplificationParams { pub algorithm: LineSimplificationAlgorithm, /// The epsilon parameter is used to determine the maximum distance between the original and the simplified geometry. - /// If `None` is provided, the epsilon is derived by the query's [`SpatialResolution`]. - pub epsilon: Option, + pub epsilon: f64, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] @@ -102,7 +98,7 @@ pub struct InitializedLineSimplification { result_descriptor: VectorResultDescriptor, source: Box, algorithm: LineSimplificationAlgorithm, - epsilon: Option, + epsilon: f64, } impl InitializedVectorOperator for InitializedLineSimplification { @@ -174,6 +170,22 @@ impl InitializedVectorOperator for InitializedLineSimplification { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(LineSimplification { + params: LineSimplificationParams { + algorithm: self.algorithm, + epsilon: self.epsilon, + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } struct LineSimplificationProcessor @@ -188,15 +200,11 @@ where { source: P, _algorithm: A, - epsilon: Option, + epsilon: f64, } pub trait LineSimplificationAlgorithmImpl: Send + Sync { fn simplify(geometry_ref: In, epsilon: f64) -> Out; - - fn derive_epsilon(spatial_resolution: SpatialResolution) -> f64 { - f64::sqrt(spatial_resolution.x.powi(2) + spatial_resolution.y.powi(2)) / f64::sqrt(2.) - } } struct DouglasPeucker; @@ -232,12 +240,6 @@ impl<'c> LineSimplificationAlgorithmImpl, MultiLineString let geo_geometry = geo_geometry.simplify_vw_preserve(epsilon); geo_geometry.into() } - - fn derive_epsilon(spatial_resolution: SpatialResolution) -> f64 { - // for visvalingam, the epsilon is squared since it reflects some triangle area - // this is a heuristic, though - spatial_resolution.x * spatial_resolution.y - } } impl<'c> LineSimplificationAlgorithmImpl, MultiPolygon> for Visvalingam { @@ -248,12 +250,6 @@ impl<'c> LineSimplificationAlgorithmImpl, MultiPolygon> for let geo_geometry = geo_geometry.simplify_vw_preserve(epsilon); geo_geometry.into() } - - fn derive_epsilon(spatial_resolution: SpatialResolution) -> f64 { - // for visvalingam, the epsilon is squared since it reflects some triangle area - // this is a heuristic, though - spatial_resolution.x * spatial_resolution.y - } } impl LineSimplificationProcessor @@ -309,9 +305,10 @@ where ) -> Result>> { let chunks = self.source.query(query.clone(), ctx).await?; - let epsilon = self - .epsilon - .unwrap_or_else(|| A::derive_epsilon(query.spatial_resolution)); + let epsilon = self.epsilon; + if epsilon <= 0.0 || !epsilon.is_finite() { + return Err(LineSimplificationError::InvalidEpsilon.into()); + } let simplified_chunks = chunks.and_then(move |chunk| async move { crate::util::spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || { @@ -346,7 +343,7 @@ pub enum LineSimplificationError { mod tests { use super::*; use crate::{ - engine::{MockExecutionContext, MockQueryContext, StaticMetaData}, + engine::{MockExecutionContext, StaticMetaData}, mock::MockFeatureCollectionSource, source::{ OgrSource, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, @@ -360,7 +357,8 @@ mod tests { }, dataset::{DataId, DatasetId, NamedData}, primitives::{ - FeatureData, MultiLineString, MultiPoint, TimeInterval, {CacheHint, CacheTtlSeconds}, + BoundingBox2D, CacheHint, CacheTtlSeconds, FeatureData, MultiLineString, MultiPoint, + TimeInterval, }, spatial_reference::SpatialReference, test_data, @@ -371,7 +369,7 @@ mod tests { async fn test_ser_de() { let operator = LineSimplification { params: LineSimplificationParams { - epsilon: Some(1.0), + epsilon: 1.0, algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::multiple(vec![]) @@ -412,7 +410,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: Some(0.0), + epsilon: 0.0, algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::single( @@ -434,7 +432,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: Some(f64::NAN), + epsilon: f64::NAN, algorithm: LineSimplificationAlgorithm::Visvalingam, }, sources: MockFeatureCollectionSource::::single( @@ -456,7 +454,7 @@ mod tests { assert!( LineSimplification { params: LineSimplificationParams { - epsilon: None, + epsilon: 0.1, algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: MockFeatureCollectionSource::::single( @@ -502,7 +500,7 @@ mod tests { let simplification = LineSimplification { params: LineSimplificationParams { - epsilon: Some(1.0), + epsilon: 1.0, algorithm: LineSimplificationAlgorithm::DouglasPeucker, }, sources: source.into(), @@ -523,14 +521,14 @@ mod tests { .multi_line_string() .unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (4., 4.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let stream = processor.query(query_rectangle, &query_ctx).await.unwrap(); @@ -608,7 +606,7 @@ mod tests { let simplification = LineSimplification { params: LineSimplificationParams { - epsilon: None, + epsilon: 1., algorithm: LineSimplificationAlgorithm::Visvalingam, }, sources: OgrSource { @@ -639,15 +637,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = exe_ctx.mock_query_context_test_default(); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &query_context, ) .await diff --git a/operators/src/processing/map_query.rs b/operators/src/processing/map_query.rs index 534b1b662..719d2b84a 100644 --- a/operators/src/processing/map_query.rs +++ b/operators/src/processing/map_query.rs @@ -1,78 +1,24 @@ -use crate::adapters::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, -}; -use crate::engine::{ - QueryContext, RasterQueryProcessor, RasterResultDescriptor, VectorQueryProcessor, - VectorResultDescriptor, -}; +use crate::engine::{QueryContext, VectorQueryProcessor, VectorResultDescriptor}; use crate::util::Result; use async_trait::async_trait; -use futures::StreamExt; + use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{RasterQueryRectangle, VectorQueryRectangle}; -use geoengine_datatypes::raster::{RasterTile2D, TilingSpecification}; +use geoengine_datatypes::primitives::VectorQueryRectangle; /// This `QueryProcessor` allows to rewrite a query. It does not change the data. Results of the children are forwarded. -pub(crate) struct MapQueryProcessor { +pub(crate) struct MapQueryProcessor { source: S, - result_descriptor: R, query_fn: Q, - additional_data: A, -} - -impl MapQueryProcessor { - pub fn new(source: S, result_descriptor: R, query_fn: Q, additional_data: A) -> Self { - Self { - source, - result_descriptor, - query_fn, - additional_data, - } - } } -#[async_trait] -impl RasterQueryProcessor - for MapQueryProcessor -where - S: RasterQueryProcessor, - Q: Fn(RasterQueryRectangle) -> Result> + Sync + Send, -{ - type RasterType = S::RasterType; - async fn raster_query<'a>( - &'a self, - query: RasterQueryRectangle, - ctx: &'a dyn QueryContext, - ) -> Result>>> { - let rewritten_query = (self.query_fn)(query.clone())?; - - if let Some(rewritten_query) = rewritten_query { - self.source.raster_query(rewritten_query, ctx).await - } else { - tracing::debug!("Query was rewritten to empty query. Returning empty / filled stream."); - let s = futures::stream::empty(); - - // TODO: The input of the `SparseTilesFillAdapter` is empty here, so we can't derive the expiration, as there are no tiles to derive them from. - // As this is the result of the query not being rewritten, we should check if the expiration could also be `max`, because this error - // will be persistent and we might as well cache the empty stream. - Ok(SparseTilesFillAdapter::new_like_subquery( - s, - &query, - self.additional_data, - FillerTileCacheExpirationStrategy::NoCache, - FillerTimeBounds::from(query.time_interval), // TODO: derive this from the query once the child query can provide this. - ) - .boxed()) - } - } - - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor +impl MapQueryProcessor { + pub fn new(source: S, query_fn: Q) -> Self { + Self { source, query_fn } } } #[async_trait] -impl VectorQueryProcessor for MapQueryProcessor +impl VectorQueryProcessor for MapQueryProcessor where S: VectorQueryProcessor, Q: Fn(VectorQueryRectangle) -> Result> + Sync + Send, diff --git a/operators/src/processing/meteosat/mod.rs b/operators/src/processing/meteosat/mod.rs index 246f0f7df..a5f5aee30 100644 --- a/operators/src/processing/meteosat/mod.rs +++ b/operators/src/processing/meteosat/mod.rs @@ -38,27 +38,29 @@ mod test_util { use std::str::FromStr; use futures::StreamExt; - use geoengine_datatypes::hashmap; - use geoengine_datatypes::primitives::{BandSelection, CacheHint, CacheTtlSeconds}; - use geoengine_datatypes::util::test::TestDefault; - use num_traits::AsPrimitive; - use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + use geoengine_datatypes::hashmap; + use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, CacheTtlSeconds, Coordinate2D, + }; use geoengine_datatypes::primitives::{ ContinuousMeasurement, DateTime, DateTimeParseFormat, Measurement, RasterQueryRectangle, - SpatialPartition2D, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, - TimeStep, + TimeGranularity, TimeInstance, TimeInterval, TimeStep, }; use geoengine_datatypes::raster::{ - Grid2D, GridOrEmpty, GridOrEmpty2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties, - RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, TileInformation, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, GridOrEmpty2D, + GridShape2D, MaskedGrid2D, Pixel, RasterDataType, RasterProperties, RasterPropertiesEntry, + RasterPropertiesEntryType, RasterTile2D, TileInformation, }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; use geoengine_datatypes::util::Identifier; + use geoengine_datatypes::util::test::TestDefault; + use num_traits::AsPrimitive; use crate::engine::{ - MockExecutionContext, MockQueryContext, QueryProcessor, RasterBandDescriptor, - RasterBandDescriptors, RasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + MockExecutionContext, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, + WorkflowOperatorPath, }; use crate::mock::{MockRasterSource, MockRasterSourceParams}; use crate::processing::meteosat::{ @@ -88,7 +90,7 @@ mod test_util { let processor = op.query_processor().unwrap().get_f32().unwrap(); - let ctx = MockQueryContext::test_default(); + let ctx = ctx.mock_query_context_test_default(); let result_stream = processor.query(query, &ctx).await.unwrap(); let mut result: Vec>> = result_stream.collect().await; assert_eq!(1, result.len()); @@ -122,27 +124,22 @@ mod test_util { } pub(crate) fn _create_gdal_query() -> RasterQueryRectangle { - let sr = SpatialResolution::new_unchecked(3_000.403_165_817_261, 3_000.403_165_817_261); - let ul = (0., 0.).into(); - let lr = (599. * sr.x, -599. * sr.y).into(); - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked(ul, lr), - time_interval: TimeInterval::new_unchecked( + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(0, 599, 0, 599).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 0, 0)), TimeInstance::from(DateTime::new_utc(2012, 12, 12, 12, 15, 0)), ), - spatial_resolution: sr, - attributes: BandSelection::first(), - } + BandSelection::first(), + ) } pub(crate) fn create_mock_query() -> RasterQueryRectangle { - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - } + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + Default::default(), + BandSelection::first(), + ) } pub(crate) fn create_mock_source( @@ -191,9 +188,11 @@ mod test_util { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::F32, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., -3.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 2]).unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), measurement.unwrap_or_else(|| { @@ -214,6 +213,11 @@ mod test_util { let dataset_name = NamedData::with_system_name("gdal-ds"); let no_data_value = Some(0.); + let origin_coordinate: Coordinate2D = + (-5_570_248.477_339_745, 5_570_248.477_339_745).into(); + let x_pixel_size = 3_000.403_165_817_261; + let y_pixel_size = -3_000.403_165_817_261; + let meta = GdalMetaDataRegular { data_time: TimeInterval::new_unchecked( TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(), @@ -233,9 +237,9 @@ mod test_util { file_path: test_data!("raster/msg/%_START_TIME_%.tif").into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-5_570_248.477_339_745, 5_570_248.477_339_745).into(), - x_pixel_size: 3_000.403_165_817_261, - y_pixel_size: -3_000.403_165_817_261, + origin_coordinate, + x_pixel_size, + y_pixel_size, }, width: 3712, height: 3712, @@ -268,9 +272,20 @@ mod test_util { data_type: RasterDataType::I16, spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81) .into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_origin_at_start( + TimeInterval::new_unchecked( + TimeInstance::from_str("2012-12-12T12:00:00.000Z").unwrap(), + TimeInstance::from_str("2020-12-12T12:00:00.000Z").unwrap(), + ), + TimeStep { + granularity: TimeGranularity::Minutes, + step: 15, + }, + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(origin_coordinate, x_pixel_size, y_pixel_size), + GridShape2D::new_2d(3712, 3712).bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), Measurement::Continuous(ContinuousMeasurement { @@ -285,7 +300,7 @@ mod test_util { ctx.add_meta_data(dataset_id, dataset_name.clone(), Box::new(meta)); GdalSource { - params: GdalSourceParameters { data: dataset_name }, + params: GdalSourceParameters::new(dataset_name), } } } diff --git a/operators/src/processing/meteosat/radiance.rs b/operators/src/processing/meteosat/radiance.rs index db3dbae9e..915171e3c 100644 --- a/operators/src/processing/meteosat/radiance.rs +++ b/operators/src/processing/meteosat/radiance.rs @@ -2,20 +2,21 @@ use std::sync::Arc; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, - TypedRasterQueryProcessor, WorkflowOperatorPath, + OperatorName, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, + WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; -use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, SpatialResolution, }; use geoengine_datatypes::raster::{ - MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, RasterTile2D, + GridBoundingBox2D, MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, + RasterTile2D, }; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; @@ -112,8 +113,7 @@ impl RasterOperator for Radiance { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -195,6 +195,19 @@ impl InitializedRasterOperator for InitializedRadiance { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Radiance { + params: RadianceParams {}, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } struct RadianceProcessor @@ -248,24 +261,19 @@ where #[async_trait] impl QueryProcessor for RadianceProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor, P: Pixel, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; - type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; async fn _query<'a>( &'a self, query: RasterQueryRectangle, - ctx: &'a dyn QueryContext, - ) -> Result>> { + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { let src = self.source.query(query, ctx).await?; let rs = src.and_then(move |tile| self.process_tile_async(tile, ctx.thread_pool().clone())); Ok(rs.boxed()) @@ -276,6 +284,24 @@ where } } +#[async_trait] +impl RasterQueryProcessor for RadianceProcessor +where + Q: RasterQueryProcessor, + P: Pixel, +{ + type RasterType = PixelOut; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use crate::engine::{MockExecutionContext, RasterOperator, SingleRasterSource}; @@ -285,7 +311,7 @@ mod tests { ClassificationMeasurement, ContinuousMeasurement, Measurement, }; use geoengine_datatypes::raster::{EmptyGrid2D, Grid2D, MaskedGrid2D, TilingSpecification}; - use std::collections::HashMap; + use std::collections::BTreeMap; // #[tokio::test] // async fn test_msg_raster() { @@ -312,7 +338,6 @@ mod tests { async fn test_ok() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -351,7 +376,6 @@ mod tests { async fn test_empty_raster() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -387,7 +411,6 @@ mod tests { async fn test_missing_offset() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -415,7 +438,6 @@ mod tests { async fn test_missing_slope() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -444,7 +466,6 @@ mod tests { async fn test_invalid_measurement_unitless() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -474,7 +495,6 @@ mod tests { async fn test_invalid_measurement_continuous() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -511,7 +531,6 @@ mod tests { async fn test_invalid_measurement_classification() { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -525,7 +544,7 @@ mod tests { None, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ); diff --git a/operators/src/processing/meteosat/reflectance.rs b/operators/src/processing/meteosat/reflectance.rs index 1aa12a99c..9685ee343 100644 --- a/operators/src/processing/meteosat/reflectance.rs +++ b/operators/src/processing/meteosat/reflectance.rs @@ -2,10 +2,11 @@ use std::sync::Arc; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, - TypedRasterQueryProcessor, WorkflowOperatorPath, + OperatorName, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, + WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use TypedRasterQueryProcessor::F32 as QueryProcessorOut; use async_trait::async_trait; @@ -13,14 +14,14 @@ use num_traits::AsPrimitive; use rayon::ThreadPool; use crate::error::Error; -use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, DateTime, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, SpatialResolution, }; use geoengine_datatypes::raster::{ - GridIdx2D, MapIndexedElementsParallel, RasterDataType, RasterPropertiesKey, RasterTile2D, + GridBoundingBox2D, GridIdx2D, MapIndexedElementsParallel, RasterDataType, RasterPropertiesKey, + RasterTile2D, }; use serde::{Deserialize, Serialize}; @@ -117,8 +118,7 @@ impl RasterOperator for Reflectance { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -179,6 +179,19 @@ impl InitializedRasterOperator for InitializedReflectance { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Reflectance { + params: self.params, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } struct ReflectanceProcessor @@ -296,23 +309,18 @@ fn calculate_esd(timestamp: &DateTime) -> f64 { #[async_trait] impl QueryProcessor for ReflectanceProcessor where - Q: QueryProcessor< - Output = RasterTile2D, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; - type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; async fn _query<'a>( &'a self, query: RasterQueryRectangle, - ctx: &'a dyn QueryContext, - ) -> Result>> { + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { let src = self.source.query(query, ctx).await?; let rs = src.and_then(move |tile| self.process_tile_async(tile, ctx.thread_pool().clone())); Ok(rs.boxed()) @@ -323,6 +331,23 @@ where } } +#[async_trait] +impl RasterQueryProcessor for ReflectanceProcessor +where + Q: RasterQueryProcessor, +{ + type RasterType = PixelOut; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use crate::engine::{MockExecutionContext, RasterOperator, SingleRasterSource}; @@ -335,7 +360,7 @@ mod tests { use geoengine_datatypes::raster::{ EmptyGrid2D, Grid2D, GridOrEmpty, MaskedGrid2D, RasterTile2D, TilingSpecification, }; - use std::collections::HashMap; + use std::collections::BTreeMap; async fn process_mock( params: ReflectanceParams, @@ -346,7 +371,6 @@ mod tests { ) -> Result> { let tile_size_in_pixels = [3, 2].into(); let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), tile_size_in_pixels, }; @@ -617,7 +641,7 @@ mod tests { false, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ) .await; diff --git a/operators/src/processing/meteosat/temperature.rs b/operators/src/processing/meteosat/temperature.rs index e693f1bab..10699e386 100644 --- a/operators/src/processing/meteosat/temperature.rs +++ b/operators/src/processing/meteosat/temperature.rs @@ -1,28 +1,26 @@ -use std::sync::Arc; - use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, - TypedRasterQueryProcessor, WorkflowOperatorPath, + OperatorName, QueryProcessor, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, + WorkflowOperatorPath, }; +use crate::error::Error; +use crate::optimization::OptimizationError; use crate::util::Result; -use async_trait::async_trait; -use rayon::ThreadPool; - use TypedRasterQueryProcessor::F32 as QueryProcessorOut; - -use crate::error::Error; -use futures::stream::BoxStream; +use async_trait::async_trait; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::primitives::{ BandSelection, ClassificationMeasurement, ContinuousMeasurement, Measurement, - RasterQueryRectangle, SpatialPartition2D, + RasterQueryRectangle, SpatialResolution, }; use geoengine_datatypes::raster::{ - MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, RasterTile2D, + GridBoundingBox2D, MapElementsParallel, Pixel, RasterDataType, RasterPropertiesKey, + RasterTile2D, }; +use rayon::ThreadPool; use serde::{Deserialize, Serialize}; +use std::sync::Arc; // Output type is always f32 type PixelOut = f32; @@ -112,8 +110,7 @@ impl RasterOperator for Temperature { spatial_reference: in_desc.spatial_reference, data_type: RasterOut, time: in_desc.time, - bbox: in_desc.bbox, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: RasterBandDescriptors::new( in_desc .bands @@ -196,6 +193,19 @@ impl InitializedRasterOperator for InitializedTemperature { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Temperature { + params: self.params.clone(), + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } struct TemperatureProcessor @@ -295,27 +305,25 @@ fn create_lookup_table(channel: &Channel, offset: f64, slope: f64, _pool: &Threa #[async_trait] impl QueryProcessor for TemperatureProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, P: Pixel, + Q: RasterQueryProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; - type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; async fn _query<'a>( &'a self, query: RasterQueryRectangle, - ctx: &'a dyn QueryContext, - ) -> Result>> { - let src = self.source.query(query, ctx).await?; - let rs = src.and_then(move |tile| self.process_tile_async(tile, ctx.thread_pool().clone())); - Ok(rs.boxed()) + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { + let src = self + .source + .raster_query(query, ctx) + .await? + .and_then(move |tile| self.process_tile_async(tile, ctx.thread_pool().clone())); + Ok(src.boxed()) } fn result_descriptor(&self) -> &Self::ResultDescription { @@ -323,6 +331,24 @@ where } } +#[async_trait] +impl RasterQueryProcessor for TemperatureProcessor +where + Q: RasterQueryProcessor, + P: Pixel, +{ + type RasterType = PixelOut; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use crate::engine::{MockExecutionContext, RasterOperator, SingleRasterSource}; @@ -332,7 +358,7 @@ mod tests { ClassificationMeasurement, ContinuousMeasurement, Measurement, }; use geoengine_datatypes::raster::{EmptyGrid2D, Grid2D, MaskedGrid2D, TilingSpecification}; - use std::collections::HashMap; + use std::collections::BTreeMap; // #[tokio::test] // async fn test_msg_raster() { @@ -357,7 +383,7 @@ mod tests { #[tokio::test] async fn test_empty_ok() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -390,7 +416,7 @@ mod tests { #[tokio::test] async fn test_ok() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -432,7 +458,7 @@ mod tests { #[tokio::test] async fn test_ok_force_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -472,7 +498,7 @@ mod tests { #[tokio::test] async fn test_ok_illegal_input_to_masked() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -522,7 +548,7 @@ mod tests { #[tokio::test] async fn test_invalid_force_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -548,7 +574,7 @@ mod tests { #[tokio::test] async fn test_missing_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -572,7 +598,7 @@ mod tests { #[tokio::test] async fn test_invalid_satellite() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -596,7 +622,7 @@ mod tests { #[tokio::test] async fn test_missing_channel() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -620,7 +646,7 @@ mod tests { #[tokio::test] async fn test_invalid_channel() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -644,7 +670,7 @@ mod tests { #[tokio::test] async fn test_missing_slope() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -668,7 +694,7 @@ mod tests { #[tokio::test] async fn test_missing_offset() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -692,7 +718,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_unitless() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -717,7 +743,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_continuous() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let res = test_util::process( @@ -749,7 +775,7 @@ mod tests { #[tokio::test] async fn test_invalid_measurement_classification() { - let tiling_specification = TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()); + let tiling_specification = TilingSpecification::new([3, 2].into()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); @@ -761,7 +787,7 @@ mod tests { None, Some(Measurement::Classification(ClassificationMeasurement { measurement: "invalid".into(), - classes: HashMap::new(), + classes: BTreeMap::new(), })), ); diff --git a/operators/src/processing/mod.rs b/operators/src/processing/mod.rs index d80848b55..dfde49cab 100644 --- a/operators/src/processing/mod.rs +++ b/operators/src/processing/mod.rs @@ -2,6 +2,7 @@ mod band_neighborhood_aggregate; mod bandwise_expression; mod circle_merging_quadtree; mod column_range_filter; +mod downsample; mod expression; mod interpolation; mod line_simplification; @@ -26,11 +27,19 @@ pub use band_neighborhood_aggregate::{ pub use circle_merging_quadtree::{ InitializedVisualPointClustering, VisualPointClustering, VisualPointClusteringParams, }; +pub use column_range_filter::{ColumnRangeFilter, ColumnRangeFilterParams}; +pub use downsample::{ + Downsampling, DownsamplingError, DownsamplingMethod, DownsamplingParams, + DownsamplingResolution, InitializedDownsampling, +}; pub use expression::{ Expression, ExpressionParams, RasterExpressionError, VectorExpression, VectorExpressionError, VectorExpressionParams, initialize_expression_dependencies, }; -pub use interpolation::{Interpolation, InterpolationError, InterpolationParams}; +pub use interpolation::{ + InitializedInterpolation, Interpolation, InterpolationError, InterpolationMethod, + InterpolationParams, InterpolationResolution, +}; pub use line_simplification::{ LineSimplification, LineSimplificationError, LineSimplificationParams, }; @@ -50,8 +59,10 @@ pub use raster_vector_join::{ ColumnNames, FeatureAggregationMethod, RasterVectorJoin, RasterVectorJoinParams, TemporalAggregationMethod, }; +pub use rasterization::{Rasterization, RasterizationParams}; pub use reprojection::{ - InitializedRasterReprojection, InitializedVectorReprojection, Reprojection, ReprojectionParams, + DeriveOutRasterSpecsSource, InitializedRasterReprojection, InitializedVectorReprojection, + Reprojection, ReprojectionParams, }; pub use temporal_raster_aggregation::{ Aggregation, TemporalRasterAggregation, TemporalRasterAggregationParameters, diff --git a/operators/src/processing/neighborhood_aggregate/aggregate.rs b/operators/src/processing/neighborhood_aggregate/aggregate.rs index 27cb3676d..ad853c04c 100644 --- a/operators/src/processing/neighborhood_aggregate/aggregate.rs +++ b/operators/src/processing/neighborhood_aggregate/aggregate.rs @@ -53,15 +53,6 @@ impl Neighborhood { self.matrix.axis_size_x() / 2 } - pub fn x_width(&self) -> usize { - self.matrix.axis_size_x() - } - - /// Specifies the y extent beginning from the center pixel - pub fn y_width(&self) -> usize { - self.matrix.axis_size_y() - } - /// Specifies the x extent right of one pixel pub fn y_radius(&self) -> usize { self.matrix.axis_size_y() / 2 diff --git a/operators/src/processing/neighborhood_aggregate/mod.rs b/operators/src/processing/neighborhood_aggregate/mod.rs index 5214b6248..e4c08730d 100644 --- a/operators/src/processing/neighborhood_aggregate/mod.rs +++ b/operators/src/processing/neighborhood_aggregate/mod.rs @@ -10,12 +10,13 @@ use crate::engine::{ OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D}; +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; use geoengine_datatypes::raster::{ - Grid2D, GridShape2D, GridSize, Pixel, RasterTile2D, TilingSpecification, + Grid2D, GridBoundingBox2D, GridShape2D, GridSize, Pixel, RasterTile2D, TilingSpecification, }; use num::Integer; use num_traits::AsPrimitive; @@ -157,10 +158,10 @@ impl RasterOperator for NeighborhoodAggregate { let initialized_operator = InitializedNeighborhoodAggregate { name, path, + params: self.params.clone(), result_descriptor: raster_source.result_descriptor().clone(), raster_source, neighborhood: self.params.neighborhood.try_into()?, - aggregate_function: self.params.aggregate_function, tiling_specification, }; @@ -173,10 +174,10 @@ impl RasterOperator for NeighborhoodAggregate { pub struct InitializedNeighborhoodAggregate { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: NeighborhoodAggregateParams, result_descriptor: RasterResultDescriptor, raster_source: Box, neighborhood: Neighborhood, - aggregate_function: AggregateFunctionParams, tiling_specification: TilingSpecification, } @@ -185,7 +186,7 @@ impl InitializedRasterOperator for InitializedNeighborhoodAggregate { let source_processor = self.raster_source.query_processor()?; let res = call_on_generic_raster_processor!( - source_processor, p => match &self.aggregate_function { + source_processor, p => match &self.params.aggregate_function { AggregateFunctionParams::Sum => NeighborhoodAggregateProcessor::<_,_, Sum>::new( p, self.tiling_specification, @@ -219,6 +220,19 @@ impl InitializedRasterOperator for InitializedNeighborhoodAggregate { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(NeighborhoodAggregate { + params: self.params.clone(), + sources: SingleRasterSource { + raster: self.raster_source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct NeighborhoodAggregateProcessor { @@ -250,18 +264,13 @@ where #[async_trait] impl QueryProcessor for NeighborhoodAggregateProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor + Send + Sync, P: Pixel, f64: AsPrimitive

, A: AggregateFunction + 'static, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -271,21 +280,26 @@ where ctx: &'a dyn QueryContext, ) -> Result>> { stack_individual_aligned_raster_bands(&query, ctx, |query, ctx| async move { - let sub_query = NeighborhoodAggregateTileNeighborhood::::new( - self.neighborhood.clone(), - self.tiling_specification, - ); + let sub_query = + NeighborhoodAggregateTileNeighborhood::::new(self.neighborhood.clone()); + + let time_stream = self.source.time_query(query.time_interval(), ctx).await?; - Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( + let tiling_strat = self + .source + .result_descriptor() + .tiling_grid_definition(self.tiling_specification) + .generate_data_tiling_strategy(); + + let sq = RasterSubQueryAdapter::<'a, P, _, _, _>::new( &self.source, query, - self.tiling_specification, + tiling_strat, ctx, sub_query, - ) - .filter_and_fill( - crate::adapters::FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - )) + time_stream, + ); + Ok(sq.box_pin()) }) .await } @@ -295,15 +309,34 @@ where } } +#[async_trait] +impl RasterQueryProcessor for NeighborhoodAggregateProcessor +where + P: Pixel, + f64: AsPrimitive

, + A: AggregateFunction + 'static, + Q: RasterQueryProcessor + Send + Sync, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use super::*; - use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - RasterOperator, RasterResultDescriptor, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, RasterOperator, + RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{RasterStacker, RasterStackerParams}, @@ -315,12 +348,12 @@ mod tests { dataset::NamedData, operations::image::{Colorizer, RgbaColor}, primitives::{ - CacheHint, DateTime, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, - TimeInstance, TimeInterval, + CacheHint, Coordinate2D, DateTime, RasterQueryRectangle, SpatialPartition2D, + TimeInstance, TimeInterval, TimeStep, }, raster::{ - Grid2D, GridOrEmpty, RasterDataType, RasterTile2D, RenameBands, TileInformation, - TilesEqualIgnoringCacheHint, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, RasterDataType, RasterTile2D, + RenameBands, TileInformation, TilesEqualIgnoringCacheHint, TilingSpecification, }, spatial_reference::SpatialReference, test_data, @@ -338,9 +371,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("matrix-input"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("matrix-input")), } .boxed(), }, @@ -367,7 +398,8 @@ mod tests { "raster": { "type": "GdalSource", "params": { - "data": "matrix-input" + "data": "matrix-input", + "overviewLevel": null } } } @@ -387,9 +419,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("matrix-input"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name("matrix-input")), } .boxed(), }, @@ -412,7 +442,8 @@ mod tests { "raster": { "type": "GdalSource", "params": { - "data": "matrix-input" + "data": "matrix-input", + "overviewLevel": null } } } @@ -439,10 +470,8 @@ mod tests { #[tokio::test] async fn test_mean_convolution() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let raster = make_raster(); @@ -462,13 +491,12 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); @@ -502,10 +530,8 @@ mod tests { #[tokio::test] async fn check_make_raster() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let raster = make_raster(); @@ -516,13 +542,12 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); @@ -656,17 +681,27 @@ mod tests { ), ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::I8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + raster_tiles.first().unwrap().time.start(), + raster_tiles.last().unwrap().time.end(), + )), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 6]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::I8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() @@ -690,7 +725,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new_with_overview_level(ndvi_id, 10), } .boxed(), }, @@ -701,15 +736,19 @@ mod tests { .unwrap(); let processor = operator.query_processor().unwrap().get_u8().unwrap(); - - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let result_descriptor = processor.result_descriptor(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + + let query_rect = RasterQueryRectangle::new( + result_descriptor + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds( + &SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + ), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); let colorizer = Colorizer::linear_gradient( vec![ @@ -736,13 +775,13 @@ mod tests { .await .unwrap(); + // Use for getting the image to compare against + // geoengine_datatypes::util::test::save_test_bytes(&bytes, "gaussian_blur.png"); + assert_eq!( bytes, include_bytes!("../../../../test_data/wms/gaussian_blur.png") ); - - // Use for getting the image to compare against - // save_test_bytes(&bytes, "gaussian_blur.png"); } #[tokio::test] @@ -759,7 +798,7 @@ mod tests { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new_with_overview_level(ndvi_id, 10), } .boxed(), }, @@ -770,15 +809,22 @@ mod tests { .unwrap(); let processor = operator.query_processor().unwrap().get_u8().unwrap(); + let result_descriptor = processor.result_descriptor(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); + + let query_rect = RasterQueryRectangle::new( + result_descriptor + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds( + &SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(), + ), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); + + // let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); let colorizer = Colorizer::linear_gradient( vec![ @@ -805,18 +851,16 @@ mod tests { .await .unwrap(); - assert_image_equals(test_data!("wms/partial_derivative.png"), &bytes); - // Use for getting the image to compare against // save_test_bytes(&bytes, "sobel_filter.png"); + + assert_image_equals(test_data!("wms/partial_derivative.png"), &bytes); } #[tokio::test] async fn test_mean_convolution_multi_bands() { - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 3].into(), - )); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 3].into())); let operator = NeighborhoodAggregate { params: NeighborhoodAggregateParams { @@ -844,13 +888,12 @@ mod tests { let processor = operator.query_processor().unwrap().get_i8().unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 3.).into(), (6., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::new(vec![0, 2]).unwrap(), - }; - let query_ctx = MockQueryContext::test_default(); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 5]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::new(vec![0, 2]).unwrap(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let result_stream = processor.query(query_rect, &query_ctx).await.unwrap(); diff --git a/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs b/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs index b81d9f528..36cbdd04e 100644 --- a/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs +++ b/operators/src/processing/neighborhood_aggregate/tile_sub_query.rs @@ -5,16 +5,13 @@ use async_trait::async_trait; use futures::future::BoxFuture; use futures::{FutureExt, TryFutureExt}; use geoengine_datatypes::primitives::CacheHint; -use geoengine_datatypes::primitives::{AxisAlignedRectangle, SpatialPartitioned}; use geoengine_datatypes::raster::{ - Blit, EmptyGrid, EmptyGrid2D, FromIndexFnParallel, GeoTransform, GridIdx, GridIdx2D, - GridIndexAccess, GridOrEmpty, GridSize, + ChangeGridBounds, FromIndexFnParallel, GridBlit, GridBoundingBox2D, GridContains, GridIdx, + GridIdx2D, GridIndexAccess, GridOrEmpty, GridSize, }; use geoengine_datatypes::{ - primitives::{ - Coordinate2D, RasterQueryRectangle, SpatialPartition2D, TimeInstance, TimeInterval, - }, - raster::{Pixel, RasterTile2D, TileInformation, TilingSpecification}, + primitives::{RasterQueryRectangle, TimeInterval}, + raster::{Pixel, RasterTile2D, TileInformation}, }; use num_traits::AsPrimitive; use rayon::ThreadPool; @@ -42,15 +39,13 @@ use tokio::task::JoinHandle; #[derive(Debug, Clone)] pub struct NeighborhoodAggregateTileNeighborhood { neighborhood: Neighborhood, - tiling_specification: TilingSpecification, _phantom_types: PhantomData<(P, A)>, } impl NeighborhoodAggregateTileNeighborhood { - pub fn new(neighborhood: Neighborhood, tiling_specification: TilingSpecification) -> Self { + pub fn new(neighborhood: Neighborhood) -> Self { Self { neighborhood, - tiling_specification, _phantom_types: PhantomData, } } @@ -77,16 +72,9 @@ where pool: &Arc, ) -> Self::TileAccuFuture { let pool = pool.clone(); - let tiling_specification = self.tiling_specification; let neighborhood = self.neighborhood.clone(); crate::util::spawn_blocking(move || { - create_enlarged_tile( - tile_info, - &query_rect, - pool, - tiling_specification, - neighborhood, - ) + create_enlarged_tile(tile_info, &query_rect, pool, neighborhood) }) .map_err(From::from) .boxed() @@ -96,28 +84,27 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, + _query_rect: RasterQueryRectangle, + time: TimeInterval, band_idx: u32, ) -> Result> { - let spatial_bounds = tile_info.spatial_partition(); + let pixel_bounds = tile_info.global_pixel_bounds(); - let margin_pixels = Coordinate2D::from(( - self.neighborhood.x_radius() as f64 * tile_info.global_geo_transform.x_pixel_size(), - self.neighborhood.y_radius() as f64 * tile_info.global_geo_transform.y_pixel_size(), - )); + let margin_y = self.neighborhood.y_radius() as isize; + let margin_x = self.neighborhood.x_radius() as isize; - let enlarged_spatial_bounds = SpatialPartition2D::new( - spatial_bounds.upper_left() - margin_pixels, - spatial_bounds.lower_right() + margin_pixels, + let larger_bounds = GridBoundingBox2D::new_min_max( + pixel_bounds.y_min() - margin_y, + pixel_bounds.y_max() + margin_y, + pixel_bounds.x_min() - margin_x, + pixel_bounds.x_max() + margin_x, )?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: enlarged_spatial_bounds, - time_interval: TimeInterval::new_instant(start_time)?, - spatial_resolution: query_rect.spatial_resolution, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new( + larger_bounds, + time, + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -131,7 +118,10 @@ where #[derive(Clone, Debug)] pub struct NeighborhoodAggregateAccu { pub output_info: TileInformation, - pub input_tile: RasterTile2D

, + pub accu_grid: GridOrEmpty, + pub accu_time: TimeInterval, + pub accu_cache_hint: CacheHint, + pub accu_band: u32, pub pool: Arc, pub neighborhood: Neighborhood, phantom_aggregate_fn: PhantomData, @@ -139,14 +129,20 @@ pub struct NeighborhoodAggregateAccu { impl NeighborhoodAggregateAccu { pub fn new( - input_tile: RasterTile2D

, + accu_grid: GridOrEmpty, + accu_time: TimeInterval, + accu_cache_hint: CacheHint, + accu_band: u32, output_info: TileInformation, pool: Arc, neighborhood: Neighborhood, ) -> Self { NeighborhoodAggregateAccu { output_info, - input_tile, + accu_grid, + accu_time, + accu_cache_hint, + accu_band, pool, neighborhood, phantom_aggregate_fn: PhantomData, @@ -168,7 +164,10 @@ where let neighborhood = self.neighborhood.clone(); let output_tile = crate::util::spawn_blocking_with_thread_pool(self.pool, move || { apply_kernel_for_each_inner_pixel::( - &self.input_tile, + &self.accu_grid, + &self.accu_time, + self.accu_cache_hint, + self.accu_band, &self.output_info, &neighborhood, ) @@ -185,7 +184,10 @@ where /// Apply kernel function to all pixels of the inner input tile in the 9x9 grid fn apply_kernel_for_each_inner_pixel( - input: &RasterTile2D

, + accu_grid: &GridOrEmpty, + accu_time: &TimeInterval, + accu_cache_hint: CacheHint, + accu_band: u32, info_out: &TileInformation, neighborhood: &Neighborhood, ) -> RasterTile2D

@@ -194,13 +196,13 @@ where f64: AsPrimitive

, A: AggregateFunction, { - if input.is_empty() { + if accu_grid.is_empty() { return RasterTile2D::new_with_tile_info( - input.time, + *accu_time, *info_out, 0, // TODO - EmptyGrid::new(info_out.tile_size_in_pixels).into(), - CacheHint::max_duration(), + GridOrEmpty::new_empty_shape(info_out.tile_size_in_pixels), + accu_cache_hint, // TODO: is this correct? Was CacheHint::max_duration() before ); } @@ -210,13 +212,16 @@ where let mut neighborhood_matrix = Vec::>::with_capacity(neighborhood.matrix().number_of_elements()); - let y_stop = y + neighborhood.y_width() as isize; - let x_stop = x + neighborhood.x_width() as isize; + let y_start = y - neighborhood.y_radius() as isize; + let x_start = x - neighborhood.x_radius() as isize; + + let y_stop = y + neighborhood.y_radius() as isize; + let x_stop = x + neighborhood.x_radius() as isize; // copy row-by-row all pixels in x direction into kernel matrix - for y_index in y..y_stop { - for x_index in x..x_stop { + for y_index in y_start..=y_stop { + for x_index in x_start..=x_stop { neighborhood_matrix.push( - input + accu_grid .get_at_grid_index_unchecked([y_index, x_index]) .map(AsPrimitive::as_), ); @@ -226,16 +231,25 @@ where A::apply(&neighborhood.apply(neighborhood_matrix)) }; + let out_pixel_bounds = info_out.global_pixel_bounds(); + + debug_assert!(accu_grid.shape_ref().contains(&out_pixel_bounds)); + // TODO: this will check for empty tiles. Change to MaskedGrid::from(…) to avoid this. - let out_data = GridOrEmpty::from_index_fn_parallel(&info_out.tile_size_in_pixels, map_fn); + let out_data = GridOrEmpty::from_index_fn_parallel(&out_pixel_bounds, map_fn); + + debug_assert_eq!( + out_data.shape_ref().axis_size(), + info_out.tile_size_in_pixels.axis_size() + ); RasterTile2D::new( - input.time, + *accu_time, info_out.global_tile_position, - input.band, + accu_band, info_out.global_geo_transform, - out_data, - input.cache_hint.clone_with_current_datetime(), + out_data.unbounded(), + accu_cache_hint.clone_with_current_datetime(), ) } @@ -243,59 +257,59 @@ fn create_enlarged_tile( tile_info: TileInformation, query_rect: &RasterQueryRectangle, pool: Arc, - tiling_specification: TilingSpecification, neighborhood: Neighborhood, ) -> NeighborhoodAggregateAccu { // create an accumulator as a single tile that fits all the input tiles + some margin for the kernel size - let tiling = tiling_specification.strategy( - query_rect.spatial_resolution.x, - -query_rect.spatial_resolution.y, - ); - - let origin_coordinate = query_rect.spatial_bounds.upper_left(); - - let geo_transform = GeoTransform::new( - origin_coordinate, - query_rect.spatial_resolution.x, - -query_rect.spatial_resolution.y, - ); + let tiling_strategy = tile_info.tiling_strategy(); + + let target_tile_start = + tiling_strategy.tile_idx_to_global_pixel_idx(tile_info.global_tile_position); + let accu_start = target_tile_start + - GridIdx([ + neighborhood.y_radius() as isize, + neighborhood.x_radius() as isize, + ]); + let accu_end = accu_start + + GridIdx2D::new_y_x( + tiling_strategy.tile_size_in_pixels.y() as isize + 2 * neighborhood.y_radius() as isize + - 1, // -1 because the end is inclusive + tiling_strategy.tile_size_in_pixels.x() as isize + 2 * neighborhood.x_radius() as isize + - 1, + ); - let shape = [ - tiling.tile_size_in_pixels.axis_size_y() + 2 * neighborhood.y_radius(), - tiling.tile_size_in_pixels.axis_size_x() + 2 * neighborhood.x_radius(), - ]; + let accu_bounds = GridBoundingBox2D::new(accu_start, accu_end) + .expect("accu bounds must be valid because they are calculated from valid bounds"); // create a non-aligned (w.r.t. the tiling specification) grid by setting the origin to the top-left of the tile and the tile-index to [0, 0] - let grid = EmptyGrid2D::new(shape.into()); - - let input_tile = RasterTile2D::new( - query_rect.time_interval, - [0, 0].into(), - 0, // TODO - geo_transform, - GridOrEmpty::from(grid), - CacheHint::max_duration(), - ); + let grid = GridOrEmpty::new_empty_shape(accu_bounds); - NeighborhoodAggregateAccu::new(input_tile, tile_info, pool, neighborhood) + NeighborhoodAggregateAccu::new( + grid, + query_rect.time_interval(), + CacheHint::max_duration(), + 0, + tile_info, + pool, + neighborhood, + ) } type FoldFutureFn = fn( - Result>, tokio::task::JoinError>, + Result, tokio::task::JoinError>, ) -> Result>; type FoldFuture = - futures::future::Map>>, FoldFutureFn>; + futures::future::Map>, FoldFutureFn>; /// Turn a result of results into a result fn flatten_result( - result: Result>, tokio::task::JoinError>, + result: Result, tokio::task::JoinError>, ) -> Result> where f64: AsPrimitive

, { match result { - Ok(r) => r, + Ok(r) => Ok(r), Err(e) => Err(e.into()), } } @@ -304,30 +318,25 @@ where pub fn merge_tile_into_enlarged_tile( mut accu: NeighborhoodAggregateAccu, tile: RasterTile2D

, -) -> Result> +) -> NeighborhoodAggregateAccu where f64: AsPrimitive

, { // get the time now because it is not known when the accu was created - accu.input_tile.time = tile.time; + accu.accu_time = tile.time; + accu.accu_cache_hint = tile.cache_hint; // if the tile is empty, we can skip it if tile.is_empty() { - return Ok(accu); + return accu; } // copy all input tiles into the accu to have all data for raster kernel - let mut accu_input_tile = accu.input_tile.into_materialized_tile(); - accu_input_tile.blit(tile)?; + let x = tile.into_inner_positioned_grid(); - let accu_input_tile: RasterTile2D

= accu_input_tile.into(); + accu.accu_grid.grid_blit_from(&x); - Ok(NeighborhoodAggregateAccu::new( - accu_input_tile, - accu.output_info, - accu.pool, - accu.neighborhood, - )) + accu } #[cfg(test)] @@ -342,72 +351,73 @@ mod tests { }, }; use geoengine_datatypes::{ - primitives::{BandSelection, SpatialResolution}, - raster::TilingStrategy, + primitives::{BandSelection, TimeInstance}, + raster::{ + GeoTransform, GridBoundingBox2D, SpatialGridDefinition, TilingSpecification, + TilingStrategy, + }, util::test::TestDefault, }; #[test] #[allow(clippy::float_cmp)] fn test_create_enlarged_tile() { - let execution_context = MockExecutionContext::test_default(); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::test_default()); - let spatial_resolution = SpatialResolution::one(); - let tiling_strategy = TilingStrategy::new_with_tiling_spec( - execution_context.tiling_specification, - spatial_resolution.x, - -spatial_resolution.y, + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new_with_coordinate_x_y(0., 1., 0., -1.), + GridBoundingBox2D::new([-2, 0], [-1, 1]).unwrap(), + ); + + let tiling_strategy = TilingStrategy::new( + execution_context.tiling_specification.tile_size_in_pixels, + spatial_grid.geo_transform(), ); - let spatial_partition = SpatialPartition2D::new((0., 1.).into(), (1., 0.).into()).unwrap(); let tile_info = tiling_strategy - .tile_information_iterator(spatial_partition) + .tile_information_iterator_from_pixel_bounds(spatial_grid.grid_bounds()) .next() .unwrap(); - let qrect = RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - time_interval: TimeInstance::from_millis(0).unwrap().into(), - spatial_resolution, - attributes: BandSelection::first(), - }; + let qrect = RasterQueryRectangle::new( + tile_info.global_pixel_bounds(), + TimeInstance::from_millis(0).unwrap().into(), + BandSelection::first(), + ); let aggregator = NeighborhoodAggregateTileNeighborhood::::new( NeighborhoodParams::Rectangle { dimensions: [5, 5] } .try_into() .unwrap(), - execution_context.tiling_specification, ); let tile_query_rectangle = aggregator - .tile_query_rectangle(tile_info, qrect.clone(), qrect.time_interval.start(), 0) + .tile_query_rectangle(tile_info, qrect.clone(), qrect.time_interval(), 0) .unwrap() .unwrap(); assert_eq!( - tile_info.spatial_partition(), - SpatialPartition2D::new((0., 512.).into(), (512., 0.).into()).unwrap() + tile_info.global_pixel_bounds(), + GridBoundingBox2D::new([-512, 0], [-1, 511]).unwrap() ); + assert_eq!( - tile_query_rectangle.spatial_bounds, - SpatialPartition2D::new((-2., 514.).into(), (514., -2.).into()).unwrap() + tile_query_rectangle.spatial_bounds(), + GridBoundingBox2D::new([-514, -2], [1, 513]).unwrap() ); let accu = create_enlarged_tile::( tile_info, &tile_query_rectangle, execution_context.thread_pool.clone(), - execution_context.tiling_specification, aggregator.neighborhood, ); assert_eq!(tile_info.tile_size_in_pixels.axis_size(), [512, 512]); assert_eq!( - accu.input_tile.grid_array.shape_ref().axis_size(), + accu.accu_grid.shape_ref().axis_size(), [512 + 2 + 2, 512 + 2 + 2] ); - - assert_eq!(accu.input_tile.tile_geo_transform().x_pixel_size(), 1.); - assert_eq!(accu.input_tile.tile_geo_transform().y_pixel_size(), -1.); } } diff --git a/operators/src/processing/point_in_polygon.rs b/operators/src/processing/point_in_polygon.rs index d63fec3e1..fbc08b1f3 100644 --- a/operators/src/processing/point_in_polygon.rs +++ b/operators/src/processing/point_in_polygon.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use futures::stream::BoxStream; use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::VectorQueryRectangle; +use geoengine_datatypes::primitives::{CacheHint, SpatialResolution}; use rayon::ThreadPool; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -21,6 +21,7 @@ use crate::engine::{ }; use crate::engine::{OperatorData, QueryProcessor}; use crate::error::{self, Error}; +use crate::optimization::OptimizationError; use crate::util::Result; use arrow::array::BooleanArray; use async_trait::async_trait; @@ -180,6 +181,20 @@ impl InitializedVectorOperator for InitializedPointInPolygonFilter { self.name.clone() } + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(PointInPolygonFilter { + params: PointInPolygonFilterParams {}, + sources: PointInPolygonFilterSource { + points: self.points.optimize(target_resolution)?, + polygons: self.polygons.optimize(target_resolution)?, + }, + } + .boxed()) + } + fn name(&self) -> &'static str { PointInPolygonFilter::TYPE_NAME } @@ -301,10 +316,7 @@ impl VectorQueryProcessor for PointInPolygonFilterProcessor { .query(query.clone(), ctx) .await? .and_then(move |points| { - let query: geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - geoengine_datatypes::primitives::ColumnSelection, - > = query.clone(); + let query: VectorQueryRectangle = query.clone(); async move { if points.is_empty() { return Ok(points); @@ -392,13 +404,13 @@ mod tests { use geoengine_datatypes::collections::ChunksEqualIgnoringCacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, MultiPoint, MultiPolygon, SpatialResolution, TimeInterval, + BoundingBox2D, Coordinate2D, MultiPoint, MultiPolygon, TimeInterval, }; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; - use crate::engine::{ChunkByteSize, MockExecutionContext, MockQueryContext}; + use crate::engine::{ChunkByteSize, MockExecutionContext}; use crate::error::Error; use crate::mock::MockFeatureCollectionSource; @@ -505,6 +517,8 @@ mod tests { let point_source = MockFeatureCollectionSource::single(points.clone()).boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let polygon_source = MockFeatureCollectionSource::single(MultiPolygonCollection::from_data( vec![MultiPolygon::new(vec![vec![vec![ @@ -528,21 +542,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -578,6 +588,8 @@ mod tests { )?) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -586,21 +598,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -647,6 +655,8 @@ mod tests { )?) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -655,21 +665,17 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -737,6 +743,8 @@ mod tests { ]) .boxed(); + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); + let operator = PointInPolygonFilter { params: PointInPolygonFilterParams {}, sources: PointInPolygonFilterSource { @@ -745,23 +753,19 @@ mod tests { }, } .boxed() - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = operator.query_processor()?.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx_one_chunk = MockQueryContext::new(ChunkByteSize::MAX); - let ctx_minimal_chunks = MockQueryContext::new(ChunkByteSize::MIN); + let ctx_one_chunk = exe_ctx.mock_query_context(ChunkByteSize::MAX); + let ctx_minimal_chunks = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query(query_rectangle.clone(), &ctx_minimal_chunks) @@ -799,6 +803,7 @@ mod tests { #[tokio::test] async fn empty_points() { + let exe_ctx: MockExecutionContext = MockExecutionContext::test_default(); let point_collection = MultiPointCollection::from_data( vec![], vec![], @@ -839,16 +844,15 @@ mod tests { .await .unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-10., -10.).into(), (10., 10.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-10., -10.).into(), (10., 10.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let query_processor = operator.query_processor().unwrap().multi_point().unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = exe_ctx.mock_query_context_test_default(); let query = query_processor .query(query_rectangle, &query_context) diff --git a/operators/src/processing/raster_scaling.rs b/operators/src/processing/raster_scaling.rs index e494f9ad2..2223644ef 100644 --- a/operators/src/processing/raster_scaling.rs +++ b/operators/src/processing/raster_scaling.rs @@ -1,12 +1,15 @@ use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, RasterBandDescriptor, RasterOperator, RasterQueryProcessor, - RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, Operator, OperatorName, QueryProcessor, RasterBandDescriptor, + RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, + TypedRasterQueryProcessor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::{StreamExt, TryStreamExt}; +use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::raster::{ CheckedMulThenAddTransformation, CheckedSubThenDivTransformation, ElementScaling, ScalingTransformation, @@ -101,10 +104,8 @@ impl RasterOperator for RasterScaling { let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: in_desc.data_type, - - bbox: in_desc.bbox, time: in_desc.time, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: in_desc .bands .iter() @@ -171,6 +172,24 @@ impl InitializedRasterOperator for InitializedRasterScalingOperator { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(RasterScaling { + params: RasterScalingParams { + slope: self.slope.clone(), + offset: self.offset.clone(), + output_measurement: Some(self.result_descriptor.bands[0].measurement.clone()), + scaling_mode: self.scaling_mode, + }, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } struct RasterTransformationProcessor @@ -189,7 +208,7 @@ fn create_boxed_processor( slope: SlopeOffsetSelection, offset: SlopeOffsetSelection, source: Q, -) -> Box> +) -> BoxRasterQueryProcessor

where Q: RasterQueryProcessor + 'static, P: Pixel + FromPrimitive + 'static + Default, @@ -249,67 +268,93 @@ where } #[async_trait] -impl RasterQueryProcessor for RasterTransformationProcessor +impl QueryProcessor for RasterTransformationProcessor where P: Pixel + FromPrimitive + 'static + Default, f64: AsPrimitive

, Q: RasterQueryProcessor + 'static, S: Send + Sync + 'static + ScalingTransformation

, { - type RasterType = P; + type Output = RasterTile2D

; + type SpatialBounds = Q::SpatialBounds; + type ResultDescription = RasterResultDescriptor; + type Selection = Q::Selection; - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: geoengine_datatypes::primitives::RasterQueryRectangle, ctx: &'a dyn crate::engine::QueryContext, - ) -> Result< - futures::stream::BoxStream< - 'a, - Result>, - >, - > { + ) -> Result>> { let src = self.source.raster_query(query, ctx).await?; let rs = src.and_then(move |tile| self.scale_tile_async(tile, ctx.thread_pool().clone())); Ok(rs.boxed()) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for RasterTransformationProcessor +where + P: Pixel + FromPrimitive + 'static + Default, + f64: AsPrimitive

, + Q: RasterQueryProcessor + 'static, + S: Send + Sync + 'static + ScalingTransformation

, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { - use geoengine_datatypes::{ - primitives::{ - BandSelection, CacheHint, SpatialPartition2D, SpatialResolution, TimeInterval, + use crate::{ + engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, + TimeDescriptor, }, + mock::{MockRasterSource, MockRasterSourceParams}, + }; + use geoengine_datatypes::{ + primitives::{BandSelection, CacheHint, Coordinate2D, TimeInterval}, raster::{ - Grid2D, GridOrEmpty2D, GridShape, MaskedGrid2D, RasterDataType, RasterProperties, - TileInformation, TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty2D, GridShape, + GridShape2D, MaskedGrid2D, RasterDataType, RasterProperties, TileInformation, + TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, }; - use crate::{ - engine::{ChunkByteSize, MockExecutionContext, RasterBandDescriptors}, - mock::{MockRasterSource, MockRasterSourceParams}, - }; - use super::*; #[tokio::test] async fn test_unscale() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; - let raster = MaskedGrid2D::from(Grid2D::new(grid_shape, vec![7_u8, 7, 7, 6]).unwrap()); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let raster = + MaskedGrid2D::from(Grid2D::new(tile_size_in_pixels, vec![7_u8, 7, 7, 6]).unwrap()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -323,7 +368,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -331,19 +376,10 @@ mod tests { CacheHint::default(), ); - let spatial_resolution = raster_tile.spatial_resolution(); - let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: Some(spatial_resolution), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -378,12 +414,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::U8(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::U8"); @@ -416,14 +451,22 @@ mod tests { #[tokio::test] async fn test_scale() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; - let raster = MaskedGrid2D::from(Grid2D::new(grid_shape, vec![15_u8, 15, 15, 13]).unwrap()); + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let raster = + MaskedGrid2D::from(Grid2D::new(tile_size_in_pixels, vec![15_u8, 15, 15, 13]).unwrap()); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -437,7 +480,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -445,19 +488,10 @@ mod tests { CacheHint::default(), ); - let spatial_resolution = raster_tile.spatial_resolution(); - let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: Some(spatial_resolution), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -494,12 +528,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::U8(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::U8"); diff --git a/operators/src/processing/raster_stacker.rs b/operators/src/processing/raster_stacker.rs index b20ebea46..15b792b07 100644 --- a/operators/src/processing/raster_stacker.rs +++ b/operators/src/processing/raster_stacker.rs @@ -1,20 +1,21 @@ use crate::adapters::{QueryWrapper, RasterStackerAdapter, RasterStackerSource}; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, - MultipleRasterSources, Operator, OperatorName, QueryContext, RasterBandDescriptor, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, TypedRasterQueryProcessor, - WorkflowOperatorPath, + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, MultipleRasterSources, Operator, OperatorName, QueryContext, + QueryProcessor, RasterBandDescriptor, RasterOperator, RasterQueryProcessor, + RasterResultDescriptor, TypedRasterQueryProcessor, WorkflowOperatorPath, }; use crate::error::{ InvalidNumberOfRasterStackerInputs, RasterInputsMustHaveSameSpatialReferenceAndDatatype, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; -use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialResolution, partitions_extent, time_interval_extent, +use geoengine_datatypes::primitives::{BandSelection, RasterQueryRectangle, SpatialResolution}; +use geoengine_datatypes::raster::{ + DynamicRasterDataType, GridBoundingBox2D, Pixel, RasterTile2D, RenameBands, }; -use geoengine_datatypes::raster::{DynamicRasterDataType, Pixel, RasterTile2D, RenameBands}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -77,19 +78,23 @@ impl RasterOperator for RasterStacker { } ); - let time = time_interval_extent(in_descriptors.iter().map(|d| d.time)); - let bbox = partitions_extent(in_descriptors.iter().map(|d| d.bbox)); - - let resolution = in_descriptors + let first_spatial_grid = in_descriptors[0].spatial_grid; + let result_spatial_grid = in_descriptors .iter() - .map(|d| d.resolution) - .reduce(|a, b| match (a, b) { - (Some(a), Some(b)) => { - Some(SpatialResolution::new_unchecked(a.x.min(b.x), a.y.min(b.y))) - } - _ => None, - }) - .flatten(); + .skip(1) + .map(|x| x.spatial_grid_descriptor()) + .try_fold(first_spatial_grid, |a, &b| { + a.merge(&b) + .ok_or(crate::error::Error::CantMergeSpatialGridDescriptor { a, b }) + })?; + + let time = in_descriptors.iter().skip(1).map(|rd| rd.time).fold( + in_descriptors + .first() + .expect("There must be at least one input") + .time, + |a, b| a.merge(b), + ); let data_type = in_descriptors[0].data_type; let spatial_reference = in_descriptors[0].spatial_reference; @@ -118,8 +123,7 @@ impl RasterOperator for RasterStacker { data_type, spatial_reference, time, - bbox, - resolution, + spatial_grid: result_spatial_grid, bands: output_band_descriptors, }; @@ -127,6 +131,7 @@ impl RasterOperator for RasterStacker { name, path, result_descriptor, + rename_bands: self.params.rename_bands.clone(), raster_sources, bands_per_source, })) @@ -139,6 +144,7 @@ pub struct InitializedRasterStacker { name: CanonicOperatorName, path: WorkflowOperatorPath, result_descriptor: RasterResultDescriptor, + rename_bands: RenameBands, raster_sources: Vec>, bands_per_source: Vec, } @@ -266,17 +272,36 @@ impl InitializedRasterOperator for InitializedRasterStacker { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(RasterStacker { + params: RasterStackerParams { + rename_bands: self.rename_bands.clone(), + }, + sources: MultipleRasterSources { + rasters: self + .raster_sources + .iter() + .map(|s| s.optimize(resolution)) + .collect::, _>>()?, + }, + } + .boxed()) + } } pub(crate) struct RasterStackerProcessor { - sources: Vec>>, + sources: Vec>, result_descriptor: RasterResultDescriptor, bands_per_source: Vec, } impl RasterStackerProcessor { pub fn new( - sources: Vec>>, + sources: Vec>, result_descriptor: RasterResultDescriptor, bands_per_source: Vec, ) -> Self { @@ -313,12 +338,16 @@ fn map_query_bands_to_source_bands( } #[async_trait] -impl RasterQueryProcessor for RasterStackerProcessor +impl QueryProcessor for RasterStackerProcessor where T: Pixel, { - type RasterType = T; - async fn raster_query<'a>( + type Output = RasterTile2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; + type SpatialBounds = GridBoundingBox2D; + + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, @@ -327,13 +356,11 @@ where for (idx, source) in self.sources.iter().enumerate() { let Some(bands) = - map_query_bands_to_source_bands(&query.attributes, &self.bands_per_source, idx) + map_query_bands_to_source_bands(query.attributes(), &self.bands_per_source, idx) else { continue; }; - let mut source_query = query.clone(); - source_query.attributes = bands.clone(); sources.push(RasterStackerSource { queryable: QueryWrapper { p: source, ctx }, band_idxs: bands.as_vec(), @@ -345,27 +372,53 @@ where Ok(Box::pin(output)) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &Self::ResultDescription { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for RasterStackerProcessor +where + T: Pixel, +{ + type RasterType = T; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + let mut time_sources = Vec::with_capacity(self.sources.len()); + for source in &self.sources { + let s = source.time_query(query, ctx).await?; + time_sources.push(s); + } + let output = crate::adapters::TimeIntervalStreamMerge::new(time_sources); + Ok(Box::pin(output)) + } +} + #[cfg(test)] mod tests { use std::str::FromStr; use futures::StreamExt; use geoengine_datatypes::{ - primitives::{CacheHint, SpatialPartition2D, TimeInstance, TimeInterval}, - raster::{Grid, GridShape, RasterDataType, TilesEqualIgnoringCacheHint}, + primitives::{CacheHint, TimeInstance, TimeInterval, TimeStep}, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, + TilesEqualIgnoringCacheHint, + }, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ engine::{ - MockExecutionContext, MockQueryContext, RasterBandDescriptor, RasterBandDescriptors, - SingleRasterSource, + MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, SingleRasterSource, + SpatialGridDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{Expression, ExpressionParams}, @@ -488,17 +541,24 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 5)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -506,14 +566,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -533,14 +586,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = stacker .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -742,21 +794,28 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 10)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new(vec![ + RasterBandDescriptor::new_unitless("band_0".into()), + RasterBandDescriptor::new_unitless("band_1".into()), + ]) + .unwrap(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![ - RasterBandDescriptor::new_unitless("band_0".into()), - RasterBandDescriptor::new_unitless("band_1".into()), - ]) - .unwrap(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -764,18 +823,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![ - RasterBandDescriptor::new_unitless("band_0".into()), - RasterBandDescriptor::new_unitless("band_1".into()), - ]) - .unwrap(), - }, + result_descriptor, }, } .boxed(); @@ -795,14 +843,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1, 2, 3].try_into().unwrap(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 2]).unwrap(), + TimeInterval::new_unchecked(0, 10), + [0, 1, 2, 3].try_into().unwrap(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = stacker .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -929,17 +976,24 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 10)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-2, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); @@ -947,14 +1001,7 @@ mod tests { let mrs2 = MockRasterSource { params: MockRasterSourceParams { data: data2.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -974,14 +1021,13 @@ mod tests { shape_array: [2, 2], }; - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: 1.into(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 2]).unwrap(), + TimeInterval::new_unchecked(0, 10), + 1.into(), + ); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let op = stacker .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1019,6 +1065,7 @@ mod tests { raster: GdalSource { params: GdalSourceParameters { data: ndvi_id.clone(), + overview_level: None, }, } .boxed(), @@ -1033,7 +1080,7 @@ mod tests { sources: MultipleRasterSources { rasters: vec![ GdalSource { - params: GdalSourceParameters { data: ndvi_id }, + params: GdalSourceParameters::new(ndvi_id), } .boxed(), expression, @@ -1054,21 +1101,17 @@ mod tests { shape_array: [2, 2], }; - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); // query both bands - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; + [0, 1].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) @@ -1081,18 +1124,14 @@ mod tests { assert!(!result.is_empty()); // query only first band - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [0].try_into().unwrap(), - }; + [0].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) @@ -1105,18 +1144,14 @@ mod tests { assert!(!result.is_empty()); // query only second band - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::one(), - attributes: [1].try_into().unwrap(), - }; + [1].try_into().unwrap(), + ); let result = processor .raster_query(query_rect, &query_ctx) diff --git a/operators/src/processing/raster_type_conversion.rs b/operators/src/processing/raster_type_conversion.rs index fcce1107b..8e691cdf4 100644 --- a/operators/src/processing/raster_type_conversion.rs +++ b/operators/src/processing/raster_type_conversion.rs @@ -1,18 +1,20 @@ +use crate::engine::{ + BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, + InitializedSources, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, + WorkflowOperatorPath, +}; +use crate::optimization::OptimizationError; +use crate::util::Result; use async_trait::async_trait; use futures::{StreamExt, TryFutureExt, TryStreamExt, stream::BoxStream}; +use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::{ - primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D}, - raster::{ConvertDataType, Pixel, RasterDataType, RasterTile2D}, + primitives::{BandSelection, RasterQueryRectangle}, + raster::{ConvertDataType, GridBoundingBox2D, Pixel, RasterDataType, RasterTile2D}, }; use serde::{Deserialize, Serialize}; -use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, - RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, -}; -use crate::util::Result; - #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] pub struct RasterTypeConversionParams { @@ -55,9 +57,8 @@ impl RasterOperator for RasterTypeConversion { let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: out_data_type, - bbox: in_desc.bbox, time: in_desc.time, - resolution: in_desc.resolution, + spatial_grid: in_desc.spatial_grid, bands: in_desc.bands.clone(), }; @@ -103,6 +104,21 @@ impl InitializedRasterOperator for InitializedRasterTypeConversionOperator { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(RasterTypeConversion { + params: RasterTypeConversionParams { + output_data_type: self.result_descriptor.data_type, + }, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct RasterTypeConversionQueryProcessor< @@ -128,7 +144,7 @@ where } } - pub fn create_boxed(source: Q) -> Box> { + pub fn create_boxed(source: Q) -> BoxRasterQueryProcessor { RasterTypeConversionQueryProcessor::new(source).boxed() } } @@ -140,7 +156,7 @@ where Q: RasterQueryProcessor, { type Output = RasterTile2D; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -162,20 +178,43 @@ where } } +#[async_trait] +impl RasterQueryProcessor for RasterTypeConversionQueryProcessor +where + Q: RasterQueryProcessor, + RasterTile2D: ConvertDataType>, + POut: Pixel, + PIn: Pixel, +{ + type RasterType = POut; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + self.query_processor.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use geoengine_datatypes::{ - primitives::{CacheHint, Measurement, SpatialPartition2D, SpatialResolution, TimeInterval}, + primitives::{CacheHint, Coordinate2D, Measurement, TimeInterval}, raster::{ - Grid2D, GridOrEmpty2D, MaskedGrid2D, RasterDataType, TileInformation, - TilingSpecification, + BoundedGrid, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty2D, GridShape2D, + MaskedGrid2D, RasterDataType, TileInformation, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::{ChunkByteSize, MockExecutionContext, RasterBandDescriptors}, + engine::{ + ChunkByteSize, MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor, + TimeDescriptor, + }, mock::{MockRasterSource, MockRasterSourceParams}, }; @@ -184,14 +223,22 @@ mod tests { #[tokio::test] #[allow(clippy::float_cmp)] async fn test_type_conversion() { - let grid_shape = [2, 2].into(); - - let tiling_specification = TilingSpecification { - origin_coordinate: [0.0, 0.0].into(), - tile_size_in_pixels: grid_shape, + let tile_size_in_pixels = GridShape2D::new_2d(2, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + tile_size_in_pixels.bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); - let raster: MaskedGrid2D = Grid2D::new(grid_shape, vec![7_u8, 7, 7, 6]).unwrap().into(); + let raster: MaskedGrid2D = Grid2D::new(tile_size_in_pixels, vec![7_u8, 7, 7, 6]) + .unwrap() + .into(); let ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); let query_ctx = ctx.mock_query_context(ChunkByteSize::test_default()); @@ -201,7 +248,7 @@ mod tests { TileInformation { global_geo_transform: TestDefault::test_default(), global_tile_position: [0, 0].into(), - tile_size_in_pixels: grid_shape, + tile_size_in_pixels, }, 0, raster.into(), @@ -211,14 +258,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - bbox: None, - time: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -246,12 +286,11 @@ mod tests { let query_processor = initialized_op.query_processor().unwrap(); - let query = geoengine_datatypes::primitives::RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 0.).into(), (2., -2.).into()).unwrap(), - spatial_resolution: SpatialResolution::one(), - time_interval: TimeInterval::default(), - attributes: BandSelection::first(), - }; + let query = geoengine_datatypes::primitives::RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + TimeInterval::default(), + BandSelection::first(), + ); let TypedRasterQueryProcessor::F32(typed_processor) = query_processor else { panic!("expected TypedRasterQueryProcessor::F32"); diff --git a/operators/src/processing/raster_vector_join/aggregated.rs b/operators/src/processing/raster_vector_join/aggregated.rs index 612736603..165e97974 100644 --- a/operators/src/processing/raster_vector_join/aggregated.rs +++ b/operators/src/processing/raster_vector_join/aggregated.rs @@ -1,30 +1,31 @@ -use futures::stream::BoxStream; -use futures::{StreamExt, TryStreamExt}; - -use geoengine_datatypes::collections::{ - FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, -}; -use geoengine_datatypes::primitives::{ - BandSelection, CacheHint, ColumnSelection, RasterQueryRectangle, -}; -use geoengine_datatypes::raster::{GridIndexAccess, Pixel, RasterDataType}; -use geoengine_datatypes::util::arrow::ArrowTyped; - -use crate::engine::{ - QueryContext, QueryProcessor, RasterQueryProcessor, VectorQueryProcessor, - VectorResultDescriptor, -}; -use crate::processing::raster_vector_join::TemporalAggregationMethod; -use crate::processing::raster_vector_join::aggregator::{ - Aggregator, FirstValueFloatAggregator, FirstValueIntAggregator, MeanValueAggregator, - TypedAggregator, -}; -use crate::util::Result; -use async_trait::async_trait; -use geoengine_datatypes::primitives::{BoundingBox2D, Geometry, VectorQueryRectangle}; - use super::util::{CoveredPixels, FeatureTimeSpanIter, PixelCoverCreator}; use super::{FeatureAggregationMethod, RasterInput, create_feature_aggregator}; +use crate::{ + engine::{ + QueryContext, QueryProcessor, RasterQueryProcessor, VectorQueryProcessor, + VectorResultDescriptor, + }, + processing::raster_vector_join::{ + TemporalAggregationMethod, + aggregator::{ + Aggregator, FirstValueFloatAggregator, FirstValueIntAggregator, MeanValueAggregator, + TypedAggregator, + }, + }, + util::Result, +}; +use async_trait::async_trait; +use futures::{StreamExt, TryStreamExt, stream::BoxStream}; +use geoengine_datatypes::raster::RasterTile2D; +use geoengine_datatypes::{ + collections::{FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications}, + primitives::{ + BandSelection, BoundingBox2D, CacheHint, ColumnSelection, Geometry, RasterQueryRectangle, + VectorQueryRectangle, + }, + raster::{GridIndexAccess, Pixel, RasterDataType}, + util::arrow::ArrowTyped, +}; pub struct RasterVectorAggregateJoinProcessor { collection: Box>>, @@ -65,7 +66,7 @@ where #[allow(clippy::too_many_arguments, clippy::too_many_lines)] // TODO: refactor to reduce arguments async fn extract_raster_values( collection: &FeatureCollection, - raster_processor: &dyn RasterQueryProcessor, + raster_processor: &dyn RasterQueryProcessor>, column_names: &[String], feature_aggreation: FeatureAggregationMethod, feature_aggregation_ignore_no_data: bool, @@ -92,20 +93,23 @@ where let mut cache_hint = CacheHint::max_duration(); + let rd = raster_processor.raster_result_descriptor(); + for time_span in FeatureTimeSpanIter::new(collection.time_intervals()) { - let query = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: time_span.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; - - let mut rasters = raster_processor - .raster_query( - RasterQueryRectangle::from_qrect_and_bands(&query, BandSelection::first()), - ctx, - ) - .await?; + let spatial_bounds = query.spatial_bounds(); + + let pixel_bounds = rd + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform() + .bounding_box_2d_to_intersecting_grid_bounds(&spatial_bounds); + + let raster_query = RasterQueryRectangle::new( + pixel_bounds, + time_span.time_interval, + BandSelection::first(), // FIXME: this should prop. use all bands? + ); + + let mut rasters = raster_processor.raster_query(raster_query, ctx).await?; // TODO: optimize geo access (only specific tiles, etc.) @@ -298,22 +302,23 @@ mod tests { use crate::engine::{ ChunkByteSize, MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, - RasterResultDescriptor, VectorColumnInfo, VectorOperator, WorkflowOperatorPath, + RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, TimeDescriptor, + VectorColumnInfo, VectorOperator, WorkflowOperatorPath, }; - use crate::engine::{MockQueryContext, RasterOperator}; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, MultiPolygonCollection, VectorDataType, }; - use geoengine_datatypes::primitives::MultiPolygon; - use geoengine_datatypes::primitives::{CacheHint, FeatureData, FeatureDataType, Measurement}; - use geoengine_datatypes::raster::{Grid2D, RasterTile2D, TileInformation}; + + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, Coordinate2D, FeatureData, FeatureDataRef, FeatureDataType, + Measurement, MultiPoint, MultiPolygon, TimeInterval, TimeStep, + }; + use geoengine_datatypes::raster::{ + GeoTransform, Grid2D, GridBoundingBox2D, RasterTile2D, TileInformation, TilingSpecification, + }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::test::TestDefault; - use geoengine_datatypes::{ - primitives::{BoundingBox2D, FeatureDataRef, MultiPoint, SpatialResolution, TimeInterval}, - raster::TilingSpecification, - }; #[tokio::test] async fn extract_raster_values_single_raster() { @@ -331,24 +336,27 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 1]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -379,13 +387,12 @@ mod tests { false, TemporalAggregationMethod::First, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -427,24 +434,30 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 1]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![raster_tile_a, raster_tile_b], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -475,13 +488,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (2.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -549,6 +561,20 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -557,21 +583,13 @@ mod tests { raster_tile_b_0, raster_tile_b_1, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -598,13 +616,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -702,6 +719,20 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 5]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -712,21 +743,13 @@ mod tests { raster_tile_b_1, raster_tile_b_2, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster_source = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -757,13 +780,12 @@ mod tests { false, TemporalAggregationMethod::Mean, false, - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), - time_interval: TimeInterval::new(0, 20).unwrap(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new(0, 20).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap(); @@ -962,9 +984,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U16, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + TestDefault::test_default(), + GridBoundingBox2D::new_min_max(0, 2, 0, 5).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -975,9 +1002,8 @@ mod tests { } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1042,14 +1068,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() diff --git a/operators/src/processing/raster_vector_join/mod.rs b/operators/src/processing/raster_vector_join/mod.rs index 4088db2ba..d8bb727c6 100644 --- a/operators/src/processing/raster_vector_join/mod.rs +++ b/operators/src/processing/raster_vector_join/mod.rs @@ -16,7 +16,7 @@ use crate::util::Result; use crate::processing::raster_vector_join::aggregated::RasterVectorAggregateJoinProcessor; use async_trait::async_trait; use geoengine_datatypes::collections::VectorDataType; -use geoengine_datatypes::primitives::FeatureDataType; +use geoengine_datatypes::primitives::{FeatureDataType, find_next_best_overview_level_resolution}; use geoengine_datatypes::raster::{Pixel, RasterDataType, RenameBands}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -350,6 +350,34 @@ impl InitializedVectorOperator for InitializedRasterVectorJoin { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: geoengine_datatypes::primitives::SpatialResolution, + ) -> Result, crate::optimization::OptimizationError> { + Ok(RasterVectorJoin { + params: self.state.clone(), + sources: SingleVectorMultipleRasterSources { + vector: self.vector_source.optimize(target_resolution)?, + rasters: self + .raster_sources + .iter() + .map(|r| { + // optimize the raster source to the next best overview level w.r.t. the target resolution + let rd = r.result_descriptor(); + let mut res = rd.spatial_grid.spatial_resolution(); + + if res < target_resolution { + res = find_next_best_overview_level_resolution(res, target_resolution); + } + + r.optimize(res) + }) + .collect::, crate::optimization::OptimizationError>>()?, + }, + } + .boxed()) + } } pub fn create_feature_aggregator( @@ -385,8 +413,9 @@ mod tests { use std::str::FromStr; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, QueryProcessor, - RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + ChunkByteSize, MockExecutionContext, QueryProcessor, RasterBandDescriptor, + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, + TimeDescriptor, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{GdalSource, GdalSourceParameters}; @@ -396,10 +425,10 @@ mod tests { use geoengine_datatypes::dataset::NamedData; use geoengine_datatypes::primitives::{ BoundingBox2D, ColumnSelection, DataRef, DateTime, FeatureDataRef, MultiPoint, - SpatialResolution, TimeInterval, VectorQueryRectangle, + TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::primitives::{CacheHint, Measurement}; - use geoengine_datatypes::raster::RasterTile2D; + use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D, RasterTile2D}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::{gdal::hide_gdal_errors, test::TestDefault}; use serde_json::json; @@ -451,7 +480,7 @@ mod tests { fn ndvi_source(name: NamedData) -> Box { let gdal_source = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), }; gdal_source.boxed() @@ -488,7 +517,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::First, @@ -510,14 +539,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -567,7 +594,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::Mean, @@ -589,14 +616,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -649,7 +674,7 @@ mod tests { let operator = RasterVectorJoin { params: RasterVectorJoinParams { - names: ColumnNames::Default, + names: ColumnNames::Names(vec!["ndvi".to_owned()]), feature_aggregation: FeatureAggregationMethod::First, feature_aggregation_ignore_no_data: false, temporal_aggregation: TemporalAggregationMethod::Mean, @@ -671,14 +696,12 @@ mod tests { let result = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MIN), + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ), + &exe_ctc.mock_query_context(ChunkByteSize::MIN), ) .await .unwrap() @@ -777,9 +800,13 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: crate::engine::TimeDescriptor::new_irregular(Some( + TimeInterval::default(), + )), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(0, 0, 2, 2).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -796,9 +823,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(0, 0, 2, 2).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), diff --git a/operators/src/processing/raster_vector_join/non_aggregated.rs b/operators/src/processing/raster_vector_join/non_aggregated.rs index bc0e5bfc4..064af12f7 100644 --- a/operators/src/processing/raster_vector_join/non_aggregated.rs +++ b/operators/src/processing/raster_vector_join/non_aggregated.rs @@ -1,35 +1,32 @@ +use super::aggregator::TypedAggregator; +use super::util::{CoveredPixels, PixelCoverCreator}; +use super::{FeatureAggregationMethod, RasterInput}; use crate::adapters::FeatureCollectionStreamExt; +use crate::engine::{ + QueryContext, QueryProcessor, RasterQueryProcessor, TypedRasterQueryProcessor, + VectorQueryProcessor, VectorResultDescriptor, +}; use crate::processing::raster_vector_join::create_feature_aggregator; +use crate::util::Result; +use crate::{adapters::RasterStreamExt, error::Error}; +use async_trait::async_trait; use futures::stream::{BoxStream, once as once_stream}; use futures::{StreamExt, TryStreamExt}; +use geoengine_datatypes::collections::GeometryCollection; +use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; use geoengine_datatypes::primitives::{ BandSelection, BoundingBox2D, CacheHint, ColumnSelection, FeatureDataType, Geometry, RasterQueryRectangle, VectorQueryRectangle, }; -use geoengine_datatypes::util::arrow::ArrowTyped; -use std::marker::PhantomData; -use std::sync::Arc; - use geoengine_datatypes::raster::{ DynamicRasterDataType, GridIdx2D, GridIndexAccess, RasterTile2D, }; +use geoengine_datatypes::util::arrow::ArrowTyped; use geoengine_datatypes::{ collections::FeatureCollectionModifications, primitives::TimeInterval, raster::Pixel, }; - -use super::util::{CoveredPixels, PixelCoverCreator}; -use crate::engine::{ - QueryContext, QueryProcessor, RasterQueryProcessor, TypedRasterQueryProcessor, - VectorQueryProcessor, VectorResultDescriptor, -}; -use crate::util::Result; -use crate::{adapters::RasterStreamExt, error::Error}; -use async_trait::async_trait; -use geoengine_datatypes::collections::GeometryCollection; -use geoengine_datatypes::collections::{FeatureCollection, FeatureCollectionInfos}; - -use super::aggregator::TypedAggregator; -use super::{FeatureAggregationMethod, RasterInput}; +use std::marker::PhantomData; +use std::sync::Arc; pub struct RasterVectorJoinProcessor { collection: Box>>, @@ -112,15 +109,15 @@ where let bbox = collection .bbox() - .and_then(|bbox| bbox.intersection(&query.spatial_bounds)); + .and_then(|bbox| bbox.intersection(&query.spatial_bounds())); let time = collection .time_bounds() - .and_then(|time| time.intersect(&query.time_interval)); + .and_then(|time| time.intersect(&query.time_interval())); // TODO: also intersect with raster spatial / time bounds - let (Some(_spatial_bounds), Some(_time_interval)) = (bbox, time) else { + let (Some(spatial_bounds), Some(time_interval)) = (bbox, time) else { tracing::debug!( "spatial or temporal intersection is empty, returning the same collection, skipping raster query" ); @@ -132,8 +129,16 @@ where ); }; - let query = RasterQueryRectangle::from_qrect_and_bands( - &query, + let rd = raster_processor.result_descriptor(); + + let pixel_bounds = rd + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform() + .bounding_box_2d_to_intersecting_grid_bounds(&spatial_bounds); + + let query = RasterQueryRectangle::new( + pixel_bounds, + time_interval, BandSelection::first_n(column_names.len() as u32), ); @@ -175,7 +180,7 @@ where #[allow(clippy::too_many_arguments)] async fn process_typed_collection_chunk<'a, P: Pixel>( collection: FeatureCollection, - raster_processor: &'a dyn RasterQueryProcessor, + raster_processor: &'a dyn RasterQueryProcessor>, column_names: &'a [String], query: RasterQueryRectangle, ctx: &'a dyn QueryContext, @@ -426,9 +431,9 @@ mod tests { use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, QueryProcessor, - RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, - VectorColumnInfo, VectorOperator, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, QueryProcessor, RasterBandDescriptor, + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, SpatialGridDescriptor, + TimeDescriptor, VectorColumnInfo, VectorOperator, WorkflowOperatorPath, }; use crate::mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}; use crate::source::{GdalSource, GdalSourceParameters}; @@ -436,12 +441,13 @@ mod tests { use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, MultiPolygonCollection, VectorDataType, }; - use geoengine_datatypes::primitives::SpatialResolution; - use geoengine_datatypes::primitives::{BoundingBox2D, DateTime, FeatureData, MultiPolygon}; - use geoengine_datatypes::primitives::{CacheHint, Measurement}; - use geoengine_datatypes::primitives::{MultiPoint, TimeInterval}; + use geoengine_datatypes::primitives::{ + BoundingBox2D, CacheHint, Coordinate2D, DateTime, FeatureData, Measurement, MultiPoint, + MultiPolygon, TimeInterval, TimeStep, + }; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, TileInformation, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, TileInformation, + TilingSpecification, }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::test::TestDefault; @@ -472,9 +478,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -521,14 +525,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: time_instant, - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + time_instant, + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -582,9 +584,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -631,18 +631,16 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2014, 1, 1, 0, 0, 0), DateTime::new_utc(2014, 3, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -704,9 +702,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -753,17 +749,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 1, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).unwrap(), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -828,9 +819,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let raster_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -877,18 +866,16 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2014, 1, 1, 0, 0, 0), DateTime::new_utc(2014, 3, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::new(0.1, 0.1).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -993,6 +980,20 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new(0, 20).unwrap()), + TimeStep::seconds(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -1001,21 +1002,13 @@ mod tests { raster_tile_b_0, raster_tile_b_1, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1074,14 +1067,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -1206,6 +1197,20 @@ mod tests { CacheHint::default(), ); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data: vec![ @@ -1216,21 +1221,13 @@ mod tests { raster_tile_b_1, raster_tile_b_2, ], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U16, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), }, } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1293,14 +1290,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() @@ -1532,9 +1527,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U16, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 20)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([0, 0], [2, 3]).unwrap(), + ), bands: RasterBandDescriptors::new(vec![ RasterBandDescriptor::new_unitless("band_0".into()), RasterBandDescriptor::new_unitless("band_1".into()), @@ -1545,9 +1545,8 @@ mod tests { } .boxed(); - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); let raster = raster_source .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1610,14 +1609,12 @@ mod tests { let mut result = processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()) - .unwrap(), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &MockQueryContext::new(ChunkByteSize::MAX), + VectorQueryRectangle::new( + BoundingBox2D::new((0.0, -3.0).into(), (4.0, 0.0).into()).unwrap(), + TimeInterval::new_unchecked(0, 20), + ColumnSelection::all(), + ), + &execution_context.mock_query_context(ChunkByteSize::MAX), ) .await .unwrap() diff --git a/operators/src/processing/raster_vector_join/util.rs b/operators/src/processing/raster_vector_join/util.rs index d8d975f4d..a66c8c271 100644 --- a/operators/src/processing/raster_vector_join/util.rs +++ b/operators/src/processing/raster_vector_join/util.rs @@ -165,7 +165,7 @@ impl CoveredPixels for MultiPolygonCoveredPixels { for row in 0..height { for col in 0..width { let idx = [row as isize, col as isize].into(); - let coordinate = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); + let coordinate = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); // FIXME: should be pixel center? if tester.multi_polygon_contains_coordinate(coordinate, feature_index) { pixels.push(idx); diff --git a/operators/src/processing/rasterization/mod.rs b/operators/src/processing/rasterization/mod.rs index f2e70804c..f7a71bce1 100644 --- a/operators/src/processing/rasterization/mod.rs +++ b/operators/src/processing/rasterization/mod.rs @@ -3,66 +3,41 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, InitializedVectorOperator, Operator, OperatorName, QueryContext, QueryProcessor, RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - SingleVectorSource, TypedRasterQueryProcessor, TypedVectorQueryProcessor, WorkflowOperatorPath, + ResultDescriptor, SingleVectorSource, SpatialGridDescriptor, TypedRasterQueryProcessor, + TypedVectorQueryProcessor, WorkflowOperatorPath, }; -use arrow::datatypes::ArrowNativeTypeOp; -use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; - use crate::error; -use crate::processing::rasterization::GridOrDensity::Grid; -use crate::util; - +use crate::optimization::{OptimizableOperator, OptimizationError}; +use crate::util::spawn_blocking; +use crate::util::{self, spawn_blocking_with_thread_pool}; use async_trait::async_trait; - use futures::stream::BoxStream; use futures::{StreamExt, stream}; use geoengine_datatypes::collections::GeometryCollection; - use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, - SpatialPartitioned, SpatialResolution, VectorQueryRectangle, + AxisAlignedRectangle, BandSelection, BoundingBox2D, Coordinate2D, RasterQueryRectangle, + SpatialPartition2D, SpatialPartitioned, SpatialResolution, TimeFilledItem, + VectorQueryRectangle, find_next_best_overview_level_resolution, }; +use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; use geoengine_datatypes::raster::{ - GeoTransform, Grid2D, GridOrEmpty, GridSize, GridSpaceToLinearSpace, RasterDataType, - RasterTile2D, TilingSpecification, + ChangeGridBounds, GeoTransform, Grid as GridWithFlexibleBoundType, Grid2D, GridBoundingBox2D, + GridIdx, GridOrEmpty, GridSize, RasterDataType, RasterTile2D, TilingSpecification, + TilingStrategy, UpdateIndexedElementsParallel, }; - +use geoengine_datatypes::spatial_reference::SpatialReference; use num_traits::FloatConst; -use rayon::prelude::*; - use serde::{Deserialize, Serialize}; use snafu::ensure; - -use crate::util::{spawn_blocking, spawn_blocking_with_thread_pool}; - +use tracing::warn; use typetag::serde; /// An operator that rasterizes vector data -pub type Rasterization = Operator; +pub type Rasterization = Operator; impl OperatorName for Rasterization { const TYPE_NAME: &'static str = "Rasterization"; } - -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum GridSizeMode { - /// The spatial resolution is interpreted as a fixed size in coordinate units - Fixed, - /// The spatial resolution is interpreted as a multiplier for the query pixel size - Relative, -} - -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -#[serde(tag = "type")] -pub enum GridOrDensity { - /// A grid which summarizes points in cells (2D histogram) - Grid(GridParams), - /// A heatmap calculated from a gaussian density function - Density(DensityParams), -} - #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] pub struct DensityParams { /// Defines the cutoff (as percentage of maximum density) down to which a point is taken @@ -74,13 +49,13 @@ pub struct DensityParams { #[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] -pub struct GridParams { +pub struct RasterizationParams { /// The size of grid cells, interpreted depending on the chosen grid size mode - spatial_resolution: SpatialResolution, + pub spatial_resolution: SpatialResolution, /// The origin coordinate which aligns the grid bounds - origin_coordinate: Coordinate2D, - /// Determines how to interpret the grid resolution - grid_size_mode: GridSizeMode, + pub origin_coordinate: Coordinate2D, + // Heatmap calculated from a gaussian density function + pub density_params: Option, } #[typetag::serde] @@ -102,38 +77,56 @@ impl RasterOperator for Rasterization { let tiling_specification = context.tiling_specification(); + let resolution = self.params.spatial_resolution; + let origin = self.params.origin_coordinate; + + let geo_transform = GeoTransform::new(origin, resolution.x, -resolution.y); + + let spatial_bounds = in_desc + .bbox + .ok_or_else(|| { + in_desc + .spatial_reference() + .as_option() + .map(SpatialReference::area_of_use_projected::) + }) + .map_err(|_| error::Error::NoSpatialBoundsAvailable)?; + + let pixel_bounds = + geo_transform.spatial_to_grid_bounds(&SpatialPartition2D::new_unchecked( + spatial_bounds.upper_left(), + spatial_bounds.lower_right(), + )); + let out_desc = RasterResultDescriptor { spatial_reference: in_desc.spatial_reference, data_type: RasterDataType::F64, - bbox: None, - time: in_desc.time, - resolution: None, + time: crate::engine::TimeDescriptor::new_irregular(in_desc.time), // FIXME: the operator really should use a regular time axis + spatial_grid: SpatialGridDescriptor::source_from_parts(geo_transform, pixel_bounds), bands: RasterBandDescriptors::new_single_band(), }; - match self.params { - Grid(params) => Ok(InitializedGridRasterization { - name, - path, - source: vector_source, - result_descriptor: out_desc, - spatial_resolution: params.spatial_resolution, - grid_size_mode: params.grid_size_mode, - tiling_specification, - origin_coordinate: params.origin_coordinate, - } - .boxed()), - GridOrDensity::Density(params) => InitializedDensityRasterization::new( + if let Some(density_params) = self.params.density_params { + return InitializedDensityRasterization::new( name, path, vector_source, out_desc, tiling_specification, - params.cutoff, - params.stddev, + density_params.cutoff, + density_params.stddev, ) - .map(InitializedRasterOperator::boxed), + .map(InitializedRasterOperator::boxed); + } + + Ok(InitializedGridRasterization { + name, + path, + source: vector_source, + result_descriptor: out_desc, + tiling_specification, } + .boxed()) } span_fn!(Rasterization); @@ -144,10 +137,7 @@ pub struct InitializedGridRasterization { path: WorkflowOperatorPath, source: Box, result_descriptor: RasterResultDescriptor, - spatial_resolution: SpatialResolution, - grid_size_mode: GridSizeMode, tiling_specification: TilingSpecification, - origin_coordinate: Coordinate2D, } impl InitializedRasterOperator for InitializedGridRasterization { @@ -160,10 +150,7 @@ impl InitializedRasterOperator for InitializedGridRasterization { GridRasterizationQueryProcessor { input: self.source.query_processor()?, result_descriptor: self.result_descriptor.clone(), - spatial_resolution: self.spatial_resolution, - grid_size_mode: self.grid_size_mode, tiling_specification: self.tiling_specification, - origin_coordinate: self.origin_coordinate, } .boxed(), )) @@ -180,6 +167,40 @@ impl InitializedRasterOperator for InitializedGridRasterization { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let rasterization_resolution = self.result_descriptor.spatial_grid.spatial_resolution(); + + let spatial_resolution = if target_resolution <= rasterization_resolution { + // target resolution is finer than the rasterization resolution, no need to change the resolution + rasterization_resolution + } else { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + // TODO: use `target_resolution` here? Due to the `ensure_resolution_is_compatible_for_optimization` call we know that the target resolution is a multiple of the rasterization resolution + find_next_best_overview_level_resolution(rasterization_resolution, target_resolution) + }; + + Ok(Rasterization { + params: RasterizationParams { + spatial_resolution, + origin_coordinate: self + .result_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate, + density_params: None, + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, // TODO: use `rasterization_resolution` here instead? + }, + } + .boxed()) + } } pub struct InitializedDensityRasterization { @@ -189,6 +210,7 @@ pub struct InitializedDensityRasterization { result_descriptor: RasterResultDescriptor, tiling_specification: TilingSpecification, radius: f64, + cutoff: f64, stddev: f64, } @@ -226,6 +248,7 @@ impl InitializedDensityRasterization { result_descriptor, tiling_specification, radius, + cutoff, stddev, }) } @@ -260,20 +283,57 @@ impl InitializedRasterOperator for InitializedDensityRasterization { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let rasterization_resolution = self.result_descriptor.spatial_grid.spatial_resolution(); + + let spatial_resolution = if target_resolution <= rasterization_resolution { + // target resolution is finer than the rasterization resolution, no need to change the resolution + rasterization_resolution + } else { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + // TODO: use `target_resolution` here? Due to the `ensure_resolution_is_compatible_for_optimization` call we know that the target resolution is a multiple of the rasterization resolution + find_next_best_overview_level_resolution(rasterization_resolution, target_resolution) + }; + + Ok(Rasterization { + params: RasterizationParams { + spatial_resolution, + origin_coordinate: self + .result_descriptor + .spatial_grid + .geo_transform() + .origin_coordinate, + density_params: Some(DensityParams { + cutoff: self.cutoff, + stddev: self.stddev, + }), + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, // TODO: use `rasterization_resolution` here instead? + }, + } + .boxed()) + } } pub struct GridRasterizationQueryProcessor { input: TypedVectorQueryProcessor, result_descriptor: RasterResultDescriptor, - spatial_resolution: SpatialResolution, - grid_size_mode: GridSizeMode, tiling_specification: TilingSpecification, - origin_coordinate: Coordinate2D, } #[async_trait] -impl RasterQueryProcessor for GridRasterizationQueryProcessor { - type RasterType = f64; +impl QueryProcessor for GridRasterizationQueryProcessor { + type Output = RasterTile2D; + type SpatialBounds = GridBoundingBox2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; /// Performs a grid rasterization by first determining the grid resolution to use. /// The grid resolution is limited to the query resolution, because a finer granularity @@ -282,65 +342,45 @@ impl RasterQueryProcessor for GridRasterizationQueryProcessor { /// All points within the spatial bounds of the grid are queried and counted in the /// grid cells. /// Finally, the grid resolution is upsampled (if necessary) to the tile resolution. - async fn raster_query<'a>( + #[allow(clippy::too_many_lines)] + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, - ) -> util::Result>>> { - if let MultiPoint(points_processor) = &self.input { - let grid_resolution = match self.grid_size_mode { - GridSizeMode::Fixed => SpatialResolution { - x: f64::max(self.spatial_resolution.x, query.spatial_resolution.x), - y: f64::max(self.spatial_resolution.y, query.spatial_resolution.y), - }, - GridSizeMode::Relative => SpatialResolution { - x: f64::max( - self.spatial_resolution.x * query.spatial_resolution.x, - query.spatial_resolution.x, - ), - y: f64::max( - self.spatial_resolution.y * query.spatial_resolution.y, - query.spatial_resolution.y, - ), - }, - }; + ) -> util::Result>> { + let spatial_grid_desc = self + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()); - let tiling_strategy = self - .tiling_specification - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - let tile_shape = tiling_strategy.tile_size_in_pixels; + let tiling_strategy = spatial_grid_desc.generate_data_tiling_strategy(); + let tiling_geo_transform = spatial_grid_desc.tiling_geo_transform(); + let query_time = query.time_interval(); + + if let MultiPoint(points_processor) = &self.input { + let query_grid_bounds = query.spatial_bounds(); + let query_spatial_partition = + tiling_geo_transform.grid_to_spatial_bounds(&query_grid_bounds); let tiles = stream::iter( - tiling_strategy.tile_information_iterator(query.spatial_bounds), + tiling_strategy.tile_information_iterator_from_pixel_bounds(query.spatial_bounds()), ) .then(move |tile_info| async move { - let grid_spatial_bounds = tile_info - .spatial_partition() - .snap_to_grid(self.origin_coordinate, grid_resolution); - - let grid_size_x = - f64::ceil(grid_spatial_bounds.size_x() / grid_resolution.x) as usize; - let grid_size_y = - f64::ceil(grid_spatial_bounds.size_y() / grid_resolution.y) as usize; - - let vector_query = VectorQueryRectangle { - spatial_bounds: grid_spatial_bounds.as_bbox(), - time_interval: query.time_interval, - spatial_resolution: grid_resolution, - attributes: ColumnSelection::all(), - }; - - let grid_geo_transform = GeoTransform::new( - grid_spatial_bounds.upper_left(), - grid_resolution.x, - -grid_resolution.y, + let tile_spatial_bounds = tile_info.spatial_partition(); + + let grid_size_x = tile_info.tile_size_in_pixels().axis_size_x(); + + let vector_query = VectorQueryRectangle::new( + tile_spatial_bounds.as_bbox(), + query_time, + ColumnSelection::all(), // FIXME: should be configurable ); let mut chunks = points_processor.query(vector_query, ctx).await?; let mut cache_hint = CacheHint::max_duration(); - let mut grid_data = vec![0.; grid_size_x * grid_size_y]; + let mut grid_data = + GridWithFlexibleBoundType::new_filled(tile_info.global_pixel_bounds(), 0.); while let Some(chunk) = chunks.next().await { let chunk = chunk?; @@ -348,11 +388,16 @@ impl RasterQueryProcessor for GridRasterizationQueryProcessor { grid_data = spawn_blocking(move || { for &coord in chunk.coordinates() { - if !grid_spatial_bounds.contains_coordinate(&coord) { + if !tile_spatial_bounds.contains_coordinate(&coord) + || !query_spatial_partition.contains_coordinate(&coord) + // TODO: old code checks if the pixel center is in the query bounds. + { continue; } - let [y, x] = grid_geo_transform.coordinate_to_grid_idx_2d(coord).0; - grid_data[x as usize + y as usize * grid_size_x] += 1.; + let GridIdx([y, x]) = tiling_geo_transform + .coordinate_to_grid_idx_2d(coord) + - tile_info.global_upper_left_pixel_idx(); + grid_data.data[x as usize + y as usize * grid_size_x] += 1.; } grid_data }) @@ -360,34 +405,10 @@ impl RasterQueryProcessor for GridRasterizationQueryProcessor { .expect("Should only forward panics from spawned task"); } - let tile_data = spawn_blocking(move || { - let mut tile_data = Vec::with_capacity(tile_shape.number_of_elements()); - for tile_y in 0..tile_shape.axis_size_y() as isize { - for tile_x in 0..tile_shape.axis_size_x() as isize { - let pixel_coordinate = tile_info - .tile_geo_transform() - .grid_idx_to_pixel_center_coordinate_2d([tile_y, tile_x].into()); - if query.spatial_bounds.contains_coordinate(&pixel_coordinate) { - let [grid_y, grid_x] = grid_geo_transform - .coordinate_to_grid_idx_2d(pixel_coordinate) - .0; - tile_data.push( - grid_data[grid_x as usize + grid_y as usize * grid_size_x], - ); - } else { - tile_data.push(0.); - } - } - } - tile_data - }) - .await - .expect("Should only forward panics from spawned task"); - let tile_grid = Grid2D::new(tile_shape, tile_data) - .expect("Data vector length should match the number of pixels in the tile"); + let tile_grid = grid_data.unbounded(); Ok(RasterTile2D::new_with_tile_info( - query.time_interval, + query_time, tile_info, 0, GridOrEmpty::Grid(tile_grid.into()), @@ -396,15 +417,40 @@ impl RasterQueryProcessor for GridRasterizationQueryProcessor { }); Ok(tiles.boxed()) } else { - Ok(generate_zeroed_tiles(self.tiling_specification, &query)) + Ok(generate_zeroed_tiles( + tiling_geo_transform, + self.tiling_specification, + &query, + )) } } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for GridRasterizationQueryProcessor { + type RasterType = f64; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + _ctx: &'a dyn crate::engine::QueryContext, + ) -> crate::util::Result< + futures::stream::BoxStream< + 'a, + crate::util::Result, + >, + > { + warn!( + "Time query called on Rasterization operator. This operator does not comply with the time semantics and needs to be fixed." + ); + Ok(stream::iter(vec![Ok(query.time())]).boxed()) + } +} + pub struct DensityRasterizationQueryProcessor { input: TypedVectorQueryProcessor, result_descriptor: RasterResultDescriptor, @@ -414,8 +460,11 @@ pub struct DensityRasterizationQueryProcessor { } #[async_trait] -impl RasterQueryProcessor for DensityRasterizationQueryProcessor { - type RasterType = f64; +impl QueryProcessor for DensityRasterizationQueryProcessor { + type Output = RasterTile2D; + type SpatialBounds = GridBoundingBox2D; + type ResultDescription = RasterResultDescriptor; + type Selection = BandSelection; /// Performs a gaussian density rasterization. /// For each tile, the spatial bounds are extended by `radius` in x and y direction. @@ -423,47 +472,48 @@ impl RasterQueryProcessor for DensityRasterizationQueryProcessor { /// its surrounding tile pixels (up to `radius` distance) is measured and input into the /// gaussian density function with the configured standard deviation. The density values /// for each pixel are then summed to result in the tile pixel grid. - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, - ) -> util::Result>>> { - if let MultiPoint(points_processor) = &self.input { - let tiling_strategy = self - .tiling_specification - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); + ) -> util::Result>> { + let spatial_grid_desc = self + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()); - let tile_size_x = tiling_strategy.tile_size_in_pixels.axis_size_x(); - let tile_size_y = tiling_strategy.tile_size_in_pixels.axis_size_y(); + let tiling_strategy = spatial_grid_desc.generate_data_tiling_strategy(); + let tiling_geo_transform = spatial_grid_desc.tiling_geo_transform(); + let query_time = query.time_interval(); + if let MultiPoint(points_processor) = &self.input { // Use rounding factor calculated from query resolution to extend in full pixel units let rounding_factor = f64::max( - 1. / query.spatial_resolution.x, - 1. / query.spatial_resolution.y, + 1. / tiling_geo_transform.x_pixel_size(), + 1. / tiling_geo_transform.y_pixel_size(), ); let radius = (self.radius * rounding_factor).ceil() / rounding_factor; + let query_grid_bounds = query.spatial_bounds(); + let query_spatial_partition = + tiling_geo_transform.grid_to_spatial_bounds(&query_grid_bounds); + let tiles = stream::iter( - tiling_strategy.tile_information_iterator(query.spatial_bounds), + tiling_strategy.tile_information_iterator_from_pixel_bounds(query.spatial_bounds()), ) .then(move |tile_info| async move { - let tile_bounds = tile_info.spatial_partition(); + let tile_spatial_bounds = tile_info.spatial_partition(); - let vector_query = VectorQueryRectangle { - spatial_bounds: extended_bounding_box_from_spatial_partition( - tile_bounds, - radius, - ), - time_interval: query.time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let vector_query = VectorQueryRectangle::new( + extended_bounding_box_from_spatial_partition(tile_spatial_bounds, radius), + query_time, + ColumnSelection::all(), // FIXME: should be configurable + ); let tile_geo_transform = tile_info.tile_geo_transform(); let mut chunks = points_processor.query(vector_query, ctx).await?; - let mut tile_data = vec![0.; tile_size_x * tile_size_y]; + let mut tile_data = Grid2D::new_filled(tiling_strategy.tile_size_in_pixels, 0.0); let mut cache_hint = CacheHint::max_duration(); @@ -475,24 +525,27 @@ impl RasterQueryProcessor for DensityRasterizationQueryProcessor { let stddev = self.stddev; tile_data = spawn_blocking_with_thread_pool(ctx.thread_pool().clone(), move || { - tile_data.par_iter_mut().enumerate().for_each( - |(linear_index, pixel)| { - let pixel_coordinate = tile_geo_transform - .grid_idx_to_pixel_center_coordinate_2d( - tile_geo_transform - .spatial_to_grid_bounds(&tile_bounds) - .grid_idx_unchecked(linear_index), - ); - - for coord in chunk.coordinates() { - let distance = coord.euclidean_distance(&pixel_coordinate); - - if distance <= radius { - *pixel += gaussian(distance, stddev); - } + tile_data.update_indexed_elements_parallel(|pixel_idx, pixel| { + let pixel_coordinate = tile_geo_transform + .grid_idx_to_pixel_center_coordinate_2d(pixel_idx); + + if !query_spatial_partition.contains_coordinate(&pixel_coordinate) { + // TODO: this is propably wrong to do since it produces different tiles depending on the query bounds + return pixel; + } + + let mut pixel_tmp = pixel; + + for coord in chunk.coordinates() { + let distance = coord.euclidean_distance(&pixel_coordinate); + + if distance <= radius { + pixel_tmp += gaussian(distance, stddev); } - }, - ); + } + + pixel_tmp + }); tile_data }) @@ -500,43 +553,63 @@ impl RasterQueryProcessor for DensityRasterizationQueryProcessor { } Ok(RasterTile2D::new_with_tile_info( - query.time_interval, + query_time, // FIXME this breaks the semantics we usually require tile_info, 0, - GridOrEmpty::Grid( - Grid2D::new(tiling_strategy.tile_size_in_pixels, tile_data) - .expect( - "Data vector length should match the number of pixels in the tile", - ) - .into(), - ), + tile_data.into(), cache_hint, )) }); Ok(tiles.boxed()) } else { - Ok(generate_zeroed_tiles(self.tiling_specification, &query)) + Ok(generate_zeroed_tiles( + tiling_geo_transform, + self.tiling_specification, + &query, + )) } } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for DensityRasterizationQueryProcessor { + type RasterType = f64; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + _ctx: &'a dyn crate::engine::QueryContext, + ) -> crate::util::Result< + futures::stream::BoxStream< + 'a, + crate::util::Result, + >, + > { + warn!( + "Time query called on Rasterization operator. This operator does not comply with the time semantics and needs to be fixed." + ); + Ok(stream::iter(vec![Ok(query.time())]).boxed()) + } +} + fn generate_zeroed_tiles<'a>( + tiling_geo_transform: GeoTransform, tiling_specification: TilingSpecification, query: &RasterQueryRectangle, ) -> BoxStream<'a, util::Result>> { - let tiling_strategy = - tiling_specification.strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - let tile_shape = tiling_strategy.tile_size_in_pixels; - let time_interval = query.time_interval; + let tile_shape = tiling_specification.tile_size_in_pixels; + let time_interval = query.time_interval(); + + let tiling_strategy = TilingStrategy::new(tile_shape, tiling_geo_transform); stream::iter( tiling_strategy - .tile_information_iterator(query.spatial_bounds) + .tile_information_iterator_from_pixel_bounds(query.spatial_bounds()) .map(move |tile_info| { let tile_data = vec![0.; tile_shape.number_of_elements()]; let tile_grid = Grid2D::new(tile_shape, tile_data) @@ -560,28 +633,12 @@ fn extended_bounding_box_from_spatial_partition( ) -> BoundingBox2D { BoundingBox2D::new_unchecked( Coordinate2D::new( - spatial_partition - .lower_left() - .x - .sub_checked(extent) - .unwrap_or(f64::MIN), - spatial_partition - .lower_left() - .y - .sub_checked(extent) - .unwrap_or(f64::MIN), + spatial_partition.lower_left().x - extent, + spatial_partition.lower_left().y - extent, ), Coordinate2D::new( - spatial_partition - .upper_right() - .x - .add_checked(extent) - .unwrap_or(f64::MAX), - spatial_partition - .upper_right() - .y - .add_checked(extent) - .unwrap_or(f64::MAX), + spatial_partition.upper_right().x + extent, + spatial_partition.upper_right().y + extent, ), ) } @@ -606,27 +663,27 @@ mod tests { RasterOperator, SingleVectorSource, VectorOperator, WorkflowOperatorPath, }; use crate::mock::{MockPointSource, MockPointSourceParams}; - use crate::processing::rasterization::GridSizeMode::{Fixed, Relative}; use crate::processing::rasterization::{ - DensityParams, GridOrDensity, GridParams, Rasterization, gaussian, + DensityParams, Rasterization, RasterizationParams, gaussian, }; use futures::StreamExt; use geoengine_datatypes::primitives::{ - BandSelection, Coordinate2D, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, + BandSelection, BoundingBox2D, Coordinate2D, RasterQueryRectangle, SpatialResolution, }; - use geoengine_datatypes::raster::TilingSpecification; + use geoengine_datatypes::raster::{GridBoundingBox2D, TilingSpecification}; use geoengine_datatypes::util::test::TestDefault; async fn get_results( rasterization: Box, query: RasterQueryRectangle, + query_ctx: &MockQueryContext, ) -> Vec> { rasterization .query_processor() .unwrap() .get_f64() .unwrap() - .query(query, &MockQueryContext::test_default()) + .query(query, query_ctx) .await .unwrap() .map(|res| { @@ -642,25 +699,31 @@ mod tests { #[tokio::test] async fn fixed_grid_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { + params: RasterizationParams { spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, origin_coordinate: [0., 0.].into(), - grid_size_mode: Fixed, - }), + density_params: None, + }, sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ + params: MockPointSourceParams::new_with_bounds( + vec![ (-1., 1.).into(), (1., 1.).into(), (-1., -1.).into(), (1., -1.).into(), ], - }, + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), + ), } .boxed(), }, @@ -670,14 +733,18 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., -2.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, -2], [1, 1]).unwrap(), + Default::default(), + BandSelection::first(), + ); - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context(TestDefault::test_default()), + ) + .await; assert_eq!( res, @@ -692,75 +759,31 @@ mod tests { #[tokio::test] async fn fixed_grid_with_shift() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { + params: RasterizationParams { spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0.5, -0.5].into(), - grid_size_mode: Fixed, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), + origin_coordinate: [0.0, 0.0].into(), + density_params: None, }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., -2.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - vec![1., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn fixed_grid_with_upsampling() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 2.0, y: 2.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Fixed, - }), sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ + params: MockPointSourceParams::new_with_bounds( + vec![ (-1., 1.).into(), (1., 1.).into(), (-1., -1.).into(), (1., -1.).into(), ], - }, + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), + ), } .boxed(), }, @@ -770,149 +793,55 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-3., 3.].into(), [3., -3.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![0., 0., 0., 0., 1., 1., 0., 1., 1.], - vec![0., 0., 0., 1., 1., 0., 1., 1., 0.], - vec![0., 1., 1., 0., 1., 1., 0., 0., 0.], - vec![1., 1., 0., 1., 1., 0., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn relative_grid_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, 1, -2, 1).unwrap(), + Default::default(), + BandSelection::first(), ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Relative, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1.5, 1.5].into(), [1.5, -1.5].into()) - .unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context(TestDefault::test_default()), + ) + .await; assert_eq!( res, vec![ - vec![0., 0., 0., 0., 1., 0., 0., 0., 0.], - vec![0., 0., 0., 0., 0., 1., 0., 0., 0.], - vec![0., 0., 0., 0., 0., 0., 0., 1., 0.], - vec![0., 0., 0., 0., 0., 0., 0., 0., 1.], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], + vec![0., 0., 0., 1.], ] ); } #[tokio::test] - async fn relative_grid_with_shift() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [3, 3].into()), - ); + async fn density_basic() { + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { + params: RasterizationParams { spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - origin_coordinate: [0.25, -0.25].into(), - grid_size_mode: Relative, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, - } - .boxed(), + origin_coordinate: [0.0, 0.0].into(), + density_params: Some(DensityParams { + cutoff: gaussian(0.99, 1.0) / gaussian(0., 1.0), + stddev: 1.0, + }), }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1.5, 1.5].into(), [1.5, -1.5].into()) - .unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 0., 0., 0., 0., 0., 0., 0., 0.], - vec![0., 1., 0., 0., 0., 0., 0., 0., 0.], - vec![0., 0., 0., 1., 0., 0., 0., 0., 0.], - vec![0., 0., 0., 0., 1., 0., 0., 0., 0.], - ] - ); - } - - #[tokio::test] - async fn relative_grid_with_upsampling() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Grid(GridParams { - spatial_resolution: SpatialResolution { x: 2.0, y: 2.0 }, - origin_coordinate: [0., 0.].into(), - grid_size_mode: Relative, - }), sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![ - (-1., 1.).into(), - (1., 1.).into(), - (-1., -1.).into(), - (1., -1.).into(), - ], - }, + params: MockPointSourceParams::new_with_bounds( + vec![(-1., 1.).into(), (1., 1.).into()], + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), + ), } .boxed(), }, @@ -922,58 +851,18 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-1., 1.].into(), [1., -1.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 0.5, y: 0.5 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; - - assert_eq!( - res, - vec![ - vec![1., 1., 1., 1.], - vec![0., 0., 0., 0.], - vec![0., 0., 0., 0.], - vec![0., 0., 0., 0.] - ] + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, -2, 1).unwrap(), + Default::default(), + BandSelection::first(), ); - } - #[tokio::test] - async fn density_basic() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); - let rasterization = Rasterization { - params: GridOrDensity::Density(DensityParams { - cutoff: gaussian(0.99, 1.0) / gaussian(0., 1.0), - stddev: 1.0, - }), - sources: SingleVectorSource { - vector: MockPointSource { - params: MockPointSourceParams { - points: vec![(-1., 1.).into(), (1., 1.).into()], - }, - } - .boxed(), - }, - } - .boxed() - .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) - .await - .unwrap(); - - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., 0.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; - - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context_test_default(), + ) + .await; assert_eq!( res, @@ -1024,19 +913,29 @@ mod tests { #[tokio::test] async fn density_radius_overlap() { - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new([0., 0.].into(), [2, 2].into()), - ); + let execution_context = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); let rasterization = Rasterization { - params: GridOrDensity::Density(DensityParams { - cutoff: gaussian(1.99, 1.0) / gaussian(0., 1.0), - stddev: 1.0, - }), + params: RasterizationParams { + spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, + origin_coordinate: [0.0, 0.0].into(), + density_params: Some(DensityParams { + cutoff: gaussian(1.99, 1.0) / gaussian(0., 1.0), + stddev: 1.0, + }), + }, sources: SingleVectorSource { vector: MockPointSource { - params: MockPointSourceParams { - points: vec![(-1., 1.).into(), (1., 1.).into()], - }, + params: MockPointSourceParams::new_with_bounds( + vec![(-1., 1.).into(), (1., 1.).into()], + crate::mock::SpatialBoundsDerive::Bounds( + BoundingBox2D::new( + Coordinate2D::new(-2., -2.), + Coordinate2D::new(2., 2.), + ) + .unwrap(), + ), + ), } .boxed(), }, @@ -1046,14 +945,18 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new([-2., 2.].into(), [2., 0.].into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution { x: 1.0, y: 1.0 }, - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, -2, 1).unwrap(), + Default::default(), + BandSelection::first(), + ); - let res = get_results(rasterization, query).await; + let res = get_results( + rasterization, + query, + &execution_context.mock_query_context_test_default(), + ) + .await; assert_eq!( res, diff --git a/operators/src/processing/reprojection.rs b/operators/src/processing/reprojection.rs index 5882905f3..7d925ec8b 100644 --- a/operators/src/processing/reprojection.rs +++ b/operators/src/processing/reprojection.rs @@ -3,48 +3,61 @@ use std::marker::PhantomData; use super::map_query::MapQueryProcessor; use crate::{ adapters::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, RasterSubQueryAdapter, - SparseTilesFillAdapter, TileReprojectionSubQuery, fold_by_coordinate_lookup_future, + RasterSubQueryAdapter, TileReprojectionSubQuery, TileReprojectionSubqueryGridInfo, + fold_by_coordinate_lookup_future, }, engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, InitializedVectorOperator, Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterOrVectorSource, - TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, - VectorResultDescriptor, WorkflowOperatorPath, + SingleRasterSource, SpatialGridDescriptor, TypedRasterQueryProcessor, + TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, + WorkflowOperatorPath, }, error::{self, Error}, - util::Result, + optimization::{OptimizableOperator, OptimizationError, ProjectionOptimizationFailed}, + processing::{ + Downsampling, DownsamplingMethod, DownsamplingParams, DownsamplingResolution, + Interpolation, InterpolationMethod, InterpolationParams, InterpolationResolution, + }, + util::{Result, input::RasterOrVectorOperator}, }; use async_trait::async_trait; use futures::stream::BoxStream; use futures::{StreamExt, stream}; use geoengine_datatypes::{ collections::FeatureCollection, + error::BoxedResultExt, operations::reproject::{ CoordinateProjection, CoordinateProjector, Reproject, ReprojectClipped, - reproject_and_unify_bbox, reproject_query, suggest_pixel_size_from_diag_cross_projected, + reproject_spatial_query, }, primitives::{ BandSelection, BoundingBox2D, ColumnSelection, Geometry, RasterQueryRectangle, - SpatialPartition2D, SpatialPartitioned, SpatialResolution, VectorQueryRectangle, + SpatialPartition2D, SpatialResolution, VectorQueryRectangle, }, - raster::{Pixel, RasterTile2D, TilingSpecification}, + raster::{GridBoundingBox2D, Pixel, RasterTile2D, TilingSpecification}, spatial_reference::SpatialReference, util::arrow::ArrowTyped, }; use serde::{Deserialize, Serialize}; +use tracing::trace; #[derive(Debug, Serialize, Deserialize, Clone, Copy)] #[serde(rename_all = "camelCase")] -pub struct ReprojectionParams { - pub target_spatial_reference: SpatialReference, +#[derive(Default)] +pub enum DeriveOutRasterSpecsSource { + DataBounds, + #[default] + ProjectionBounds, } #[derive(Debug, Serialize, Deserialize, Clone, Copy)] -pub struct ReprojectionBounds { - valid_in_bounds: SpatialPartition2D, - valid_out_bounds: SpatialPartition2D, +#[serde(rename_all = "camelCase")] +pub struct ReprojectionParams { + pub target_spatial_reference: SpatialReference, + #[serde(default)] + pub derive_out_spec: DeriveOutRasterSpecsSource, } pub type Reprojection = Operator; @@ -59,20 +72,21 @@ pub struct InitializedVectorReprojection { name: CanonicOperatorName, path: WorkflowOperatorPath, result_descriptor: VectorResultDescriptor, + params: ReprojectionParams, source: Box, source_srs: SpatialReference, target_srs: SpatialReference, } -pub struct InitializedRasterReprojection { +pub struct InitializedRasterReprojection { name: CanonicOperatorName, path: WorkflowOperatorPath, result_descriptor: RasterResultDescriptor, - source: Box, - state: Option, + params: ReprojectionParams, + source: O, + state: TileReprojectionSubqueryGridInfo, source_srs: SpatialReference, target_srs: SpatialReference, - tiling_spec: TilingSpecification, } impl InitializedVectorReprojection { @@ -114,6 +128,7 @@ impl InitializedVectorReprojection { name, path, result_descriptor: out_desc, + params, source: source_vector_operator, source_srs: in_srs, target_srs: params.target_spatial_reference, @@ -121,85 +136,101 @@ impl InitializedVectorReprojection { } } -impl InitializedRasterReprojection { +impl InitializedRasterReprojection { pub fn try_new_with_input( name: CanonicOperatorName, path: WorkflowOperatorPath, params: ReprojectionParams, - source_raster_operator: Box, + source_raster_operator: O, tiling_spec: TilingSpecification, ) -> Result { let in_desc: RasterResultDescriptor = source_raster_operator.result_descriptor().clone(); - let in_srs = Into::>::into(in_desc.spatial_reference) .ok_or(Error::AllSourcesMustHaveSameSpatialReference)?; // calculate the intersection of input and output srs in both coordinate systems - let (in_bounds, out_bounds, out_res) = Self::derive_raster_in_bounds_out_bounds_out_res( - in_srs, - params.target_spatial_reference, - in_desc.resolution, - in_desc.bbox, - )?; + let proj_from_to = + CoordinateProjector::from_known_srs(in_srs, params.target_spatial_reference)?; + + let out_spatial_grid = match params.derive_out_spec { + DeriveOutRasterSpecsSource::DataBounds => in_desc + .spatial_grid_descriptor() + .reproject_clipped(&proj_from_to)?, // TODO: we could skip the intersection (clipped) and try to project larger area + DeriveOutRasterSpecsSource::ProjectionBounds => { + let in_srs_area: SpatialPartition2D = in_srs.area_of_use_projected()?; // TODO: since we clip in projection anyway, we could use the AOU of the source projection? + let target_proj_total_grid = in_desc + .spatial_grid_descriptor() + .spatial_bounds_to_compatible_spatial_grid(in_srs_area) + .reproject_clipped(&proj_from_to)?; + // TODO: we could skip the intersection and try to project larger area + let spatial_bounds_proj = + in_desc.spatial_bounds().reproject_clipped(&proj_from_to)?; + target_proj_total_grid.and_then(|x| { + spatial_bounds_proj.map(|spb| x.spatial_bounds_to_compatible_spatial_grid(spb)) + }) + } + }; - let result_descriptor = RasterResultDescriptor { + // Operator will return an error when there is no intersection between data and output projection bounds! + let out_spatial_grid = out_spatial_grid.ok_or(error::Error::ReprojectionFailed)?; // TODO: better error! + + let out_desc = RasterResultDescriptor { spatial_reference: params.target_spatial_reference.into(), data_type: in_desc.data_type, time: in_desc.time, - bbox: out_bounds, - resolution: out_res, + spatial_grid: out_spatial_grid, bands: in_desc.bands.clone(), }; - let state = match (in_bounds, out_bounds) { - (Some(in_bounds), Some(out_bounds)) => Some(ReprojectionBounds { - valid_in_bounds: in_bounds, - valid_out_bounds: out_bounds, - }), - _ => None, + let state = TileReprojectionSubqueryGridInfo { + in_spatial_grid: in_desc + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition(), + out_spatial_grid: out_spatial_grid + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition(), }; Ok(InitializedRasterReprojection { name, path, - result_descriptor, + params, + result_descriptor: out_desc, source: source_raster_operator, state, source_srs: in_srs, target_srs: params.target_spatial_reference, - tiling_spec, }) } +} - fn derive_raster_in_bounds_out_bounds_out_res( - source_srs: SpatialReference, - target_srs: SpatialReference, - source_spatial_resolution: Option, - source_bbox: Option, - ) -> Result<( - Option, - Option, - Option, - )> { - let (in_bbox, out_bbox) = if let Some(bbox) = source_bbox { - reproject_and_unify_bbox(bbox, source_srs, target_srs)? - } else { - // use the parts of the area of use that are valid in both spatial references - let valid_bounds_in = source_srs.area_of_use_intersection(&target_srs)?; - let valid_bounds_out = target_srs.area_of_use_intersection(&source_srs)?; - - (valid_bounds_in, valid_bounds_out) - }; - - let out_res = match (source_spatial_resolution, in_bbox, out_bbox) { - (Some(in_res), Some(in_bbox), Some(out_bbox)) => { - suggest_pixel_size_from_diag_cross_projected(in_bbox, out_bbox, in_res).ok() - } - _ => None, - }; - - Ok((in_bbox, out_bbox, out_res)) - } +fn compute_output_spatial_grid( + in_spatial_grid_descriptor: SpatialGridDescriptor, + in_spatial_bounds: SpatialPartition2D, + in_srs: SpatialReference, + params: ReprojectionParams, +) -> Result { + let proj_from_to = + CoordinateProjector::from_known_srs(in_srs, params.target_spatial_reference)?; + let out_spatial_grid = match params.derive_out_spec { + DeriveOutRasterSpecsSource::DataBounds => { + in_spatial_grid_descriptor.reproject_clipped(&proj_from_to)? + } + DeriveOutRasterSpecsSource::ProjectionBounds => { + let in_srs_area: SpatialPartition2D = in_srs.area_of_use_projected()?; // TODO: since we clip in projection anyway, we could use the AOU of the source projection? + let target_proj_total_grid = in_spatial_grid_descriptor + .spatial_bounds_to_compatible_spatial_grid(in_srs_area) + .reproject_clipped(&proj_from_to)?; + // jetzt grid mit origin (tl) auf grid vom dataset. dann umprojeziren. Dann intersection mit boundingbox in dataset + let spatial_bounds_proj = in_spatial_bounds.reproject_clipped(&proj_from_to)?; + target_proj_total_grid.and_then(|x| { + spatial_bounds_proj.map(|spb| x.spatial_bounds_to_compatible_spatial_grid(spb)) + }) + } + }; + let out_spatial_grid = out_spatial_grid.ok_or(error::Error::ReprojectionFailed)?; + Ok(out_spatial_grid) } #[typetag::serde] @@ -247,14 +278,19 @@ impl InitializedVectorOperator for InitializedVectorReprojection { let target_srs = self.target_srs; match self.source.query_processor()? { TypedVectorQueryProcessor::Data(source) => Ok(TypedVectorQueryProcessor::Data( - MapQueryProcessor::new( - source, - self.result_descriptor.clone(), - move |query| { - reproject_query(query, source_srs, target_srs, false).map_err(From::from) - }, - (), - ) + MapQueryProcessor::new(source, move |query: VectorQueryRectangle| { + reproject_spatial_query(query.spatial_bounds(), source_srs, target_srs, false) + .map(|sqr| { + sqr.map(|x| { + VectorQueryRectangle::new( + x, + query.time_interval(), + *query.attributes(), + ) + }) + }) + .map_err(From::from) + }) .boxed(), )), TypedVectorQueryProcessor::MultiPoint(source) => { @@ -304,6 +340,22 @@ impl InitializedVectorOperator for InitializedVectorReprojection { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.source + .optimize(target_resolution) + .map(|optimized_source| { + Box::new(Reprojection { + params: self.params, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Vector(optimized_source), + }, + }) as Box + }) + } } struct VectorReprojectionProcessor @@ -357,7 +409,10 @@ where query: VectorQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - let rewritten_query = reproject_query(query, self.from, self.to, false)?; + let rewritten_spatial_query = + reproject_spatial_query(query.spatial_bounds(), self.from, self.to, false)?; + + let rewritten_query = rewritten_spatial_query.map(|rwq| query.select_spatial_bounds(rwq)); if let Some(rewritten_query) = rewritten_query { Ok(self @@ -419,7 +474,7 @@ impl RasterOperator for Reprojection { span_fn!(Reprojection); } -impl InitializedRasterOperator for InitializedRasterReprojection { +impl InitializedRasterOperator for InitializedRasterReprojection { fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } @@ -439,7 +494,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -452,7 +506,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -466,7 +519,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -479,7 +531,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -492,7 +543,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -505,7 +555,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -518,7 +567,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -531,7 +579,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -544,7 +591,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -557,7 +603,6 @@ impl InitializedRasterOperator for InitializedRasterReprojection { self.result_descriptor.clone(), self.source_srs, self.target_srs, - self.tiling_spec, self.state, ))) } @@ -575,6 +620,106 @@ impl InitializedRasterOperator for InitializedRasterReprojection { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + // adjust input resolution relative to change in target resolution + // idea: optimization of reprojection snaps from one overview level to another, so do the same in the source + // if the result does not match, we need to interpolate or resample the result to match the target resolution + + // TODO: validate the quality of this approach, especially if the output resolution is actually the target resolution + + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + let current_resolution = self.result_descriptor.spatial_grid.spatial_resolution(); + + // TODO: use `find_next_best_resolution_overview_level` instead? + let output_overview_level_factor = + (target_resolution.x / current_resolution.x).round() as u32; + // must be the same in x and y + debug_assert_eq!( + output_overview_level_factor, + (target_resolution.y / current_resolution.y).round() as u32 + ); + // must be a power of 2 + debug_assert!(output_overview_level_factor.is_power_of_two()); + + let input_descriptor = self.source.result_descriptor(); + + let input_resolution = input_descriptor.spatial_grid.spatial_resolution(); + let scaled_input_resolution = input_resolution * f64::from(output_overview_level_factor); + + let source_optimized = self.source.optimize(scaled_input_resolution)?; + + // given the input resolution, derive_out_spec, input and output spatial reference, calculate the output spatial grid + // TODO: instead of repeating the logic here, we could attach the result descriptor to the result of the `optimize` method + let optimized_spatial_grid_descriptor = input_descriptor + .spatial_grid_descriptor() + .with_changed_resolution(scaled_input_resolution); + + // TODO: check if these are the correct bounds + let optimized_spatial_bounds = optimized_spatial_grid_descriptor.spatial_partition(); + + let output_spatial_grid = compute_output_spatial_grid( + optimized_spatial_grid_descriptor, + optimized_spatial_bounds, + self.source_srs, + self.params, + ) + .boxed_context(ProjectionOptimizationFailed)?; + + let output_spatial_resolution = output_spatial_grid.spatial_resolution(); + trace!( + "output_spatial_resolution: {output_spatial_resolution:?}, target_resolution: {target_resolution:?}" + ); + + let optimized_reprojection = Box::new(Reprojection { + params: ReprojectionParams { + target_spatial_reference: self.target_srs, + derive_out_spec: self.params.derive_out_spec, + }, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(source_optimized), + }, + }); + + if output_spatial_resolution == target_resolution { + return Ok(optimized_reprojection); + } + + // depending on the reprojection, the reprojection of the optimized input might not perfectly match the target resolution, so we may need to fix it + + if output_spatial_resolution > target_resolution { + // reprojected raster is too coarse, we need to interpolate + // TODO: in this case we could (should?) also decrease the `scaled_input_resolution` to the previous overview level to get a finer result + return Ok(Interpolation { + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: InterpolationResolution::Resolution(target_resolution), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: optimized_reprojection, + }, + } + .boxed()); + } + + // reprojected raster is too fine, we need to downsample + Ok(Downsampling { + params: DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution(target_resolution), + output_origin_reference: None, + }, + sources: SingleRasterSource { + raster: optimized_reprojection, + }, + } + .boxed()) + } } pub struct RasterReprojectionProcessor @@ -585,19 +730,13 @@ where result_descriptor: RasterResultDescriptor, from: SpatialReference, to: SpatialReference, - tiling_spec: TilingSpecification, - state: Option, + state: TileReprojectionSubqueryGridInfo, _phantom_data: PhantomData

, } impl RasterReprojectionProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor, P: Pixel, { pub fn new( @@ -605,15 +744,13 @@ where result_descriptor: RasterResultDescriptor, from: SpatialReference, to: SpatialReference, - tiling_spec: TilingSpecification, - state: Option, + state: TileReprojectionSubqueryGridInfo, ) -> Self { Self { source, result_descriptor, from, to, - tiling_spec, state, _phantom_data: PhantomData, } @@ -623,16 +760,11 @@ where #[async_trait] impl QueryProcessor for RasterReprojectionProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor + Send + Sync, P: Pixel, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -641,56 +773,35 @@ where query: RasterQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - if let Some(state) = &self.state { - let valid_bounds_in = state.valid_in_bounds; - let valid_bounds_out = state.valid_out_bounds; - - // calculate the spatial resolution the input data should have using the intersection and the requested resolution - let in_spatial_res = suggest_pixel_size_from_diag_cross_projected( - valid_bounds_out, - valid_bounds_in, - query.spatial_resolution, - )?; - - // setup the subquery - let sub_query_spec = TileReprojectionSubQuery { - in_srs: self.from, - out_srs: self.to, - fold_fn: fold_by_coordinate_lookup_future, - in_spatial_res, - valid_bounds_in, - valid_bounds_out, - _phantom_data: PhantomData, - }; - - // return the adapter which will reproject the tiles and uses the fill adapter to inject missing tiles - Ok(RasterSubQueryAdapter::<'a, P, _, _>::new( - &self.source, - query, - self.tiling_spec, - ctx, - sub_query_spec, - ) - .filter_and_fill(FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles)) - } else { - tracing::debug!("No intersection between source data / srs and target srs"); - - let tiling_strat = self - .tiling_spec - .strategy(query.spatial_resolution.x, -query.spatial_resolution.y); - - let grid_bounds = tiling_strat.tile_grid_box(query.spatial_partition()); - Ok(Box::pin(SparseTilesFillAdapter::new( - stream::empty(), - grid_bounds, - query.attributes.count(), - tiling_strat.geo_transform, - self.tiling_spec.tile_size_in_pixels, - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - query.time_interval, - FillerTimeBounds::from(query.time_interval), // TODO: derive this from the query once the child query can provide this. - ))) - } + let state = self.state; + + // setup the subquery + let sub_query_spec = TileReprojectionSubQuery { + in_srs: self.from, + out_srs: self.to, + fold_fn: fold_by_coordinate_lookup_future, + state, + _phantom_data: PhantomData, + }; + + let tiling_strat = self + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(ctx.tiling_specification()) + .generate_data_tiling_strategy(); + + let time_stream = self.time_query(query.time_interval(), ctx).await?; + + // return the adapter which will reproject the tiles and uses the fill adapter to inject missing tiles + Ok(RasterSubQueryAdapter::<'a, P, _, _, _>::new( + &self.source, + query, + tiling_strat, + ctx, + sub_query_spec, + time_stream, + ) + .box_pin()) } fn result_descriptor(&self) -> &RasterResultDescriptor { @@ -698,43 +809,60 @@ where } } +#[async_trait] +impl RasterQueryProcessor for RasterReprojectionProcessor +where + Q: RasterQueryProcessor + Send + Sync, + P: Pixel, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + self.source.time_query(query, ctx).await + } +} + #[cfg(test)] mod tests { use super::*; - use crate::engine::{MockExecutionContext, MockQueryContext, RasterBandDescriptors}; + use crate::engine::{MockExecutionContext, RasterBandDescriptors, SpatialGridDescriptor}; use crate::mock::MockFeatureCollectionSource; use crate::mock::{MockRasterSource, MockRasterSourceParams}; + use crate::source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, + GdalMetaDataStatic, GdalSourceTimePlaceholder, TimeReference, + }; + use crate::util::gdal::add_ndvi_dataset; use crate::{ engine::{ChunkByteSize, VectorOperator}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, - GdalMetaDataRegular, GdalMetaDataStatic, GdalSource, GdalSourceParameters, - GdalSourceTimePlaceholder, TimeReference, - }, + source::{GdalSource, GdalSourceParameters}, test_data, - util::gdal::{add_ndvi_dataset, gdal_open_dataset}, }; - use float_cmp::approx_eq; + use float_cmp::{approx_eq, assert_approx_eq}; use futures::StreamExt; use geoengine_datatypes::collections::IntoGeometryIterator; - use geoengine_datatypes::dataset::NamedData; + use geoengine_datatypes::dataset::{DataId, DatasetId, NamedData}; + use geoengine_datatypes::hashmap; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, DateTimeParseFormat, + CacheHint, CacheTtlSeconds, DateTimeParseFormat, SpatialResolution, TimeGranularity, + TimeInstance, + }; + use geoengine_datatypes::primitives::{Coordinate2D, TimeStep}; + use geoengine_datatypes::raster::{ + GeoTransform, GridBoundingBox2D, GridShape2D, GridSize, SpatialGridDefinition, + TilesEqualIgnoringCacheHint, }; - use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; - use geoengine_datatypes::raster::TilesEqualIgnoringCacheHint; use geoengine_datatypes::{ collections::{ GeometryCollection, MultiLineStringCollection, MultiPointCollection, MultiPolygonCollection, }, - dataset::{DataId, DatasetId}, - hashmap, - primitives::{ - BoundingBox2D, MultiLineString, MultiPoint, MultiPolygon, QueryRectangle, - SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, TimeStep, - }, - raster::{Grid, GridShape, GridShape2D, GridSize, RasterDataType, RasterTile2D}, + primitives::{BoundingBox2D, MultiLineString, MultiPoint, MultiPolygon, TimeInterval}, + raster::{Grid, RasterDataType, RasterTile2D}, spatial_reference::SpatialReferenceAuthority, util::{ Identifier, @@ -775,35 +903,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_point().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -852,35 +979,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: lines_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_line_string().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -936,35 +1062,34 @@ mod tests { let target_spatial_reference = SpatialReference::new(SpatialReferenceAuthority::Epsg, 900_913); + let exe_ctx = MockExecutionContext::test_default(); + let initialized_operator = VectorOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: polygon_source.into(), }, }) - .initialize( - WorkflowOperatorPath::initialize_root(), - &MockExecutionContext::test_default(), - ) + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; let query_processor = initialized_operator.query_processor()?; let query_processor = query_processor.multi_polygon().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (COLOGNE_EPSG_4326.x, MARBURG_EPSG_4326.y).into(), (MARBURG_EPSG_4326.x, HAMBURG_EPSG_4326.y).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor.query(query_rectangle, &ctx).await.unwrap(); @@ -985,12 +1110,10 @@ mod tests { Ok(()) } + #[allow(clippy::too_many_lines)] #[tokio::test] async fn raster_identity() -> Result<()> { - let projection = SpatialReference::new( - geoengine_datatypes::spatial_reference::SpatialReferenceAuthority::Epsg, - 4326, - ); + let projection = SpatialReference::epsg_4326(); let data = vec![ RasterTile2D { @@ -998,7 +1121,9 @@ mod tests { tile_position: [-1, 0].into(), band: 0, global_geo_transform: TestDefault::test_default(), - grid_array: Grid::new([2, 2].into(), vec![1, 2, 3, 4]).unwrap().into(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), properties: Default::default(), cache_hint: CacheHint::default(), }, @@ -1011,6 +1136,26 @@ mod tests { properties: Default::default(), cache_hint: CacheHint::default(), }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![1_u8, 2, 3, 4]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(0, 5), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![7, 8, 9, 10]).unwrap().into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, RasterTile2D { time: TimeInterval::new_unchecked(5, 10), tile_position: [-1, 0].into(), @@ -1033,34 +1178,63 @@ mod tests { properties: Default::default(), cache_hint: CacheHint::default(), }, + RasterTile2D { + time: TimeInterval::new_unchecked(5, 10), + tile_position: [0, 0].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![13, 14, 15, 16]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, + RasterTile2D { + time: TimeInterval::new_unchecked(5, 10), + tile_position: [0, 1].into(), + band: 0, + global_geo_transform: TestDefault::test_default(), + grid_array: Grid::new([2, 2].into(), vec![19, 20, 21, 22]) + .unwrap() + .into(), + properties: Default::default(), + cache_hint: CacheHint::default(), + }, ]; + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 10)), + TimeStep::millis(5).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + geo_transform, + GridBoundingBox2D::new([-2, 0], [1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([2, 2].into())); + + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let mrs1 = MockRasterSource { params: MockRasterSourceParams { data: data.clone(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: Some(SpatialResolution::one()), - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let mut exe_ctx = MockExecutionContext::test_default(); - exe_ctx.tiling_specification.tile_size_in_pixels = GridShape { - // we need a smaller tile size - shape_array: [2, 2], - }; - - let query_ctx = MockQueryContext::test_default(); - let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: projection, // This test will do a identity reprojection + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: mrs1.into(), @@ -1075,12 +1249,11 @@ mod tests { .get_u8() .unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 1.).into(), (3., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 10), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, 0], [1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 10), + BandSelection::first(), + ); let a = qp.raster_query(query_rect, &query_ctx).await?; @@ -1096,19 +1269,18 @@ mod tests { #[tokio::test] async fn raster_ndvi_3857() -> Result<()> { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - exe_ctx.tiling_specification = - TilingSpecification::new((0.0, 0.0).into(), [450, 450].into()); - let output_shape: GridShape2D = [900, 1800].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((0., 20_000_000.).into(), (20_000_000., 0.).into()); - let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_001); - // 2014-01-01 + let tile_size = GridShape2D::new_2d(512, 512); + exe_ctx.tiling_specification = TilingSpecification::new(tile_size); + + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + + let time_interval = TimeInterval::new_unchecked(1_396_303_200_000, 1_396_389_600_000); + // 2014-04-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: id.clone() }, + params: GdalSourceParameters::new(id.clone()), } .boxed(); @@ -1120,6 +1292,7 @@ mod tests { let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: projection, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1128,48 +1301,80 @@ mod tests { .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / (output_shape.axis_size_y() * 2) as f64; // *2 to account for the dataset aspect ratio 2:1 - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() .get_u8() .unwrap(); - let qs = qp - .raster_query( - RasterQueryRectangle { - spatial_bounds: output_bounds, - time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, - &query_ctx, - ) - .await - .unwrap(); + let result_descritptor = qp.result_descriptor(); + + assert_approx_eq!( + f64, + 14_255.015_508_816_849, // TODO: GDAL output is 14228.560819126376373 + result_descritptor + .spatial_grid_descriptor() + .spatial_resolution() + .x, + epsilon = 0.000_001 + ); + + assert_approx_eq!( + f64, + 14_255.015_508_816_849, // TODO: GDAL output is -14233.615370039031404 + result_descritptor + .spatial_grid_descriptor() + .spatial_resolution() + .y, + epsilon = 0.000_001 + ); + + let tlz = result_descritptor + .spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .generate_data_tiling_strategy(); + let query_tl_pixel = tlz.tile_idx_to_global_pixel_idx([-1, 0].into()); + let query_bounds = + GridBoundingBox2D::new(query_tl_pixel, query_tl_pixel + [511, 511]).unwrap(); + + let qrect = RasterQueryRectangle::new(query_bounds, time_interval, BandSelection::first()); + + let qs = qp.raster_query(qrect.clone(), &query_ctx).await.unwrap(); let res = qs .map(Result::unwrap) .collect::>>() .await; - // Write the tiles to a file - // let mut buffer = File::create("MOD13A2_M_NDVI_2014-04-01_tile-20.rst")?; - // buffer.write(res[8].clone().into_materialized_tile().grid_array.data.as_slice())?; + // get the worldfile + // println!("{}", res[0].tile_geo_transform().worldfile_string()); + + // Write the tile to a file + + /* + let mut buffer = std::fs::File::create("MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst")?; + + std::io::Write::write( + &mut buffer, + res[0] + .clone() + .into_materialized_tile() + .grid_array + .inner_grid + .data + .as_slice(), + )?; + */ // This check is against a tile produced by the operator itself. It was visually validated. TODO: rebuild when open issues are solved. // A perfect validation would be against a GDAL output generated like this: - // gdalwarp -t_srs EPSG:3857 -tr 11111.11111111 11111.11111111 -r near -te 0.0 5011111.111111112 5000000.0 10011111.111111112 -te_srs EPSG:3857 -of GTiff ./MOD13A2_M_NDVI_2014-04-01.TIFF ./MOD13A2_M_NDVI_2014-04-01_tile-20.rst + // gdalwarp -t_srs EPSG:3857 -r near -te_srs EPSG:3857 -of GTiff ./MOD13A2_M_NDVI_2014-04-01.TIFF ./MOD13A2_M_NDVI_2014-04-01.TIFF assert_eq!( include_bytes!( - "../../../test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20.rst" + "../../../test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst" ) as &[u8], - res[8] + res[0] .clone() .into_materialized_tile() .grid_array @@ -1183,20 +1388,19 @@ mod tests { #[test] fn query_rewrite_4326_3857() { - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + ); let expected = BoundingBox2D::new_unchecked( (-20_037_508.342_789_244, -20_048_966.104_014_594).into(), (20_037_508.342_789_244, 20_048_966.104_014_594).into(), ); - let reprojected = reproject_query( - query, + let reprojected = reproject_spatial_query( + query.spatial_bounds(), SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857), SpatialReference::epsg_4326(), true, @@ -1207,15 +1411,34 @@ mod tests { assert!(approx_eq!( BoundingBox2D, expected, - reprojected.spatial_bounds, + reprojected, epsilon = 0.000_001 )); } + #[allow(clippy::too_many_lines)] #[tokio::test] async fn raster_ndvi_3857_to_4326() -> Result<()> { - let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let tile_size_in_pixels = [200, 200].into(); + let data_geo_transform = GeoTransform::new( + Coordinate2D::new(-20_037_508.342_789_244, 19_971_868.880_408_562), + 14_052.950_258_048_738, + -14_057.881_117_788_405, + ); + let data_bounds = GridBoundingBox2D::new([0, 0], [2840, 2850]).unwrap(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857).into(), + time: crate::engine::TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z").unwrap(), + TimeInstance::from_str("2014-07-01T00:00:00.000Z").unwrap(), + )), + TimeStep::months(1).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts(data_geo_transform, data_bounds), + bands: RasterBandDescriptors::new_single_band(), + }; let m = GdalMetaDataRegular { data_time: TimeInterval::new_unchecked( @@ -1239,12 +1462,12 @@ mod tests { .into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-20_037_508.342_789_244, 19_971_868.880_408_563).into(), - x_pixel_size: 14_052.950_258_048_739, - y_pixel_size: -14_057.881_117_788_405, + origin_coordinate: data_geo_transform.origin_coordinate, + x_pixel_size: data_geo_transform.x_pixel_size(), + y_pixel_size: data_geo_transform.y_pixel_size(), }, - width: 2851, - height: 2841, + width: data_bounds.axis_size_x(), + height: data_bounds.axis_size_y(), file_not_found_handling: FileNotFoundHandling::Error, no_data_value: Some(0.), properties_mapping: None, @@ -1253,37 +1476,29 @@ mod tests { allow_alphaband_as_mask: true, retry: None, }, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857) - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), cache_ttl: CacheTtlSeconds::default(), }; + let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( + tile_size_in_pixels, + )); + let id: DataId = DatasetId::new().into(); let name = NamedData::with_system_name("ndvi"); exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); - exe_ctx.tiling_specification = TilingSpecification::new((0.0, 0.0).into(), [60, 60].into()); - - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let time_interval = TimeInterval::new_unchecked(1_396_310_400_000, 1_396_310_400_000); - // 2014-04-01 + let time_interval = TimeInterval::new_unchecked(1_396_310_400_000, 1_396_310_400_000); // 2014-04-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed(); let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::DataBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1292,25 +1507,24 @@ mod tests { .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; - let x_query_resolution = output_bounds.size_x() / 480.; // since we request x -180 to 180 and y -90 to 90 with 60x60 tiles this will result in 8 x 4 tiles - let y_query_resolution = output_bounds.size_y() / 240.; // *2 to account for the dataset aspect ratio 2:1 - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() .get_u8() .unwrap(); + let qr = qp.result_descriptor(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let qs = qp .raster_query( - QueryRectangle { - spatial_bounds: output_bounds, + RasterQueryRectangle::new( + qr.spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .tiling_grid_bounds(), time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_ctx, ) .await @@ -1321,51 +1535,31 @@ mod tests { .collect::>>() .await; - // the test must generate 8x4 tiles - assert_eq!(tiles.len(), 32); + // the test should generate 18x10 tiles. However, since the real procucrd pixel size is < 0.1 we will get 20 tiles on the x-axis + assert_eq!(tiles.len(), /*18*/ 20 * 10); // none of the tiles should be empty assert!(tiles.iter().all(|t| !t.is_empty())); - Ok(()) } - #[test] - fn source_resolution() { - let epsg_4326 = SpatialReference::epsg_4326(); - let epsg_3857 = SpatialReference::new(SpatialReferenceAuthority::Epsg, 3857); - - // use ndvi dataset that was reprojected using gdal as ground truth - let dataset_4326 = gdal_open_dataset(test_data!( - "raster/modis_ndvi/MOD13A2_M_NDVI_2014-04-01.TIFF" - )) - .unwrap(); - let geotransform_4326 = dataset_4326.geo_transform().unwrap(); - let res_4326 = SpatialResolution::new(geotransform_4326[1], -geotransform_4326[5]).unwrap(); - - let dataset_3857 = gdal_open_dataset(test_data!( - "raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01.TIFF" - )) - .unwrap(); - let geotransform_3857 = dataset_3857.geo_transform().unwrap(); - let res_3857 = SpatialResolution::new(geotransform_3857[1], -geotransform_3857[5]).unwrap(); - - // ndvi was projected from 4326 to 3857. The calculated source_resolution for getting the raster in 3857 with `res_3857` - // should thus roughly be like the original `res_4326` - let result_res = suggest_pixel_size_from_diag_cross_projected::( - epsg_3857.area_of_use_projected().unwrap(), - epsg_4326.area_of_use_projected().unwrap(), - res_3857, - ) - .unwrap(); - assert!(1. - (result_res.x / res_4326.x).abs() < 0.02); - assert!(1. - (result_res.y / res_4326.y).abs() < 0.02); - } - #[tokio::test] async fn query_outside_projection_area_of_use_produces_empty_tiles() { - let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let tile_size_in_pixels = [600, 600].into(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32636).into(), + time: crate::engine::TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new( + Coordinate2D::new(166_021.44, 9_329_005.188), + 534_994.66 - 166_021.444, + -9_329_005.18, + ), + GridBoundingBox2D::new_min_max(0, 100, 0, 100).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; let m = GdalMetaDataStatic { time: Some(TimeInterval::default()), @@ -1387,38 +1581,30 @@ mod tests { allow_alphaband_as_mask: true, retry: None, }, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 32636) - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor: result_descriptor.clone(), cache_ttl: CacheTtlSeconds::default(), }; + let mut exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( + tile_size_in_pixels, + )); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); + let id: DataId = DatasetId::new().into(); let name = NamedData::with_system_name("ndvi"); exe_ctx.add_meta_data(id.clone(), name.clone(), Box::new(m)); - exe_ctx.tiling_specification = - TilingSpecification::new((0.0, 0.0).into(), [600, 600].into()); - - let output_shape: GridShape2D = [1000, 1000].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 0.).into(), (180., -90.).into()); let time_interval = TimeInterval::new_instant(1_388_534_400_000).unwrap(); // 2014-01-01 let gdal_op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed(); let initialized_operator = RasterOperator::boxed(Reprojection { params: ReprojectionParams { target_spatial_reference: SpatialReference::epsg_4326(), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: gdal_op.into(), @@ -1428,11 +1614,6 @@ mod tests { .await .unwrap(); - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / (output_shape.axis_size_y()) as f64; - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let qp = initialized_operator .query_processor() .unwrap() @@ -1441,12 +1622,11 @@ mod tests { let result = qp .raster_query( - QueryRectangle { - spatial_bounds: output_bounds, + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(500, 1000, 500, 1000).unwrap(), time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_ctx, ) .await @@ -1465,7 +1645,7 @@ mod tests { #[tokio::test] async fn points_from_wgs84_to_utm36n() { let exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let point_source = MockFeatureCollectionSource::single( MultiPointCollection::from_data( @@ -1489,6 +1669,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 32636, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1512,12 +1693,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::new( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1542,7 +1722,7 @@ mod tests { #[tokio::test] async fn points_from_utm36n_to_wgs84() { let exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context(TestDefault::test_default()); let point_source = MockFeatureCollectionSource::with_collections_and_sref( vec![ @@ -1569,6 +1749,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 4326, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1592,12 +1773,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::new( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1627,7 +1807,7 @@ mod tests { // This test checks that points that are outside the area of use of the target spatial reference are not projected and an empty collection is returned let exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let point_source = MockFeatureCollectionSource::with_collections_and_sref( vec![ @@ -1652,6 +1832,7 @@ mod tests { SpatialReferenceAuthority::Epsg, 4326, // utm36n ), + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }, sources: SingleRasterOrVectorSource { source: point_source.into(), @@ -1675,12 +1856,11 @@ mod tests { let qs = qp .vector_query( - QueryRectangle { + VectorQueryRectangle::new( spatial_bounds, - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }, + TimeInterval::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -1696,39 +1876,40 @@ mod tests { //assert!(points.coordinates().is_empty()); } + /* TODO resolve the problem with empty intersections + */ #[test] fn it_derives_raster_result_descriptor() { let in_proj = SpatialReference::epsg_4326(); let out_proj = SpatialReference::from_str("EPSG:3857").unwrap(); - let bbox = Some(SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - )); - let resolution = Some(SpatialResolution::new_unchecked(0.1, 0.1)); + let geo_transform = GeoTransform::new(Coordinate2D::new(0., 0.), 0.1, -0.1); + let grid_bounds = GridBoundingBox2D::new_min_max(-850, 849, -1800, 1799).unwrap(); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); - let (in_bounds, out_bounds, out_res) = - InitializedRasterReprojection::derive_raster_in_bounds_out_bounds_out_res( - in_proj, out_proj, resolution, bbox, - ) - .unwrap(); + let projector = CoordinateProjector::from_known_srs(in_proj, out_proj).unwrap(); + + let out_spatial_grid = spatial_grid.reproject(&projector).unwrap(); assert_eq!( - in_bounds.unwrap(), - SpatialPartition2D::new_unchecked((-180., 85.06).into(), (180., -85.06).into(),) + out_spatial_grid.geo_transform.origin_coordinate(), + Coordinate2D::new(0., 0.) ); assert_eq!( - out_bounds.unwrap(), - out_proj - .area_of_use_projected::() - .unwrap() + out_spatial_grid.geo_transform.spatial_resolution(), + SpatialResolution::new_unchecked(14_212.246_793_017_477, 14_212.246_793_017_477) ); - // TODO: y resolution should be double the x resolution, but currently we only compute a uniform resolution + /* + Projected bounds: + -20037508.34 -20048966.1 + 20037508.34 20048966.1 + */ + assert_eq!( - out_res.unwrap(), - SpatialResolution::new_unchecked(14_237.781_884_528_267, 14_237.781_884_528_267), + out_spatial_grid.grid_bounds, + GridBoundingBox2D::new_min_max(-1405, 1405, -1410, 1409).unwrap() ); } } diff --git a/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs b/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs index fba77e322..46ad74040 100644 --- a/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs +++ b/operators/src/processing/temporal_raster_aggregation/first_last_subquery.rs @@ -5,10 +5,7 @@ use crate::{ use async_trait::async_trait; use futures::{Future, FutureExt, TryFuture, TryFutureExt, future::BoxFuture}; use geoengine_datatypes::{ - primitives::{ - CacheHint, QueryRectangle, RasterQueryRectangle, SpatialPartitioned, TimeInstance, - TimeInterval, TimeStep, - }, + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, raster::{EmptyGrid2D, Pixel, RasterTile2D, TileInformation}, }; use rayon::ThreadPool; @@ -106,16 +103,18 @@ impl FoldTileAccu for TemporalRasterAggregationTileAccu { } impl FoldTileAccuMut for TemporalRasterAggregationTileAccu { - fn tile_mut(&mut self) -> &mut RasterTile2D { - &mut self.accu_tile + fn set_time(&mut self, time: TimeInterval) { + self.accu_tile.time = time; + } + + fn set_cache_hint(&mut self, new_cache_hint: CacheHint) { + self.accu_tile.cache_hint = new_cache_hint; } } #[derive(Debug, Clone)] pub struct TemporalRasterAggregationSubQueryNoDataOnly { pub fold_fn: F, - pub step: TimeStep, - pub step_reference: TimeInstance, pub _phantom_pixel_type: PhantomData, } @@ -148,17 +147,15 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, + _query_rect: RasterQueryRectangle, + time: TimeInterval, band_idx: u32, ) -> Result> { - let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(QueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new( + tile_info.global_pixel_bounds(), + time, // The time is already snapped by the operator where the time stream is created. + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -171,7 +168,7 @@ fn build_temporal_no_data_accu( tile_info: TileInformation, pool: Arc, ) -> impl Future>> + use { - let time_interval = query_rect.time_interval; + let time_interval = query_rect.time_interval(); crate::util::spawn_blocking(move || { let output_raster = EmptyGrid2D::new(tile_info.tile_size_in_pixels).into(); diff --git a/operators/src/processing/temporal_raster_aggregation/subquery.rs b/operators/src/processing/temporal_raster_aggregation/subquery.rs index 9ee9dae39..fa9a2a561 100644 --- a/operators/src/processing/temporal_raster_aggregation/subquery.rs +++ b/operators/src/processing/temporal_raster_aggregation/subquery.rs @@ -6,9 +6,7 @@ use crate::{ use async_trait::async_trait; use futures::TryFuture; use geoengine_datatypes::{ - primitives::{ - CacheHint, RasterQueryRectangle, SpatialPartitioned, TimeInstance, TimeInterval, TimeStep, - }, + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, raster::{ EmptyGrid2D, GeoTransform, GridIdx2D, GridIndexAccess, GridOrEmpty, GridOrEmpty2D, GridShapeAccess, Pixel, RasterTile2D, TileInformation, UpdateIndexedElementsParallel, @@ -267,8 +265,7 @@ where pub struct TemporalRasterAggregationSubQuery> { pub fold_fn: FoldFn, - pub step: TimeStep, - pub step_reference: TimeInstance, + pub _phantom_pixel_type: PhantomData<(P, F)>, } @@ -281,8 +278,7 @@ pub struct GlobalStateTemporalRasterAggregationSubQuery< > { pub aggregator: Arc, pub fold_fn: FoldFn, - pub step: TimeStep, - pub step_reference: TimeInstance, + pub _phantom_pixel_type: PhantomData<(P, F)>, } @@ -308,7 +304,7 @@ where pool: &Arc, ) -> Self::TileAccuFuture { let accu = TileAccumulator { - time: query_rect.time_interval, + time: query_rect.time_interval(), tile_position: tile_info.global_tile_position, global_geo_transform: tile_info.global_geo_transform, state_grid: EmptyGrid2D::new(tile_info.tile_size_in_pixels).into(), @@ -323,17 +319,15 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, + _query_rect: RasterQueryRectangle, + time: TimeInterval, band_idx: u32, ) -> Result> { - let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new( + tile_info.global_pixel_bounds(), + time, // The time is already snapped by the operator where the time stream is created. + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { @@ -368,7 +362,7 @@ where ) -> Self::TileAccuFuture { let accu = GlobalStateTileAccumulator { aggregator: self.aggregator.clone(), - time: query_rect.time_interval, + time: query_rect.time_interval(), tile_position: tile_info.global_tile_position, global_geo_transform: tile_info.global_geo_transform, state_grid: EmptyGrid2D::new(tile_info.tile_size_in_pixels).into(), @@ -383,17 +377,15 @@ where fn tile_query_rectangle( &self, tile_info: TileInformation, - query_rect: RasterQueryRectangle, - start_time: TimeInstance, + _query_rect: RasterQueryRectangle, + time: TimeInterval, band_idx: u32, ) -> Result> { - let snapped_start = self.step.snap_relative(self.step_reference, start_time)?; - Ok(Some(RasterQueryRectangle { - spatial_bounds: tile_info.spatial_partition(), - spatial_resolution: query_rect.spatial_resolution, - time_interval: TimeInterval::new(snapped_start, (snapped_start + self.step)?)?, - attributes: band_idx.into(), - })) + Ok(Some(RasterQueryRectangle::new( + tile_info.global_pixel_bounds(), + time, // The time is already snapped by the operator where the time stream is created. + band_idx.into(), + ))) } fn fold_method(&self) -> Self::FoldMethod { diff --git a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs index 9d0c2e0b6..81b4ecf66 100644 --- a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs +++ b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs @@ -12,8 +12,9 @@ use super::subquery::GlobalStateTemporalRasterAggregationSubQuery; use crate::adapters::stack_individual_aligned_raster_bands; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedSources, Operator, QueryProcessor, - RasterOperator, SingleRasterSource, WorkflowOperatorPath, + RasterOperator, SingleRasterSource, TimeDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::processing::temporal_raster_aggregation::aggregators::PercentileEstimateAggregator; use crate::{ adapters::SubQueryTileAggregator, @@ -25,10 +26,12 @@ use crate::{ util::Result, }; use async_trait::async_trait; +use futures::{Stream, StreamExt}; use geoengine_datatypes::primitives::{ - BandSelection, RasterQueryRectangle, SpatialPartition2D, TimeInstance, + BandSelection, RasterQueryRectangle, RegularTimeDimension, SpatialResolution, TimeFilledItem, + TimeInstance, }; -use geoengine_datatypes::raster::{Pixel, RasterDataType, RasterTile2D}; +use geoengine_datatypes::raster::{GridBoundingBox2D, Pixel, RasterDataType, RasterTile2D}; use geoengine_datatypes::{primitives::TimeStep, raster::TilingSpecification}; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -109,6 +112,17 @@ impl RasterOperator for TemporalRasterAggregation { let mut out_result_descriptor = source.result_descriptor().clone(); + out_result_descriptor.time = TimeDescriptor::new( + source.result_descriptor().time.bounds, + geoengine_datatypes::primitives::TimeDimension::Regular(RegularTimeDimension { + step: self.params.window, + origin: self + .params + .window_reference + .unwrap_or(TimeInstance::from_millis_unchecked(0)), + }), + ); + if let Some(output_type) = self.params.output_type { out_result_descriptor.data_type = output_type; } @@ -174,8 +188,6 @@ impl InitializedRasterOperator for InitializedTemporalRasterAggregation { TemporalRasterAggregationProcessor::new( self.result_descriptor.clone(), self.aggregation_type, - self.window, - self.window_reference, p, self.tiling_specification, ).boxed() @@ -196,6 +208,24 @@ impl InitializedRasterOperator for InitializedTemporalRasterAggregation { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(TemporalRasterAggregation { + params: TemporalRasterAggregationParameters { + aggregation: self.aggregation_type, + window: self.window, + window_reference: Some(self.window_reference), + output_type: self.output_type, + }, + sources: SingleRasterSource { + raster: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct TemporalRasterAggregationProcessor @@ -205,8 +235,6 @@ where { result_descriptor: RasterResultDescriptor, aggregation_type: Aggregation, - window: TimeStep, - window_reference: TimeInstance, source: Q, tiling_specification: TilingSpecification, } @@ -216,7 +244,7 @@ where Q: RasterQueryProcessor + QueryProcessor< Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, + SpatialBounds = GridBoundingBox2D, Selection = BandSelection, ResultDescription = RasterResultDescriptor, >, @@ -225,29 +253,22 @@ where fn new( result_descriptor: RasterResultDescriptor, aggregation_type: Aggregation, - window: TimeStep, - window_reference: TimeInstance, source: Q, tiling_specification: TilingSpecification, ) -> Self { Self { result_descriptor, aggregation_type, - window, - window_reference, source, tiling_specification, } } fn create_subquery + 'static, FoldFn>( - &self, fold_fn: FoldFn, ) -> super::subquery::TemporalRasterAggregationSubQuery { super::subquery::TemporalRasterAggregationSubQuery { fold_fn, - step: self.window, - step_reference: self.window_reference, _phantom_pixel_type: PhantomData, } } @@ -256,212 +277,212 @@ where F: GlobalStateTemporalRasterPixelAggregator

+ 'static, FoldFn, >( - &self, aggregator: F, fold_fn: FoldFn, ) -> GlobalStateTemporalRasterAggregationSubQuery { GlobalStateTemporalRasterAggregationSubQuery { aggregator: Arc::new(aggregator), fold_fn, - step: self.window, - step_reference: self.window_reference, + _phantom_pixel_type: PhantomData, } } - fn create_subquery_first( - &self, - fold_fn: F, - ) -> TemporalRasterAggregationSubQueryNoDataOnly { + fn create_subquery_first(fold_fn: F) -> TemporalRasterAggregationSubQueryNoDataOnly { TemporalRasterAggregationSubQueryNoDataOnly { fold_fn, - step: self.window, - step_reference: self.window_reference, + _phantom_pixel_type: PhantomData, } } - fn create_subquery_last( - &self, - fold_fn: F, - ) -> TemporalRasterAggregationSubQueryNoDataOnly { + fn create_subquery_last(fold_fn: F) -> TemporalRasterAggregationSubQueryNoDataOnly { TemporalRasterAggregationSubQueryNoDataOnly { fold_fn, - step: self.window, - step_reference: self.window_reference, + _phantom_pixel_type: PhantomData, } } #[allow(clippy::too_many_lines)] - fn create_subquery_adapter_stream_for_single_band<'a>( + async fn create_subquery_adapter_stream_for_single_band<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn crate::engine::QueryContext, ) -> Result>>> { ensure!( - query.attributes.count() == 1, + query.attributes().count() == 1, error::InvalidBandCount { expected: 1u32, - found: query.attributes.count() + found: query.attributes().count() } ); + let grid_desc = self.result_descriptor.spatial_grid_descriptor(); + let tiling_strategy = grid_desc + .tiling_grid_definition(self.tiling_specification) + .generate_data_tiling_strategy(); + + let time_stream: std::pin::Pin< + Box< + dyn Stream< + Item = std::result::Result< + geoengine_datatypes::primitives::TimeInterval, + error::Error, + >, + > + Send, + >, + > = self.time_query(query.time_interval(), ctx).await?; + Ok(match self.aggregation_type { Aggregation::Min { ignore_no_data: true, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::< - P, - MinPixelAggregatorIngoringNoData, - >, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Min"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Min { ignore_no_data: false, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Min"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Max { ignore_no_data: true, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::< - P, - MaxPixelAggregatorIngoringNoData, - >, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Max"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Max { ignore_no_data: false, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Max"), - + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::First { ignore_no_data: true, - } => self - .create_subquery( + } => { + Self::create_subquery( super::subquery::subquery_all_tiles_fold_fn::< P, FirstPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::First"), + .into_raster_subquery_adapter( + &self.source, + query, + ctx, + tiling_strategy, + time_stream, + ) + .box_pin() + } Aggregation::First { ignore_no_data: false, - } => self - .create_subquery_first(first_tile_fold_future::

) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::First"), + } => Self::create_subquery_first(first_tile_fold_future::

) + .into_raster_subquery_adapter( + &self.source, + query, + ctx, + tiling_strategy, + time_stream, + ) + .box_pin(), Aggregation::Last { ignore_no_data: true, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::< - P, - LastPixelAggregatorIngoringNoData, - >, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Last"), - + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Last { ignore_no_data: false, - } => self - .create_subquery_last(last_tile_fold_future::

) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Last"), - + } => Self::create_subquery_last(last_tile_fold_future::

) + .into_raster_subquery_adapter( + &self.source, + query, + ctx, + tiling_strategy, + time_stream, + ) + .box_pin(), Aggregation::Mean { ignore_no_data: true, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::>, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Mean"), - + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::>, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Mean { ignore_no_data: false, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::>, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Mean"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::>, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Sum { ignore_no_data: true, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::< - P, - SumPixelAggregatorIngoringNoData, - >, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Sum"), - + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Sum { ignore_no_data: false, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Sum"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::Count { ignore_no_data: true, - } => self - .create_subquery( + } => { + Self::create_subquery( super::subquery::subquery_all_tiles_fold_fn::< P, CountPixelAggregatorIngoringNoData, >, ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Sum"), - + .into_raster_subquery_adapter( + &self.source, + query, + ctx, + tiling_strategy, + time_stream, + ) + .box_pin() + } Aggregation::Count { ignore_no_data: false, - } => self - .create_subquery( - super::subquery::subquery_all_tiles_fold_fn::, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::Sum"), + } => Self::create_subquery( + super::subquery::subquery_all_tiles_fold_fn::, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::PercentileEstimate { ignore_no_data: true, percentile, - } => self - .create_global_state_subquery( - PercentileEstimateAggregator::::new(percentile), - super::subquery::subquery_all_tiles_global_state_fold_fn, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::PercentileEstimate"), + } => Self::create_global_state_subquery( + PercentileEstimateAggregator::::new(percentile), + super::subquery::subquery_all_tiles_global_state_fold_fn, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), Aggregation::PercentileEstimate { ignore_no_data: false, percentile, - } => self - .create_global_state_subquery( - PercentileEstimateAggregator::::new(percentile), - super::subquery::subquery_all_tiles_global_state_fold_fn, - ) - .into_raster_subquery_adapter(&self.source, query, ctx, self.tiling_specification) - .expect("no tiles must be skipped in Aggregation::PercentileEstimate"), + } => Self::create_global_state_subquery( + PercentileEstimateAggregator::::new(percentile), + super::subquery::subquery_all_tiles_global_state_fold_fn, + ) + .into_raster_subquery_adapter(&self.source, query, ctx, tiling_strategy, time_stream) + .box_pin(), }) } } @@ -469,16 +490,11 @@ where #[async_trait] impl QueryProcessor for TemporalRasterAggregationProcessor where - Q: QueryProcessor< - Output = RasterTile2D

, - SpatialBounds = SpatialPartition2D, - Selection = BandSelection, - ResultDescription = RasterResultDescriptor, - >, + Q: RasterQueryProcessor + Send + Sync, P: Pixel, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -489,6 +505,7 @@ where ) -> Result>> { stack_individual_aligned_raster_bands(&query, ctx, |query, ctx| async { self.create_subquery_adapter_stream_for_single_band(query, ctx) + .await }) .await } @@ -498,22 +515,39 @@ where } } +#[async_trait] +impl RasterQueryProcessor for TemporalRasterAggregationProcessor +where + Q: RasterQueryProcessor, + P: Pixel, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: geoengine_datatypes::primitives::TimeInterval, + _ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> + { + let ti_iter = self + .result_descriptor + .time + .dimension + .unwrap_regular() + .expect("must be regular as this operator only creates regular time dimensions") + .intersecting_intervals(query.time())? + .map(Ok); + Ok(futures::stream::iter(ti_iter).boxed()) + } +} + #[cfg(test)] mod tests { - use futures::stream::StreamExt; - use geoengine_datatypes::{ - primitives::{CacheHint, SpatialResolution, TimeInterval}, - raster::{ - EmptyGrid, EmptyGrid2D, Grid2D, GridOrEmpty, MaskedGrid2D, RasterDataType, RenameBands, - TileInformation, TilesEqualIgnoringCacheHint, - }, - spatial_reference::SpatialReference, - util::test::TestDefault, - }; - + use super::*; use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, + SpatialGridDescriptor, TimeDescriptor, }, mock::{MockRasterSource, MockRasterSourceParams}, processing::{ @@ -521,25 +555,43 @@ mod tests { raster_stacker::{RasterStacker, RasterStackerParams}, }, }; - - use super::*; + use futures::stream::StreamExt; + use geoengine_datatypes::{ + primitives::{CacheHint, Coordinate2D, TimeInterval}, + raster::{ + EmptyGrid, EmptyGrid2D, GeoTransform, Grid2D, GridBoundingBox2D, GridOrEmpty, + GridShape2D, MaskedGrid2D, RasterDataType, RenameBands, TileInformation, + TilesEqualIgnoringCacheHint, + }, + spatial_reference::SpatialReference, + util::test::TestDefault, + }; #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_min() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 2]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -560,17 +612,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -652,17 +700,26 @@ mod tests { async fn test_max() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -683,17 +740,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -775,17 +828,26 @@ mod tests { async fn test_max_with_no_data() { let raster_tiles = make_raster(); // TODO: switch to make_raster_with_no_data? + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -806,17 +868,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -898,17 +956,26 @@ mod tests { async fn test_max_with_no_data_but_ignoring_it() { let raster_tiles = make_raster(); // TODO: switch to make_raster_with_no_data? + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -929,17 +996,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1019,6 +1082,22 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_only_no_data() { + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new_min_max(-3, -1, 0, 2).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -1032,14 +1111,7 @@ mod tests { GridOrEmpty::from(EmptyGrid2D::::new([3, 2].into())), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1060,17 +1132,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (2., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-3, -1, 0, 1).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1109,17 +1177,26 @@ mod tests { async fn test_first_with_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1140,17 +1217,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1212,21 +1285,29 @@ mod tests { async fn test_last_with_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); - let agg = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Last { @@ -1243,17 +1324,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1315,17 +1392,26 @@ mod tests { async fn test_last() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1346,17 +1432,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1418,17 +1500,26 @@ mod tests { async fn test_first() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1449,17 +1540,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1521,17 +1608,26 @@ mod tests { async fn test_mean_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1552,17 +1648,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1624,17 +1716,26 @@ mod tests { async fn test_mean_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1655,17 +1756,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1726,6 +1823,32 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_sum_without_nodata() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor, + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -1738,36 +1861,17 @@ mod tests { window_reference: Some(TimeInstance::from_millis(0).unwrap()), output_type: None, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let query_processor = operator .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1848,17 +1952,26 @@ mod tests { async fn test_sum_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1879,17 +1992,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -1951,17 +2060,26 @@ mod tests { async fn test_sum_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1982,17 +2100,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2053,6 +2167,32 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_sum_with_larger_data_type() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor: result_descriptor.clone(), + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -2073,39 +2213,20 @@ mod tests { output_band: None, map_no_data: true, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(), }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let query_processor = operator .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2197,6 +2318,32 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_count_without_nodata() { + let raster_tiles = make_raster(); + + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let mrs = MockRasterSource { + params: MockRasterSourceParams { + data: raster_tiles, + result_descriptor, + }, + } + .boxed(); + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Count { @@ -2209,36 +2356,17 @@ mod tests { window_reference: Some(TimeInstance::from_millis(0).unwrap()), output_type: None, }, - sources: SingleRasterSource { - raster: MockRasterSource { - params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, - }, - } - .boxed(), - }, + sources: SingleRasterSource { raster: mrs }, } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let query_processor = operator .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2319,17 +2447,26 @@ mod tests { async fn test_count_nodata() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2350,17 +2487,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2422,17 +2555,26 @@ mod tests { async fn test_count_ignore_no_data() { let raster_tiles = make_raster_with_no_data(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2453,17 +2595,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2525,17 +2663,26 @@ mod tests { async fn test_query_not_aligned_with_window_reference() { let raster_tiles = make_raster(); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2556,17 +2703,13 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(5, 5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(5, 5), + BandSelection::first(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -2804,6 +2947,21 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn it_sums_multiple_bands() { + let data = make_raster(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let operator = TemporalRasterAggregation { params: TemporalRasterAggregationParameters { aggregation: Aggregation::Sum { @@ -2825,29 +2983,15 @@ mod tests { rasters: vec![ MockRasterSource { params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + data: data.clone(), + result_descriptor: result_descriptor.clone(), }, } .boxed(), MockRasterSource { params: MockRasterSourceParams { - data: make_raster(), - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + data: data.clone(), + result_descriptor, }, } .boxed(), @@ -2859,17 +3003,14 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: [0, 1].try_into().unwrap(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 30), + [0, 1].try_into().unwrap(), + ); + let query_ctx = exe_ctx.mock_query_context_test_default(); let query_processor = operator .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) @@ -3002,17 +3143,24 @@ mod tests { async fn it_estimates_a_median() { let raster_tiles = make_raster(); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked(0, 40)), + TimeStep::millis(10).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -3034,17 +3182,15 @@ mod tests { } .boxed(); - let exe_ctx = MockExecutionContext::new_with_tiling_spec(TilingSpecification::new( - (0., 0.).into(), - [3, 2].into(), - )); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 3.).into(), (4., 0.).into()), - time_interval: TimeInterval::new_unchecked(0, 40), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; - let query_ctx = MockQueryContext::test_default(); + let exe_ctx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([3, 2].into())); + let query_rect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, -0], [-1, 3]).unwrap(), + TimeInterval::new_unchecked(0, 40), + BandSelection::first(), + ); + + let query_ctx = exe_ctx.mock_query_context_test_default(); let qp = agg .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) diff --git a/operators/src/processing/time_projection/mod.rs b/operators/src/processing/time_projection/mod.rs index 25284e1f0..2b3ade8e8 100644 --- a/operators/src/processing/time_projection/mod.rs +++ b/operators/src/processing/time_projection/mod.rs @@ -5,6 +5,7 @@ use crate::engine::{ OperatorName, QueryContext, SingleVectorSource, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; use async_trait::async_trait; use futures::stream::BoxStream; @@ -12,7 +13,7 @@ use futures::{StreamExt, TryStreamExt}; use geoengine_datatypes::collections::{ FeatureCollection, FeatureCollectionInfos, FeatureCollectionModifications, }; -use geoengine_datatypes::primitives::{ColumnSelection, Geometry, TimeInterval}; +use geoengine_datatypes::primitives::{ColumnSelection, Geometry, SpatialResolution, TimeInterval}; use geoengine_datatypes::primitives::{TimeInstance, TimeStep, VectorQueryRectangle}; use geoengine_datatypes::util::arrow::ArrowTyped; use rayon::ThreadPool; @@ -153,6 +154,22 @@ impl InitializedVectorOperator for InitializedVectorTimeProjection { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(TimeProjection { + params: TimeProjectionParams { + step: self.step, + step_reference: Some(self.step_reference), + }, + sources: SingleVectorSource { + vector: self.source.optimize(target_resolution)?, + }, + } + .boxed()) + } } pub struct VectorTimeProjectionProcessor @@ -254,12 +271,11 @@ fn expand_query_rectangle( step_reference: TimeInstance, query: &VectorQueryRectangle, ) -> Result { - Ok(VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: expand_time_interval(step, step_reference, query.time_interval)?, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }) + Ok(VectorQueryRectangle::new( + query.spatial_bounds(), + expand_time_interval(step, step_reference, query.time_interval())?, + ColumnSelection::all(), + )) } fn expand_time_interval( @@ -286,15 +302,11 @@ fn expand_time_interval( mod tests { use super::*; - use crate::{ - engine::{MockExecutionContext, MockQueryContext}, - mock::MockFeatureCollectionSource, - }; + use crate::{engine::MockExecutionContext, mock::MockFeatureCollectionSource}; use geoengine_datatypes::{ collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection, VectorDataType}, primitives::{ - BoundingBox2D, CacheHint, DateTime, MultiPoint, SpatialResolution, TimeGranularity, - TimeInterval, + BoundingBox2D, CacheHint, DateTime, MultiPoint, TimeGranularity, TimeInterval, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -425,7 +437,7 @@ mod tests { #[tokio::test] async fn single_year() { let execution_context = MockExecutionContext::test_default(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let source = MockFeatureCollectionSource::single( MultiPointCollection::from_data( @@ -478,16 +490,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 4, 3, 0, 0, 0), DateTime::new_utc(2010, 5, 14, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -530,7 +541,7 @@ mod tests { #[tokio::test] async fn over_a_year() { let execution_context = MockExecutionContext::test_default(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let source = MockFeatureCollectionSource::single( MultiPointCollection::from_data( @@ -583,16 +594,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 4, 3, 0, 0, 0), DateTime::new_utc(2010, 5, 14, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await diff --git a/operators/src/processing/time_shift.rs b/operators/src/processing/time_shift.rs index 8f295b2a4..3d1457719 100644 --- a/operators/src/processing/time_shift.rs +++ b/operators/src/processing/time_shift.rs @@ -1,12 +1,14 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSingleRasterOrVectorOperator, InitializedSources, InitializedVectorOperator, - Operator, OperatorName, QueryContext, RasterOperator, RasterQueryProcessor, + Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, ResultDescriptor, SingleRasterOrVectorSource, TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; +use crate::optimization::OptimizationError; use crate::util::Result; +use crate::util::input::RasterOrVectorOperator; use async_trait::async_trait; use futures::StreamExt; use futures::stream::BoxStream; @@ -15,8 +17,8 @@ use geoengine_datatypes::collections::{ }; use geoengine_datatypes::error::{BoxedResultExt, ErrorSource}; use geoengine_datatypes::primitives::{ - ColumnSelection, Duration, Geometry, RasterQueryRectangle, TimeGranularity, TimeInstance, - TimeInterval, + ColumnSelection, Duration, Geometry, RasterQueryRectangle, SpatialResolution, TimeGranularity, + TimeInstance, TimeInterval, }; use geoengine_datatypes::primitives::{TimeStep, VectorQueryRectangle}; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; @@ -178,7 +180,7 @@ impl VectorOperator for TimeShift { .initialize_sources(path.clone(), context) .await?; - match (init_sources.source, self.params) { + match (init_sources.source, self.params.clone()) { ( InitializedSingleRasterOrVectorOperator::Vector(source), TimeShiftParams::Relative { granularity, value }, @@ -195,6 +197,7 @@ impl VectorOperator for TimeShift { Ok(Box::new(InitializedVectorTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -216,6 +219,7 @@ impl VectorOperator for TimeShift { Ok(Box::new(InitializedVectorTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -232,6 +236,7 @@ impl VectorOperator for TimeShift { Ok(Box::new(InitializedVectorTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -261,7 +266,7 @@ impl RasterOperator for TimeShift { .initialize_sources(path.clone(), context) .await?; - match (init_sources.source, self.params) { + match (init_sources.source, self.params.clone()) { ( InitializedSingleRasterOrVectorOperator::Raster(source), TimeShiftParams::Relative { granularity, value }, @@ -278,6 +283,7 @@ impl RasterOperator for TimeShift { Ok(Box::new(InitializedRasterTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -299,6 +305,7 @@ impl RasterOperator for TimeShift { Ok(Box::new(InitializedRasterTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -315,6 +322,7 @@ impl RasterOperator for TimeShift { Ok(Box::new(InitializedRasterTimeShift { name, path, + params: self.params.clone(), source, result_descriptor, shift, @@ -345,6 +353,7 @@ fn shift_result_descriptor( pub struct InitializedVectorTimeShift { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: TimeShiftParams, source: Box, result_descriptor: VectorResultDescriptor, shift: Shift, @@ -353,6 +362,7 @@ pub struct InitializedVectorTimeShift { pub struct InitializedRasterTimeShift { name: CanonicOperatorName, path: WorkflowOperatorPath, + params: TimeShiftParams, source: Box, result_descriptor: RasterResultDescriptor, shift: Shift, @@ -388,6 +398,18 @@ impl InitializedVectorOperator fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Box::new(TimeShift { + params: self.params.clone(), + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Vector(self.source.optimize(target_resolution)?), + }, + })) + } } impl InitializedRasterOperator @@ -420,6 +442,18 @@ impl InitializedRasterOperator fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(Box::new(TimeShift { + params: self.params.clone(), + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(self.source.optimize(target_resolution)?), + }, + })) + } } pub struct RasterTimeShiftProcessor @@ -455,14 +489,13 @@ where query: VectorQueryRectangle, ctx: &'a dyn QueryContext, ) -> Result>> { - let (time_interval, state) = self.shift.shift(query.time_interval)?; + let (time_interval, state) = self.shift.shift(query.time_interval())?; - let query = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, + let query = VectorQueryRectangle::new( + query.spatial_bounds(), time_interval, - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; + ColumnSelection::all(), + ); let stream = self.processor.vector_query(query, ctx).await?; let stream = stream.then(move |collection| async move { @@ -493,26 +526,24 @@ where } #[async_trait] -impl RasterQueryProcessor for RasterTimeShiftProcessor +impl QueryProcessor for RasterTimeShiftProcessor where Q: RasterQueryProcessor, P: Pixel, Shift: TimeShiftOperation, { - type RasterType = P; + type Output = RasterTile2D

; + type SpatialBounds = Q::SpatialBounds; + type ResultDescription = RasterResultDescriptor; + type Selection = Q::Selection; - async fn raster_query<'a>( + async fn _query<'a>( &'a self, query: RasterQueryRectangle, ctx: &'a dyn QueryContext, - ) -> Result>>> { - let (time_interval, state) = self.shift.shift(query.time_interval)?; - let query = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval, - spatial_resolution: query.spatial_resolution, - attributes: query.attributes, - }; + ) -> Result>> { + let (time_interval, state) = self.shift.shift(query.time_interval())?; + let query = query.select_time_interval(time_interval); let stream = self.processor.raster_query(query, ctx).await?; let stream = stream.map(move |raster| { @@ -527,19 +558,47 @@ where Ok(Box::pin(stream)) } - fn raster_result_descriptor(&self) -> &RasterResultDescriptor { + fn result_descriptor(&self) -> &RasterResultDescriptor { &self.result_descriptor } } +#[async_trait] +impl RasterQueryProcessor for RasterTimeShiftProcessor +where + P: Pixel, + Q: RasterQueryProcessor, + Shift: TimeShiftOperation, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + let (time_interval, state) = self.shift.shift(query)?; + let stream = self.processor.time_query(time_interval, ctx).await?; + + let stream = stream + .map(move |ti| { + // reverse time shift for results + ti.and_then(|t| self.shift.reverse_shift(t, state).map_err(Into::into)) // TODO: maybe we need a time_query error... + }) + .boxed(); + + Ok(stream) + } +} + #[cfg(test)] mod tests { use super::*; use crate::{ engine::{ - MockExecutionContext, MockQueryContext, MultipleRasterSources, RasterBandDescriptors, - SingleRasterSource, + MockExecutionContext, MultipleRasterSources, RasterBandDescriptors, SingleRasterSource, + SpatialGridDescriptor, TimeDescriptor, }, mock::{MockFeatureCollectionSource, MockRasterSource, MockRasterSourceParams}, processing::{Expression, ExpressionParams, RasterStacker, RasterStackerParams}, @@ -551,12 +610,12 @@ mod tests { collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}, dataset::NamedData, primitives::{ - BandSelection, BoundingBox2D, CacheHint, DateTime, MultiPoint, SpatialPartition2D, - SpatialResolution, TimeGranularity, + BandSelection, BoundingBox2D, CacheHint, Coordinate2D, DateTime, MultiPoint, + TimeGranularity, }, raster::{ - EmptyGrid2D, GridOrEmpty, RasterDataType, RenameBands, TileInformation, - TilingSpecification, + BoundedGrid, EmptyGrid2D, GeoTransform, GridBoundingBox2D, GridOrEmpty, GridShape2D, + RasterDataType, RenameBands, TileInformation, TilingSpecification, }, spatial_reference::SpatialReference, util::test::TestDefault, @@ -568,9 +627,9 @@ mod tests { sources: SingleRasterOrVectorSource { source: RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test-raster"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name( + "test-raster", + )), } .boxed(), ), @@ -599,7 +658,8 @@ mod tests { "source": { "type": "GdalSource", "params": { - "data": "test-raster" + "data": "test-raster", + "overviewLevel": null } } } @@ -617,9 +677,9 @@ mod tests { sources: SingleRasterOrVectorSource { source: RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_system_name("test-raster"), - }, + params: GdalSourceParameters::new(NamedData::with_system_name( + "test-raster", + )), } .boxed(), ), @@ -644,7 +704,8 @@ mod tests { "source": { "type": "GdalSource", "params": { - "data": "test-raster" + "data": "test-raster", + "overviewLevel": null } } } @@ -659,7 +720,7 @@ mod tests { #[tokio::test] async fn test_absolute_vector_shift() { let execution_context = MockExecutionContext::test_default(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let source = MockFeatureCollectionSource::single( MultiPointCollection::from_data( @@ -711,16 +772,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2009, 1, 1, 0, 0, 0), DateTime::new_utc(2012, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -753,7 +813,7 @@ mod tests { #[tokio::test] async fn test_relative_vector_shift() { let execution_context = MockExecutionContext::test_default(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let source = MockFeatureCollectionSource::single( MultiPointCollection::from_data( @@ -802,16 +862,15 @@ mod tests { let mut stream = query_processor .vector_query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), - time_interval: TimeInterval::new( + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (2., 2.).into()).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: ColumnSelection::all(), - }, + ColumnSelection::all(), + ), &query_context, ) .await @@ -849,7 +908,26 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_absolute_raster_shift() { - let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new([3, 2].into())); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + DateTime::new_utc(2010, 1, 1, 0, 0, 0), + DateTime::new_utc(2013, 1, 1, 0, 0, 0), + )), + TimeStep::years(1).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., -3.), 1., -1.), + GridShape2D::new_2d(3, 4).bounding_box(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new(tile_size_in_pixels)); let raster_tiles = vec![ RasterTile2D::new_with_tile_info( TimeInterval::new_unchecked( @@ -940,14 +1018,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -964,10 +1035,9 @@ mod tests { }, }; - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); - let query_context = MockQueryContext::test_default(); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); + + let query_context = execution_context.mock_query_context_test_default(); let query_processor = RasterOperator::boxed(time_shift) .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -980,19 +1050,15 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (4., 0.).into(), - ), - time_interval: TimeInterval::new( + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_context, ) .await @@ -1024,7 +1090,26 @@ mod tests { #[tokio::test] #[allow(clippy::too_many_lines)] async fn test_relative_raster_shift() { - let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new([3, 2].into())); + let tile_size_in_pixels = GridShape2D::new_2d(3, 2); + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + DateTime::new_utc(2010, 1, 1, 0, 0, 0), + DateTime::new_utc(2013, 1, 1, 0, 0, 0), + )), + TimeStep::years(1).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([-3, 0], [0, 4]).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let tiling_specification = TilingSpecification::new(tile_size_in_pixels); + + let empty_grid = GridOrEmpty::Empty(EmptyGrid2D::::new(tile_size_in_pixels)); let raster_tiles = vec![ RasterTile2D::new_with_tile_info( TimeInterval::new_unchecked( @@ -1115,14 +1200,7 @@ mod tests { let mrs = MockRasterSource { params: MockRasterSourceParams { data: raster_tiles, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -1137,10 +1215,8 @@ mod tests { }, }; - let execution_context = MockExecutionContext::new_with_tiling_spec( - TilingSpecification::new((0., 0.).into(), [3, 2].into()), - ); - let query_context = MockQueryContext::test_default(); + let execution_context = MockExecutionContext::new_with_tiling_spec(tiling_specification); + let query_context = execution_context.mock_query_context_test_default(); let query_processor = RasterOperator::boxed(time_shift) .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1153,19 +1229,15 @@ mod tests { let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 3.).into(), - (4., 0.).into(), - ), - time_interval: TimeInterval::new( + RasterQueryRectangle::new( + GridBoundingBox2D::new([-3, 0], [-1, 3]).unwrap(), + TimeInterval::new( DateTime::new_utc(2010, 1, 1, 0, 0, 0), DateTime::new_utc(2011, 1, 1, 0, 0, 0), ) .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + BandSelection::first(), + ), &query_context, ) .await @@ -1199,9 +1271,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let ndvi_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -1245,22 +1315,15 @@ mod tests { .get_f64() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), // Note: this is not the actual bounding box of the NDVI dataset. The pixel size is 0.1! + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await @@ -1287,9 +1350,7 @@ mod tests { let mut execution_context = MockExecutionContext::test_default(); let ndvi_source = GdalSource { - params: GdalSourceParameters { - data: add_ndvi_dataset(&mut execution_context), - }, + params: GdalSourceParameters::new(add_ndvi_dataset(&mut execution_context)), } .boxed(); @@ -1312,22 +1373,15 @@ mod tests { .get_u8() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); let mut stream = query_processor .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-90, 89, -180, 179).unwrap(), // Note: this is not the actual bounding box of the NDVI dataset. The pixel size is 0.1! + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await diff --git a/operators/src/processing/vector_join/equi_data_join.rs b/operators/src/processing/vector_join/equi_data_join.rs index 79b95740f..85a114732 100644 --- a/operators/src/processing/vector_join/equi_data_join.rs +++ b/operators/src/processing/vector_join/equi_data_join.rs @@ -409,23 +409,20 @@ where #[cfg(test)] mod tests { use futures::executor::block_on_stream; - use geoengine_datatypes::collections::{ ChunksEqualIgnoringCacheHint, MultiPointCollection, VectorDataType, }; - use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; + use super::*; use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, VectorOperator, WorkflowOperatorPath, + ChunkByteSize, MockExecutionContext, VectorOperator, WorkflowOperatorPath, }; use crate::mock::MockFeatureCollectionSource; - - use super::*; use crate::processing::vector_join::util::translation_table; async fn join_mock_collections( @@ -451,18 +448,13 @@ mod tests { let left_processor = left.query_processor().unwrap().multi_point().unwrap(); let right_processor = right.query_processor().unwrap().data().unwrap(); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( - (f64::MIN, f64::MIN).into(), - (f64::MAX, f64::MAX).into(), - ) - .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((f64::MIN, f64::MIN).into(), (f64::MAX, f64::MAX).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ctx = execution_context.mock_query_context(ChunkByteSize::MAX); let processor = EquiGeoToDataJoinProcessor::new( VectorResultDescriptor { diff --git a/operators/src/processing/vector_join/mod.rs b/operators/src/processing/vector_join/mod.rs index 58511e613..365e2b4d0 100644 --- a/operators/src/processing/vector_join/mod.rs +++ b/operators/src/processing/vector_join/mod.rs @@ -1,4 +1,5 @@ use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::SpatialResolution; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -10,6 +11,7 @@ use crate::engine::{ VectorResultDescriptor, WorkflowOperatorPath, }; use crate::error; +use crate::optimization::OptimizationError; use crate::util::Result; use self::equi_data_join::EquiGeoToDataJoinProcessor; @@ -294,6 +296,22 @@ impl InitializedVectorOperator for InitializedVectorJoin { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(VectorJoin { + params: VectorJoinParams { + join_type: self.state.join_type.clone(), + }, + sources: VectorJoinSources { + left: self.left.optimize(target_resolution)?, + right: self.right.optimize(target_resolution)?, + }, + } + .boxed()) + } } #[cfg(test)] diff --git a/operators/src/source/csv.rs b/operators/src/source/csv.rs index 8a3d43a10..96cd081d0 100644 --- a/operators/src/source/csv.rs +++ b/operators/src/source/csv.rs @@ -1,35 +1,35 @@ -use std::path::PathBuf; -use std::pin::Pin; -use std::sync::{Arc, Mutex}; -use std::{fs::File, sync::atomic::AtomicBool}; - +use crate::engine::{ + CanonicOperatorName, InitializedVectorOperator, OperatorData, OperatorName, QueryContext, + SourceOperator, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, + VectorResultDescriptor, +}; +use crate::engine::{QueryProcessor, WorkflowOperatorPath}; +use crate::error; +use crate::optimization::OptimizationError; +use crate::util::{Result, safe_lock_mutex}; +use async_trait::async_trait; use csv::{Position, Reader, StringRecord}; use futures::stream::BoxStream; use futures::task::{Context, Poll}; use futures::{Stream, StreamExt}; -use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{ColumnSelection, VectorQueryRectangle}; -use serde::{Deserialize, Serialize}; -use snafu::{OptionExt, ResultExt, ensure}; - use geoengine_datatypes::collections::{ BuilderProvider, GeoFeatureCollectionRowBuilder, MultiPointCollection, VectorDataType, }; +use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::{ - primitives::{BoundingBox2D, Coordinate2D, TimeInterval}, + primitives::{ + BoundingBox2D, ColumnSelection, Coordinate2D, TimeInterval, VectorQueryRectangle, + }, spatial_reference::SpatialReference, }; - -use crate::engine::{ - CanonicOperatorName, InitializedVectorOperator, OperatorData, OperatorName, QueryContext, - SourceOperator, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, - VectorResultDescriptor, -}; -use crate::engine::{QueryProcessor, WorkflowOperatorPath}; -use crate::error; -use crate::util::{Result, safe_lock_mutex}; -use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use snafu::{OptionExt, ResultExt, ensure}; +use std::path::PathBuf; +use std::pin::Pin; use std::sync::atomic::Ordering; +use std::sync::{Arc, Mutex}; +use std::{fs::File, sync::atomic::AtomicBool}; /// Parameters for the CSV Source Operator #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)] @@ -172,6 +172,16 @@ impl InitializedVectorOperator for InitializedCsvSource { fn path(&self) -> WorkflowOperatorPath { self.path.clone() } + + fn optimize( + &self, + _target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + Ok(CsvSource { + params: self.state.clone(), + } + .boxed()) + } } impl CsvSourceStream { @@ -379,7 +389,7 @@ impl QueryProcessor for CsvSourceProcessor { _ctx: &'a dyn QueryContext, ) -> Result>> { // TODO: properly handle chunk_size - Ok(CsvSourceStream::new(self.params.clone(), query.spatial_bounds, 10)?.boxed()) + Ok(CsvSourceStream::new(self.params.clone(), query.spatial_bounds(), 10)?.boxed()) } fn result_descriptor(&self) -> &VectorResultDescriptor { @@ -402,13 +412,13 @@ struct ParsedRow { #[cfg(test)] mod tests { - use std::io::{Seek, SeekFrom, Write}; - - use geoengine_datatypes::primitives::SpatialResolution; - use super::*; - use crate::engine::MockQueryContext; - use geoengine_datatypes::collections::{FeatureCollectionInfos, ToGeoJson}; + use crate::engine::MockExecutionContext; + use geoengine_datatypes::{ + collections::{FeatureCollectionInfos, ToGeoJson}, + util::test::TestDefault, + }; + use std::io::{Seek, SeekFrom, Write}; #[test] fn it_deserializes() { @@ -585,16 +595,13 @@ x,y }, }; - let query = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - Coordinate2D::new(0., 0.), - Coordinate2D::new(3., 3.), - ), - time_interval: TimeInterval::new_unchecked(0, 1), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::new((10 * 8 * 2).into()); + let query = VectorQueryRectangle::new( + BoundingBox2D::new_unchecked(Coordinate2D::new(0., 0.), Coordinate2D::new(3., 3.)), + TimeInterval::new_unchecked(0, 1), + ColumnSelection::all(), + ); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context((10 * 8 * 2).into()); let r: Vec> = p.query(query, &ctx).await.unwrap().collect().await; diff --git a/operators/src/source/gdal_source/error.rs b/operators/src/source/gdal_source/error.rs index b7958c52b..ca9411e61 100644 --- a/operators/src/source/gdal_source/error.rs +++ b/operators/src/source/gdal_source/error.rs @@ -1,4 +1,4 @@ -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::raster::{GridBoundingBox2D, RasterDataType}; use snafu::Snafu; #[derive(Debug, Snafu)] @@ -7,4 +7,7 @@ use snafu::Snafu; pub enum GdalSourceError { #[snafu(display("Unsupported raster type: {raster_type:?}"))] UnsupportedRasterType { raster_type: RasterDataType }, + + #[snafu(display("Unsupported spatial query: {spatial_query:?}"))] + IncompatibleSpatialQuery { spatial_query: GridBoundingBox2D }, } diff --git a/operators/src/source/gdal_source/loading_info.rs b/operators/src/source/gdal_source/loading_info.rs index c03d0c053..8ae2cf528 100644 --- a/operators/src/source/gdal_source/loading_info.rs +++ b/operators/src/source/gdal_source/loading_info.rs @@ -6,13 +6,14 @@ use crate::{ }; use async_trait::async_trait; use geoengine_datatypes::primitives::{ - CacheTtlSeconds, RasterQueryRectangle, TimeInstance, TimeInterval, TimeStep, TimeStepIter, + CacheTtlSeconds, RasterQueryRectangle, TimeFilledItem, TimeInstance, TimeInterval, TimeStep, + TimeStepIter, }; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, FromSql, ToSql)] +#[derive(Serialize, Deserialize, Debug, Clone, FromSql, ToSql, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataStatic { pub time: Option, @@ -29,7 +30,7 @@ impl MetaData async fn loading_info(&self, query: RasterQueryRectangle) -> Result { let valid = self.time.unwrap_or_default(); - let parts = if valid.intersects(&query.time_interval) { + let parts = if valid.intersects(&query.time_interval()) { vec![GdalLoadingInfoTemporalSlice { time: valid, params: Some(self.params.clone()), @@ -39,17 +40,17 @@ impl MetaData vec![] }; - let known_time_before = if query.time_interval.start() < valid.start() { + let known_time_before = if query.time_interval().start() < valid.start() { TimeInstance::MIN - } else if query.time_interval.start() < valid.end() { + } else if query.time_interval().start() < valid.end() { valid.start() } else { valid.end() }; - let known_time_after = if query.time_interval.end() <= valid.start() { + let known_time_after = if query.time_interval().end() <= valid.start() { valid.start() - } else if query.time_interval.end() <= valid.end() { + } else if query.time_interval().end() <= valid.end() { valid.end() } else { TimeInstance::MAX @@ -79,7 +80,7 @@ impl MetaData /// sets `step` time apart. The `time_placeholders` in the file path of the dataset are replaced with the /// specified time `reference` in specified time `format`. Inside the `data_time` the gdal source will load the data /// from the files and outside it will create nodata. -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataRegular { pub result_descriptor: RasterResultDescriptor, @@ -106,22 +107,28 @@ impl MetaData TimeStepIter::new_with_interval(data_time, step)? .into_intervals(step, data_time.end()) .for_each(|time_interval| { - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + if time_interval.contains(&query.time_interval()) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() <= query.time_interval().start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval().start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } + + if time_interval.start() >= query.time_interval().end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval().end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }); @@ -134,7 +141,7 @@ impl MetaData self.params.clone(), self.time_placeholders.clone(), self.step, - query.time_interval, + query.time_interval(), self.data_time, self.cache_ttl, )?), @@ -155,7 +162,7 @@ impl MetaData } /// Meta data for 4D `NetCDF` CF datasets -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetadataNetCdfCf { pub result_descriptor: RasterResultDescriptor, @@ -198,10 +205,10 @@ impl MetaData let snapped_start = self .step - .snap_relative(self.start, query.time_interval.start())?; + .snap_relative(self.start, query.time_interval().start())?; let snapped_interval = - TimeInterval::new_unchecked(snapped_start, query.time_interval.end()); // TODO: snap end? + TimeInterval::new_unchecked(snapped_start, query.time_interval().end()); // TODO: snap end? let time_iterator = TimeStepIter::new_with_interval(snapped_interval, self.step)?; @@ -232,7 +239,7 @@ impl MetaData } // TODO: custom deserializer that checks that that params are sorted and do not overlap -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, FromSql, ToSql)] +#[derive(Serialize, Deserialize, Debug, Clone, FromSql, ToSql, PartialEq)] #[serde(rename_all = "camelCase")] pub struct GdalMetaDataList { pub result_descriptor: RasterResultDescriptor, @@ -251,25 +258,45 @@ impl MetaData for .inspect(|m| { let time_interval = m.time; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + debug_assert!( + !time_interval.is_instant(), + "time_interval {time_interval} is an instant!" + ); + + if time_interval.contains(&query.time_interval()) { + let t1 = time_interval.start(); + let t2 = time_interval.end(); + known_time_start = Some(t1); + known_time_end = Some(t2); + return; } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() <= query.time_interval().start() { + let t1 = time_interval.end(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } else if time_interval.start() <= query.time_interval().start() { + let t1 = time_interval.start(); + known_time_start = known_time_start.map(|old| old.max(t1)).or(Some(t1)); + } + + if query.time_interval().is_instant() { + // be carefull not to use instant ends... + if time_interval.start() > query.time_interval().end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() > query.time_interval().end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } + } else if time_interval.start() >= query.time_interval().end() { + let t2 = time_interval.start(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); + } else if time_interval.end() >= query.time_interval().end() { + let t2 = time_interval.end(); + known_time_end = known_time_end.map(|old| old.min(t2)).or(Some(t2)); } }) - .filter(|m| m.time.intersects(&query.time_interval)) + .filter(|m| m.time.intersects(&query.time_interval())) .cloned() .collect::>(); @@ -612,21 +639,32 @@ pub struct GdalLoadingInfoTemporalSlice { pub cache_ttl: CacheTtlSeconds, } +impl TimeFilledItem for GdalLoadingInfoTemporalSlice { + fn create_fill_element(time: TimeInterval) -> Self { + Self { + time, + params: None, + cache_ttl: CacheTtlSeconds::max(), // TODO: is this ok? + } + } + + fn time(&self) -> TimeInterval { + self.time + } +} + #[cfg(test)] mod tests { use geoengine_datatypes::{ hashmap, - primitives::{ - BandSelection, DateTime, DateTimeParseFormat, SpatialPartition2D, SpatialResolution, - TimeGranularity, - }, - raster::RasterDataType, + primitives::{BandSelection, DateTime, DateTimeParseFormat, TimeGranularity}, + raster::{BoundedGrid, GeoTransform, GridBoundingBox2D, GridShape2D, RasterDataType}, spatial_reference::SpatialReference, util::test::TestDefault, }; use crate::{ - engine::RasterBandDescriptors, + engine::{RasterBandDescriptors, SpatialGridDescriptor, TimeDescriptor}, source::{FileNotFoundHandling, GdalDatasetGeoTransform, TimeReference}, }; @@ -639,9 +677,17 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + TimeInstance::from_millis_unchecked(0), + TimeInstance::from_millis_unchecked(33), + )), + TimeStep::millis(11).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -668,10 +714,7 @@ mod tests { TimeInstance::from_millis_unchecked(0), TimeInstance::from_millis_unchecked(33), ), - step: TimeStep { - granularity: TimeGranularity::Millis, - step: 11, - }, + step: TimeStep::millis(11).unwrap(), cache_ttl: CacheTtlSeconds::default(), } } @@ -685,10 +728,18 @@ mod tests { RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band() + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new_unchecked( + TimeInstance::from_millis_unchecked(0), + TimeInstance::from_millis_unchecked(33), + )), + TimeStep::millis(11).unwrap(), + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box() + ), + bands: RasterBandDescriptors::new_single_band(), } ); } @@ -699,15 +750,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 30), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 30), + BandSelection::first() + )) .await .unwrap() .info @@ -742,15 +789,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::default(), + BandSelection::first() + )) .await .unwrap() .info @@ -787,15 +830,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(-10, -5), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(-10, -5), + BandSelection::first() + )) .await .unwrap() .info @@ -817,15 +856,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(50, 55), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(50, 55), + BandSelection::first() + )) .await .unwrap() .info @@ -847,15 +882,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 22), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 22), + BandSelection::first() + )) .await .unwrap() .info @@ -886,15 +917,11 @@ mod tests { assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 20), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 20), + BandSelection::first() + )) .await .unwrap() .info @@ -928,9 +955,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::new_unchecked(0, 6))), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ @@ -996,24 +1025,22 @@ mod tests { RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(Some(TimeInterval::new_unchecked(0, 6))), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box() + ), bands: RasterBandDescriptors::new_single_band() } ); assert_eq!( meta_data - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 1.).into(), - (1., 0.).into() - ), - time_interval: TimeInterval::new_unchecked(0, 3), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first() - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([-1, 0], [-1, 0]).unwrap(), + TimeInterval::new_unchecked(0, 3), + BandSelection::first() + )) .await .unwrap() .info @@ -1043,9 +1070,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), // FIXME: the regular time step settings are crazy + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridShape2D::new_2d(128, 128).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1073,12 +1102,11 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new(time_start, time_end).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new(time_start, time_end).unwrap(), + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; @@ -1111,9 +1139,11 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), // FIXME: the regular time step settings are crazy + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1141,12 +1171,11 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new(time_start, time_end).unwrap(), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new(time_start, time_end).unwrap(), + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; @@ -1179,9 +1208,14 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_regular_with_epoch( + Some(TimeInterval::new(time_start, time_end).unwrap()), + time_step, + ), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., -90.).into(), 1., -1.), + GridShape2D::new_2d(180, 360).bounding_box(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: GdalDatasetParameters { @@ -1209,15 +1243,14 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked((0., 128.).into(), (128., 0.).into()), - time_interval: TimeInterval::new_unchecked( + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-128, 0], [-1, 127]).unwrap(), + TimeInterval::new_unchecked( TimeInstance::from(DateTime::new_utc(2009, 7, 1, 0, 0, 0)), TimeInstance::from(DateTime::new_utc(2013, 3, 1, 0, 0, 0)), ), - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }; + BandSelection::first(), + ); let loading_info = metadata.loading_info(query).await.unwrap(); let mut iter = loading_info.info; diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index 029fd98b1..9b2ae7269 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -1,9 +1,9 @@ -use crate::adapters::{ - FillerTileCacheExpirationStrategy, FillerTimeBounds, SparseTilesFillAdapter, -}; use crate::engine::{ - CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, + CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryProcessor, + SpatialGridDescriptor, WorkflowOperatorPath, }; +use crate::optimization::{OptimizableOperator, OptimizationError, SourcesMustNotUseOverviews}; +use crate::source::gdal_source::reader::ReaderState; use crate::util::TemporaryGdalThreadLocalConfigOptions; use crate::util::gdal::gdal_open_dataset_ex; use crate::util::input::float_option_with_nan; @@ -28,23 +28,21 @@ use gdal::errors::GdalError; use gdal::raster::{GdalType, RasterBand as GdalRasterBand}; use gdal::{Dataset as GdalDataset, DatasetOptions, GdalOpenFlags, Metadata as GdalMetadata}; use gdal_sys::VSICurlPartialClearCache; -use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, Coordinate2D, DateTimeParseFormat, RasterQueryRectangle, - SpatialPartition2D, SpatialPartitioned, TimeInstance, -}; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_datatypes::raster::TileInformation; -use geoengine_datatypes::raster::{ - EmptyGrid, GeoTransform, GridIdx2D, GridOrEmpty, GridOrEmpty2D, GridShape2D, GridShapeAccess, - MapElements, MaskedGrid, NoDataValueGrid, Pixel, RasterDataType, RasterProperties, - RasterPropertiesEntry, RasterPropertiesEntryType, RasterPropertiesKey, RasterTile2D, - TilingStrategy, -}; -use geoengine_datatypes::util::test::TestDefault; +use geoengine_datatypes::primitives::{SpatialResolution, find_next_best_overview_level}; use geoengine_datatypes::{ - primitives::TimeInterval, - raster::{Grid, GridBlit, GridBoundingBox2D, GridIdx, GridSize, TilingSpecification}, + dataset::NamedData, + primitives::{ + BandSelection, CacheHint, Coordinate2D, DateTimeParseFormat, RasterQueryRectangle, + TimeInterval, TryIrregularTimeFillIterExt, TryRegularTimeFillIterExt, + }, + raster::{ + ChangeGridBounds, EmptyGrid, GeoTransform, Grid, GridBlit, GridBoundingBox2D, GridOrEmpty, + GridOrEmpty2D, GridShapeAccess, GridSize, MapElements, MaskedGrid, NoDataValueGrid, Pixel, + RasterDataType, RasterProperties, RasterPropertiesEntry, RasterPropertiesEntryType, + RasterPropertiesKey, RasterTile2D, SpatialGridDefinition, TileInformation, + TilingSpecification, TilingStrategy, + }, + util::test::TestDefault, }; use itertools::Itertools; pub use loading_info::{ @@ -52,7 +50,11 @@ pub use loading_info::{ GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf, }; use num::FromPrimitive; +use num::integer::{div_ceil, div_floor}; use postgres_types::{FromSql, ToSql}; +use reader::{ + GdalReadAdvise, GdalReadWindow, GdalReaderMode, GridAndProperties, OverviewReaderState, +}; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use std::collections::HashMap; @@ -66,6 +68,7 @@ use tracing::debug; mod db_types; mod error; mod loading_info; +mod reader; static GDAL_RETRY_INITIAL_BACKOFF_MS: u64 = 1000; static GDAL_RETRY_MAX_BACKOFF_MS: u64 = 60 * 60 * 1000; @@ -73,8 +76,35 @@ static GDAL_RETRY_EXPONENTIAL_BACKOFF_FACTOR: f64 = 2.; /// Parameters for the GDAL Source Operator #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct GdalSourceParameters { pub data: NamedData, + #[serde(default)] + pub overview_level: Option, // TODO: should also allow a resolution? Add resample method? +} + +impl GdalSourceParameters { + #[must_use] + pub fn new(data: NamedData) -> Self { + Self { + data, + overview_level: None, + } + } + + #[must_use] + pub fn new_with_overview_level(data: NamedData, overview_level: u32) -> Self { + Self { + data, + overview_level: Some(overview_level), + } + } + + #[must_use] + pub fn with_overview_level(mut self, overview_level: Option) -> Self { + self.overview_level = overview_level; + self + } } impl OperatorData for GdalSourceParameters { @@ -125,30 +155,41 @@ pub struct GdalDatasetParameters { pub retry: Option, } -#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, Copy)] -#[serde(rename_all = "camelCase")] -pub struct GdalRetryOptions { - pub max_retries: usize, -} - -#[derive(Debug, PartialEq, Eq)] -struct GdalReadWindow { - start_x: isize, // pixelspace origin - start_y: isize, - size_x: usize, // pixelspace size - size_y: usize, -} +impl GdalDatasetParameters { + pub fn dataset_bounds(&self) -> GridBoundingBox2D { + GridBoundingBox2D::new_unchecked( + [0, 0], + [self.height as isize - 1, self.width as isize - 1], + ) + } -impl GdalReadWindow { - fn gdal_window_start(&self) -> (isize, isize) { - (self.start_x, self.start_y) + pub fn gdal_geo_transform(&self) -> GdalDatasetGeoTransform { + self.geo_transform } - fn gdal_window_size(&self) -> (usize, usize) { - (self.size_x, self.size_y) + /// Returns the `SpatialGridDefinition` of the Gdal dataset. + /// + /// Note: This allows upside down datasets (where `GeoTransform` `y_pixel_size` is positive)! + /// + /// # Panics + /// Panics if the `GdalDatasetParameters` are faulty. + pub fn spatial_grid_definition(&self) -> SpatialGridDefinition { + let gdal_geo_transform = GeoTransform::new( + self.gdal_geo_transform().origin_coordinate, + self.gdal_geo_transform().x_pixel_size, + self.gdal_geo_transform().y_pixel_size, + ); + + SpatialGridDefinition::new(gdal_geo_transform, self.dataset_bounds()) } } +#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, Copy)] +#[serde(rename_all = "camelCase")] +pub struct GdalRetryOptions { + pub max_retries: usize, +} + /// A user friendly representation of Gdal's geo transform. In contrast to [`GeoTransform`] this /// geo transform allows arbitrary pixel sizes and can thus also represent rasters where the origin is not located /// in the upper left corner. It should only be used for loading rasters with Gdal and not internally. @@ -162,120 +203,6 @@ pub struct GdalDatasetGeoTransform { pub y_pixel_size: f64, } -impl GdalDatasetGeoTransform { - /// Produce the `SpatialPartition` anchored at the datasets origin with a size of x * y pixels. This method handles non-standard pixel sizes. - pub fn spatial_partition(&self, x_size: usize, y_size: usize) -> SpatialPartition2D { - // the opposite y value (y value of the non origin edge) - let opposite_coord_y = self.origin_coordinate.y + self.y_pixel_size * y_size as f64; - - // if the y-axis is negative then the origin is on the upper side. - let (upper_y, lower_y) = if self.y_pixel_size.is_sign_negative() { - (self.origin_coordinate.y, opposite_coord_y) - } else { - (opposite_coord_y, self.origin_coordinate.y) - }; - - let opposite_coord_x = self.origin_coordinate.x + self.x_pixel_size * x_size as f64; - - // if the y-axis is negative then the origin is on the upper side. - let (left_x, right_x) = if self.x_pixel_size.is_sign_positive() { - (self.origin_coordinate.x, opposite_coord_x) - } else { - (opposite_coord_x, self.origin_coordinate.x) - }; - - SpatialPartition2D::new_unchecked( - Coordinate2D::new(left_x, upper_y), - Coordinate2D::new(right_x, lower_y), - ) - } - - /// Transform a `Coordinate2D` into a `GridIdx2D` - #[inline] - pub fn coordinate_to_grid_idx_2d(&self, coord: Coordinate2D) -> GridIdx2D { - // TODO: use an epsilon here? - let grid_x_index = - ((coord.x - self.origin_coordinate.x) / self.x_pixel_size).floor() as isize; - let grid_y_index = - ((coord.y - self.origin_coordinate.y) / self.y_pixel_size).floor() as isize; - - [grid_y_index, grid_x_index].into() - } - - fn spatial_partition_to_read_window( - &self, - spatial_partition: &SpatialPartition2D, - ) -> GdalReadWindow { - // World coordinates and pixel sizes use float values. Since the float imprecision might cause overflowing into the next pixel we use an epsilon to correct values very close the pixel borders. This logic is the same as used in [`GeoTransform::grid_idx_to_pixel_upper_left_coordinate_2d`]. - const EPSILON: f64 = 0.000_001; - let epsilon: Coordinate2D = - (self.x_pixel_size * EPSILON, self.y_pixel_size * EPSILON).into(); - - /* - The read window is relative to the transform of the gdal dataset. The `SpatialPartition` is oriented at axis of the spatial SRS. This usually causes this situation: - - The gdal data is stored with negative pixel size. The "ul" coordinate of the `SpatialPartition` is neareest to the origin of the gdal raster data. - ul ur - +_______________________+ - |_|_ row 1 | - | |_|_ row 2 | - | |_|_ row ... | - | |_| | - |_______________________| - + * - ll lr - - However, sometimes the data is stored up-side down. Like this: - - The gdal data is stored with a positive pixel size. So the "ll" coordinate is nearest to the reading the raster data needs to starts at this anchor. - - ul ur - +_______________________+ - | _ | - | _|_| row ... | - | _|_| row 3 | - | |_| row 2 | - |_______________________| - + * - ll lr - - Therefore we need to select the raster read start based on the coordinate next to the raster data origin. From there we then calculate the size of the window to read. - */ - let (near_origin_coord, far_origin_coord) = if self.y_pixel_size.is_sign_negative() { - ( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ) - } else { - ( - spatial_partition.lower_left(), - spatial_partition.upper_right(), - ) - }; - - // Move the coordinate near the origin a bit inside the bbox by adding an epsilon of the pixel size. - let safe_near_coord = near_origin_coord + epsilon; - // Move the coordinate far from the origin a bit inside the bbox by subtracting an epsilon of the pixel size - let safe_far_coord = far_origin_coord - epsilon; - - let GridIdx([near_idx_y, near_idx_x]) = self.coordinate_to_grid_idx_2d(safe_near_coord); - let GridIdx([far_idx_y, far_idx_x]) = self.coordinate_to_grid_idx_2d(safe_far_coord); - - debug_assert!(near_idx_x <= far_idx_x); - debug_assert!(near_idx_y <= far_idx_y); - - let read_size_x = (far_idx_x - near_idx_x) as usize + 1; - let read_size_y = (far_idx_y - near_idx_y) as usize + 1; - - GdalReadWindow { - start_x: near_idx_x, - start_y: near_idx_y, - size_x: read_size_x, - size_y: read_size_y, - } - } -} - /// Default implementation for testing purposes where geo transform doesn't matter impl TestDefault for GdalDatasetGeoTransform { fn test_default() -> Self { @@ -307,8 +234,8 @@ impl TryFrom for GeoTransform { fn try_from(dataset_geo_transform: GdalDatasetGeoTransform) -> Result { ensure!( - dataset_geo_transform.x_pixel_size > 0.0 && dataset_geo_transform.y_pixel_size < 0.0, - crate::error::GeoTransformOrigin + dataset_geo_transform.x_pixel_size != 0.0 && dataset_geo_transform.y_pixel_size != 0.0, + crate::error::GeoTransformOrigin // TODO new name? ); Ok(GeoTransform::new( @@ -329,13 +256,6 @@ impl From for GdalDatasetGeoTransform { } } -impl SpatialPartitioned for GdalDatasetParameters { - fn spatial_partition(&self) -> SpatialPartition2D { - self.geo_transform - .spatial_partition(self.width, self.height) - } -} - impl GridShapeAccess for GdalDatasetParameters { type ShapeArray = [usize; 2]; @@ -390,9 +310,11 @@ pub struct GdalSourceProcessor where T: Pixel, { - pub result_descriptor: RasterResultDescriptor, + pub produced_result_descriptor: RasterResultDescriptor, pub tiling_specification: TilingSpecification, pub meta_data: GdalMetaData, + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, pub _phantom_data: PhantomData, } @@ -404,10 +326,8 @@ impl GdalRasterLoader { /// async fn load_tile_data_async( dataset_params: GdalDatasetParameters, - tile_information: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, - ) -> Result> { + read_advise: GdalReadAdvise, + ) -> Result>> { // TODO: detect usage of vsi curl properly, e.g. also check for `/vsicurl_streaming` and combinations with `/vsizip` let is_vsi_curl = dataset_params.file_path.starts_with("/vsicurl/"); @@ -424,11 +344,10 @@ impl GdalRasterLoader { let file_path = ds.file_path.clone(); async move { - let load_tile_result = crate::util::spawn_blocking(move || { - Self::load_tile_data(&ds, tile_information, tile_time, cache_hint) - }) - .await - .context(crate::error::TokioJoin); + let load_tile_result = + crate::util::spawn_blocking(move || Self::load_tile_data(&ds, read_advise)) + .await + .context(crate::error::TokioJoin); match load_tile_result { Ok(Ok(r)) => Ok(r), @@ -450,31 +369,51 @@ impl GdalRasterLoader { async fn load_tile_async( dataset_params: Option, + reader_mode: GdalReaderMode, tile_information: TileInformation, tile_time: TimeInterval, cache_hint: CacheHint, ) -> Result> { + let tile_spatial_grid = tile_information.spatial_grid_definition(); + match dataset_params { // TODO: discuss if we need this check here. The metadata provider should only pass on loading infos if the query intersects the datasets bounds! And the tiling strategy should only generate tiles that intersect the querys bbox. - Some(ds) - if tile_information - .spatial_partition() - .intersects(&ds.spatial_partition()) => - { + Some(ds) => { debug!( "Loading tile {:?}, from {}, band: {}", &tile_information, ds.file_path.display(), ds.rasterband_channel ); - Self::load_tile_data_async(ds, tile_information, tile_time, cache_hint).await - } - Some(_) => { - debug!("Skipping tile not in query rect {:?}", &tile_information); - - Ok(create_no_data_tile(tile_information, tile_time, cache_hint)) + let gdal_read_advise: Option = reader_mode + .tiling_to_dataset_read_advise( + &ds.spatial_grid_definition(), + &tile_spatial_grid, + ); + + let Some(gdal_read_advise) = gdal_read_advise else { + debug!( + "Tile {:?} not intersecting dataset grid or gdal grid {:?}", + &tile_information, ds.file_path + ); + return Ok(create_no_data_tile(tile_information, tile_time, cache_hint)); + }; + + let grid = Self::load_tile_data_async(ds, gdal_read_advise).await?; + + match grid { + Some(grid) => Ok(RasterTile2D::new_with_properties( + tile_time, + tile_information.global_tile_position, + 0, + tile_information.global_geo_transform, + grid.grid, + grid.properties, + cache_hint, + )), + None => Ok(create_no_data_tile(tile_information, tile_time, cache_hint)), + } } - _ => { debug!( "Skipping tile without GdalDatasetParameters {:?}", @@ -491,16 +430,14 @@ impl GdalRasterLoader { /// fn load_tile_data( dataset_params: &GdalDatasetParameters, - tile_information: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, - ) -> Result> { + read_advise: GdalReadAdvise, + ) -> Result>> { let start = Instant::now(); debug!( "GridOrEmpty2D<{:?}> requested for {:?}.", T::TYPE, - &tile_information.spatial_partition() + &read_advise.bounds_of_target, ); let options = dataset_params @@ -527,9 +464,7 @@ impl GdalRasterLoader { let is_file_not_found = error_is_gdal_file_not_found(error); let err_result = match dataset_params.file_not_found_handling { - FileNotFoundHandling::NoData if is_file_not_found => { - Ok(create_no_data_tile(tile_information, tile_time, cache_hint)) - } + FileNotFoundHandling::NoData if is_file_not_found => Ok(None), _ => Err(crate::error::Error::CouldNotOpenGdalDataset { file_path: dataset_params.file_path.to_string_lossy().to_string(), }), @@ -547,36 +482,69 @@ impl GdalRasterLoader { let dataset = dataset_result.expect("checked"); - let result_tile = read_raster_tile_with_properties( - &dataset, - dataset_params, - tile_information, - tile_time, - cache_hint, - )?; + let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; + + let gdal_dataset_geotransform = GdalDatasetGeoTransform::from(dataset.geo_transform()?); + // check that the dataset geo transform is the same as the one we get from GDAL + debug_assert!(approx_eq!( + Coordinate2D, + gdal_dataset_geotransform.origin_coordinate, + dataset_params.geo_transform.origin_coordinate + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.x_pixel_size, + dataset_params.geo_transform.x_pixel_size + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.y_pixel_size, + dataset_params.geo_transform.y_pixel_size + )); + + let (gdal_dataset_pixels_x, gdal_dataset_pixels_y) = dataset.raster_size(); + // check that the dataset pixel size is the same as the one we get from GDAL + debug_assert_eq!(gdal_dataset_pixels_x, dataset_params.width); + debug_assert_eq!(gdal_dataset_pixels_y, dataset_params.height); + + let result_grid = + read_grid_and_handle_edges(&dataset, &rasterband, dataset_params, read_advise)?; + + let properties = read_raster_properties(&dataset, dataset_params, &rasterband); let elapsed = start.elapsed(); debug!("data loaded -> returning data grid, took {elapsed:?}"); - Ok(result_tile) + Ok(Some(GridAndProperties { + grid: result_grid, + properties, + })) } /// /// A stream of futures producing `RasterTile2D` for a single slice in time /// fn temporal_slice_tile_future_stream( - spatial_bounds: SpatialPartition2D, + spatial_query: GridBoundingBox2D, info: GdalLoadingInfoTemporalSlice, tiling_strategy: TilingStrategy, + reader_mode: GdalReaderMode, ) -> impl Stream>>> + use { - stream::iter(tiling_strategy.tile_information_iterator(spatial_bounds)).map(move |tile| { - GdalRasterLoader::load_tile_async( - info.params.clone(), - tile, - info.time, - info.cache_ttl.into(), - ) - }) + stream::iter( + tiling_strategy + .tile_information_iterator_from_pixel_bounds(spatial_query) + .map(move |tile| { + GdalRasterLoader::load_tile_async( + info.params.clone(), + reader_mode, + tile, + info.time, + info.cache_ttl.into(), + ) + }), + ) } fn loading_info_to_tile_stream< @@ -584,16 +552,17 @@ impl GdalRasterLoader { S: Stream>, >( loading_info_stream: S, - query: &RasterQueryRectangle, + spatial_query: GridBoundingBox2D, tiling_strategy: TilingStrategy, + reader_mode: GdalReaderMode, ) -> impl Stream>> + use { - let spatial_bounds = query.spatial_bounds; loading_info_stream .map_ok(move |info| { GdalRasterLoader::temporal_slice_tile_future_stream( - spatial_bounds, + spatial_query, info, tiling_strategy, + reader_mode, ) .map(Result::Ok) }) @@ -630,7 +599,7 @@ where P: Pixel + gdal::raster::GdalType + FromPrimitive, { type Output = RasterTile2D

; - type SpatialBounds = SpatialPartition2D; + type SpatialBounds = GridBoundingBox2D; type Selection = BandSelection; type ResultDescription = RasterResultDescriptor; @@ -640,129 +609,152 @@ where _ctx: &'a dyn crate::engine::QueryContext, ) -> Result>> { ensure!( - query.attributes.as_slice() == [0], + query.attributes().as_slice() == [0], crate::error::GdalSourceDoesNotSupportQueryingOtherBandsThanTheFirstOneYet ); - - let start = Instant::now(); - debug!( + tracing::debug!( "Querying GdalSourceProcessor<{:?}> with: {:?}.", P::TYPE, &query ); + // this is the result descriptor of the operator. It already incorporates the overview level AND shifts the origin to the tiling origin + let result_descriptor = self.result_descriptor(); - debug!( - "GdalSourceProcessor<{:?}> meta data loaded, took {:?}.", - P::TYPE, - start.elapsed() - ); - - let spatial_resolution = query.spatial_resolution; - + let grid_produced_by_source_desc = result_descriptor.spatial_grid; + let grid_produced_by_source = grid_produced_by_source_desc + .source_spatial_grid_definition() + .expect("the source grid definition should be present in a source..."); // A `GeoTransform` maps pixel space to world space. // Usually a SRS has axis directions pointing "up" (y-axis) and "up" (y-axis). // We are not aware of spatial reference systems where the x-axis points to the right. // However, there are spatial reference systems where the y-axis points downwards. // The standard "pixel-space" starts at the top-left corner of a `GeoTransform` and points down-right. // Therefore, the pixel size on the x-axis is always increasing - let pixel_size_x = spatial_resolution.x; + let pixel_size_x = grid_produced_by_source.geo_transform().x_pixel_size(); debug_assert!(pixel_size_x.is_sign_positive()); // and the y-axis should only be positive if the y-axis of the spatial reference system also "points down". // NOTE: at the moment we do not allow "down pointing" y-axis. - let pixel_size_y = -spatial_resolution.y; + let pixel_size_y = grid_produced_by_source.geo_transform().y_pixel_size(); debug_assert!(pixel_size_y.is_sign_negative()); - let tiling_strategy = self - .tiling_specification - .strategy(pixel_size_x, pixel_size_y); - - let result_descriptor = self.meta_data.result_descriptor().await?; + // The data origin is not neccessarily the origin of the tileing we want to use. + // TODO: maybe derive tilling origin reference from the data projection + let produced_tiling_grid = + grid_produced_by_source_desc.tiling_grid_definition(self.tiling_specification); - let mut empty = false; - debug!("result descr bbox: {:?}", result_descriptor.bbox); - debug!("query bbox: {:?}", query.spatial_bounds); + let tiling_strategy = produced_tiling_grid.generate_data_tiling_strategy(); - if let Some(data_spatial_bounds) = result_descriptor.bbox - && !data_spatial_bounds.intersects(&query.spatial_bounds) - { - debug!("query does not intersect spatial data bounds"); - empty = true; - } - - // TODO: use the time bounds to early return. - /* - if let Some(data_time_bounds) = result_descriptor.time { - if !data_time_bounds.intersects(&query.time_interval) { - debug!("query does not intersect temporal data bounds"); - empty = true; + let reader_mode = match self.original_resolution_spatial_grid { + None => GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: grid_produced_by_source, + }), + Some(original_resolution_spatial_grid) => { + GdalReaderMode::OverviewLevel(OverviewReaderState { + original_dataset_grid: original_resolution_spatial_grid, + }) } - } - */ - - let loading_info = if empty { - // TODO: using this shortcut will insert one no-data element with max time validity. However, this does not honor time intervals of data in other areas! - GdalLoadingInfo::new( - GdalLoadingInfoTemporalSliceIterator::Static { - parts: vec![].into_iter(), - }, - TimeInstance::MIN, - TimeInstance::MAX, - ) - } else { - self.meta_data.loading_info(query.clone()).await? }; - let time_bounds = match ( + let loading_info = self.meta_data.loading_info(query.clone()).await?; + + debug_assert!( + loading_info.start_time_of_output_stream < loading_info.end_time_of_output_stream, + "Data time validity must not be a TimeInstance. Is ({:?}, {:?}]", loading_info.start_time_of_output_stream, - loading_info.end_time_of_output_stream, - ) { - (Some(start), Some(end)) => FillerTimeBounds::new(start, end), - (None, None) => { - tracing::debug!( - "The provider did not provide a time range that covers the query. Falling back to query time range. " - ); - FillerTimeBounds::new(query.time_interval.start(), query.time_interval.end()) - } - (Some(start), None) => { - tracing::debug!( - "The provider did only provide a time range start that covers the query. Falling back to query time end. " - ); - FillerTimeBounds::new(start, query.time_interval.end()) - } - (None, Some(end)) => { - tracing::debug!( - "The provider did only provide a time range end that covers the query. Falling back to query time start. " - ); - FillerTimeBounds::new(query.time_interval.start(), end) - } - }; + loading_info.end_time_of_output_stream + ); + + let time_bounds = TimeInterval::new_unchecked( + loading_info + .start_time_of_output_stream + .expect("must exist"), + loading_info.end_time_of_output_stream.expect("must exist"), + ); - let query_time = query.time_interval; + let query_time = query.time_interval(); let skipping_loading_info = loading_info .info - .filter_ok(move |s: &GdalLoadingInfoTemporalSlice| s.time.intersects(&query_time)); + .filter_ok(move |s: &GdalLoadingInfoTemporalSlice| s.time.intersects(&query_time)); // Check that the time slice intersects the query time - let source_stream = stream::iter(skipping_loading_info); - - let source_stream = - GdalRasterLoader::loading_info_to_tile_stream(source_stream, &query, tiling_strategy); + let filled_loading_info_stream = match self.result_descriptor().time.dimension { + geoengine_datatypes::primitives::TimeDimension::Regular(regular_time_dimension) => { + let times_fill_iter = skipping_loading_info + .try_time_regular_range_fill(regular_time_dimension, time_bounds); + stream::iter(times_fill_iter).boxed() + } + geoengine_datatypes::primitives::TimeDimension::Irregular => { + let times_fill_iter = + skipping_loading_info.try_time_irregular_range_fill(time_bounds); + stream::iter(times_fill_iter).boxed() + } + }; - // use SparseTilesFillAdapter to fill all the gaps - let filled_stream = SparseTilesFillAdapter::new( - source_stream, - tiling_strategy.tile_grid_box(query.spatial_partition()), - query.attributes.count(), - tiling_strategy.geo_transform, - tiling_strategy.tile_size_in_pixels, - FillerTileCacheExpirationStrategy::DerivedFromSurroundingTiles, - query.time_interval, - time_bounds, + let source_stream = GdalRasterLoader::loading_info_to_tile_stream( + filled_loading_info_stream + .inspect_ok(|r| debug!("GdalSource _query now producing time slice: {:?}", r.time)), + query.spatial_bounds(), + tiling_strategy, + reader_mode, ); - Ok(filled_stream.boxed()) + + Ok(source_stream.boxed()) } fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.produced_result_descriptor + } +} + +#[async_trait] +impl RasterQueryProcessor for GdalSourceProcessor +where + T: Pixel + gdal::raster::GdalType + FromPrimitive, +{ + type RasterType = T; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { + let rdt = self.raster_result_descriptor().time; + + let q_bounds = self + .raster_result_descriptor() + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_grid_bounds(); + let q_rect = RasterQueryRectangle::new(q_bounds, query, BandSelection::first()); + let ldif = self.meta_data.loading_info(q_rect).await?; + let unique_times = ldif.info.map(|s| s.map(|s| s.time)); + + let res_stream = match rdt.dimension { + geoengine_datatypes::primitives::TimeDimension::Regular(regular_time_dimension) => { + let times_fill_iter = unique_times.try_time_regular_range_fill( + regular_time_dimension, + TimeInterval::new_unchecked( + ldif.start_time_of_output_stream + .expect("start time must be set for irregular time dimension"), // TODO: make this a requirement of the meta data trait + ldif.end_time_of_output_stream + .expect("end time must be set for irregular time dimension"), + ), + ); + stream::iter(times_fill_iter).boxed() + } + geoengine_datatypes::primitives::TimeDimension::Irregular => { + let times_fill_iter = + unique_times.try_time_irregular_range_fill(TimeInterval::new_unchecked( + ldif.start_time_of_output_stream + .expect("start time must be set for regular time dimension"), // TODO: make this a requirement of the meta data trait + ldif.end_time_of_output_stream + .expect("end time must be set for regular time dimension"), + )); + stream::iter(times_fill_iter).boxed() + } + }; + + Ok(res_stream + .inspect_ok(|ti| debug!("GdalSource time_query producing time interval: {:?}", ti)) + .boxed()) } } @@ -772,6 +764,44 @@ impl OperatorName for GdalSource { const TYPE_NAME: &'static str = "GdalSource"; } +fn overview_level_spatial_grid( + source_spatial_grid: SpatialGridDefinition, + overview_level: u32, +) -> Option { + if overview_level > 0 { + debug!("Using overview level {overview_level}"); + let geo_transform = GeoTransform::new( + source_spatial_grid.geo_transform.origin_coordinate, + source_spatial_grid.geo_transform.x_pixel_size() * f64::from(overview_level), + source_spatial_grid.geo_transform.y_pixel_size() * f64::from(overview_level), + ); + let grid_bounds = GridBoundingBox2D::new_min_max( + div_floor( + source_spatial_grid.grid_bounds.y_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.y_max(), + overview_level as isize, + ), + div_floor( + source_spatial_grid.grid_bounds.x_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.x_max(), + overview_level as isize, + ), + ) + .expect("overview level must be a positive integer"); + + Some(SpatialGridDefinition::new(geo_transform, grid_bounds)) + } else { + debug!("Using original resolution (ov = 0)"); + None + } +} + #[typetag::serde] #[async_trait] impl RasterOperator for GdalSource { @@ -786,13 +816,29 @@ impl RasterOperator for GdalSource { debug!("Initializing GdalSource for {:?}.", &self.params.data); debug!("GdalSource path: {:?}", path); - let op = InitializedGdalSourceOperator { - name: CanonicOperatorName::from(&self), - path, - data: self.params.data.to_string(), - result_descriptor: meta_data.result_descriptor().await?, - meta_data, - tiling_specification: context.tiling_specification(), + let meta_data_result_descriptor = meta_data.result_descriptor().await?; + + let op_name = CanonicOperatorName::from(&self); + let op = if self.params.overview_level.is_none() { + InitializedGdalSourceOperator::initialize_original_resolution( + op_name, + path, + self.params.data, + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + ) + } else { + // generate a result descriptor with the overview level + InitializedGdalSourceOperator::initialize_with_overview_level( + op_name, + path, + self.params.data, + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + self.params.overview_level.unwrap_or(0), + ) }; Ok(op.boxed()) @@ -801,45 +847,115 @@ impl RasterOperator for GdalSource { span_fn!(GdalSource); } +#[derive(Clone)] pub struct InitializedGdalSourceOperator { - name: CanonicOperatorName, + pub name: CanonicOperatorName, path: WorkflowOperatorPath, - data: String, pub meta_data: GdalMetaData, - pub result_descriptor: RasterResultDescriptor, + pub produced_result_descriptor: RasterResultDescriptor, pub tiling_specification: TilingSpecification, + pub data_name: NamedData, + // the overview level to use. 0/1 means the highest resolution + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, +} + +impl InitializedGdalSourceOperator { + pub fn initialize_original_resolution( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data_name: NamedData, + meta_data: GdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + ) -> Self { + InitializedGdalSourceOperator { + name, + path, + data_name, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + overview_level: 0, + original_resolution_spatial_grid: None, + } + } + + pub fn initialize_with_overview_level( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data_name: NamedData, + meta_data: GdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + overview_level: u32, + ) -> Self { + let source_resolution_spatial_grid = result_descriptor + .spatial_grid_descriptor() + .source_spatial_grid_definition() + .expect("Source data must be a source grid definition..."); + + let (result_descriptor, original_grid) = if let Some(ovr_spatial_grid) = + overview_level_spatial_grid(source_resolution_spatial_grid, overview_level) + { + let ovr_res = RasterResultDescriptor { + spatial_grid: SpatialGridDescriptor::new_source(ovr_spatial_grid), + ..result_descriptor + }; + (ovr_res, Some(source_resolution_spatial_grid)) + } else { + (result_descriptor, None) + }; + + InitializedGdalSourceOperator { + name, + path, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + data_name, + overview_level, + original_resolution_spatial_grid: original_grid, + } + } } impl InitializedRasterOperator for InitializedGdalSourceOperator { fn result_descriptor(&self) -> &RasterResultDescriptor { - &self.result_descriptor + &self.produced_result_descriptor } fn query_processor(&self) -> Result { Ok(match self.result_descriptor().data_type { RasterDataType::U8 => TypedRasterQueryProcessor::U8( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::U16 => TypedRasterQueryProcessor::U16( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::U32 => TypedRasterQueryProcessor::U32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -856,18 +972,22 @@ impl InitializedRasterOperator for InitializedGdalSourceOperator { } RasterDataType::I16 => TypedRasterQueryProcessor::I16( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::I32 => TypedRasterQueryProcessor::I32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -879,18 +999,22 @@ impl InitializedRasterOperator for InitializedGdalSourceOperator { } RasterDataType::F32 => TypedRasterQueryProcessor::F32( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), ), RasterDataType::F64 => TypedRasterQueryProcessor::F64( GdalSourceProcessor { - result_descriptor: self.result_descriptor.clone(), + produced_result_descriptor: self.produced_result_descriptor.clone(), tiling_specification: self.tiling_specification, meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, _phantom_data: PhantomData, } .boxed(), @@ -911,7 +1035,44 @@ impl InitializedRasterOperator for InitializedGdalSourceOperator { } fn data(&self) -> Option { - Some(self.data.clone()) + Some(self.data_name.to_string()) + } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + // TODO: handle cases where the original workflow explicitly loads overviews in the source + ensure!( + self.overview_level == 0, + SourcesMustNotUseOverviews { + data: self.data_name.to_string(), + oveview_level: self.overview_level + } + ); + + // as overview level is always 0 for now, the result descriptor contains the native resolution + // TODO: when allowing to optimize upon overview levels, compute the native resolution first + let native_resolution = self + .produced_result_descriptor + .spatial_grid + .spatial_resolution(); + + // TODO: get available overviews levels from the dataset metadata (not available yet) and only load these. + // Then, we might have to prepend a Resampling operator to match the target resolution. + // For now, we just load the overview level regardless and let gdal handle the resamṕling. + let next_best_overview_level = + find_next_best_overview_level(native_resolution, target_resolution); + + Ok(GdalSource { + params: GdalSourceParameters { + data: self.data_name.clone(), + overview_level: Some(next_best_overview_level), + }, + } + .boxed()) } } @@ -993,144 +1154,61 @@ where Ok(GridOrEmpty::from(masked_grid)) } -/// This method reads the data for a single grid with a specified size from the GDAL dataset. -/// If the tile overlaps the borders of the dataset only the data in the dataset bounds is read. -/// The data read from the dataset is clipped into a grid with the requested size filled with the `no_data_value`. -fn read_partial_grid_from_raster( - rasterband: &GdalRasterBand, - gdal_read_window: &GdalReadWindow, - out_tile_read_bounds: GridBoundingBox2D, - out_tile_shape: GridShape2D, - dataset_params: &GdalDatasetParameters, - flip_y_axis: bool, -) -> Result> -where - T: Pixel + GdalType + Default + FromPrimitive, -{ - let dataset_raster = read_grid_from_raster( - rasterband, - gdal_read_window, - out_tile_read_bounds, - dataset_params, - flip_y_axis, - )?; - - let mut tile_raster = GridOrEmpty::from(EmptyGrid::new(out_tile_shape)); - tile_raster.grid_blit_from(&dataset_raster); - Ok(tile_raster) -} - /// This method reads the data for a single tile with a specified size from the GDAL dataset. /// It handles conversion to grid coordinates. /// If the tile is inside the dataset it uses the `read_grid_from_raster` method. /// If the tile overlaps the borders of the dataset it uses the `read_partial_grid_from_raster` method. fn read_grid_and_handle_edges( - tile_info: TileInformation, - dataset: &GdalDataset, + _dataset: &GdalDataset, rasterband: &GdalRasterBand, dataset_params: &GdalDatasetParameters, + gdal_read_advice: GdalReadAdvise, ) -> Result> where T: Pixel + GdalType + Default + FromPrimitive, { - let gdal_dataset_geotransform = GdalDatasetGeoTransform::from(dataset.geo_transform()?); - let (gdal_dataset_pixels_x, gdal_dataset_pixels_y) = dataset.raster_size(); - - if !approx_eq!( - GdalDatasetGeoTransform, - gdal_dataset_geotransform, - dataset_params.geo_transform - ) { - tracing::debug!( - "GdalDatasetParameters geo transform is different to the one retrieved from GDAL dataset: {:?} != {:?}", - dataset_params.geo_transform, - gdal_dataset_geotransform, - ); - } - - debug_assert_eq!(gdal_dataset_pixels_x, dataset_params.width); - debug_assert_eq!(gdal_dataset_pixels_y, dataset_params.height); - - let gdal_dataset_bounds = - gdal_dataset_geotransform.spatial_partition(gdal_dataset_pixels_x, gdal_dataset_pixels_y); - - let output_bounds = tile_info.spatial_partition(); - let dataset_intersects_tile = gdal_dataset_bounds.intersection(&output_bounds); - let output_shape = tile_info.tile_size_in_pixels(); - - let Some(dataset_intersection_area) = dataset_intersects_tile else { - return Ok(GridOrEmpty::from(EmptyGrid::new(output_shape))); - }; - - let tile_geo_transform = tile_info.tile_geo_transform(); - - let gdal_read_window = - gdal_dataset_geotransform.spatial_partition_to_read_window(&dataset_intersection_area); - - let is_y_axis_flipped = tile_geo_transform.y_pixel_size().is_sign_negative() - != gdal_dataset_geotransform.y_pixel_size.is_sign_negative(); - - if is_y_axis_flipped { - debug!("The GDAL data has a flipped y-axis. Need to unflip it!"); - } - - let result_grid = if dataset_intersection_area == output_bounds { + let result_grid = if gdal_read_advice.direct_read() { read_grid_from_raster( rasterband, - &gdal_read_window, - output_shape, + &gdal_read_advice.gdal_read_widow, + gdal_read_advice.read_window_bounds.grid_shape(), dataset_params, - is_y_axis_flipped, + gdal_read_advice.flip_y, )? } else { - let partial_tile_grid_bounds = - tile_geo_transform.spatial_to_grid_bounds(&dataset_intersection_area); - - read_partial_grid_from_raster( + let r: GridOrEmpty = read_grid_from_raster( rasterband, - &gdal_read_window, - partial_tile_grid_bounds, - output_shape, + &gdal_read_advice.gdal_read_widow, + gdal_read_advice.read_window_bounds, dataset_params, - is_y_axis_flipped, - )? + gdal_read_advice.flip_y, + )?; + let mut tile_raster = GridOrEmpty::from(EmptyGrid::new(gdal_read_advice.bounds_of_target)); + tile_raster.grid_blit_from(&r); + tile_raster.unbounded() }; Ok(result_grid) } /// This method reads the data for a single tile with a specified size from the GDAL dataset and adds the requested metadata as properties to the tile. -fn read_raster_tile_with_properties( +fn read_raster_properties( dataset: &GdalDataset, dataset_params: &GdalDatasetParameters, - tile_info: TileInformation, - tile_time: TimeInterval, - cache_hint: CacheHint, -) -> Result> { - let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; - - let result_grid = read_grid_and_handle_edges(tile_info, dataset, &rasterband, dataset_params)?; - + rasterband: &GdalRasterBand, +) -> RasterProperties { let mut properties = RasterProperties::default(); // always read the scale and offset values from the rasterband - properties_from_band(&mut properties, &rasterband); + properties_from_band(&mut properties, rasterband); // read the properties from the dataset and rasterband metadata if let Some(properties_mapping) = dataset_params.properties_mapping.as_ref() { properties_from_gdal_metadata(&mut properties, dataset, properties_mapping); - properties_from_gdal_metadata(&mut properties, &rasterband, properties_mapping); + properties_from_gdal_metadata(&mut properties, rasterband, properties_mapping); } - // TODO: add cache_hint - Ok(RasterTile2D::new_with_tile_info_and_properties( - tile_time, - tile_info, - 0, - result_grid, - properties, - cache_hint, - )) + properties } fn create_no_data_tile( @@ -1226,35 +1304,48 @@ mod tests { use crate::test_data; use crate::util::Result; use crate::util::gdal::add_ndvi_dataset; + use float_cmp::assert_approx_eq; use geoengine_datatypes::hashmap; use geoengine_datatypes::primitives::{AxisAlignedRectangle, SpatialPartition2D, TimeInstance}; + use geoengine_datatypes::raster::{BoundedGrid, GridShape2D, SpatialGridDefinition}; use geoengine_datatypes::raster::{ EmptyGrid2D, GridBounds, GridIdx2D, TilesEqualIgnoringCacheHint, }; use geoengine_datatypes::raster::{TileInformation, TilingStrategy}; use geoengine_datatypes::util::gdal::hide_gdal_errors; - use geoengine_datatypes::{primitives::SpatialResolution, raster::GridShape2D}; use httptest::matchers::request; use httptest::{Expectation, Server, responders}; + use reader::{GdalReadAdvise, GdalReadWindow}; + + fn tile_information_with_partition_and_shape( + partition: SpatialPartition2D, + shape: GridShape2D, + ) -> TileInformation { + let real_geotransform = GeoTransform::new( + partition.upper_left(), + partition.size_x() / shape.axis_size_x() as f64, + -partition.size_y() / shape.axis_size_y() as f64, + ); + + TileInformation { + tile_size_in_pixels: shape, + global_tile_position: [0, 0].into(), + global_geo_transform: real_geotransform, + } + } async fn query_gdal_source( exe_ctx: &MockExecutionContext, query_ctx: &MockQueryContext, name: NamedData, - output_shape: GridShape2D, - output_bounds: SpatialPartition2D, + spatial_query: GridBoundingBox2D, time_interval: TimeInterval, ) -> Vec>> { let op = GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(); - let x_query_resolution = output_bounds.size_x() / output_shape.axis_size_x() as f64; - let y_query_resolution = output_bounds.size_y() / output_shape.axis_size_y() as f64; - let spatial_resolution = - SpatialResolution::new_unchecked(x_query_resolution, y_query_resolution); - let o = op .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) .await @@ -1265,12 +1356,7 @@ mod tests { .get_u8() .unwrap() .raster_query( - RasterQueryRectangle { - spatial_bounds: output_bounds, - time_interval, - spatial_resolution, - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new(spatial_query, time_interval, BandSelection::first()), query_ctx, ) .await @@ -1279,56 +1365,45 @@ mod tests { .await } - fn load_ndvi_jan_2014( - output_shape: GridShape2D, - output_bounds: SpatialPartition2D, - ) -> Result> { - GdalRasterLoader::load_tile_data::( - &GdalDatasetParameters { - file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_2014-01-01.TIFF").into(), - rasterband_channel: 1, - geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-180., 90.).into(), - x_pixel_size: 0.1, - y_pixel_size: -0.1, - }, - width: 3600, - height: 1800, - file_not_found_handling: FileNotFoundHandling::NoData, - no_data_value: Some(0.), - properties_mapping: Some(vec![ - GdalMetadataMapping { - source_key: RasterPropertiesKey { - domain: None, - key: "AREA_OR_POINT".to_string(), - }, - target_type: RasterPropertiesEntryType::String, - target_key: RasterPropertiesKey { - domain: None, - key: "AREA_OR_POINT".to_string(), - }, - }, - GdalMetadataMapping { - source_key: RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE".to_string()), - key: "COMPRESSION".to_string(), - }, - target_type: RasterPropertiesEntryType::String, - target_key: RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE_INFO".to_string()), - key: "COMPRESSION".to_string(), - }, - }, - ]), - gdal_open_options: None, - gdal_config_options: None, - allow_alphaband_as_mask: true, - retry: None, + // This method loads raster data from a cropped MODIS NDVI raster. + // To inspect the byte values first convert the file to XYZ with GDAL: + // 'gdal_translate -of xyz MOD13A2_M_NDVI_2014-04-01_30X30.tif MOD13A2_M_NDVI_2014-04-01_30x30.xyz' + // Then you can convert them to gruped bytes: + // 'cut -d ' ' -f 1,2 --complement MOD13A2_M_NDVI_2014-04-01_30x30.xyz | xargs -n 30 > MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt'. + fn load_ndvi_apr_2014_cropped( + gdal_read_advice: GdalReadAdvise, + ) -> Result>> { + let dataset_params = GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif") + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (8.0, 57.4).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, }, - TileInformation::with_partition_and_shape(output_bounds, output_shape), - TimeInterval::default(), - CacheHint::default(), - ) + width: 30, + height: 30, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: Some(255.), + properties_mapping: Some(vec![GdalMetadataMapping { + source_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + target_type: RasterPropertiesEntryType::String, + target_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + }]), + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + GdalRasterLoader::load_tile_data::(&dataset_params, gdal_read_advice) } #[test] @@ -1346,9 +1421,7 @@ mod tests { assert_eq!( operator, GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("ns", "dataset"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("ns", "dataset")), } ); } @@ -1441,7 +1514,7 @@ mod tests { dataset_y_pixel_size, ); - let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); let origin_split_tileing_strategy = TilingStrategy { tile_size_in_pixels: tile_size_in_pixels.into(), @@ -1449,7 +1522,7 @@ mod tests { }; let vres: Vec = origin_split_tileing_strategy - .tile_idx_iterator(partition) + .tile_idx_iterator_from_grid_bounds(grid_bounds) .collect(); assert_eq!(vres.len(), 4 * 6); assert_eq!(vres[0], [-2, -3].into()); @@ -1471,7 +1544,7 @@ mod tests { dataset_y_pixel_size, ); - let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); let origin_split_tileing_strategy = TilingStrategy { tile_size_in_pixels: tile_size_in_pixels.into(), @@ -1479,7 +1552,7 @@ mod tests { }; let vres: Vec = origin_split_tileing_strategy - .tile_information_iterator(partition) + .tile_information_iterator_from_pixel_bounds(grid_bounds) .collect(); assert_eq!(vres.len(), 4 * 6); assert_eq!( @@ -1558,41 +1631,48 @@ mod tests { } #[test] - fn test_load_tile_data() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + fn test_load_tile_data_top_left() { + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = load_ndvi_jan_2014(output_shape, output_bounds).unwrap(); + let GridAndProperties { grid, properties } = load_ndvi_apr_2014_cropped(gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); + // pixel value are the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 127, 107, 255, 255, 255, 255, 255, 164, 185, 182, + 255, 255, 255, 175, 186, 190, 167, 140, 255, 255, 161, 175, 184, 173, 170, 188, + 255, 255, 128, 177, 165, 145, 191, 174, 255, 117, 100, 174, 159, 147, 99, 135 ] ); assert_eq!(grid.validity_mask.data.len(), 64); - assert_eq!(grid.validity_mask.data, &[true; 64]); + // pixel mask is pixel > 0 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, true, true, + false, false, false, false, false, true, true, true, false, false, false, true, + true, true, true, true, false, false, true, true, true, true, true, true, false, + false, true, true, true, true, true, true, false, true, true, true, true, true, + true, true + ] + ); - assert!((properties.scale_option()).is_none()); - assert!(properties.offset_option().is_none()); assert_eq!( properties.get_property(&RasterPropertiesKey { key: "AREA_OR_POINT".to_string(), @@ -1600,34 +1680,24 @@ mod tests { }), Some(&RasterPropertiesEntry::String("Area".to_string())) ); - assert_eq!( - properties.get_property(&RasterPropertiesKey { - domain: Some("IMAGE_STRUCTURE_INFO".to_string()), - key: "COMPRESSION".to_string(), - }), - Some(&RasterPropertiesEntry::String("LZW".to_string())) - ); } #[test] - fn test_load_tile_data_overlaps_dataset_bounds() { - let output_shape: GridShape2D = [8, 8].into(); + fn test_load_tile_data_overlaps_dataset_bounds_top_left_out1() { // shift world bbox one pixel up and to the left - let (x_size, y_size) = (45., 22.5); - let output_bounds = SpatialPartition2D::new_unchecked( - (-180. - x_size, 90. + y_size).into(), - (180. - x_size, -90. + y_size).into(), - ); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [7, 7].into()), // this is the data we can read + read_window_bounds: GridBoundingBox2D::new([1, 1], [7, 7]).unwrap(), // this is the area we can fill in target + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), // this is the area of the target + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties: _, - cache_hint: _, - } = load_ndvi_jan_2014(output_shape, output_bounds).unwrap(); + let GridAndProperties { + grid, + properties: _properties, + } = load_ndvi_apr_2014_cropped(gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); @@ -1637,19 +1707,34 @@ mod tests { assert_eq!( x.inner_grid.data, &[ - 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 0, 255, 75, 37, 255, - 44, 34, 39, 0, 255, 86, 255, 255, 255, 30, 96, 0, 255, 255, 255, 255, 90, 255, 255, - 0, 255, 255, 202, 255, 193, 255, 255, 0, 255, 255, 89, 255, 111, 255, 255, 0, 255, - 255, 255, 255, 255, 255, 255 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 255, 255, 255, 255, 255, 255, 0, 255, 255, 255, + 255, 255, 255, 255, 0, 255, 255, 255, 255, 255, 255, 127, 0, 255, 255, 255, 255, + 255, 164, 185, 0, 255, 255, 255, 175, 186, 190, 167, 0, 255, 255, 161, 175, 184, + 173, 170, 0, 255, 255, 128, 177, 165, 145, 191, + ] + ); + + assert_eq!(x.validity_mask.data.len(), 64); + // pixel mask is pixel == 255 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + x.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, true, false, false, false, false, + false, false, true, true, false, false, false, false, true, true, true, true, + false, false, false, true, true, true, true, true, false, false, false, true, true, + true, true, true, ] ); } + /* This test no longer works since we now employ a clipping strategy and this makes us read a lot more data? #[test] fn test_load_tile_data_is_inside_single_pixel() { let output_shape: GridShape2D = [8, 8].into(); // shift world bbox one pixel up and to the left - let (x_size, y_size) = (0.000_000_000_01, 0.000_000_000_01); + let (x_size, y_size) = (0.001, 0.001); let output_bounds = SpatialPartition2D::new( (-116.22222, 66.66666).into(), (-116.22222 + x_size, 66.66666 - y_size).into(), @@ -1673,27 +1758,18 @@ mod tests { assert_eq!(x.inner_grid.data.len(), 64); assert_eq!(x.inner_grid.data, &[1; 64]); } + */ #[tokio::test] async fn test_query_single_time_slice() { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let id = add_ndvi_dataset(&mut exe_ctx); + let spatial_query = GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_001); // 2014-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1727,23 +1803,14 @@ mod tests { #[tokio::test] async fn test_query_multi_time_slices() { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(); + let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_393_632_000_000); // 2014-01-01 - 2014-03-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 8); @@ -1762,23 +1829,13 @@ mod tests { #[tokio::test] async fn test_query_before_data() { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(); let time_interval = TimeInterval::new_unchecked(1_380_585_600_000, 1_380_585_600_000); // 2013-10-01 - 2013-10-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1792,23 +1849,13 @@ mod tests { #[tokio::test] async fn test_query_after_data() { let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(); let time_interval = TimeInterval::new_unchecked(1_420_074_000_000, 1_420_074_000_000); // 2015-01-01 - 2015-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1824,23 +1871,13 @@ mod tests { hide_gdal_errors(); let mut exe_ctx = MockExecutionContext::test_default(); - let query_ctx = MockQueryContext::test_default(); + let query_ctx = exe_ctx.mock_query_context_test_default(); let id = add_ndvi_dataset(&mut exe_ctx); - let output_shape: GridShape2D = [256, 256].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); + let spatial_query = GridBoundingBox2D::new([-256, -256], [255, 255]).unwrap(); let time_interval = TimeInterval::new_unchecked(1_385_856_000_000, 1_388_534_400_000); // 2013-12-01 - 2014-01-01 - let c = query_gdal_source( - &exe_ctx, - &query_ctx, - id, - output_shape, - output_bounds, - time_interval, - ) - .await; + let c = query_gdal_source(&exe_ctx, &query_ctx, id, spatial_query, time_interval).await; let c: Vec> = c.into_iter().map(Result::unwrap).collect(); assert_eq!(c.len(), 4); @@ -1861,12 +1898,19 @@ mod tests { SpatialPartition2D::new_unchecked((-90., 90.).into(), (90., -90.).into()); let output_shape: GridShape2D = [256, 256].into(); - let tile_info = TileInformation::with_partition_and_shape(output_bounds, output_shape); + let tile_info = tile_information_with_partition_and_shape(output_bounds, output_shape); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_391_212_800_000); // 2014-01-01 - 2014-01-15 let params = None; + let reader_mode = GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + tile_info.global_geo_transform, + GridShape2D::new([3600, 1800]).bounding_box(), + ), + }); let tile = GdalRasterLoader::load_tile_async::( params, + reader_mode, tile_info, time_interval, CacheHint::default(), @@ -2042,121 +2086,8 @@ mod tests { } #[test] - fn gdal_geotransform_to_bounds_neg_y_0() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: -1., - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(0., 0.), Coordinate2D::new(10., -10.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_neg_y_5() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., 5.), - x_pixel_size: 0.5, - y_pixel_size: -0.5, - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = - SpatialPartition2D::new(Coordinate2D::new(5., 5.), Coordinate2D::new(10., 0.)).unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_pos_y_0() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: 1., - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(0., 10.), Coordinate2D::new(10., 0.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_geotransform_to_bounds_pos_y_5() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., -5.), - x_pixel_size: 0.5, - y_pixel_size: 0.5, - }; - - let sb = gt.spatial_partition(10, 10); - - let exp = SpatialPartition2D::new(Coordinate2D::new(5., 0.), Coordinate2D::new(10., -5.)) - .unwrap(); - - assert_eq!(sb, exp); - } - - #[test] - fn gdal_read_window_data_origin_upper_left() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(5., -5.), - x_pixel_size: 0.5, - y_pixel_size: -0.5, - }; - - let sb = SpatialPartition2D::new(Coordinate2D::new(8., -7.), Coordinate2D::new(10., -10.)) - .unwrap(); - - let rw = gt.spatial_partition_to_read_window(&sb); - - let exp = GdalReadWindow { - size_x: 4, - size_y: 6, - start_x: 6, - start_y: 4, - }; - - assert_eq!(rw, exp); - } - - #[test] - fn gdal_read_window_data_origin_lower_left() { - let gt = GdalDatasetGeoTransform { - origin_coordinate: Coordinate2D::new(0., 0.), - x_pixel_size: 1., - y_pixel_size: 1., - }; - - let sb = SpatialPartition2D::new(Coordinate2D::new(0., 10.), Coordinate2D::new(10., 0.)) - .unwrap(); - - let rw = gt.spatial_partition_to_read_window(&sb); - - let exp = GdalReadWindow { - size_x: 10, - size_y: 10, - start_x: 0, - start_y: 0, - }; - - assert_eq!(rw, exp); - } - - #[test] + #[allow(clippy::too_many_lines)] fn read_up_side_down_raster() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let up_side_down_params = GdalDatasetParameters { file_path: test_data!( "raster/modis_ndvi/flipped_axis_y/MOD13A2_M_NDVI_2014-01-01_flipped_y.tiff" @@ -2202,37 +2133,78 @@ mod tests { retry: None, }; - let tile_information = - TileInformation::with_partition_and_shape(output_bounds, output_shape); + let ge_global_dataset_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 0.1, -0.1), + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap(), + ); - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = GdalRasterLoader::load_tile_data::( - &up_side_down_params, - tile_information, - TimeInterval::default(), - CacheHint::default(), - ) - .unwrap(); + let gdal_dataset_grid = ge_global_dataset_grid.flip_axis_y(); // first, flip axis + assert_approx_eq!( + Coordinate2D, + gdal_dataset_grid.geo_transform().origin_coordinate, + Coordinate2D::new(-180., 90.) + ); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.y_pixel_size(), 0.1); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.x_pixel_size(), 0.1); + assert_eq!( + gdal_dataset_grid.grid_bounds, + GridBoundingBox2D::new_min_max(-1800, -1, 0, 3599).unwrap() + ); - assert!(!grid.is_empty()); + let gdal_dataset_grid = gdal_dataset_grid + .with_moved_origin_exact_grid(Coordinate2D::new(-180., -90.)) + .unwrap(); // second, move origin (to other side of axis) + assert_approx_eq!( + Coordinate2D, + gdal_dataset_grid.geo_transform().origin_coordinate, + Coordinate2D::new(-180., -90.) + ); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.y_pixel_size(), 0.1); + assert_approx_eq!(f64, gdal_dataset_grid.geo_transform.x_pixel_size(), 0.1); + assert_eq!( + gdal_dataset_grid.grid_bounds, + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap() + ); + + let ovr = OverviewReaderState { + original_dataset_grid: ge_global_dataset_grid, + }; + + let tile = SpatialGridDefinition::new( + ge_global_dataset_grid.geo_transform, + GridBoundingBox2D::new_min_max(326, 326 + 7, 1880, 1880 + 7).unwrap(), + ); + let gdal_read_advice = ovr + .tiling_to_dataset_read_advise(&gdal_dataset_grid, &tile) + .unwrap(); + + let exp_gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([1466, 1880].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([326, 1880], [326 + 7, 1880 + 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([326, 1880], [326 + 7, 1880 + 7]).unwrap(), + flip_y: true, + }; + + assert_eq!(gdal_read_advice, exp_gdal_read_advice); + + let GridAndProperties { grid, properties } = + GdalRasterLoader::load_tile_data::(&up_side_down_params, gdal_read_advice) + .unwrap() + .unwrap(); + + assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + // TODO: check in tiff! + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 53, 47, 255, 255, 255, 255, 255, 68, 81, 93, 255, 255, + 255, 97, 102, 91, 73, 72, 255, 255, 91, 97, 100, 86, 78, 106, 255, 255, 59, 95, 85, + 66, 105, 104, 255, 47, 42, 82, 81, 76, 73, 98 ] ); @@ -2245,25 +2217,19 @@ mod tests { #[test] fn read_raster_and_offset_scale() { - let output_shape: GridShape2D = [8, 8].into(); - let output_bounds = - SpatialPartition2D::new_unchecked((-180., 90.).into(), (180., -90.).into()); - let up_side_down_params = GdalDatasetParameters { - file_path: test_data!( - "raster/modis_ndvi/with_offset_scale/MOD13A2_M_NDVI_2014-01-01.TIFF" - ) - .into(), + file_path: test_data!("raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif") + .into(), rasterband_channel: 1, geo_transform: GdalDatasetGeoTransform { - origin_coordinate: (-180., -90.).into(), + origin_coordinate: (8.0, 57.4).into(), x_pixel_size: 0.1, - y_pixel_size: 0.1, + y_pixel_size: -0.1, }, - width: 3600, - height: 1800, + width: 30, + height: 30, file_not_found_handling: FileNotFoundHandling::NoData, - no_data_value: Some(0.), + no_data_value: Some(255.), properties_mapping: None, gdal_open_options: None, gdal_config_options: None, @@ -2271,52 +2237,56 @@ mod tests { retry: None, }; - let tile_information = - TileInformation::with_partition_and_shape(output_bounds, output_shape); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; - let RasterTile2D { - global_geo_transform: _, - grid_array: grid, - tile_position: _, - band: _, - time: _, - properties, - cache_hint: _, - } = GdalRasterLoader::load_tile_data::( - &up_side_down_params, - tile_information, - TimeInterval::default(), - CacheHint::default(), - ) - .unwrap(); + let GridAndProperties { grid, properties } = + GdalRasterLoader::load_tile_data::(&up_side_down_params, gdal_read_advice) + .unwrap() + .unwrap(); assert!(!grid.is_empty()); let grid = grid.into_materialized_masked_grid(); assert_eq!(grid.inner_grid.data.len(), 64); + // pixel value are the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt assert_eq!( grid.inner_grid.data, &[ - 255, 255, 255, 255, 255, 255, 255, 255, 255, 75, 37, 255, 44, 34, 39, 32, 255, 86, - 255, 255, 255, 30, 96, 255, 255, 255, 255, 255, 90, 255, 255, 255, 255, 255, 202, - 255, 193, 255, 255, 255, 255, 255, 89, 255, 111, 255, 255, 255, 255, 255, 255, 255, - 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 127, 107, 255, 255, 255, 255, 255, 164, 185, 182, + 255, 255, 255, 175, 186, 190, 167, 140, 255, 255, 161, 175, 184, 173, 170, 188, + 255, 255, 128, 177, 165, 145, 191, 174, 255, 117, 100, 174, 159, 147, 99, 135 ] ); assert_eq!(grid.validity_mask.data.len(), 64); - assert_eq!(grid.validity_mask.data, &[true; 64]); + // pixel mask is pixel > 0 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, true, true, + false, false, false, false, false, true, true, true, false, false, false, true, + true, true, true, true, false, false, true, true, true, true, true, true, false, + false, true, true, true, true, true, true, false, true, true, true, true, true, + true, true + ] + ); - assert_eq!(properties.offset_option(), Some(37.)); - assert_eq!(properties.scale_option(), Some(3.7)); + assert_eq!(properties.offset_option(), Some(1.)); + assert_eq!(properties.scale_option(), Some(2.)); - assert!(approx_eq!(f64, properties.offset(), 37.)); - assert!(approx_eq!(f64, properties.scale(), 3.7)); + assert!(approx_eq!(f64, properties.offset(), 1.)); + assert!(approx_eq!(f64, properties.scale(), 2.)); } #[test] - #[allow(clippy::too_many_lines)] fn it_creates_no_data_only_for_missing_files() { hide_gdal_errors(); @@ -2335,19 +2305,20 @@ mod tests { retry: None, }; - let tile_info = TileInformation { - tile_size_in_pixels: [100, 100].into(), - global_tile_position: [0, 0].into(), - global_geo_transform: TestDefault::test_default(), + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, }; - let tile_time = TimeInterval::default(); + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); - // file doesn't exist => no data - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()) - .unwrap(); - assert!(matches!(result.grid_array, GridOrEmpty::Empty(_))); + assert!(res.is_ok()); + + let res = res.unwrap(); + + assert!(res.is_none()); let ds = GdalDatasetParameters { file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_2014-01-01.TIFF").into(), @@ -2365,10 +2336,12 @@ mod tests { }; // invalid channel => error - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()); + let result = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); assert!(result.is_err()); + } + #[test] + fn it_creates_no_data_only_for_http_404() { let server = Server::run(); server.expect( @@ -2410,10 +2383,17 @@ mod tests { }; // 404 => no data - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()) - .unwrap(); - assert!(matches!(result.grid_array, GridOrEmpty::Empty(_))); + let gdal_read_advice = GdalReadAdvise { + gdal_read_widow: GdalReadWindow::new([0, 0].into(), [8, 8].into()), + read_window_bounds: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + bounds_of_target: GridBoundingBox2D::new([0, 0], [7, 7]).unwrap(), + flip_y: false, + }; + + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); + assert!(res.is_ok()); + let res = res.unwrap(); + assert!(res.is_none()); let ds = GdalDatasetParameters { file_path: format!("/vsicurl/{}", server.url_str("/internal_error.tif")).into(), @@ -2442,9 +2422,8 @@ mod tests { }; // 500 => error - let result = - GdalRasterLoader::load_tile_data::(&ds, tile_info, tile_time, CacheHint::default()); - assert!(result.is_err()); + let res = GdalRasterLoader::load_tile_data::(&ds, gdal_read_advice); + assert!(res.is_err()); } #[test] @@ -2536,12 +2515,18 @@ mod tests { SpatialPartition2D::new_unchecked((-90., 90.).into(), (90., -90.).into()); let output_shape: GridShape2D = [256, 256].into(); - let tile_info = TileInformation::with_partition_and_shape(output_bounds, output_shape); + let tile_info = tile_information_with_partition_and_shape(output_bounds, output_shape); let time_interval = TimeInterval::new_unchecked(1_388_534_400_000, 1_391_212_800_000); // 2014-01-01 - 2014-01-15 let params = None; let tile = GdalRasterLoader::load_tile_async::( params, + GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + tile_info.global_geo_transform, + GridShape2D::new([3600, 1800]).bounding_box(), + ), + }), tile_info, time_interval, CacheHint::seconds(1234), diff --git a/operators/src/source/gdal_source/reader.rs b/operators/src/source/gdal_source/reader.rs new file mode 100644 index 000000000..b03d47e2b --- /dev/null +++ b/operators/src/source/gdal_source/reader.rs @@ -0,0 +1,1153 @@ +use geoengine_datatypes::raster::{ + GridBoundingBox2D, GridBounds, GridContains, GridIdx2D, GridOrEmpty2D, GridShape2D, + GridShapeAccess, GridSize, RasterProperties, SpatialGridDefinition, +}; + +/// This struct is used to advise the GDAL reader how to read the data from the dataset. +/// The Workflow is as follows: +/// 1. The `gdal_read_window` is the window in the pixel space of the dataset that should be read. +/// 2. The `read_window_bounds` is the area in the target pixel space where the data should be placed. +/// 2.1 The data read in step one is read to the width and height of the `read_window_bounds`. +/// 2.2 if `flip_y` is true the data is flipped in the y direction. And should be unflipped after reading. +/// 3. The `bounds_of_target` is the area in the target pixel space where the data should be placed. +/// 3.1 The `read_window_bounds` might be offset from the `bounds_of_target` or might have a different size. +/// Then, the data needs to be placed in the target pixel space accordingly. Other parts of the target pixel space should be filled with nodata. +#[allow(dead_code)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct GdalReadAdvise { + pub gdal_read_widow: GdalReadWindow, + pub read_window_bounds: GridBoundingBox2D, + pub bounds_of_target: GridBoundingBox2D, + pub flip_y: bool, +} + +impl GdalReadAdvise { + pub fn direct_read(&self) -> bool { + self.read_window_bounds == self.bounds_of_target + } +} + +#[allow(dead_code)] +#[derive(Copy, Clone, Debug)] +pub enum GdalReaderMode { + // read the original resolution + OriginalResolution(ReaderState), + // read an overview level of the dataset + OverviewLevel(OverviewReaderState), +} + +impl GdalReaderMode { + /// Returns the read advise for the tiling based bounds + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + match self { + GdalReaderMode::OriginalResolution(re) => { + re.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + GdalReaderMode::OverviewLevel(rs) => { + rs.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ReaderState { + pub dataset_spatial_grid: SpatialGridDefinition, +} + +impl ReaderState { + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.dataset_spatial_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + }; + + // Now we can work with a matching dataset. However, we need to reverse the read window later! + + // let's only look at data in the geo engine dataset definition! The intersection is relative to the first elements origin coordinate. + let dataset_gdal_data_intersection = + actual_gdal_dataset_spatial_grid_definition.intersection(&self.dataset_spatial_grid)?; + + // Now, we need the tile in the gdal dataset bounds to identify readable areas + let tile_in_gdal_dataset_bounds = tile.with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate, + )?; // TODO: raise error if this fails! + + // Then, calculate the intersection between the datataset and the tile. Again, the intersection is relative to the first elements orrigin coordinate. + let tile_gdal_dataset_intersection = + dataset_gdal_data_intersection.intersection(&tile_in_gdal_dataset_bounds)?; + + // if we need to unflip the dataset grid now is the time to do this. + let tile_intersection_for_read_window = if flip_y { + tile_gdal_dataset_intersection + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + tile_gdal_dataset_intersection + }; + + // generate the read window for GDAL + + let gdal_read_window = GdalReadWindow::new( + tile_intersection_for_read_window.grid_bounds.min_index(), + tile_intersection_for_read_window.grid_bounds.grid_shape(), + ); + + // if the read window has the same shape as the tiling based bounds we can fill that completely + if tile_in_gdal_dataset_bounds == tile_gdal_dataset_intersection { + return Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: tile.grid_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }); + } + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = + tile_gdal_dataset_intersection.min_index() - tile_in_gdal_dataset_bounds.min_index(); + let crop_lr = + tile_gdal_dataset_intersection.max_index() - tile_in_gdal_dataset_bounds.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: shifted_readable_bounds, + bounds_of_target: tile.grid_bounds, + flip_y: false, + }) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct OverviewReaderState { + pub original_dataset_grid: SpatialGridDefinition, +} + +impl OverviewReaderState { + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, // This is the spatial grid of an actual gdal file + tile: &SpatialGridDefinition, // This is a tile inside the grid we use for the global dataset consisting of potentially many gdal files... + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.original_dataset_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + }; + + // This is the intersection of grid of the gdal file and the global grid we use. Usually the dataset is inside the global dataset grid. + // IF the intersection is empty the we return early and load nothing + // The intersection uses the geo_transform of the gdal dataset which enables us to adress gdal pixels starting at 0,0 + let actual_bounds_to_use_original_resolution = actual_gdal_dataset_spatial_grid_definition + .intersection(&self.original_dataset_grid)?; + + // now we map the tile we want to fill to the original grid. First, we set the tile to use the same origin coordinate as the gdal file/dataset + let tile_with_overview_resolution_in_actual_space = tile + .with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate(), + ) + .expect("The overview level grid must map to pixel coordinates in the original grid"); // TODO: maybe relax this? + let tile_with_original_resolution_in_actual_space = + tile_with_overview_resolution_in_actual_space.with_changed_resolution( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .spatial_resolution(), + ); + + // Now we need to intersect the tile and the actual bounds to use to identify what we can really read + let tile_intersection_original_resolution_actual_space = + &tile_with_original_resolution_in_actual_space + .intersection(&actual_bounds_to_use_original_resolution)?; + + // if we need to unflip the dataset grid now is the time to do this. + let tile_intersection_for_read_window = if flip_y { + tile_intersection_original_resolution_actual_space + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + *tile_intersection_original_resolution_actual_space + }; + + // generate the read window for GDAL --> This is what we can read in any case. + let read_window = GdalReadWindow::new( + tile_intersection_for_read_window.min_index(), + tile_intersection_for_read_window.grid_bounds().grid_shape(), + ); + + let is_tile_contained = tile_intersection_for_read_window.grid_bounds() + == tile_with_original_resolution_in_actual_space.grid_bounds(); + + if is_tile_contained { + return Some(GdalReadAdvise { + gdal_read_widow: read_window, + read_window_bounds: tile.grid_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }); + } + + // IF we can't read the whole tile, we have to find out which area of the tile we can fill. + let readble_area_in_overview_res = tile_intersection_original_resolution_actual_space + .with_changed_resolution(tile.geo_transform().spatial_resolution()); + + // Calculate the intersection of the readable area and the tile, result is in geotransform of the tile! + let readable_tile_area = tile.intersection(&readble_area_in_overview_res).expect( + "Since there was an intersection earlyer, there must be a part of data to read.", + ); + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = readable_tile_area.min_index() - tile.min_index(); + let crop_lr = readable_tile_area.max_index() - tile.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + Some(GdalReadAdvise { + gdal_read_widow: read_window, + read_window_bounds: shifted_readable_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }) + } +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct GdalReadWindow { + start_x: isize, // pixelspace origin + start_y: isize, + size_x: usize, // pixelspace size + size_y: usize, +} + +impl GdalReadWindow { + pub fn new(start: GridIdx2D, size: GridShape2D) -> Self { + Self { + start_x: start.x(), + start_y: start.y(), + size_x: size.axis_size_x(), + size_y: size.axis_size_y(), + } + } + + pub fn gdal_window_start(&self) -> (isize, isize) { + (self.start_x, self.start_y) + } + + pub fn gdal_window_size(&self) -> (usize, usize) { + (self.size_x, self.size_y) + } +} + +pub struct GridAndProperties { + pub grid: GridOrEmpty2D, + pub properties: RasterProperties, +} + +#[cfg(test)] +mod tests { + use geoengine_datatypes::{ + primitives::Coordinate2D, + raster::{ + BoundedGrid, GeoTransform, GridBoundingBox2D, GridIdx2D, GridShape2D, + SpatialGridDefinition, + }, + }; + + use crate::source::gdal_source::reader::{GdalReadWindow, OverviewReaderState, ReaderState}; + + #[test] + fn reader_state_dataset_geo_transform() { + let reader_state = ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ), + }; + + assert_eq!( + reader_state.dataset_spatial_grid.geo_transform(), + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.) + ); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_no_change() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 512, + size_y: 512, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 90, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_and_clipped() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 190, + start_y: 100, + size_x: 170, + size_y: 80, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_flipy() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let spatial_grid_flipy = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., -90.), 1., 1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid_flipy, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 0, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(tiling_to_dataset_read_advise.flip_y); + } + + /* + #[test] + fn gdal_geotransform_to_read_bounds() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([1, 1]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 512, + start_y: 512, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1023, 1023])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_half_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([0, 0]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 2., -2.), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 1024, + size_y: 1024, + start_x: 0, + start_y: 0, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_2x_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(0., 0.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + + let ti: TileInformation = TileInformation::new( + GridIdx([0, 0]), + GridShape2D::new([512, 512]), + GeoTransform::new(Coordinate2D::new(0., 0.), 0.5, -0.5), + ); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 256, + size_y: 256, + start_x: 0, + start_y: 0, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_out() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(-3., 3.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // since the origin of the tile is at -3,3 and the "coordinate nearest to zero" is 0,0 the tile at tile position 0,0 maps to the read window starting at 3,3 with 512x512 pixels + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 3, + start_y: 3, + } + ); + + // the data maps to the complete tile + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([511, 511])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // since the origin of the tile is at -3,3 and the "coordinate nearest to zero" is 0,0 the tile at tile position 1,1 maps to the read window starting at 515,515 (512+3, 512+3) with 512x512 pixels + assert_eq!( + read_window, + GdalReadWindow { + size_x: 509, + size_y: 509, + start_x: 515, + start_y: 515, + } + ); + + // the data maps only to a part of the tile since the data is only 1024x1024 pixels in size. So the tile at tile position 1,1 maps to the data starting at 515,515 (512+3, 512+3) with 509x509 pixels left. + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1020, 1020])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_in() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(3., -3.), + x_pixel_size: 1., + y_pixel_size: -1., + }; + + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + // in this case the data origin is at 3,-3 which is inside the tile at tile position 0,0. Since the tile starts at the "coordinate nearest to zero, which is 0.0,0.0" we need to read the data starting at data 0,0 with 509x509 pixels (512-3, 512-3). + assert_eq!( + read_window, + GdalReadWindow { + size_x: 509, + size_y: 509, + start_x: 0, + start_y: 0, + } + ); + + // in this case, the data only maps to the last 509x509 pixels of the tile. So the data we read does not fill a whole tile. + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([3, 3]), GridIdx([511, 511])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 512, + size_y: 512, + start_x: 509, + start_y: 509, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([512, 512]), GridIdx([1023, 1023])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([2, 2]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 3, + size_y: 3, + start_x: 1021, + start_y: 1021, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([1024, 1024]), GridIdx([1026, 1026])).unwrap() + ); + } + + #[test] + fn gdal_geotransform_to_read_bounds_ul_out_frac_res() { + let gdal_geo_transform: GdalDatasetGeoTransform = GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D::new(-9., 9.), + x_pixel_size: 9., + y_pixel_size: -9., + }; + let gdal_data_size = GridShape2D::new([1024, 1024]); + let tile_grid_shape = GridShape2D::new([512, 512]); + let tiling_global_geo_transfom = GeoTransform::new(Coordinate2D::new(-0., 0.), 3., -3.); + + let ti: TileInformation = + TileInformation::new(GridIdx([0, 0]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 170, // + size_y: 170, + start_x: 1, + start_y: 1, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([0, 0]), GridIdx([512, 512])).unwrap() + ); // we need to read 683 pixels but we only want 682.6666666666666 pixels. + + let ti: TileInformation = + TileInformation::new(GridIdx([1, 1]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 171, + size_y: 171, + start_x: 171, + start_y: 171, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([510, 510]), GridIdx([1025, 1025])).unwrap() + ); + + let ti: TileInformation = + TileInformation::new(GridIdx([2, 2]), tile_grid_shape, tiling_global_geo_transfom); + + let (read_window, target_bounds) = gdal_geo_transform + .grid_bounds_resolution_to_read_window_and_target_grid(gdal_data_size, &ti) + .unwrap(); + + assert_eq!( + read_window, + GdalReadWindow { + size_x: 171, + size_y: 171, + start_x: 342, + start_y: 342, + } + ); + + assert_eq!( + target_bounds, + GridBoundingBox2D::new(GridIdx([1023, 1023]), GridIdx([1535, 1535])).unwrap() + ); + } + */ + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_2() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 2., -2.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 1024, + size_y: 1024, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([2048, 2048]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096, 4096]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_lrcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new( + GridIdx2D::new([512, 512]), + GridIdx2D::new([1023 - 4, 1023 - 4]) + ) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(16., -16.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop_numbers() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(17.123_456, -17.123_456), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(1.123_456, -1.123_456), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } +} diff --git a/operators/src/source/mod.rs b/operators/src/source/mod.rs index c93efd259..696879eb7 100755 --- a/operators/src/source/mod.rs +++ b/operators/src/source/mod.rs @@ -1,5 +1,6 @@ mod csv; mod gdal_source; +mod multi_band_gdal_source; mod ogr_source; pub use self::csv::{ @@ -12,6 +13,11 @@ pub use self::gdal_source::{ GdalRetryOptions, GdalSource, GdalSourceError, GdalSourceParameters, GdalSourceProcessor, GdalSourceTimePlaceholder, TimeReference, }; +pub use self::multi_band_gdal_source::{ + GdalMultiBand, GdalSourceError as MultiBandGdalSourceError, + GdalSourceParameters as MultiBandGdalSourceParameters, MultiBandGdalLoadingInfo, + MultiBandGdalLoadingInfoQueryRectangle, MultiBandGdalSource, TileFile, +}; pub use self::ogr_source::{ AttributeFilter, CsvHeader, FormatSpecifics, OgrSource, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, OgrSourceParameters, diff --git a/operators/src/source/multi_band_gdal_source/error.rs b/operators/src/source/multi_band_gdal_source/error.rs new file mode 100644 index 000000000..ca9411e61 --- /dev/null +++ b/operators/src/source/multi_band_gdal_source/error.rs @@ -0,0 +1,13 @@ +use geoengine_datatypes::raster::{GridBoundingBox2D, RasterDataType}; +use snafu::Snafu; + +#[derive(Debug, Snafu)] +#[snafu(visibility(pub(crate)))] +#[snafu(context(suffix(false)))] // disables default `Snafu` suffix +pub enum GdalSourceError { + #[snafu(display("Unsupported raster type: {raster_type:?}"))] + UnsupportedRasterType { raster_type: RasterDataType }, + + #[snafu(display("Unsupported spatial query: {spatial_query:?}"))] + IncompatibleSpatialQuery { spatial_query: GridBoundingBox2D }, +} diff --git a/operators/src/source/multi_band_gdal_source/loading_info.rs b/operators/src/source/multi_band_gdal_source/loading_info.rs new file mode 100644 index 000000000..cc75a0d77 --- /dev/null +++ b/operators/src/source/multi_band_gdal_source/loading_info.rs @@ -0,0 +1,97 @@ +use super::GdalDatasetParameters; +use crate::engine::RasterResultDescriptor; +use geoengine_datatypes::{ + primitives::{CacheHint, SpatialPartition2D, SpatialPartitioned, TimeInterval}, + raster::TileInformation, +}; +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, Debug, Clone, FromSql, ToSql, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct GdalMultiBand { + pub result_descriptor: RasterResultDescriptor, +} + +#[derive(Debug, Clone)] +pub struct MultiBandGdalLoadingInfo { + files: Vec, + time_steps: Vec, + cache_hint: CacheHint, +} + +#[derive(Debug, Clone, Deserialize)] +pub struct TileFile { + pub time: TimeInterval, + pub spatial_partition: SpatialPartition2D, + pub band: u32, + pub z_index: u32, + pub params: GdalDatasetParameters, +} + +impl MultiBandGdalLoadingInfo { + pub fn new(time_steps: Vec, files: Vec, cache_hint: CacheHint) -> Self { + debug_assert!(!time_steps.is_empty(), "time_steps must not be empty"); + + debug_assert!( + time_steps.windows(2).all(|w| w[0] <= w[1]), + "time_steps must be sorted" + ); + + #[cfg(debug_assertions)] + { + let mut groups: std::collections::HashMap<(TimeInterval, u32), Vec<&TileFile>> = + std::collections::HashMap::new(); + + for file in &files { + groups.entry((file.time, file.band)).or_default().push(file); + } + + for ((time, band), group) in &groups { + debug_assert!( + group.windows(2).all(|w| w[0].z_index <= w[1].z_index), + "Files for time {time:?} and band {band} are not sorted by z_index", + ); + } + } + + Self { + files, + time_steps, + cache_hint, + } + } + + /// Return a gap-free list of time steps for the current loading info and query time. + pub fn time_steps(&self) -> &[TimeInterval] { + &self.time_steps + } + + /// Return all files necessary to load a single tile, sorted by z-index. Might be empty if no files are needed. + pub fn tile_files( + &self, + time: TimeInterval, + tile: TileInformation, + band: u32, + ) -> Vec { + let tile_partition = tile.spatial_partition(); + + let mut files = vec![]; + for file in &self.files { + if time.intersects(&file.time) + && file.spatial_partition.intersects(&tile_partition) + && file.band == band + { + debug_assert!(file.time == time, "file's time must match query time"); + + files.push(file.params.clone()); + } + } + + files + } + + pub fn cache_hint(&self) -> CacheHint { + self.cache_hint + } +} diff --git a/operators/src/source/multi_band_gdal_source/mod.rs b/operators/src/source/multi_band_gdal_source/mod.rs new file mode 100644 index 000000000..386186ba9 --- /dev/null +++ b/operators/src/source/multi_band_gdal_source/mod.rs @@ -0,0 +1,2647 @@ +use crate::engine::{ + CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryContext, QueryProcessor, + SpatialGridDescriptor, WorkflowOperatorPath, +}; +use crate::optimization::{OptimizableOperator, OptimizationError, SourcesMustNotUseOverviews}; +use crate::source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetadataMapping, +}; +use crate::util::TemporaryGdalThreadLocalConfigOptions; +use crate::util::gdal::gdal_open_dataset_ex; +use crate::util::retry::retry; +use crate::{ + engine::{ + InitializedRasterOperator, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, + SourceOperator, TypedRasterQueryProcessor, + }, + error::Error, + util::Result, +}; +use async_trait::async_trait; +pub use error::GdalSourceError; +use float_cmp::approx_eq; +use futures::stream::{self, BoxStream, StreamExt}; +use gdal::errors::GdalError; +use gdal::raster::{GdalType, RasterBand as GdalRasterBand}; +use gdal::{Dataset as GdalDataset, DatasetOptions, GdalOpenFlags, Metadata as GdalMetadata}; +use gdal_sys::VSICurlPartialClearCache; +use geoengine_datatypes::primitives::{ + QueryRectangle, SpatialPartition2D, SpatialResolution, find_next_best_overview_level, +}; +use geoengine_datatypes::raster::TilingSpatialGridDefinition; +use geoengine_datatypes::{ + dataset::NamedData, + primitives::{BandSelection, Coordinate2D, RasterQueryRectangle, TimeInterval}, + raster::{ + ChangeGridBounds, EmptyGrid, GeoTransform, Grid, GridBlit, GridBoundingBox2D, GridOrEmpty, + GridSize, MapElements, MaskedGrid, NoDataValueGrid, Pixel, RasterDataType, + RasterProperties, RasterPropertiesEntry, RasterPropertiesEntryType, RasterTile2D, + SpatialGridDefinition, TileInformation, TilingSpecification, + }, +}; +pub use loading_info::{GdalMultiBand, MultiBandGdalLoadingInfo, TileFile}; +use num::{FromPrimitive, integer::div_ceil, integer::div_floor}; +use reader::{ + GdalReadAdvise, GdalReadWindow, GdalReaderMode, GridAndProperties, OverviewReaderState, + ReaderState, +}; +use serde::{Deserialize, Serialize}; +use snafu::{ResultExt, ensure}; +use std::ffi::CString; +use std::marker::PhantomData; +use std::path::Path; +use tracing::{debug, trace}; + +mod error; +mod loading_info; +mod reader; + +static GDAL_RETRY_INITIAL_BACKOFF_MS: u64 = 1000; +static GDAL_RETRY_MAX_BACKOFF_MS: u64 = 60 * 60 * 1000; +static GDAL_RETRY_EXPONENTIAL_BACKOFF_FACTOR: f64 = 2.; + +/// Parameters for the GDAL Source Operator +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GdalSourceParameters { + pub data: NamedData, + #[serde(default)] + pub overview_level: Option, // TODO: should also allow a resolution? Add resample method? +} + +impl GdalSourceParameters { + #[must_use] + pub fn new(data: NamedData) -> Self { + Self { + data, + overview_level: None, + } + } + + #[must_use] + pub fn new_with_overview_level(data: NamedData, overview_level: u32) -> Self { + Self { + data, + overview_level: Some(overview_level), + } + } + + #[must_use] + pub fn with_overview_level(mut self, overview_level: Option) -> Self { + self.overview_level = overview_level; + self + } +} + +impl OperatorData for GdalSourceParameters { + fn data_names_collect(&self, data_names: &mut Vec) { + data_names.push(self.data.clone()); + } +} + +type MultiBandGdalMetaData = Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, +>; + +/// A query rectangle on rasters in coordinate (instead of pixel) space. This is used to request tiles that are anchored in coordinates space. +/// Additionaly you can also specify to fetch tiles (with gdal params). +/// Set `fetch_tiles` to false if you only want to access the time steps. +#[derive(Clone, Debug)] +pub struct MultiBandGdalLoadingInfoQueryRectangle { + pub query_rectangle: QueryRectangle, + pub fetch_tiles: bool, +} + +impl MultiBandGdalLoadingInfoQueryRectangle { + pub fn new( + spatial_bounds: SpatialPartition2D, + time_interval: TimeInterval, + attributes: BandSelection, + fetch_tiles: bool, + ) -> Self { + Self { + query_rectangle: QueryRectangle::new(spatial_bounds, time_interval, attributes), + fetch_tiles, + } + } + + pub fn with_query_rectangle( + query_rectangle: QueryRectangle, + fetch_tiles: bool, + ) -> Self { + Self { + query_rectangle, + fetch_tiles, + } + } +} + +fn raster_query_rectangle_to_loading_info_query_rectangle( + raster_query_rectangle: &RasterQueryRectangle, + tiling_spatial_grid: TilingSpatialGridDefinition, + fetch_tiles: bool, +) -> MultiBandGdalLoadingInfoQueryRectangle { + MultiBandGdalLoadingInfoQueryRectangle::new( + tiling_spatial_grid + .tiling_geo_transform() + .grid_to_spatial_bounds(&raster_query_rectangle.spatial_bounds()), + raster_query_rectangle.time_interval(), + raster_query_rectangle.attributes().clone(), + fetch_tiles, + ) +} + +pub struct GdalSourceProcessor +where + T: Pixel, +{ + pub produced_result_descriptor: RasterResultDescriptor, + pub tiling_specification: TilingSpecification, + pub meta_data: MultiBandGdalMetaData, + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, + pub _phantom_data: PhantomData, +} + +struct GdalRasterLoader {} + +impl GdalRasterLoader { + async fn load_tile_from_files_async( + loading_info: MultiBandGdalLoadingInfo, + reader_mode: GdalReaderMode, + tile_information: TileInformation, + time: TimeInterval, + band: u32, + ) -> Result> { + debug!( + "loading tile {:?} for time: {}, band: {band}", + tile_information.global_tile_position.inner(), + time.to_string() + ); + let tile_files = loading_info.tile_files(time, tile_information, band); + + for tile_file in &tile_files { + trace!( + "tile_file: {:?}, {:?}", + tile_file.file_path, + tile_file + .spatial_grid_definition() + .geo_transform + .origin_coordinate + ); + } + + let mut tile_raster: GridOrEmpty = + GridOrEmpty::from(EmptyGrid::new(tile_information.global_pixel_bounds())); + + let mut properties = RasterProperties::default(); + let cache_hint = loading_info.cache_hint(); + + for dataset_params in tile_files { + let Some(file_tile) = Self::retrying_load_raster_tile_from_file::( + &dataset_params, + reader_mode, + tile_information, + ) + .await? + else { + debug!("didn't load from file: {:?}", dataset_params.file_path); + continue; + }; + tile_raster.grid_blit_from(&file_tile.grid); + + properties = file_tile.properties; + } + + Ok(RasterTile2D::new_with_properties( + time, + tile_information.global_tile_position, + band, + tile_information.global_geo_transform, + tile_raster.unbounded(), + properties, + cache_hint, + )) + } + + async fn retrying_load_raster_tile_from_file( + dataset_params: &GdalDatasetParameters, + reader_mode: GdalReaderMode, + tile_information: TileInformation, + ) -> Result>> { + // TODO: detect usage of vsi curl properly, e.g. also check for `/vsicurl_streaming` and combinations with `/vsizip` + let is_vsi_curl = dataset_params.file_path.starts_with("/vsicurl/"); + + retry( + dataset_params + .retry + .map(|r| r.max_retries) + .unwrap_or_default(), + GDAL_RETRY_INITIAL_BACKOFF_MS, + GDAL_RETRY_EXPONENTIAL_BACKOFF_FACTOR, + Some(GDAL_RETRY_MAX_BACKOFF_MS), + move || { + let ds = dataset_params.clone(); + let file_path = ds.file_path.clone(); + + async move { + let load_tile_result = crate::util::spawn_blocking(move || { + Self::load_raster_tile_from_file(&ds, reader_mode, tile_information) + }) + .await + .context(crate::error::TokioJoin); + + match load_tile_result { + Ok(Ok(r)) => Ok(r), + Ok(Err(e)) | Err(e) => { + if is_vsi_curl { + // clear the VSICurl cache, to force GDAL to try to re-download the file + // otherwise it will assume any observed error will happen again + clear_gdal_vsi_cache_for_path(file_path.as_path()); + } + + Err(e) + } + } + } + }, + ) + .await + } + + fn load_raster_tile_from_file( + dataset_params: &GdalDatasetParameters, + reader_mode: GdalReaderMode, + tile_information: TileInformation, + ) -> Result>> { + debug!( + "Loading raster tile from file: {:?}", + dataset_params.file_path.file_name().unwrap_or_default() + ); + let gdal_read_advise: Option = reader_mode.tiling_to_dataset_read_advise( + &dataset_params.spatial_grid_definition(), + &tile_information.spatial_grid_definition(), + ); + + let Some(gdal_read_advise) = gdal_read_advise else { + trace!( + "no read advise returned for tile {:?}, skipping file.", + tile_information.global_tile_position.inner(), + ); + return Ok(None); + }; + + let options = dataset_params + .gdal_open_options + .as_ref() + .map(|o| o.iter().map(String::as_str).collect::>()); + + // reverts the thread local configs on drop + let _thread_local_configs = dataset_params + .gdal_config_options + .as_ref() + .map(|config_options| TemporaryGdalThreadLocalConfigOptions::new(config_options)); + + let dataset_result = gdal_open_dataset_ex( + &dataset_params.file_path, + DatasetOptions { + open_flags: GdalOpenFlags::GDAL_OF_RASTER, + open_options: options.as_deref(), + ..DatasetOptions::default() + }, + ); + + let dataset = match dataset_result { + Ok(dataset) => dataset, + Err(error) => { + let is_file_not_found = error_is_gdal_file_not_found(&error); + + match dataset_params.file_not_found_handling { + FileNotFoundHandling::NoData if is_file_not_found => return Ok(None), + _ => { + return Err(crate::error::Error::CouldNotOpenGdalDataset { + file_path: dataset_params.file_path.to_string_lossy().to_string(), + }); + } + }; + } + }; + + let rasterband = dataset.rasterband(dataset_params.rasterband_channel)?; + + // overwrite old properties with the properties of the current dataset (z-index) + let properties = read_raster_properties(&dataset, dataset_params, &rasterband); + + let gdal_dataset_geotransform = GdalDatasetGeoTransform::from(dataset.geo_transform()?); + // check that the dataset geo transform is the same as the one we get from GDAL + debug_assert!(approx_eq!( + Coordinate2D, + gdal_dataset_geotransform.origin_coordinate, + dataset_params.geo_transform.origin_coordinate + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.x_pixel_size, + dataset_params.geo_transform.x_pixel_size + )); + + debug_assert!(approx_eq!( + f64, + gdal_dataset_geotransform.y_pixel_size, + dataset_params.geo_transform.y_pixel_size + )); + + let (gdal_dataset_pixels_x, gdal_dataset_pixels_y) = dataset.raster_size(); + // check that the dataset pixel size is the same as the one we get from GDAL + debug_assert_eq!(gdal_dataset_pixels_x, dataset_params.width); + debug_assert_eq!(gdal_dataset_pixels_y, dataset_params.height); + + Ok(Some(GridAndProperties { + grid: read_grid_from_raster::( + &rasterband, + &gdal_read_advise.gdal_read_widow, + gdal_read_advise.read_window_bounds, + dataset_params, + gdal_read_advise.flip_y, + )?, + properties, + })) + } +} + +fn error_is_gdal_file_not_found(error: &Error) -> bool { + matches!( + error, + Error::Gdal { + source: GdalError::NullPointer { + method_name, + msg + }, + } if *method_name == "GDALOpenEx" && (*msg == "HTTP response code: 404" || msg.ends_with("No such file or directory")) + ) +} + +fn clear_gdal_vsi_cache_for_path(file_path: &Path) { + unsafe { + if let Some(Some(c_string)) = file_path.to_str().map(|s| CString::new(s).ok()) { + VSICurlPartialClearCache(c_string.as_ptr()); + } + } +} + +impl GdalSourceProcessor where T: gdal::raster::GdalType + Pixel {} + +#[async_trait] +impl

QueryProcessor for GdalSourceProcessor

+where + P: Pixel + gdal::raster::GdalType + FromPrimitive, +{ + type Output = RasterTile2D

; + type SpatialBounds = GridBoundingBox2D; + type Selection = BandSelection; + type ResultDescription = RasterResultDescriptor; + + async fn _query<'a>( + &'a self, + query: RasterQueryRectangle, + _ctx: &'a dyn crate::engine::QueryContext, + ) -> Result>> { + // TODO: check all bands exist + + tracing::debug!( + "Querying GdalSourceProcessor<{:?}> with: {:?}.", + P::TYPE, + &query + ); + // this is the result descriptor of the operator. It already incorporates the overview level AND shifts the origin to the tiling origin + let result_descriptor = self.result_descriptor(); + + let grid_produced_by_source_desc = result_descriptor.spatial_grid; + let grid_produced_by_source = grid_produced_by_source_desc + .source_spatial_grid_definition() + .expect("the source grid definition should be present in a source..."); + // A `GeoTransform` maps pixel space to world space. + // Usually a SRS has axis directions pointing "up" (y-axis) and "up" (y-axis). + // We are not aware of spatial reference systems where the x-axis points to the right. + // However, there are spatial reference systems where the y-axis points downwards. + // The standard "pixel-space" starts at the top-left corner of a `GeoTransform` and points down-right. + // Therefore, the pixel size on the x-axis is always increasing + let pixel_size_x = grid_produced_by_source.geo_transform().x_pixel_size(); + debug_assert!(pixel_size_x.is_sign_positive()); + // and the y-axis should only be positive if the y-axis of the spatial reference system also "points down". + // NOTE: at the moment we do not allow "down pointing" y-axis. + let pixel_size_y = grid_produced_by_source.geo_transform().y_pixel_size(); + debug_assert!(pixel_size_y.is_sign_negative()); + + // The data origin is not neccessarily the origin of the tileing we want to use. + // TODO: maybe derive tilling origin reference from the data projection + let produced_tiling_grid = + grid_produced_by_source_desc.tiling_grid_definition(self.tiling_specification); + + let tiling_strategy = produced_tiling_grid.generate_data_tiling_strategy(); + + let reader_mode = match self.original_resolution_spatial_grid { + None => GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: grid_produced_by_source, + }), + Some(original_resolution_spatial_grid) => { + GdalReaderMode::OverviewLevel(OverviewReaderState { + original_dataset_grid: original_resolution_spatial_grid, + overview_level: self.overview_level, + }) + } + }; + + let loading_info = self + .meta_data + .loading_info(raster_query_rectangle_to_loading_info_query_rectangle( + &query, + produced_tiling_grid, + true, + )) + .await?; + + let time_steps = loading_info.time_steps().to_vec(); + let bands = query.attributes().clone().as_vec(); + let spatial_tiles = tiling_strategy + .tile_information_iterator_from_pixel_bounds(query.spatial_bounds()) + .collect::>(); + + debug!( + "num timesteps: {}, num bands: {}, num spatial tiles: {}", + time_steps.len(), + bands.len(), + spatial_tiles.len() + ); + + // create a stream with a tile for each band of each spatial tile for each time step + let time_tile_band_iter = itertools::iproduct!( + time_steps.into_iter(), + spatial_tiles.into_iter(), + bands.into_iter(), + ); + + let stream = stream::iter(time_tile_band_iter) + .map(move |(time_interval, tile_info, band_idx)| { + GdalRasterLoader::load_tile_from_files_async::

( + loading_info.clone(), + reader_mode, + tile_info, + time_interval, + band_idx, + ) + }) + .buffered(16) // TODO: make configurable + .boxed(); + + return Ok(stream); + } + + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.produced_result_descriptor + } +} + +#[async_trait] +impl

RasterQueryProcessor for GdalSourceProcessor

+where + P: Pixel + gdal::raster::GdalType + FromPrimitive, +{ + type RasterType = P; + + async fn _time_query<'a>( + &'a self, + query: TimeInterval, + ctx: &'a dyn QueryContext, + ) -> Result>> { + let result_descriptor = self.result_descriptor(); + + let grid_produced_by_source_desc = result_descriptor.spatial_grid; + + let produced_tiling_grid = + grid_produced_by_source_desc.tiling_grid_definition(self.tiling_specification); + + let q_bounds = self + .raster_result_descriptor() + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_grid_bounds(); + let query = RasterQueryRectangle::new(q_bounds, query, BandSelection::first()); + + let qrect = raster_query_rectangle_to_loading_info_query_rectangle( + &query, + produced_tiling_grid, + false, + ); + + let loading_info = self.meta_data.loading_info(qrect).await?; + + let time_steps = loading_info.time_steps().to_vec(); + + let stream = stream::iter(time_steps).map(Result::Ok).boxed(); + + Ok(stream) + } +} + +pub type MultiBandGdalSource = SourceOperator; + +impl OperatorName for MultiBandGdalSource { + const TYPE_NAME: &'static str = "MultiBandGdalSource"; +} + +fn overview_level_spatial_grid( + source_spatial_grid: SpatialGridDefinition, + overview_level: u32, +) -> Option { + if overview_level > 0 { + debug!("Using overview level {overview_level}"); + let geo_transform = GeoTransform::new( + source_spatial_grid.geo_transform.origin_coordinate, + source_spatial_grid.geo_transform.x_pixel_size() * f64::from(overview_level), + source_spatial_grid.geo_transform.y_pixel_size() * f64::from(overview_level), + ); + let grid_bounds = GridBoundingBox2D::new_min_max( + div_floor( + source_spatial_grid.grid_bounds.y_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.y_max(), + overview_level as isize, + ), + div_floor( + source_spatial_grid.grid_bounds.x_min(), + overview_level as isize, + ), + div_ceil( + source_spatial_grid.grid_bounds.x_max(), + overview_level as isize, + ), + ) + .expect("overview level must be a positive integer"); + + Some(SpatialGridDefinition::new(geo_transform, grid_bounds)) + } else { + debug!("Using original resolution (ov = 0)"); + None + } +} + +#[typetag::serde] +#[async_trait] +impl RasterOperator for MultiBandGdalSource { + async fn _initialize( + self: Box, + path: WorkflowOperatorPath, + context: &dyn crate::engine::ExecutionContext, + ) -> Result> { + let data_id = context.resolve_named_data(&self.params.data).await?; + let meta_data: MultiBandGdalMetaData = context.meta_data(&data_id).await?; + + debug!( + "Initializing MultiBandGdalSource for {:?}.", + &self.params.data + ); + debug!("GdalSource path: {:?}", path); + + let meta_data_result_descriptor = meta_data.result_descriptor().await?; + + let op_name = CanonicOperatorName::from(&self); + let op = if self.params.overview_level.is_none() { + InitializedGdalSourceOperator::initialize_original_resolution( + op_name, + path, + self.params.data, + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + ) + } else { + // generate a result descriptor with the overview level + InitializedGdalSourceOperator::initialize_with_overview_level( + op_name, + path, + self.params.data, + meta_data, + meta_data_result_descriptor, + context.tiling_specification(), + self.params.overview_level.unwrap_or(0), + ) + }; + + Ok(op.boxed()) + } + + span_fn!(MultiBandGdalSource); +} + +pub struct InitializedGdalSourceOperator { + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data_name: NamedData, + pub meta_data: MultiBandGdalMetaData, + pub produced_result_descriptor: RasterResultDescriptor, + pub tiling_specification: TilingSpecification, + // the overview level to use. 0/1 means the highest resolution + pub overview_level: u32, + pub original_resolution_spatial_grid: Option, +} + +impl InitializedGdalSourceOperator { + pub fn initialize_original_resolution( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data_name: NamedData, + meta_data: MultiBandGdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + ) -> Self { + InitializedGdalSourceOperator { + name, + path, + data_name, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + overview_level: 0, + original_resolution_spatial_grid: None, + } + } + + pub fn initialize_with_overview_level( + name: CanonicOperatorName, + path: WorkflowOperatorPath, + data_name: NamedData, + meta_data: MultiBandGdalMetaData, + result_descriptor: RasterResultDescriptor, + tiling_specification: TilingSpecification, + overview_level: u32, + ) -> Self { + let source_resolution_spatial_grid = result_descriptor + .spatial_grid_descriptor() + .source_spatial_grid_definition() + .expect("Source data must be a source grid definition..."); + + let (result_descriptor, original_grid) = if let Some(ovr_spatial_grid) = + overview_level_spatial_grid(source_resolution_spatial_grid, overview_level) + { + let ovr_res = RasterResultDescriptor { + spatial_grid: SpatialGridDescriptor::new_source(ovr_spatial_grid), + ..result_descriptor + }; + (ovr_res, Some(source_resolution_spatial_grid)) + } else { + (result_descriptor, None) + }; + + InitializedGdalSourceOperator { + name, + path, + data_name, + produced_result_descriptor: result_descriptor, + meta_data, + tiling_specification, + overview_level, + original_resolution_spatial_grid: original_grid, + } + } +} + +impl InitializedRasterOperator for InitializedGdalSourceOperator { + fn result_descriptor(&self) -> &RasterResultDescriptor { + &self.produced_result_descriptor + } + + fn query_processor(&self) -> Result { + Ok(match self.result_descriptor().data_type { + RasterDataType::U8 => TypedRasterQueryProcessor::U8( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::U16 => TypedRasterQueryProcessor::U16( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::U32 => TypedRasterQueryProcessor::U32( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::U64 => { + return Err(GdalSourceError::UnsupportedRasterType { + raster_type: RasterDataType::U64, + })?; + } + RasterDataType::I8 => { + return Err(GdalSourceError::UnsupportedRasterType { + raster_type: RasterDataType::I8, + })?; + } + RasterDataType::I16 => TypedRasterQueryProcessor::I16( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::I32 => TypedRasterQueryProcessor::I32( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::I64 => { + return Err(GdalSourceError::UnsupportedRasterType { + raster_type: RasterDataType::I64, + })?; + } + RasterDataType::F32 => TypedRasterQueryProcessor::F32( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + RasterDataType::F64 => TypedRasterQueryProcessor::F64( + GdalSourceProcessor { + produced_result_descriptor: self.produced_result_descriptor.clone(), + tiling_specification: self.tiling_specification, + meta_data: self.meta_data.clone(), + overview_level: self.overview_level, + original_resolution_spatial_grid: self.original_resolution_spatial_grid, + _phantom_data: PhantomData, + } + .boxed(), + ), + }) + } + + fn canonic_name(&self) -> CanonicOperatorName { + self.name.clone() + } + + fn name(&self) -> &'static str { + MultiBandGdalSource::TYPE_NAME + } + + fn path(&self) -> WorkflowOperatorPath { + self.path.clone() + } + + fn data(&self) -> Option { + Some(self.data_name.to_string()) + } + + fn optimize( + &self, + target_resolution: SpatialResolution, + ) -> Result, OptimizationError> { + self.ensure_resolution_is_compatible_for_optimization(target_resolution)?; + + // TODO: handle cases where the original workflow explicitly loads overviews in the source + ensure!( + self.overview_level == 0, + SourcesMustNotUseOverviews { + data: self.data_name.to_string(), + oveview_level: self.overview_level + } + ); + + // as overview level is always 0 for now, the result descriptor contains the native resolution + // TODO: when allowing to optimize upon overview levels, compute the native resolution first + let native_resolution = self + .produced_result_descriptor + .spatial_grid + .spatial_resolution(); + + // TODO: get available overviews levels from the dataset metadata (not available yet) and only load these. + // Then, we might have to prepend a Resampling operator to match the target resolution. + // For now, we just load the overview level regardless and let gdal handle the resamṕling. + let next_best_overview_level = + find_next_best_overview_level(native_resolution, target_resolution); + + Ok(MultiBandGdalSource { + params: GdalSourceParameters { + data: self.data_name.clone(), + overview_level: Some(next_best_overview_level), + }, + } + .boxed()) + } +} + +/// This method reads the data for a single grid with a specified size from the GDAL dataset. +/// It fails if the tile is not within the dataset. +#[allow(clippy::float_cmp)] +fn read_grid_from_raster( + rasterband: &GdalRasterBand, + read_window: &GdalReadWindow, + out_shape: D, + dataset_params: &GdalDatasetParameters, + flip_y_axis: bool, +) -> Result> +where + T: Pixel + GdalType + Default + FromPrimitive, + D: Clone + GridSize + PartialEq, +{ + let gdal_out_shape = (out_shape.axis_size_x(), out_shape.axis_size_y()); + + let buffer = rasterband.read_as::( + read_window.gdal_window_start(), // pixelspace origin + read_window.gdal_window_size(), // pixelspace size + gdal_out_shape, // requested raster size + None, // sampling mode + )?; + let (_, buffer_data) = buffer.into_shape_and_vec(); + let data_grid = Grid::new(out_shape.clone(), buffer_data)?; + + let data_grid = if flip_y_axis { + data_grid.reversed_y_axis_grid() + } else { + data_grid + }; + + let dataset_mask_flags = rasterband.mask_flags()?; + + if dataset_mask_flags.is_all_valid() { + debug!("all pixels are valid --> skip no-data and mask handling."); + return Ok(MaskedGrid::new_with_data(data_grid).into()); + } + + if dataset_mask_flags.is_nodata() { + debug!("raster uses a no-data value --> use no-data handling."); + let no_data_value = dataset_params + .no_data_value + .or_else(|| rasterband.no_data_value()) + .and_then(FromPrimitive::from_f64); + let no_data_value_grid = NoDataValueGrid::new(data_grid, no_data_value); + let grid_or_empty = GridOrEmpty::from(no_data_value_grid); + return Ok(grid_or_empty); + } + + if dataset_mask_flags.is_alpha() { + debug!("raster uses alpha band to mask pixels."); + if !dataset_params.allow_alphaband_as_mask { + return Err(Error::AlphaBandAsMaskNotAllowed); + } + } + + debug!("use mask based no-data handling."); + + let mask_band = rasterband.open_mask_band()?; + let mask_buffer = mask_band.read_as::( + read_window.gdal_window_start(), // pixelspace origin + read_window.gdal_window_size(), // pixelspace size + gdal_out_shape, // requested raster size + None, // sampling mode + )?; + let (_, mask_buffer_data) = mask_buffer.into_shape_and_vec(); + let mask_grid = Grid::new(out_shape, mask_buffer_data)?.map_elements(|p: u8| p > 0); + + let mask_grid = if flip_y_axis { + mask_grid.reversed_y_axis_grid() + } else { + mask_grid + }; + + let masked_grid = MaskedGrid::new(data_grid, mask_grid)?; + Ok(GridOrEmpty::from(masked_grid)) +} + +/// This method reads the data for a single tile with a specified size from the GDAL dataset and adds the requested metadata as properties to the tile. +fn read_raster_properties( + dataset: &GdalDataset, + dataset_params: &GdalDatasetParameters, + rasterband: &GdalRasterBand, +) -> RasterProperties { + let mut properties = RasterProperties::default(); + + // always read the scale and offset values from the rasterband + properties_from_band(&mut properties, rasterband); + + // read the properties from the dataset and rasterband metadata + if let Some(properties_mapping) = dataset_params.properties_mapping.as_ref() { + properties_from_gdal_metadata(&mut properties, dataset, properties_mapping); + properties_from_gdal_metadata(&mut properties, rasterband, properties_mapping); + } + + properties +} + +fn properties_from_gdal_metadata<'a, I, M>( + properties: &mut RasterProperties, + gdal_dataset: &M, + properties_mapping: I, +) where + I: IntoIterator, + M: GdalMetadata, +{ + let mapping_iter = properties_mapping.into_iter(); + + for m in mapping_iter { + let data = if let Some(domain) = &m.source_key.domain { + gdal_dataset.metadata_item(&m.source_key.key, domain) + } else { + gdal_dataset.metadata_item(&m.source_key.key, "") + }; + + if let Some(d) = data { + let entry = match m.target_type { + RasterPropertiesEntryType::Number => d.parse::().map_or_else( + |_| RasterPropertiesEntry::String(d), + RasterPropertiesEntry::Number, + ), + RasterPropertiesEntryType::String => RasterPropertiesEntry::String(d), + }; + + debug!( + "gdal properties key \"{:?}\" => target key \"{:?}\". Value: {:?} ", + &m.source_key, &m.target_key, &entry + ); + + properties.insert_property(m.target_key.clone(), entry); + } + } +} + +fn properties_from_band(properties: &mut RasterProperties, gdal_dataset: &GdalRasterBand) { + if let Some(scale) = gdal_dataset.scale() { + properties.set_scale(scale); + } + if let Some(offset) = gdal_dataset.offset() { + properties.set_offset(offset); + } + + // ignore if there is no description + if let Ok(description) = gdal_dataset.description() { + properties.set_description(description); + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::str::FromStr; + + use super::*; + use crate::engine::{ + ExecutionContext, MockExecutionContext, RasterBandDescriptor, StaticMetaData, + TimeDescriptor, + }; + use crate::test_data; + use crate::util::test::raster_tile_from_file; + use futures::TryStreamExt; + use geoengine_datatypes::dataset::{DataId, DatasetId}; + use geoengine_datatypes::primitives::{ + CacheHint, Measurement, SpatialPartition2D, TimeInstance, + }; + use geoengine_datatypes::raster::{GridBounds, GridIdx2D}; + use geoengine_datatypes::raster::{RasterPropertiesKey, SpatialGridDefinition}; + use geoengine_datatypes::raster::{TileInformation, TilingStrategy}; + use geoengine_datatypes::spatial_reference::SpatialReference; + use geoengine_datatypes::util::Identifier; + use geoengine_datatypes::util::gdal::hide_gdal_errors; + use geoengine_datatypes::util::test::{TestDefault, assert_eq_two_list_of_tiles}; + use httptest::matchers::request; + use httptest::{Expectation, Server, responders}; + + #[test] + fn it_deserializes() { + let json_string = r#" + { + "type": "MultiBandGdalSource", + "params": { + "data": "ns:dataset" + } + }"#; + + let operator: MultiBandGdalSource = serde_json::from_str(json_string).unwrap(); + + assert_eq!( + operator, + MultiBandGdalSource { + params: GdalSourceParameters::new(NamedData::with_namespaced_name("ns", "dataset")), + } + ); + } + + #[test] + fn tiling_strategy_origin() { + let tile_size_in_pixels = [600, 600]; + let dataset_upper_right_coord = (-180.0, 90.0).into(); + let dataset_x_pixel_size = 0.1; + let dataset_y_pixel_size = -0.1; + let dataset_geo_transform = GeoTransform::new( + dataset_upper_right_coord, + dataset_x_pixel_size, + dataset_y_pixel_size, + ); + + let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + + let origin_split_tileing_strategy = TilingStrategy { + tile_size_in_pixels: tile_size_in_pixels.into(), + geo_transform: dataset_geo_transform, + }; + + assert_eq!( + origin_split_tileing_strategy + .geo_transform + .upper_left_pixel_idx(&partition), + [0, 0].into() + ); + assert_eq!( + origin_split_tileing_strategy + .geo_transform + .lower_right_pixel_idx(&partition), + [1800 - 1, 3600 - 1].into() + ); + + let tile_grid = origin_split_tileing_strategy.tile_grid_box(partition); + assert_eq!(tile_grid.axis_size(), [3, 6]); + assert_eq!(tile_grid.min_index(), [0, 0].into()); + assert_eq!(tile_grid.max_index(), [2, 5].into()); + } + + #[test] + fn tiling_strategy_zero() { + let tile_size_in_pixels = [600, 600]; + let dataset_x_pixel_size = 0.1; + let dataset_y_pixel_size = -0.1; + let central_geo_transform = GeoTransform::new_with_coordinate_x_y( + 0.0, + dataset_x_pixel_size, + 0.0, + dataset_y_pixel_size, + ); + + let partition = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); + + let origin_split_tileing_strategy = TilingStrategy { + tile_size_in_pixels: tile_size_in_pixels.into(), + geo_transform: central_geo_transform, + }; + + assert_eq!( + origin_split_tileing_strategy + .geo_transform + .upper_left_pixel_idx(&partition), + [-900, -1800].into() + ); + assert_eq!( + origin_split_tileing_strategy + .geo_transform + .lower_right_pixel_idx(&partition), + [1800 / 2 - 1, 3600 / 2 - 1].into() + ); + + let tile_grid = origin_split_tileing_strategy.tile_grid_box(partition); + assert_eq!(tile_grid.axis_size(), [4, 6]); + assert_eq!(tile_grid.min_index(), [-2, -3].into()); + assert_eq!(tile_grid.max_index(), [1, 2].into()); + } + + #[test] + fn tile_idx_iterator() { + let tile_size_in_pixels = [600, 600]; + let dataset_x_pixel_size = 0.1; + let dataset_y_pixel_size = -0.1; + let central_geo_transform = GeoTransform::new_with_coordinate_x_y( + 0.0, + dataset_x_pixel_size, + 0.0, + dataset_y_pixel_size, + ); + + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); + + let origin_split_tileing_strategy = TilingStrategy { + tile_size_in_pixels: tile_size_in_pixels.into(), + geo_transform: central_geo_transform, + }; + + let vres: Vec = origin_split_tileing_strategy + .tile_idx_iterator_from_grid_bounds(grid_bounds) + .collect(); + assert_eq!(vres.len(), 4 * 6); + assert_eq!(vres[0], [-2, -3].into()); + assert_eq!(vres[1], [-2, -2].into()); + assert_eq!(vres[2], [-2, -1].into()); + assert_eq!(vres[23], [1, 2].into()); + } + + #[test] + fn tile_information_iterator() { + let tile_size_in_pixels = [600, 600]; + let dataset_x_pixel_size = 0.1; + let dataset_y_pixel_size = -0.1; + + let central_geo_transform = GeoTransform::new_with_coordinate_x_y( + 0.0, + dataset_x_pixel_size, + 0.0, + dataset_y_pixel_size, + ); + + let grid_bounds = GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(); + + let origin_split_tileing_strategy = TilingStrategy { + tile_size_in_pixels: tile_size_in_pixels.into(), + geo_transform: central_geo_transform, + }; + + let vres: Vec = origin_split_tileing_strategy + .tile_information_iterator_from_pixel_bounds(grid_bounds) + .collect(); + assert_eq!(vres.len(), 4 * 6); + assert_eq!( + vres[0], + TileInformation::new( + [-2, -3].into(), + tile_size_in_pixels.into(), + central_geo_transform, + ) + ); + assert_eq!( + vres[1], + TileInformation::new( + [-2, -2].into(), + tile_size_in_pixels.into(), + central_geo_transform, + ) + ); + assert_eq!( + vres[12], + TileInformation::new( + [0, -3].into(), + tile_size_in_pixels.into(), + central_geo_transform, + ) + ); + assert_eq!( + vres[23], + TileInformation::new( + [1, 2].into(), + tile_size_in_pixels.into(), + central_geo_transform, + ) + ); + } + + #[test] + #[allow(clippy::too_many_lines)] + fn deserialize_dataset_parameters() { + let dataset_parameters = GdalDatasetParameters { + file_path: "path-to-data.tiff".into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (-180., 90.).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + }, + width: 3600, + height: 1800, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: Some(f64::NAN), + properties_mapping: Some(vec![ + GdalMetadataMapping { + source_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + target_type: RasterPropertiesEntryType::String, + target_key: RasterPropertiesKey { + domain: None, + key: "AREA_OR_POINT".to_string(), + }, + }, + GdalMetadataMapping { + source_key: RasterPropertiesKey { + domain: Some("IMAGE_STRUCTURE".to_string()), + key: "COMPRESSION".to_string(), + }, + target_type: RasterPropertiesEntryType::String, + target_key: RasterPropertiesKey { + domain: Some("IMAGE_STRUCTURE_INFO".to_string()), + key: "COMPRESSION".to_string(), + }, + }, + ]), + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + let dataset_parameters_json = serde_json::to_value(&dataset_parameters).unwrap(); + + assert_eq!( + dataset_parameters_json, + serde_json::json!({ + "filePath": "path-to-data.tiff", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180., + "y": 90. + }, + "xPixelSize": 0.1, + "yPixelSize": -0.1 + }, + "width": 3600, + "height": 1800, + "fileNotFoundHandling": "NoData", + "noDataValue": "nan", + "propertiesMapping": [{ + "source_key": { + "domain": null, + "key": "AREA_OR_POINT" + }, + "target_key": { + "domain": null, + "key": "AREA_OR_POINT" + }, + "target_type": "String" + }, + { + "source_key": { + "domain": "IMAGE_STRUCTURE", + "key": "COMPRESSION" + }, + "target_key": { + "domain": "IMAGE_STRUCTURE_INFO", + "key": "COMPRESSION" + }, + "target_type": "String" + } + ], + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true, + "retry": null, + }) + ); + + let deserialized_parameters = + serde_json::from_value::(dataset_parameters_json).unwrap(); + + // since there is NaN in the data, we can't check for equality on the whole object + + assert_eq!( + deserialized_parameters.file_path, + dataset_parameters.file_path, + ); + assert_eq!( + deserialized_parameters.rasterband_channel, + dataset_parameters.rasterband_channel, + ); + assert_eq!( + deserialized_parameters.geo_transform, + dataset_parameters.geo_transform, + ); + assert_eq!(deserialized_parameters.width, dataset_parameters.width); + assert_eq!(deserialized_parameters.height, dataset_parameters.height); + assert_eq!( + deserialized_parameters.file_not_found_handling, + dataset_parameters.file_not_found_handling, + ); + assert!( + deserialized_parameters.no_data_value.unwrap().is_nan() + && dataset_parameters.no_data_value.unwrap().is_nan() + ); + assert_eq!( + deserialized_parameters.properties_mapping, + dataset_parameters.properties_mapping, + ); + assert_eq!( + deserialized_parameters.gdal_open_options, + dataset_parameters.gdal_open_options, + ); + assert_eq!( + deserialized_parameters.gdal_config_options, + dataset_parameters.gdal_config_options, + ); + } + + #[test] + fn read_raster_and_offset_scale() { + let up_side_down_params = GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif") + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (8.0, 57.4).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + }, + width: 30, + height: 30, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: Some(255.), + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + let tile_information = TileInformation::new( + [0, 0].into(), + [8, 8].into(), + up_side_down_params.geo_transform.try_into().unwrap(), + ); + let reader_mode = GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + up_side_down_params.geo_transform.try_into().unwrap(), + GridBoundingBox2D::new([0, 0], [29, 29]).unwrap(), + ), + }); + + let GridAndProperties { grid, properties } = + GdalRasterLoader::load_raster_tile_from_file::( + &up_side_down_params, + reader_mode, + tile_information, + ) + .unwrap() + .unwrap(); + + assert!(!grid.is_empty()); + + let grid = grid.into_materialized_masked_grid(); + + assert_eq!(grid.inner_grid.data.len(), 64); + // pixel value are the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.inner_grid.data, + &[ + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 127, 107, 255, 255, 255, 255, 255, 164, 185, 182, + 255, 255, 255, 175, 186, 190, 167, 140, 255, 255, 161, 175, 184, 173, 170, 188, + 255, 255, 128, 177, 165, 145, 191, 174, 255, 117, 100, 174, 159, 147, 99, 135 + ] + ); + + assert_eq!(grid.validity_mask.data.len(), 64); + // pixel mask is pixel > 0 from the top left 8x8 block from MOD13A2_M_NDVI_2014-04-01_27x27_bytes.txt + assert_eq!( + grid.validity_mask.data, + &[ + false, false, false, false, false, false, false, false, false, false, false, false, + false, false, false, false, false, false, false, false, false, false, true, true, + false, false, false, false, false, true, true, true, false, false, false, true, + true, true, true, true, false, false, true, true, true, true, true, true, false, + false, true, true, true, true, true, true, false, true, true, true, true, true, + true, true + ] + ); + + assert_eq!(properties.offset_option(), Some(1.)); + assert_eq!(properties.scale_option(), Some(2.)); + + assert!(approx_eq!(f64, properties.offset(), 1.)); + assert!(approx_eq!(f64, properties.scale(), 2.)); + } + + #[test] + fn it_creates_no_data_only_for_missing_files() { + hide_gdal_errors(); + + let ds = GdalDatasetParameters { + file_path: "nonexisting_file.tif".into(), + rasterband_channel: 1, + geo_transform: TestDefault::test_default(), + width: 100, + height: 100, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + let tile_information = TileInformation::new( + [0, 0].into(), + [8, 8].into(), + ds.geo_transform.try_into().unwrap(), + ); + let reader_mode = GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + ds.geo_transform.try_into().unwrap(), + GridBoundingBox2D::new([0, 0], [29, 29]).unwrap(), + ), + }); + + let res = + GdalRasterLoader::load_raster_tile_from_file::(&ds, reader_mode, tile_information); + + assert!(res.is_ok()); + + let res = res.unwrap(); + + assert!(res.is_none()); + + let ds = GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_2014-01-01.TIFF").into(), + rasterband_channel: 100, // invalid channel + geo_transform: TestDefault::test_default(), + width: 100, + height: 100, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }; + + // invalid channel => error + let result = + GdalRasterLoader::load_raster_tile_from_file::(&ds, reader_mode, tile_information); + assert!(result.is_err()); + } + + #[test] + fn it_creates_no_data_only_for_http_404() { + let server = Server::run(); + + server.expect( + Expectation::matching(request::method_path("HEAD", "/non_existing.tif")) + .times(1) + .respond_with(responders::cycle![responders::status_code(404),]), + ); + + server.expect( + Expectation::matching(request::method_path("HEAD", "/internal_error.tif")) + .times(1) + .respond_with(responders::cycle![responders::status_code(500),]), + ); + + let ds = GdalDatasetParameters { + file_path: format!("/vsicurl/{}", server.url_str("/non_existing.tif")).into(), + rasterband_channel: 1, + geo_transform: TestDefault::test_default(), + width: 100, + height: 100, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: Some(vec![ + ( + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), + ".tif".to_owned(), + ), + ( + "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), + "EMPTY_DIR".to_owned(), + ), + ("GDAL_HTTP_NETRC".to_owned(), "NO".to_owned()), + ("GDAL_HTTP_MAX_RETRY".to_owned(), "0".to_string()), + ]), + allow_alphaband_as_mask: true, + retry: None, + }; + + // 404 => no data + let tile_information = TileInformation::new( + [0, 0].into(), + [8, 8].into(), + ds.geo_transform.try_into().unwrap(), + ); + let reader_mode = GdalReaderMode::OriginalResolution(ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + ds.geo_transform.try_into().unwrap(), + GridBoundingBox2D::new([0, 0], [29, 29]).unwrap(), + ), + }); + + let res = + GdalRasterLoader::load_raster_tile_from_file::(&ds, reader_mode, tile_information); + assert!(res.is_ok()); + let res = res.unwrap(); + assert!(res.is_none()); + + let ds = GdalDatasetParameters { + file_path: format!("/vsicurl/{}", server.url_str("/internal_error.tif")).into(), + rasterband_channel: 1, + geo_transform: TestDefault::test_default(), + width: 100, + height: 100, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: Some(vec![ + ( + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), + ".tif".to_owned(), + ), + ( + "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), + "EMPTY_DIR".to_owned(), + ), + ("GDAL_HTTP_NETRC".to_owned(), "NO".to_owned()), + ("GDAL_HTTP_MAX_RETRY".to_owned(), "0".to_string()), + ]), + allow_alphaband_as_mask: true, + retry: None, + }; + + // 500 => error + let res = + GdalRasterLoader::load_raster_tile_from_file::(&ds, reader_mode, tile_information); + assert!(res.is_err()); + } + + #[test] + fn it_retries_only_after_clearing_vsi_cache() { + hide_gdal_errors(); + + let server = Server::run(); + + server.expect( + Expectation::matching(request::method_path("HEAD", "/foo.tif")) + .times(2) + .respond_with(responders::cycle![ + // first generic error + responders::status_code(500), + // then 404 file not found + responders::status_code(404) + ]), + ); + + let file_path: PathBuf = format!("/vsicurl/{}", server.url_str("/foo.tif")).into(); + + let options = Some(vec![ + ( + "CPL_VSIL_CURL_ALLOWED_EXTENSIONS".to_owned(), + ".tif".to_owned(), + ), + ( + "GDAL_DISABLE_READDIR_ON_OPEN".to_owned(), + "EMPTY_DIR".to_owned(), + ), + ("GDAL_HTTP_NETRC".to_owned(), "NO".to_owned()), + ("GDAL_HTTP_MAX_RETRY".to_owned(), "0".to_string()), + ]); + + let _thread_local_configs = options + .as_ref() + .map(|config_options| TemporaryGdalThreadLocalConfigOptions::new(config_options)); + + // first fail + let result = gdal_open_dataset_ex( + file_path.as_path(), + DatasetOptions { + open_flags: GdalOpenFlags::GDAL_OF_RASTER, + ..DatasetOptions::default() + }, + ); + + // it failed, but not with file not found + assert!(result.is_err()); + if let Err(error) = result { + assert!(!error_is_gdal_file_not_found(&error)); + } + + // second fail doesn't even try, so still not "file not found", even though it should be now + let result = gdal_open_dataset_ex( + file_path.as_path(), + DatasetOptions { + open_flags: GdalOpenFlags::GDAL_OF_RASTER, + ..DatasetOptions::default() + }, + ); + + assert!(result.is_err()); + if let Err(error) = result { + assert!(!error_is_gdal_file_not_found(&error)); + } + + clear_gdal_vsi_cache_for_path(file_path.as_path()); + + // after clearing the cache, it tries again + let result = gdal_open_dataset_ex( + file_path.as_path(), + DatasetOptions { + open_flags: GdalOpenFlags::GDAL_OF_RASTER, + ..DatasetOptions::default() + }, + ); + + // now we get the file not found error + assert!(result.is_err()); + if let Err(error) = result { + assert!(error_is_gdal_file_not_found(&error)); + } + } + + fn add_multi_tile_dataset( + ctx: &mut MockExecutionContext, + mut files: Vec, + time_steps: Vec, + ) -> NamedData { + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("multi_tiles"); + + // fix the file path because the test runs with a different working directory than the server + for tile in &mut files { + tile.params.file_path = test_data!( + tile.params + .file_path + .to_string_lossy() + .replace("test_data/", "") + ) + .to_path_buf(); + } + + let time = TimeDescriptor::new_irregular(Some(TimeInterval::new_unchecked( + time_steps.first().unwrap().start(), + time_steps.last().unwrap().end(), + ))); + + let meta: MultiBandGdalMetaData = Box::new(StaticMetaData { + loading_info: MultiBandGdalLoadingInfo::new(time_steps, files, CacheHint::default()), + result_descriptor: RasterResultDescriptor { + data_type: RasterDataType::U16, + spatial_reference: SpatialReference::epsg_4326().into(), + time, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180.0, 90.0).into(), 0.2, -0.2), + GridBoundingBox2D::new([0, 0], [899, 1799]).unwrap(), + ), + bands: vec![ + RasterBandDescriptor::new("band 0".to_string(), Measurement::Unitless), + RasterBandDescriptor::new("band 1".to_string(), Measurement::Unitless), + ] + .try_into() + .unwrap(), + }, + phantom: Default::default(), + }); + + ctx.add_meta_data(id, name.clone(), meta); + + name + } + + fn tile_files() -> Vec { + serde_json::from_str( + &std::fs::read_to_string(test_data!("raster/multi_tile/metadata/loading_info.json")) + .unwrap(), + ) + .unwrap() + } + + fn tile_files_rev() -> Vec { + serde_json::from_str( + &std::fs::read_to_string(test_data!( + "raster/multi_tile/metadata/loading_info_rev.json" + )) + .unwrap(), + ) + .unwrap() + } + + fn tiles_by_file_name(file_names: &[&str]) -> Vec { + let tile_files = tile_files(); + + file_names + .iter() + .map(|file_name| { + tile_files + .iter() + .find(|tile_file| tile_file.params.file_path.ends_with(*file_name)) + .cloned() + .unwrap() + }) + .collect() + } + + fn tiles_by_file_name_rev(file_names: &[&str]) -> Vec { + let tile_files = tile_files_rev(); + + file_names + .iter() + .map(|file_name| { + tile_files + .iter() + .find(|tile_file| tile_file.params.file_path.ends_with(*file_name)) + .cloned() + .unwrap() + }) + .collect() + } + + #[tokio::test] + async fn it_loads_multi_band_multi_file_mosaics() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + )]; + + let files = tiles_by_file_name(&[ + "2025-01-01_tile_x0_y0_b0.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x1_y1_b0.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new(dataset_name), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + "2025-01-01_global_b0_tile_0.tif", + "2025-01-01_global_b0_tile_1.tif", + "2025-01-01_global_b0_tile_2.tif", + "2025-01-01_global_b0_tile_3.tif", + "2025-01-01_global_b0_tile_4.tif", + "2025-01-01_global_b0_tile_5.tif", + "2025-01-01_global_b0_tile_6.tif", + "2025-01-01_global_b0_tile_7.tif", + ]; + + let expected_time = TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + ); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|f| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + expected_time, + 0, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[tokio::test] + async fn it_loads_multi_band_multi_file_mosaics_2_bands() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + )]; + + let files = tiles_by_file_name(&[ + "2025-01-01_tile_x0_y0_b0.tif", + "2025-01-01_tile_x0_y0_b1.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x0_y1_b1.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x1_y0_b1.tif", + "2025-01-01_tile_x1_y1_b0.tif", + "2025-01-01_tile_x1_y1_b1.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new(dataset_name), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32), + ("2025-01-01_global_b1_tile_0.tif", 1), + ("2025-01-01_global_b0_tile_1.tif", 0), + ("2025-01-01_global_b1_tile_1.tif", 1), + ("2025-01-01_global_b0_tile_2.tif", 0), + ("2025-01-01_global_b1_tile_2.tif", 1), + ("2025-01-01_global_b0_tile_3.tif", 0), + ("2025-01-01_global_b1_tile_3.tif", 1), + ("2025-01-01_global_b0_tile_4.tif", 0), + ("2025-01-01_global_b1_tile_4.tif", 1), + ("2025-01-01_global_b0_tile_5.tif", 0), + ("2025-01-01_global_b1_tile_5.tif", 1), + ("2025-01-01_global_b0_tile_6.tif", 0), + ("2025-01-01_global_b1_tile_6.tif", 1), + ("2025-01-01_global_b0_tile_7.tif", 0), + ("2025-01-01_global_b1_tile_7.tif", 1), + ]; + + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + expected_time, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_loads_multi_band_multi_file_mosaics_2_bands_2_timesteps() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![ + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-03-01T00:00:00Z").unwrap(), + ), + ]; + + let files = tiles_by_file_name(&[ + "2025-01-01_tile_x0_y0_b0.tif", + "2025-01-01_tile_x0_y0_b1.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x0_y1_b1.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x1_y0_b1.tif", + "2025-01-01_tile_x1_y1_b0.tif", + "2025-01-01_tile_x1_y1_b1.tif", + "2025-02-01_tile_x0_y0_b0.tif", + "2025-02-01_tile_x0_y0_b1.tif", + "2025-02-01_tile_x0_y1_b0.tif", + "2025-02-01_tile_x0_y1_b1.tif", + "2025-02-01_tile_x1_y0_b0.tif", + "2025-02-01_tile_x1_y0_b1.tif", + "2025-02-01_tile_x1_y1_b0.tif", + "2025-02-01_tile_x1_y1_b1.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new(dataset_name), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_time1 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_time2 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32, expected_time1), + ("2025-01-01_global_b1_tile_0.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_1.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_1.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_2.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_2.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_3.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_3.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_4.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_4.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_5.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_5.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_6.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_6.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_7.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_7.tif", 1, expected_time1), + ("2025-02-01_global_b0_tile_0.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_0.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_1.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_1.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_2.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_2.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_3.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_3.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_4.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_4.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_5.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_5.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_6.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_6.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_7.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_7.tif", 1, expected_time2), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[tokio::test] + #[allow(clippy::too_many_lines)] + async fn it_loads_multi_band_multi_file_mosaics_with_time_gaps() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![ + TimeInterval::new_unchecked( + TimeInstance::MIN, + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-03-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-03-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-04-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-04-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-05-01T00:00:00Z").unwrap(), + ), + TimeInterval::new_unchecked( + TimeInstance::from_str("2025-05-01T00:00:00Z").unwrap(), + TimeInstance::MAX, + ), + ]; + + let files = tiles_by_file_name(&[ + "2025-01-01_tile_x0_y0_b0.tif", + "2025-01-01_tile_x0_y0_b1.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x0_y1_b1.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x1_y0_b1.tif", + "2025-01-01_tile_x1_y1_b0.tif", + "2025-01-01_tile_x1_y1_b1.tif", + "2025-02-01_tile_x0_y0_b0.tif", + "2025-02-01_tile_x0_y0_b1.tif", + "2025-02-01_tile_x0_y1_b0.tif", + "2025-02-01_tile_x0_y1_b1.tif", + "2025-02-01_tile_x1_y0_b0.tif", + "2025-02-01_tile_x1_y0_b1.tif", + "2025-02-01_tile_x1_y1_b0.tif", + "2025-02-01_tile_x1_y1_b1.tif", + "2025-04-01_tile_x0_y0_b0.tif", + "2025-04-01_tile_x0_y0_b1.tif", + "2025-04-01_tile_x0_y1_b0.tif", + "2025-04-01_tile_x0_y1_b1.tif", + "2025-04-01_tile_x1_y0_b0.tif", + "2025-04-01_tile_x1_y0_b1.tif", + "2025-04-01_tile_x1_y1_b0.tif", + "2025-04-01_tile_x1_y1_b1.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new(dataset_name), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + // query a time interval that is greater than the time interval of the tiles and covers a region with a temporal gap + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-12-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-15T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let mut tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + // first 8 (spatial) x 2 (bands) tiles must be no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::MIN, + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data + let expected_time1 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_time2 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32, expected_time1), + ("2025-01-01_global_b1_tile_0.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_1.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_1.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_2.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_2.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_3.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_3.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_4.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_4.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_5.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_5.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_6.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_6.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_7.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_7.tif", 1, expected_time1), + ("2025-02-01_global_b0_tile_0.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_0.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_1.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_1.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_2.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_2.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_3.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_3.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_4.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_4.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_5.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_5.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_6.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_6.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_7.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_7.tif", 1, expected_time2), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // next comes a gap of no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data again + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-04-01_global_b0_tile_0.tif", 0u32, expected_time), + ("2025-04-01_global_b1_tile_0.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_1.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_1.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_2.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_2.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_3.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_3.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_4.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_4.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_5.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_5.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_6.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_6.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_7.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_7.tif", 1, expected_time), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // last 8 (spatial) x 2 (bands) tiles must be no data + for tile in &tiles[..16] { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::MAX + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + Ok(()) + } + + #[tokio::test] + async fn it_loads_multi_band_multi_file_mosaics_reverse_z_index() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + )]; + + let files = tiles_by_file_name_rev(&[ + "2025-01-01_tile_x1_y1_b0.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x0_y0_b0.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new(dataset_name), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + "2025-01-01_global_b0_tile_0.tif", + "2025-01-01_global_b0_tile_1.tif", + "2025-01-01_global_b0_tile_2.tif", + "2025-01-01_global_b0_tile_3.tif", + "2025-01-01_global_b0_tile_4.tif", + "2025-01-01_global_b0_tile_5.tif", + "2025-01-01_global_b0_tile_6.tif", + "2025-01-01_global_b0_tile_7.tif", + ]; + + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|f| { + raster_tile_from_file::( + test_data!(format!( + "raster/multi_tile/results/z_index_reversed/tiles/{f}" + )), + tiling_spatial_grid_definition, + expected_time, + 0, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[tokio::test] + async fn it_loads_overview_level() -> Result<()> { + let mut execution_context = MockExecutionContext::test_default(); + let query_ctx = execution_context.mock_query_context_test_default(); + + let time_steps = vec![TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + )]; + + let files = tiles_by_file_name(&[ + "2025-01-01_tile_x0_y0_b0.tif", + "2025-01-01_tile_x0_y1_b0.tif", + "2025-01-01_tile_x1_y0_b0.tif", + "2025-01-01_tile_x1_y1_b0.tif", + ]); + + let dataset_name = add_multi_tile_dataset(&mut execution_context, files, time_steps); + + let operator = MultiBandGdalSource { + params: GdalSourceParameters::new_with_overview_level(dataset_name, 2), + } + .boxed(); + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?.get_u16().unwrap(); + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .query(query_rect.clone(), &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + // // Write geotiff_bytes to disk + // let mut geotiff_bytes = raster_stream_to_geotiff_bytes( + // processor, + // query_rect, + // query_ctx, + // GdalGeoTiffDatasetMetadata { + // no_data_value: Some(0.), + // spatial_reference: SpatialReference::epsg_4326(), + // }, + // GdalGeoTiffOptions { + // as_cog: false, + // compression_num_threads: GdalCompressionNumThreads::AllCpus, + // force_big_tiff: false, + // }, + // None, + // Box::pin(futures::future::pending()), + // ) + // .await?; + + // std::fs::write("overview_level_2.tif", &geotiff_bytes[0]) + // .expect("Failed to write GeoTIFF file"); + + let expected_tiles = [ + "2025-01-01_global_b0_tile_0.tif", + "2025-01-01_global_b0_tile_1.tif", + "2025-01-01_global_b0_tile_2.tif", + "2025-01-01_global_b0_tile_3.tif", + ]; + + let expected_time = TimeInterval::new_unchecked( + TimeInstance::from_str("2025-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2025-02-01T00:00:00Z").unwrap(), + ); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|f| { + raster_tile_from_file::( + test_data!(format!( + "raster/multi_tile/results/overview_level_2/tiles/{f}" + )), + tiling_spatial_grid_definition, + expected_time, + 0, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } +} diff --git a/operators/src/source/multi_band_gdal_source/reader.rs b/operators/src/source/multi_band_gdal_source/reader.rs new file mode 100644 index 000000000..daf937674 --- /dev/null +++ b/operators/src/source/multi_band_gdal_source/reader.rs @@ -0,0 +1,1099 @@ +use geoengine_datatypes::{ + primitives::AxisAlignedRectangle, + raster::{ + GridBoundingBox2D, GridBounds, GridContains, GridIdx2D, GridOrEmpty, GridShape2D, + GridShapeAccess, GridSize, RasterProperties, SpatialGridDefinition, + }, +}; +use tracing::{trace, warn}; + +/// This struct is used to advise the GDAL reader how to read the data from the dataset. +/// The Workflow is as follows: +/// 1. The `gdal_read_window` is the window in the pixel space of the dataset that should be read. +/// 2. The `read_window_bounds` is the area in the target pixel space where the data should be placed. +/// 2.1 The data read in step one is read to the width and height of the `read_window_bounds`. +/// 2.2 if `flip_y` is true the data is flipped in the y direction. And should be unflipped after reading. +/// 3. The `bounds_of_target` is the area in the target pixel space where the data should be placed. +/// 3.1 The `read_window_bounds` might be offset from the `bounds_of_target` or might have a different size. +/// Then, the data needs to be placed in the target pixel space accordingly. Other parts of the target pixel space should be filled with nodata. +#[allow(dead_code)] +#[derive(Copy, Clone, Debug, PartialEq)] +pub struct GdalReadAdvise { + pub gdal_read_widow: GdalReadWindow, + pub read_window_bounds: GridBoundingBox2D, + pub bounds_of_target: GridBoundingBox2D, + pub flip_y: bool, +} + +// TODO: re-implement direct read(?) +// impl GdalReadAdvise { +// pub fn direct_read(&self) -> bool { +// self.read_window_bounds == self.bounds_of_target +// } +// } + +#[allow(dead_code)] +#[derive(Copy, Clone, Debug)] +pub enum GdalReaderMode { + // read the original resolution + OriginalResolution(ReaderState), + // read an overview level of the dataset + OverviewLevel(OverviewReaderState), +} + +impl GdalReaderMode { + /// Returns the read advise for the tiling based bounds + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + match self { + GdalReaderMode::OriginalResolution(re) => { + re.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + GdalReaderMode::OverviewLevel(rs) => { + rs.tiling_to_dataset_read_advise(actual_gdal_dataset_spatial_grid_definition, tile) + } + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct ReaderState { + pub dataset_spatial_grid: SpatialGridDefinition, +} + +impl ReaderState { + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + tile: &SpatialGridDefinition, + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.dataset_spatial_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + }; + + // Now we can work with a matching dataset. However, we need to reverse the read window later! + + // let's only look at data in the geo engine dataset definition! The intersection is relative to the first elements origin coordinate. + let dataset_gdal_data_intersection = + actual_gdal_dataset_spatial_grid_definition.intersection(&self.dataset_spatial_grid)?; + + // Now, we need the tile in the gdal dataset bounds to identify readable areas + let tile_in_gdal_dataset_bounds = tile.with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate, + ); // TODO: raise error if this fails! + + let Some(tile_in_gdal_dataset_bounds) = tile_in_gdal_dataset_bounds else { + warn!("BUG: Tile is not in the dataset bounds, skipping read advise generation"); + return None; + }; + + // Then, calculate the intersection between the datataset and the tile. Again, the intersection is relative to the first elements orrigin coordinate. + let tile_gdal_dataset_intersection = + dataset_gdal_data_intersection.intersection(&tile_in_gdal_dataset_bounds)?; + + // if we need to unflip the dataset grid now is the time to do this. + let tile_intersection_for_read_window = if flip_y { + tile_gdal_dataset_intersection + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + tile_gdal_dataset_intersection + }; + + // generate the read window for GDAL + + let gdal_read_window = GdalReadWindow::new( + tile_intersection_for_read_window.grid_bounds.min_index(), + tile_intersection_for_read_window.grid_bounds.grid_shape(), + ); + + // if the read window has the same shape as the tiling based bounds we can fill that completely + if tile_in_gdal_dataset_bounds == tile_gdal_dataset_intersection { + return Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: tile.grid_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }); + } + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = + tile_gdal_dataset_intersection.min_index() - tile_in_gdal_dataset_bounds.min_index(); + let crop_lr = + tile_gdal_dataset_intersection.max_index() - tile_in_gdal_dataset_bounds.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + Some(GdalReadAdvise { + gdal_read_widow: gdal_read_window, + read_window_bounds: shifted_readable_bounds, + bounds_of_target: tile.grid_bounds, + flip_y: false, + }) + } +} + +#[derive(Copy, Clone, Debug)] +pub struct OverviewReaderState { + pub original_dataset_grid: SpatialGridDefinition, + pub overview_level: u32, +} + +impl OverviewReaderState { + /// Compute the `ReadAdvise` for a given input gdal dataset (file) and output tile, i.e., which pixels we need to read and where to put them in the output tile. + /// + /// There are some special cases here, because while the grid of the input file must be aligned with the output tile, + /// the overviews of (a) the geo engine raster and (b) the indiviual file at hand, must not necessarily be aligned. + /// This is, e.g., the case if the file's origin distance from the geo engine datasets's global origin is not a multiple of 2. + /// In that case, for building a pixel in the overview we would need to take one pixel from one file, and another pixel from a different file. + /// Currently, we do not do this, however we need to correctly calculate the read window for the overview that + /// (1) reads from the file s.t. the existing overviews (pyramid levels) can be used + /// (2) all pixels in the output tile that can be filled, are filled + pub fn tiling_to_dataset_read_advise( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, // This is the spatial grid of an actual gdal file + tile: &SpatialGridDefinition, // This is a tile inside the grid we use for the global dataset consisting of potentially many gdal files... + ) -> Option { + // Check if the y_axis is fliped. + let (actual_gdal_dataset_spatial_grid_definition, flip_y) = + self.normalize_y_axis(actual_gdal_dataset_spatial_grid_definition); + + let Some(( + tile_with_original_resolution_in_actual_space, + tile_intersection_original_resolution_actual_space, + tile_intersection_for_read_window, + tile_intersection_for_read_window_snapped_to_overview_level, + )) = self.intersect_tile_and_dataset( + tile, + actual_gdal_dataset_spatial_grid_definition, + flip_y, + ) + else { + trace!( + "no intersection between tile {:?} and dataset grid {:?}, skipping read advise generation, grid: {:?}, overview level: {}", + tile, + actual_gdal_dataset_spatial_grid_definition, + self.original_dataset_grid, + self.overview_level + ); + return None; + }; + + // generate the read window for GDAL --> This is what we can read in any case. + let read_window = GdalReadWindow::new( + tile_intersection_for_read_window_snapped_to_overview_level.min_index(), + tile_intersection_for_read_window_snapped_to_overview_level + .grid_bounds() + .grid_shape(), + ); + + // ensure that start and size of the read window are aligned with the overview level + // must start from overview level aligned pixel + debug_assert_eq!(read_window.start_x % self.overview_level as isize, 0); + debug_assert_eq!(read_window.start_y % self.overview_level as isize, 0); + + // must be multiple of overview level in size, unless we are at the edge of the dataset + debug_assert!( + read_window + .size_x + .is_multiple_of(self.overview_level as usize) + || read_window.start_x + read_window.size_x as isize + == actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_x() as isize + ); + debug_assert!( + read_window + .size_y + .is_multiple_of(self.overview_level as usize) + || read_window.start_y + read_window.size_y as isize + == actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize + ); + + let can_fill_whole_tile = tile_intersection_for_read_window.grid_bounds() + == tile_with_original_resolution_in_actual_space.grid_bounds(); + + let read_window_bounds = if can_fill_whole_tile { + // ensure that the read window is `overview_level` times larger than the tile + debug_assert_eq!( + read_window.size_x / tile.grid_bounds().grid_shape().x(), + self.overview_level as usize + ); + debug_assert_eq!( + read_window.size_y / tile.grid_bounds().grid_shape().y(), + self.overview_level as usize + ); + + trace!("Tile is fully contained"); + + tile.grid_bounds + } else { + // IF we can't fill the whole tile, we have to find out which area of the tile we can fill. + read_window_bounds_for_partially_tile( + tile, + actual_gdal_dataset_spatial_grid_definition, + tile_intersection_original_resolution_actual_space, + ) + }; + + Some(GdalReadAdvise { + gdal_read_widow: read_window, + read_window_bounds, + bounds_of_target: tile.grid_bounds, + flip_y, + }) + } + + fn intersect_tile_and_dataset( + &self, + tile: &SpatialGridDefinition, + actual_gdal_dataset_spatial_grid_definition: SpatialGridDefinition, + flip_y: bool, + ) -> Option<( + SpatialGridDefinition, + SpatialGridDefinition, + SpatialGridDefinition, + SpatialGridDefinition, + )> { + // This is the intersection of grid of the gdal file and the global grid we use. Usually the dataset is inside the global dataset grid. + // IF the intersection is empty the we return early and load nothing + // The intersection uses the geo_transform of the gdal dataset which enables us to adress gdal pixels starting at 0,0 + let actual_bounds_to_use_original_resolution = actual_gdal_dataset_spatial_grid_definition + .intersection(&self.original_dataset_grid)?; + + let tile_with_overview_resolution_in_actual_space = if let Some(tile) = tile + .with_moved_origin_exact_grid( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate(), + ) { + tile + } else { + // special case where the original resolution tile's origin is not a valid pixel edge in the overview + // in this case we need to snap to the nearest pixel edge in the original resolution + trace!( + "overview tile grid {:?} does not align with actual datasets grid {:?}, use nearest original edge instead", + tile, actual_gdal_dataset_spatial_grid_definition + ); + + // move to pixel edge nearest to actual grid edge, the result is within one pixel distance in overview space + let nearest_in_overview_space = tile.with_moved_origin_to_nearest_grid_edge( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate(), + ); + + // replace the origin with the desired one. this forces the overview grid on the actual grid + nearest_in_overview_space.replace_origin( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .origin_coordinate(), + ) + }; + + let tile_with_original_resolution_in_actual_space = + tile_with_overview_resolution_in_actual_space.with_changed_resolution( + actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .spatial_resolution(), + ); + + let tile_intersection_original_resolution_actual_space = + &tile_with_original_resolution_in_actual_space + .intersection(&actual_bounds_to_use_original_resolution)?; + + let tile_intersection_for_read_window = if flip_y { + tile_intersection_original_resolution_actual_space + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )) + } else { + *tile_intersection_original_resolution_actual_space + }; + + let tile_intersection_for_read_window_snapped_to_overview_level = + tile_intersection_original_resolution_actual_space.snap_to_dataset_overview_level( + actual_gdal_dataset_spatial_grid_definition, + self.overview_level, + )?; + + Some(( + tile_with_original_resolution_in_actual_space, + *tile_intersection_original_resolution_actual_space, + tile_intersection_for_read_window, + tile_intersection_for_read_window_snapped_to_overview_level, + )) + } + + fn normalize_y_axis( + &self, + actual_gdal_dataset_spatial_grid_definition: &SpatialGridDefinition, + ) -> (SpatialGridDefinition, bool) { + if actual_gdal_dataset_spatial_grid_definition + .geo_transform() + .y_axis_is_neg() + == self.original_dataset_grid.geo_transform().y_axis_is_neg() + { + (*actual_gdal_dataset_spatial_grid_definition, false) + } else { + ( + actual_gdal_dataset_spatial_grid_definition + .flip_axis_y() // first: reverse the coordinate system to match the one used by tiling + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + // second: move the origin to the other end of the y-axis + actual_gdal_dataset_spatial_grid_definition + .grid_bounds() + .axis_size_y() as isize, + 0, + )), + true, + ) + } + } +} + +fn read_window_bounds_for_partially_tile( + tile: &SpatialGridDefinition, + actual_gdal_dataset_spatial_grid_definition: SpatialGridDefinition, + tile_intersection_original_resolution_actual_space: SpatialGridDefinition, +) -> geoengine_datatypes::raster::GridBoundingBox<[isize; 2]> { + let readable_area_in_overview_res = tile_intersection_original_resolution_actual_space + .with_changed_resolution(tile.geo_transform().spatial_resolution()); + + let readable_area_in_overview_res_and_tile_space = + if tile.is_compatible_grid_generic(&readable_area_in_overview_res) { + readable_area_in_overview_res + } else { + // we need to make the readable area grid compatible to the tile in overview space, as the actual origin does not exist in overview space + let tile_edge_idx = tile.geo_transform.coordinate_to_grid_idx_2d( + readable_area_in_overview_res + .geo_transform() + .origin_coordinate(), + ); + let tile_edge_coord = tile + .geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(tile_edge_idx); + + let readable_area_in_overview_res_and_tile_space = + readable_area_in_overview_res.replace_origin(tile_edge_coord); + + // if the snapping of the origin causes us to lose a pixel, we extend the bounds by one pixel in the respective direction + // if it is still contained in the actual dataset bounds + let mut extend = [0, 0]; + if tile.geo_transform().origin_coordinate().y - tile_edge_coord.y + > tile.geo_transform().y_pixel_size() + { + let max_index = readable_area_in_overview_res_and_tile_space + .grid_bounds() + .max_index() + .y(); + + let new_border_y = readable_area_in_overview_res_and_tile_space + .geo_transform() + .origin_coordinate() + .y + + (readable_area_in_overview_res_and_tile_space + .geo_transform() + .y_pixel_size() + * (max_index + 1) as f64); + + if new_border_y + < actual_gdal_dataset_spatial_grid_definition + .spatial_partition() + .lower_right() + .y + { + extend[0] = 1; + } + } + + if tile.geo_transform().origin_coordinate().x - tile_edge_coord.x + > tile.geo_transform().x_pixel_size() + { + let max_index = readable_area_in_overview_res_and_tile_space + .grid_bounds() + .max_index() + .x(); + + let new_border_x = readable_area_in_overview_res_and_tile_space + .geo_transform() + .origin_coordinate() + .x + + (readable_area_in_overview_res_and_tile_space + .geo_transform() + .x_pixel_size() + * (max_index + 1) as f64); + + if new_border_x + < actual_gdal_dataset_spatial_grid_definition + .spatial_partition() + .lower_right() + .x + { + extend[1] = 1; + } + } + + let bounds = GridBoundingBox2D::new_unchecked( + readable_area_in_overview_res_and_tile_space + .grid_bounds() + .min_index(), + readable_area_in_overview_res_and_tile_space + .grid_bounds() + .max_index() + + GridIdx2D::new_y_x(extend[0], extend[1]), + ); + + SpatialGridDefinition::new( + readable_area_in_overview_res_and_tile_space.geo_transform(), + bounds, + ) + }; + + // Calculate the intersection of the readable area and the tile, result is in geotransform of the tile! + let readable_tile_area = tile + .intersection(&readable_area_in_overview_res_and_tile_space) + .expect("Since there was an intersection earlyer, there must be a part of data to read."); + + // we need to crop the window to the intersection of the tiling based bounds and the dataset bounds + let crop_tl = readable_tile_area.min_index() - tile.min_index(); + let crop_lr = readable_tile_area.max_index() - tile.max_index(); + + let shifted_tl = tile.grid_bounds.min_index() + crop_tl; + let shifted_lr = tile.grid_bounds.max_index() + crop_lr; + + // now we need to adapt the target pixel space read window to the clipped dataset intersection area + let shifted_readable_bounds = GridBoundingBox2D::new_unchecked(shifted_tl, shifted_lr); + debug_assert!( + tile.grid_bounds().contains(&shifted_readable_bounds), + "readable bounds must be contained in tile bounds" + ); + + shifted_readable_bounds +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub struct GdalReadWindow { + start_x: isize, // pixelspace origin + start_y: isize, + size_x: usize, // pixelspace size + size_y: usize, +} + +impl GdalReadWindow { + pub fn new(start: GridIdx2D, size: GridShape2D) -> Self { + Self { + start_x: start.x(), + start_y: start.y(), + size_x: size.axis_size_x(), + size_y: size.axis_size_y(), + } + } + + pub fn gdal_window_start(&self) -> (isize, isize) { + (self.start_x, self.start_y) + } + + pub fn gdal_window_size(&self) -> (usize, usize) { + (self.size_x, self.size_y) + } +} + +pub struct GridAndProperties { + pub grid: GridOrEmpty, + pub properties: RasterProperties, +} + +#[cfg(test)] +mod tests { + use geoengine_datatypes::{ + primitives::Coordinate2D, + raster::{ + BoundedGrid, GeoTransform, GridBoundingBox2D, GridIdx2D, GridShape2D, + SpatialGridDefinition, + }, + }; + + use crate::source::multi_band_gdal_source::reader::{ + GdalReadWindow, OverviewReaderState, ReaderState, + }; + + #[test] + fn reader_state_dataset_geo_transform() { + let reader_state = ReaderState { + dataset_spatial_grid: SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ), + }; + + assert_eq!( + reader_state.dataset_spatial_grid.geo_transform(), + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.) + ); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_no_change() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 512, + size_y: 512, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 90, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_and_clipped() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 190, + start_y: 100, + size_x: 170, + size_y: 80, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([10, 10]), GridIdx2D::new([99, 189])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_shifted_flipy() { + let spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., 90.), 1., -1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let spatial_grid_flipy = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(-180., -90.), 1., 1.), + GridShape2D::new([180, 360]).bounding_box(), + ); + + let reader_state = ReaderState { + dataset_spatial_grid: spatial_grid, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &spatial_grid_flipy, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 180, + start_y: 0, + size_x: 180, + size_y: 90, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([89, 179])).unwrap() + ); + + assert!(tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_2() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([1024, 1024]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 2, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 2., -2.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 1024, + size_y: 1024, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([2048, 2048]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 4, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096, 4096]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 4, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048, + size_y: 2048, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_lrcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 4, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 2048, + start_y: 2048, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new( + GridIdx2D::new([512, 512]), + GridIdx2D::new([1023 - 4, 1023 - 4]) + ) + .unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([512, 512]), GridIdx2D::new([1023, 1023])) + .unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(16., -16.), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 4, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(0., 0.), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn reader_state_tiling_to_dataset_read_advise_overview_4_tile_22_ulcrop_numbers() { + let original_spatial_grid = SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(17.123_456, -17.123_456), 1., -1.), + GridShape2D::new([4096 - 16, 4096 - 16]).bounding_box(), + ); + + let reader_state = OverviewReaderState { + original_dataset_grid: original_spatial_grid, + overview_level: 4, + }; + + let tiling_to_dataset_read_advise = reader_state.tiling_to_dataset_read_advise( + &original_spatial_grid, + &SpatialGridDefinition::new( + GeoTransform::new(Coordinate2D::new(1.123_456, -1.123_456), 4., -4.), + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap(), + ), + ); + + assert!(tiling_to_dataset_read_advise.is_some()); + + let tiling_to_dataset_read_advise = tiling_to_dataset_read_advise.unwrap(); + + assert_eq!( + tiling_to_dataset_read_advise.gdal_read_widow, + GdalReadWindow { + start_x: 0, + start_y: 0, + size_x: 2048 - 16, + size_y: 2048 - 16, + }, + ); + + assert_eq!( + tiling_to_dataset_read_advise.read_window_bounds, + GridBoundingBox2D::new(GridIdx2D::new([4, 4]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert_eq!( + tiling_to_dataset_read_advise.bounds_of_target, + GridBoundingBox2D::new(GridIdx2D::new([0, 0]), GridIdx2D::new([511, 511])).unwrap() + ); + + assert!(!tiling_to_dataset_read_advise.flip_y); + } + + #[test] + fn it_snaps_overview_read_window_to_tile_edge() { + let reader_state = OverviewReaderState { + original_dataset_grid: SpatialGridDefinition { + geo_transform: GeoTransform::new(Coordinate2D { x: -180.0, y: 90.0 }, 0.2, -0.2), + grid_bounds: GridBoundingBox2D::new_unchecked([0, 0], [899, 1799]), + }, + overview_level: 2, + }; + + let actual_gdal_dataset_spatial_grid_definition = SpatialGridDefinition { + geo_transform: GeoTransform::new( + Coordinate2D { + x: -45.0, + y: 22.399_999_999_999_99, + }, + 0.2, + -0.2, + ), + grid_bounds: GridBoundingBox2D::new_unchecked([0, 0], [561, 1124]), + }; + + let tile = SpatialGridDefinition { + geo_transform: GeoTransform::new(Coordinate2D { x: 0.0, y: 0.0 }, 0.4, -0.4), + grid_bounds: GridBoundingBox2D::new_unchecked([-512, -512], [-1, -1]), + }; + + let read_advise = reader_state + .tiling_to_dataset_read_advise(&actual_gdal_dataset_spatial_grid_definition, &tile) + .unwrap(); + + assert_eq!( + read_advise.read_window_bounds, + GridBoundingBox2D::new_unchecked([-56, -113], [-1, -1]) + ); + } +} diff --git a/operators/src/source/ogr_source/dataset_iterator.rs b/operators/src/source/ogr_source/dataset_iterator.rs index 38a91cca1..a9546e505 100644 --- a/operators/src/source/ogr_source/dataset_iterator.rs +++ b/operators/src/source/ogr_source/dataset_iterator.rs @@ -137,11 +137,12 @@ impl OgrDatasetIterator { let time_filter = if dataset_information.force_ogr_time_filter { debug!( "using time filter {:?} for layer {:?}", - query_rectangle.time_interval, &dataset_information.layer_name + query_rectangle.time_interval(), + &dataset_information.layer_name ); FeaturesProvider::create_time_filter_string( dataset_information.time.clone(), - query_rectangle.time_interval, + query_rectangle.time_interval(), &dataset.driver().short_name(), ) } else { @@ -199,10 +200,11 @@ impl OgrDatasetIterator { if use_ogr_spatial_filter { debug!( "using spatial filter {:?} for layer {:?}", - query_rectangle.spatial_bounds, &dataset_information.layer_name + query_rectangle.spatial_bounds(), + &dataset_information.layer_name ); // NOTE: the OGR-filter may be inaccurately allowing more features that should be returned in a "strict" fashion. - features_provider.set_spatial_filter(&query_rectangle.spatial_bounds); + features_provider.set_spatial_filter(&query_rectangle.spatial_bounds()); } Ok((features_provider, time_filter.is_some())) diff --git a/operators/src/source/ogr_source/mod.rs b/operators/src/source/ogr_source/mod.rs index c63f59c5c..75b11bf03 100644 --- a/operators/src/source/ogr_source/mod.rs +++ b/operators/src/source/ogr_source/mod.rs @@ -1,55 +1,59 @@ mod dataset_iterator; + use self::dataset_iterator::OgrDatasetIterator; -use crate::adapters::FeatureCollectionStreamExt; -use crate::engine::{ - CanonicOperatorName, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, -}; -use crate::error::Error; -use crate::util::input::StringOrNumberRange; -use crate::util::{Result, safe_lock_mutex}; use crate::{ + adapters::FeatureCollectionStreamExt, engine::{ - InitializedVectorOperator, MetaData, QueryContext, SourceOperator, - TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, + CanonicOperatorName, InitializedVectorOperator, MetaData, OperatorData, OperatorName, + QueryContext, QueryProcessor, SourceOperator, TypedVectorQueryProcessor, VectorOperator, + VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }, - error, + error::{self, Error}, + util::{Result, input::StringOrNumberRange, safe_lock_mutex}, }; use async_trait::async_trait; -use futures::future::BoxFuture; +use futures::FutureExt; +use futures::future::{BoxFuture, Future}; use futures::stream::{BoxStream, FusedStream}; use futures::task::Context; -use futures::{Future, FutureExt}; use futures::{Stream, StreamExt, ready}; -use gdal::errors::GdalError; -use gdal::vector::sql::ResultSet; -use gdal::vector::{Feature, FieldValue, Layer, LayerAccess, LayerCaps, OGRwkbGeometryType}; -use geoengine_datatypes::collections::{ - BuilderProvider, FeatureCollection, FeatureCollectionBuilder, FeatureCollectionInfos, - FeatureCollectionModifications, FeatureCollectionRowBuilder, GeoFeatureCollectionRowBuilder, - VectorDataType, +use gdal::{ + errors::GdalError, + vector::{ + Feature, FieldValue, Layer, LayerAccess, LayerCaps, OGRwkbGeometryType, sql::ResultSet, + }, }; use geoengine_datatypes::dataset::NamedData; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, Coordinate2D, DateTime, DateTimeParseFormat, - FeatureDataType, FeatureDataValue, Geometry, MultiLineString, MultiPoint, MultiPolygon, - NoGeometry, TimeInstance, TimeInterval, TimeStep, TypedGeometry, VectorQueryRectangle, -}; -use geoengine_datatypes::primitives::{CacheTtlSeconds, ColumnSelection}; +use geoengine_datatypes::primitives::CacheTtlSeconds; +use geoengine_datatypes::primitives::ColumnSelection; use geoengine_datatypes::util::arrow::ArrowTyped; +use geoengine_datatypes::{ + collections::{ + BuilderProvider, FeatureCollection, FeatureCollectionBuilder, FeatureCollectionInfos, + FeatureCollectionModifications, FeatureCollectionRowBuilder, + GeoFeatureCollectionRowBuilder, VectorDataType, + }, + primitives::{ + AxisAlignedRectangle, BoundingBox2D, Coordinate2D, DateTime, DateTimeParseFormat, + FeatureDataType, FeatureDataValue, Geometry, MultiLineString, MultiPoint, MultiPolygon, + NoGeometry, TimeInstance, TimeInterval, TimeStep, TypedGeometry, VectorQueryRectangle, + }, +}; use pin_project::pin_project; use postgres_protocol::escape::{escape_identifier, escape_literal}; use postgres_types::{FromSql, ToSql}; use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; -use std::convert::{TryFrom, TryInto}; -use std::fmt::Debug; -use std::marker::PhantomData; -use std::ops::{Add, DerefMut}; -use std::path::PathBuf; -use std::pin::Pin; -use std::str::FromStr; -use std::sync::Arc; -use std::task::Poll; +use std::{ + collections::{HashMap, HashSet}, + fmt::Debug, + marker::PhantomData, + ops::{Add, DerefMut}, + path::PathBuf, + pin::Pin, + str::FromStr, + sync::Arc, + task::Poll, +}; use tokio::sync::Mutex; use tracing::debug; @@ -342,6 +346,7 @@ pub struct InitializedOgrSource { data: String, result_descriptor: VectorResultDescriptor, state: OgrSourceState, + params: OgrSourceParameters, } #[typetag::serde] @@ -399,6 +404,7 @@ impl VectorOperator for OgrSource { path, result_descriptor, data: self.params.data.to_string(), + params: self.params.clone(), state: OgrSourceState { dataset_information: info, attribute_filters: self.params.attribute_filters.unwrap_or_default(), @@ -488,6 +494,17 @@ impl InitializedVectorOperator for InitializedOgrSource { fn data(&self) -> Option { Some(self.data.clone()) } + + fn optimize( + &self, + _resolution: geoengine_datatypes::primitives::SpatialResolution, + ) -> Result, crate::optimization::OptimizationError> { + // we do not optimize vector inputs, but it would be possible to, e.g., sample or simplify features here based on the resolution + Ok(OgrSource { + params: self.params.clone(), + } + .boxed()) + } } pub struct OgrSourceProcessor @@ -1279,7 +1296,8 @@ where let time_interval = time_extractor(feature)?; // filter out data items not in the query time interval - if !was_time_filtered_by_ogr && !time_interval.intersects(&query_rectangle.time_interval) { + if !was_time_filtered_by_ogr && !time_interval.intersects(&query_rectangle.time_interval()) + { return Ok(()); } @@ -1302,7 +1320,7 @@ where // filter out geometries that are not contained in the query's bounding box if !was_spatial_filtered_by_ogr - && !geometry.intersects_bbox(&query_rectangle.spatial_bounds) + && !geometry.intersects_bbox(&query_rectangle.spatial_bounds()) { return Ok(()); } @@ -1985,9 +2003,7 @@ mod db_types { mod tests { use super::*; - use crate::engine::{ - ChunkByteSize, MockExecutionContext, MockQueryContext, StaticMetaData, VectorColumnInfo, - }; + use crate::engine::{ChunkByteSize, MockExecutionContext, StaticMetaData, VectorColumnInfo}; use crate::source::ogr_source::FormatSpecifics::Csv; use crate::test_data; use futures::{StreamExt, TryStreamExt}; @@ -1998,7 +2014,7 @@ mod tests { use geoengine_datatypes::dataset::{DataId, DatasetId}; use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, Measurement, SpatialResolution, TimeGranularity, + BoundingBox2D, FeatureData, Measurement, TimeGranularity, }; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; use geoengine_datatypes::util::Identifier; @@ -2181,16 +2197,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -2236,16 +2252,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await; @@ -2285,16 +2301,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (5., 5.).into()).unwrap(), - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (5., 5.).into()).unwrap(), + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -2339,16 +2355,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (5., 5.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (5., 5.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -2425,15 +2441,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2525,15 +2540,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2628,15 +2642,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2782,15 +2795,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -2957,18 +2969,14 @@ mod tests { let query_processor = source.query_processor()?.multi_point().unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let context = exe_ctx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( - (-180.0, -90.0).into(), - (180.0, 90.0).into(), - )?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((-180.0, -90.0).into(), (180.0, 90.0).into())?, + Default::default(), + ColumnSelection::all(), + ), &context, ) .await @@ -4146,16 +4154,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -4268,16 +4276,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 2.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 2.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -4478,15 +4486,10 @@ mod tests { (4.824_087_161, 52.413_055_56), ])?; - let context1 = MockQueryContext::new(ChunkByteSize::MIN); + let context1 = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context1, ) .await @@ -4513,15 +4516,10 @@ mod tests { assert!(!result.last().unwrap().is_empty()); // LARGER CHUNK - let context = MockQueryContext::new((1_650).into()); + let context = exe_ctx.mock_query_context((1_650).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -4631,15 +4629,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (-180.00, -90.0).into()).unwrap(); - let context = MockQueryContext::new(ChunkByteSize::MIN); + let context = exe_ctx.mock_query_context(ChunkByteSize::MIN); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -4721,15 +4714,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -4842,15 +4830,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -4970,15 +4953,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5096,15 +5074,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5222,15 +5195,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5344,15 +5312,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5479,15 +5442,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5600,15 +5558,10 @@ mod tests { let query_bbox = BoundingBox2D::new((-180.0, -90.0).into(), (180.00, 90.0).into()).unwrap(); - let context = MockQueryContext::new((1024 * 1024).into()); + let context = exe_ctx.mock_query_context((1024 * 1024).into()); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: query_bbox, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.).unwrap(), - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new(query_bbox, Default::default(), ColumnSelection::all()), &context, ) .await @@ -5714,16 +5667,16 @@ mod tests { let query_processor = OgrSourceProcessor::::new(rd, Box::new(info), vec![]); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -5844,16 +5797,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -5963,16 +5916,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6082,16 +6035,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6205,16 +6158,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6327,16 +6280,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6464,16 +6417,16 @@ mod tests { ], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6583,16 +6536,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6694,16 +6647,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6791,16 +6744,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6885,16 +6838,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -6979,16 +6932,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); @@ -7073,16 +7026,16 @@ mod tests { }], ); - let context = MockQueryContext::new(ChunkByteSize::MAX); + let ecx = MockExecutionContext::test_default(); + let ctx = ecx.mock_query_context(ChunkByteSize::MAX); let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, - &context, + VectorQueryRectangle::new( + BoundingBox2D::new((0., 0.).into(), (1., 1.).into())?, + Default::default(), + ColumnSelection::all(), + ), + &ctx, ) .await .unwrap(); diff --git a/operators/src/util/gdal.rs b/operators/src/util/gdal.rs index f676bcea3..4d8ecdbee 100644 --- a/operators/src/util/gdal.rs +++ b/operators/src/util/gdal.rs @@ -1,11 +1,18 @@ -use std::{ - collections::HashSet, - convert::TryInto, - hash::BuildHasher, - path::{Path, PathBuf}, - str::FromStr, +use crate::{ + engine::{ + MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, + SpatialGridDescriptor, StaticMetaData, TimeDescriptor, VectorColumnInfo, + VectorResultDescriptor, + }, + error::{self, Error}, + source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, + GdalSourceTimePlaceholder, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, + OgrSourceErrorSpec, TimeReference, + }, + test_data, + util::Result, }; - use gdal::{Dataset, DatasetOptions, DriverManager}; use geoengine_datatypes::{ collections::VectorDataType, @@ -13,50 +20,50 @@ use geoengine_datatypes::{ hashmap, primitives::{ BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, DateTimeParseFormat, - FeatureDataType, Measurement, SpatialPartition2D, SpatialResolution, TimeGranularity, - TimeInstance, TimeInterval, TimeStep, VectorQueryRectangle, + FeatureDataType, Measurement, TimeGranularity, TimeInstance, TimeInterval, TimeStep, + VectorQueryRectangle, }, - raster::{GeoTransform, RasterDataType}, + raster::{BoundedGrid, GeoTransform, GridBoundingBox2D, GridShape2D, RasterDataType}, spatial_reference::SpatialReference, util::Identifier, }; use itertools::Itertools; use snafu::ResultExt; - -use crate::{ - engine::{ - MockExecutionContext, RasterBandDescriptor, RasterBandDescriptors, RasterResultDescriptor, - StaticMetaData, VectorColumnInfo, VectorResultDescriptor, - }, - error::{self, Error}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, - GdalSourceTimePlaceholder, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, - OgrSourceErrorSpec, TimeReference, - }, - test_data, - util::Result, +use std::{ + collections::HashSet, + convert::TryInto, + hash::BuildHasher, + path::{Path, PathBuf}, + str::FromStr, }; +use tracing::warn; // TODO: move test helper somewhere else? pub fn create_ndvi_meta_data() -> GdalMetaDataRegular { create_ndvi_meta_data_with_cache_ttl(CacheTtlSeconds::default()) } +pub fn create_ndvi_meta_data_cropped_to_valid_webmercator_bounds() -> GdalMetaDataRegular { + create_ndvi_meta_data_with_cache_ttl(CacheTtlSeconds::default()) +} + #[allow(clippy::missing_panics_doc)] pub fn create_ndvi_meta_data_with_cache_ttl(cache_ttl: CacheTtlSeconds) -> GdalMetaDataRegular { let no_data_value = Some(0.); // TODO: is it really 0? + let time_bounds = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("it should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z") + .expect("it should only be used in tests"), + ); + let time_step = TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }; + GdalMetaDataRegular { - data_time: TimeInterval::new_unchecked( - TimeInstance::from_str("2014-01-01T00:00:00.000Z") - .expect("it should only be used in tests"), - TimeInstance::from_str("2014-07-01T00:00:00.000Z") - .expect("it should only be used in tests"), - ), - step: TimeStep { - granularity: TimeGranularity::Months, - step: 1, - }, + data_time: time_bounds, + step: time_step, time_placeholders: hashmap! { "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder { format: DateTimeParseFormat::custom("%Y-%m-%d".to_string()), @@ -81,20 +88,96 @@ pub fn create_ndvi_meta_data_with_cache_ttl(cache_ttl: CacheTtlSeconds) -> GdalM allow_alphaband_as_mask: true, retry: None, }, + result_descriptor: create_ndvi_result_descriptor(true), + cache_ttl, + } +} + +#[allow(clippy::missing_panics_doc)] +pub fn create_ndvi_result_descriptor(as_regular_timeseries: bool) -> RasterResultDescriptor { + let time_bounds = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("it should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z") + .expect("it should only be used in tests"), + ); + let time_step = TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }; + RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: if as_regular_timeseries { + TimeDescriptor::new_regular_with_epoch(Some(time_bounds), time_step) + } else { + TimeDescriptor::new_irregular(Some(time_bounds)) + }, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., 90.).into(), 0.1, -0.1), + GridBoundingBox2D::new([0, 0], [1799, 3599]).expect("should only be used in tests"), + ), + bands: vec![RasterBandDescriptor { + name: "ndvi".to_string(), + measurement: Measurement::Continuous(ContinuousMeasurement { + measurement: "vegetation".to_string(), + unit: None, + }), + }] + .try_into() + .expect("it should only be used in tests"), + } +} + +#[allow(clippy::missing_panics_doc)] +pub fn create_ndvi_meta_data_cropped_to_valid_webmercator_bounds_with_cache_ttl( + cache_ttl: CacheTtlSeconds, +) -> GdalMetaDataRegular { + let no_data_value = Some(0.); // TODO: is it really 0? + let time_bounds = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z").expect("should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z").expect("should only be used in tests"), + ); + let time_step = TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }; + GdalMetaDataRegular { + data_time: time_bounds, + step: time_step, + time_placeholders: hashmap! { + "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder { + format: DateTimeParseFormat::custom("%Y-%m-%d".to_string()), + reference: TimeReference::Start, + }, + }, + params: GdalDatasetParameters { + file_path: test_data!("raster/modis_ndvi/MOD13A2_M_NDVI_%_START_TIME_%.TIFF").into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (-180., 85.).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + }, + width: 3600, + height: 1700, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }, result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: Some(TimeInterval::new_unchecked( - TimeInstance::from_str("2014-01-01T00:00:00.000Z") - .expect("it should only be used in tests"), - TimeInstance::from_str("2014-07-01T00:00:00.000Z") - .expect("it should only be used in tests"), - )), - bbox: Some(SpatialPartition2D::new_unchecked( - (-180., 90.).into(), - (180., -90.).into(), - )), - resolution: Some(SpatialResolution::new_unchecked(0.1, 0.1)), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 0.1, -0.1), + GridBoundingBox2D::new([-850, -1800], [-845, -1799]) + .expect("should only be used in tests"), + ), + time: TimeDescriptor::new_regular_with_epoch(Some(time_bounds), time_step), bands: vec![RasterBandDescriptor { name: "ndvi".to_string(), measurement: Measurement::Continuous(ContinuousMeasurement { @@ -117,6 +200,101 @@ pub fn add_ndvi_dataset(ctx: &mut MockExecutionContext) -> NamedData { name } +pub fn add_ndvi_dataset_cropped_to_valid_webmercator_bounds( + ctx: &mut MockExecutionContext, +) -> NamedData { + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ndvi_crop_y_85"); + ctx.add_meta_data(id, name.clone(), Box::new(create_ndvi_meta_data())); + name +} + +#[allow(clippy::missing_panics_doc)] +pub fn create_ndvi_downscaled_3x_meta_data_with_cache_ttl( + cache_ttl: CacheTtlSeconds, +) -> GdalMetaDataRegular { + let no_data_value = Some(0.); + let time_bounds = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z").expect("should only be used in tests"), + TimeInstance::from_str("2014-07-01T00:00:00.000Z").expect("should only be used in tests"), + ); + let time_step = TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }; + GdalMetaDataRegular { + data_time: TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("it should only be used in tests"), + TimeInstance::from_str("2014-01-01T00:00:00.000Z") + .expect("it should only be used in tests"), + ), + step: TimeStep { + granularity: TimeGranularity::Months, + step: 1, + }, + time_placeholders: hashmap! { + "%_START_TIME_%".to_string() => GdalSourceTimePlaceholder { + format: DateTimeParseFormat::custom("%Y-%m-%d".to_string()), + reference: TimeReference::Start, + }, + }, + params: GdalDatasetParameters { + file_path: test_data!( + "raster/modis_ndvi/downscaled_3x/MOD13A2_M_NDVI_%_START_TIME_%.TIFF" + ) + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: (-180., 90.).into(), + x_pixel_size: 0.3, + y_pixel_size: -0.3, + }, + width: 1200, + height: 600, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + retry: None, + }, + result_descriptor: RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_regular_with_epoch(Some(time_bounds), time_step), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((-180., 90.).into(), 0.3, -0.3), + GridBoundingBox2D::new([0, 0], [599, 1199]).expect("should only be used in tests"), + ), + bands: vec![RasterBandDescriptor { + name: "ndvi".to_string(), + measurement: Measurement::Continuous(ContinuousMeasurement { + measurement: "vegetation".to_string(), + unit: None, + }), + }] + .try_into() + .expect("it should only be used in tests"), + }, + cache_ttl, + } +} + +pub fn add_ndvi_downscaled_3x_dataset(ctx: &mut MockExecutionContext) -> NamedData { + let id: DataId = DatasetId::new().into(); + let name = NamedData::with_system_name("ndvi_downscaled_3x"); + ctx.add_meta_data( + id, + name.clone(), + Box::new(create_ndvi_downscaled_3x_meta_data_with_cache_ttl( + CacheTtlSeconds::default(), + )), + ); + name +} + #[allow(clippy::missing_panics_doc)] pub fn create_ports_meta_data() -> StaticMetaData { @@ -240,14 +418,17 @@ pub fn raster_descriptor_from_dataset( let data_type = RasterDataType::from_gdal_data_type(rasterband.band_type()) .map_err(|_| Error::GdalRasterDataTypeNotSupported)?; - let geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_shape = GridShape2D::new([dataset.raster_size().1, dataset.raster_size().0]); Ok(RasterResultDescriptor { data_type, spatial_reference: spatial_ref.into(), - time: None, - bbox: None, - resolution: Some(geo_transfrom.spatial_resolution()), + time: TimeDescriptor::new_irregular(None), // TODO: can we parse the time info from the dataset? + spatial_grid: SpatialGridDescriptor::source_from_parts( + data_geo_transfrom, + data_shape.bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), // TODO: derive better name? measurement_from_rasterband(dataset, band)?, @@ -266,14 +447,17 @@ pub fn raster_descriptor_from_dataset_and_sref( let data_type = RasterDataType::from_gdal_data_type(rasterband.band_type()) .map_err(|_| Error::GdalRasterDataTypeNotSupported)?; - let geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_geo_transfrom = GeoTransform::from(dataset.geo_transform()?); + let data_shape = GridShape2D::new([dataset.raster_size().1, dataset.raster_size().0]); Ok(RasterResultDescriptor { data_type, spatial_reference: spatial_ref.into(), - time: None, - bbox: None, - resolution: Some(geo_transfrom.spatial_resolution()), + time: TimeDescriptor::new_irregular(None), // TODO: can we parse the time info from the dataset? + spatial_grid: SpatialGridDescriptor::source_from_parts( + data_geo_transfrom, + data_shape.bounding_box(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), // TODO derive better name? measurement_from_rasterband(dataset, band)?, @@ -281,7 +465,7 @@ pub fn raster_descriptor_from_dataset_and_sref( }) } -fn measurement_from_rasterband(dataset: &Dataset, band: usize) -> Result { +pub fn measurement_from_rasterband(dataset: &Dataset, band: usize) -> Result { let unit = dataset.rasterband(band)?.unit(); if unit.trim().is_empty() || unit == "no unit" { @@ -352,6 +536,6 @@ pub fn register_gdal_drivers_from_list(mut drivers: HashSet = drivers.into_iter().collect(); drivers.sort(); let remaining_drivers = drivers.into_iter().join(", "); - tracing::warn!("Could not register drivers: {remaining_drivers}"); + warn!("Could not register drivers: {remaining_drivers}"); } } diff --git a/operators/src/util/input/multi_raster_or_vector.rs b/operators/src/util/input/multi_raster_or_vector.rs index 49109bd4c..95e274093 100644 --- a/operators/src/util/input/multi_raster_or_vector.rs +++ b/operators/src/util/input/multi_raster_or_vector.rs @@ -68,9 +68,7 @@ mod tests { fn it_serializes() { let operator = MultiRasterOrVectorOperator::Raster(vec![ GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("foo", "bar"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("foo", "bar")), } .boxed(), ]); @@ -80,7 +78,8 @@ mod tests { serde_json::json!([{ "type": "GdalSource", "params": { - "data": "foo:bar" + "data": "foo:bar", + "overviewLevel": null, } }]) ); diff --git a/operators/src/util/input/raster_or_vector.rs b/operators/src/util/input/raster_or_vector.rs index c05854000..7a4cc6d06 100644 --- a/operators/src/util/input/raster_or_vector.rs +++ b/operators/src/util/input/raster_or_vector.rs @@ -67,9 +67,7 @@ mod tests { fn it_serializes() { let operator = RasterOrVectorOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData::with_namespaced_name("foo", "bar"), - }, + params: GdalSourceParameters::new(NamedData::with_namespaced_name("foo", "bar")), } .boxed(), ); @@ -79,7 +77,8 @@ mod tests { serde_json::json!({ "type": "GdalSource", "params": { - "data": "foo:bar" + "data": "foo:bar", + "overviewLevel": null, } }) ); diff --git a/operators/src/util/mod.rs b/operators/src/util/mod.rs index 4dcb0e1d6..25c4ba668 100644 --- a/operators/src/util/mod.rs +++ b/operators/src/util/mod.rs @@ -12,6 +12,8 @@ pub mod stream_zip; pub mod string_token; pub mod sunpos; mod temporary_gdal_thread_local_config_options; +pub mod test; +mod wrap_with_projection_and_resample; use crate::error::Error; use std::collections::HashSet; @@ -23,6 +25,7 @@ pub use self::async_util::{ }; pub use self::rayon::create_rayon_thread_pool; pub use self::temporary_gdal_thread_local_config_options::TemporaryGdalThreadLocalConfigOptions; +pub use wrap_with_projection_and_resample::WrapWithProjectionAndResample; pub type Result = std::result::Result; diff --git a/operators/src/util/raster_stream_to_geotiff.rs b/operators/src/util/raster_stream_to_geotiff.rs index 4c0ba6fad..359a8920a 100644 --- a/operators/src/util/raster_stream_to_geotiff.rs +++ b/operators/src/util/raster_stream_to_geotiff.rs @@ -1,3 +1,4 @@ +use crate::engine::{BoxRasterQueryProcessor, QueryProcessor}; use crate::error; use crate::source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, @@ -13,15 +14,12 @@ use futures::future::BoxFuture; use futures::{StreamExt, TryFutureExt}; use gdal::raster::{Buffer, GdalType, RasterBand, RasterCreationOptions}; use gdal::{Dataset, DriverManager, Metadata}; -use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, DateTimeParseFormat, QueryRectangle, RasterQueryRectangle, - SpatialPartition2D, TimeInterval, -}; use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds}; +use geoengine_datatypes::primitives::{DateTimeParseFormat, RasterQueryRectangle, TimeInterval}; use geoengine_datatypes::raster::{ - ChangeGridBounds, EmptyGrid2D, GeoTransform, GridBlit, GridIdx, GridIdx2D, GridSize, - MapElements, MaskedGrid2D, NoDataValueGrid, Pixel, RasterTile2D, TilingSpecification, - TilingStrategy, + ChangeGridBounds, GeoTransform, GridBlit, GridBoundingBox2D, GridBounds, GridIntersection, + GridOrEmpty, GridSize, MapElements, MaskedGrid2D, NoDataValueGrid, Pixel, RasterTile2D, + TilingSpecification, TilingStrategy, }; use geoengine_datatypes::spatial_reference::SpatialReference; use serde::{Deserialize, Serialize}; @@ -39,7 +37,7 @@ use super::{abortable_query_execution, spawn_blocking}; /// time series #[allow(clippy::too_many_arguments)] pub async fn raster_stream_to_multiband_geotiff_bytes( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, mut query_ctx: C, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, @@ -51,6 +49,10 @@ pub async fn raster_stream_to_multiband_geotiff_bytes( tiles: &[RasterTile2D], - query_rect: &QueryRectangle, - tiling_specification: TilingSpecification, + query_rect: &RasterQueryRectangle, + tiling_strategy: TilingStrategy, gdal_tiff_options: GdalGeoTiffOptions, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, ) -> Result<(TimeInterval, PathBuf, Dataset, GdalDatasetWriter), Error> @@ -131,28 +133,22 @@ where geo_transform: initial_tile_info.global_geo_transform, }; let num_tiles_per_timestep = strat - .tile_grid_box(query_rect.spatial_bounds) + .global_pixel_grid_bounds_to_tile_grid_bounds(query_rect.spatial_bounds()) .number_of_elements(); let num_timesteps = tiles.len() / num_tiles_per_timestep; - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as usize; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as usize; - let output_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_pixel_size, - -y_pixel_size, - ); + let x_pixel_size = tiling_strategy.geo_transform.x_pixel_size(); + let y_pixel_size = tiling_strategy.geo_transform.y_pixel_size(); - let global_geo_transform = tiling_specification - .strategy(x_pixel_size, -y_pixel_size) - .geo_transform; - let window_start = - global_geo_transform.coordinate_to_grid_idx_2d(query_rect.spatial_bounds.upper_left()); - let window_end = window_start + GridIdx2D::from([height as isize, width as isize]); + let coordinate_of_ul_query_pixel = strat + .geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(query_rect.spatial_bounds().min_index()); + let output_geo_transform = + GeoTransform::new(coordinate_of_ul_query_pixel, x_pixel_size, y_pixel_size); + let out_pixel_bounds = query_rect.spatial_bounds(); - let uncompressed_byte_size = width * height * std::mem::size_of::(); + let uncompressed_byte_size = + query_rect.spatial_bounds().number_of_elements() * std::mem::size_of::(); let use_big_tiff = gdal_tiff_options.force_big_tiff || uncompressed_byte_size >= BIG_TIFF_BYTE_THRESHOLD; @@ -185,8 +181,8 @@ where let mut dataset = driver.create_with_band_type_with_options::( &file_path, - width, - height, + query_rect.spatial_bounds().axis_size_x(), + query_rect.spatial_bounds().axis_size_y(), num_timesteps, &options, )?; @@ -207,12 +203,10 @@ where let writer = GdalDatasetWriter:: { gdal_tiff_options, gdal_tiff_metadata, - _output_bounds: query_rect.spatial_bounds, + output_pixel_grid_bounds: out_pixel_bounds, output_geo_transform, use_big_tiff, _type: Default::default(), - window_start, - window_end, }; drop(option_vars); @@ -221,8 +215,8 @@ where } async fn consume_stream_into_vec( - processor: Box>, - query_rect: geoengine_datatypes::primitives::QueryRectangle, + processor: BoxRasterQueryProcessor, + query_rect: geoengine_datatypes::primitives::RasterQueryRectangle, query_ctx: C, tile_limit: Option, ) -> Result>> @@ -252,14 +246,13 @@ pub async fn single_timestep_raster_stream_to_geotiff_bytes< T, C: QueryContext + 'static, >( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, query_ctx: C, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, tile_limit: Option, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, progress_consumer: G, ) -> Result> where @@ -273,7 +266,6 @@ where gdal_tiff_options, tile_limit, conn_closed, - tiling_specification, progress_consumer, ) .await?; @@ -296,14 +288,13 @@ pub async fn raster_stream_to_geotiff_bytes< T, C: QueryContext + 'static, >( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, query_ctx: C, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, tile_limit: Option, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, progress_consumer: G, ) -> Result>> where @@ -320,7 +311,6 @@ where gdal_tiff_options, tile_limit, conn_closed, - tiling_specification, progress_consumer, ) .await? @@ -337,14 +327,13 @@ where #[allow(clippy::too_many_arguments, clippy::missing_panics_doc)] pub async fn raster_stream_to_geotiff( file_path: &Path, - processor: Box>, + processor: BoxRasterQueryProcessor

, query_rect: RasterQueryRectangle, mut query_ctx: C, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, tile_limit: Option, conn_closed: BoxFuture<'_, ()>, - tiling_specification: TilingSpecification, progress_consumer: G, ) -> Result> where @@ -352,7 +341,7 @@ where { // TODO: support multi band geotiffs ensure!( - query_rect.attributes.count() == 1, + query_rect.attributes().is_single() || processor.result_descriptor().bands.is_single(), crate::error::OperationDoesNotSupportMultiBandQueriesYet { operation: "raster_stream_to_geotiff" } @@ -360,6 +349,12 @@ where let query_abort_trigger = query_ctx.abort_trigger()?; + let tiling_strategy = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(query_ctx.tiling_specification()) + .generate_data_tiling_strategy(); + // TODO: create file path if it doesn't exist let file_path = file_path.to_owned(); @@ -374,8 +369,8 @@ where None }; - let mut dataset_holder: GdalDatasetHolder

= GdalDatasetHolder::new_with_tiling_spec( - tiling_specification, + let mut dataset_holder: GdalDatasetHolder

= GdalDatasetHolder::new_with_tiling_strat( + tiling_strategy, &file_path, &query_rect, gdal_tiff_metadata, @@ -388,12 +383,8 @@ where .await? .enumerate(); - let tiles_intersecting_qrect = tiling_specification - .strategy( - query_rect.spatial_resolution.x, - -query_rect.spatial_resolution.y, - ) - .num_tiles_intersecting(query_rect.spatial_bounds); + let tiles_intersecting_qrect = + tiling_strategy.num_tiles_intersecting_grid_bounds(query_rect.spatial_bounds()); let mut tile_count = 0; @@ -607,8 +598,7 @@ impl GdalDatasetHolder

{ gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, gdal_config_options: Option>, - window_start: GridIdx2D, - window_end: GridIdx2D, + tiling_strategy: TilingStrategy, ) -> Self { const INTERMEDIATE_FILE_SUFFIX: &str = "GEO-ENGINE-TMP"; @@ -627,27 +617,33 @@ impl GdalDatasetHolder

{ let file_path = file_path.join("raster.tiff"); let intermediate_file_path = file_path.with_extension(INTERMEDIATE_FILE_SUFFIX); - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as u32; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as u32; + let width = query_rect.spatial_bounds().axis_size_x(); + let height = query_rect.spatial_bounds().axis_size_y(); + + let output_pixel_grid_bounds = query_rect.spatial_bounds(); + + let out_geo_transform_origin = tiling_strategy + .geo_transform + .grid_idx_to_pixel_upper_left_coordinate_2d(query_rect.spatial_bounds().min_index()); let output_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_pixel_size, - -y_pixel_size, + out_geo_transform_origin, + tiling_strategy.geo_transform.x_pixel_size(), + tiling_strategy.geo_transform.y_pixel_size(), ); + let output_gdal_geo_transform = GdalDatasetGeoTransform { + origin_coordinate: out_geo_transform_origin, + x_pixel_size: tiling_strategy.geo_transform.x_pixel_size(), + y_pixel_size: tiling_strategy.geo_transform.y_pixel_size(), + }; + let intermediate_dataset_parameters = GdalDatasetParameters { file_path: intermediate_file_path, rasterband_channel: 1, - geo_transform: GdalDatasetGeoTransform { - origin_coordinate: query_rect.spatial_bounds.upper_left(), - x_pixel_size, - y_pixel_size: -y_pixel_size, - }, - width: width as usize, - height: height as usize, + geo_transform: output_gdal_geo_transform, + width, + height, file_not_found_handling: FileNotFoundHandling::Error, no_data_value: None, // `None` will let the GdalSource detect the correct no-data value. properties_mapping: None, // TODO: add properties @@ -674,8 +670,8 @@ impl GdalDatasetHolder

{ intermediate_dataset: None, create_meta: IntermediateDatasetMetadata { raster_band_index: rasterband_index, - width, - height, + width: width as u32, + height: height as u32, use_big_tiff, path_with_placeholder, gdal_config_options, @@ -684,12 +680,10 @@ impl GdalDatasetHolder

{ dataset_writer: GdalDatasetWriter { gdal_tiff_options, gdal_tiff_metadata, - _output_bounds: query_rect.spatial_bounds, + output_pixel_grid_bounds, output_geo_transform, use_big_tiff, _type: Default::default(), - window_start, - window_end, }, result: vec![], } @@ -766,37 +760,21 @@ impl GdalDatasetHolder

{ Ok(()) } - fn new_with_tiling_spec( - tiling_specification: TilingSpecification, + fn new_with_tiling_strat( + tiling_strategy: TilingStrategy, file_path: &Path, query_rect: &RasterQueryRectangle, gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, gdal_config_options: Option>, ) -> Self { - let x_pixel_size = query_rect.spatial_resolution.x; - let y_pixel_size = query_rect.spatial_resolution.y; - - let width = (query_rect.spatial_bounds.size_x() / x_pixel_size).ceil() as u32; - let height = (query_rect.spatial_bounds.size_y() / y_pixel_size).ceil() as u32; - - let global_geo_transform = tiling_specification - .strategy(x_pixel_size, -y_pixel_size) - .geo_transform; - - let window_start = - global_geo_transform.coordinate_to_grid_idx_2d(query_rect.spatial_bounds.upper_left()); - - let window_end = window_start + GridIdx2D::from([height as isize, width as isize]); - Self::new( file_path, query_rect, gdal_tiff_metadata, gdal_tiff_options, gdal_config_options, - window_start, - window_end, + tiling_strategy, ) } @@ -847,82 +825,43 @@ pub struct GdalGeoTiffDatasetMetadata { struct GdalDatasetWriter { gdal_tiff_metadata: GdalGeoTiffDatasetMetadata, gdal_tiff_options: GdalGeoTiffOptions, - _output_bounds: SpatialPartition2D, // currently unused due to workaround for intersection and contained because of float precision + output_pixel_grid_bounds: GridBoundingBox2D, output_geo_transform: GeoTransform, use_big_tiff: bool, _type: std::marker::PhantomData

, - window_start: GridIdx2D, - window_end: GridIdx2D, } impl GdalDatasetWriter

{ fn write_tile_into_band(&self, tile: RasterTile2D

, raster_band: RasterBand) -> Result<()> { let tile_info = tile.tile_information(); - let tile_start = tile_info.global_upper_left_pixel_idx(); - let [tile_height, tile_width] = tile_info.tile_size_in_pixels.shape_array; - let tile_end = tile_start + GridIdx2D::from([tile_height as isize, tile_width as isize]); - - let GridIdx([tile_start_y, tile_start_x]) = tile_start; - let GridIdx([tile_end_y, tile_end_x]) = tile_end; - let GridIdx([window_start_y, window_start_x]) = self.window_start; - let GridIdx([window_end_y, window_end_x]) = self.window_end; - - // compute the upper left pixel index in the output raster and extract the input data - let (GridIdx([output_ul_y, output_ul_x]), grid_array) = - // TODO: check contains on the `SpatialPartition2D`s once the float precision issue is fixed - if tile_start_x >= window_start_x && tile_start_y >= window_start_y && tile_end_x <= window_end_x && tile_end_y <= window_end_y { - // tile is completely inside the output raster - ( - tile_info.global_upper_left_pixel_idx() - self.window_start, - tile.into_materialized_tile().grid_array, - ) - } else { - // extract relevant data from tile (intersection with output_bounds) - - // TODO: compute the intersection on the `SpatialPartition2D`s once the float precision issue is fixed - - if tile_end_y < window_start_y - || tile_end_x < window_start_x - || tile_start_y >= window_end_y - || tile_start_x >= window_end_x - { - // tile is outside of output bounds - return Ok(()); - } - - let intersection_start = GridIdx2D::from([ - std::cmp::max(tile_start_y, window_start_y), - std::cmp::max(tile_start_x, window_start_x), - ]); - let GridIdx([intersection_start_y, intersection_start_x]) = intersection_start; - - let width = std::cmp::min( - tile_info.tile_size_in_pixels.axis_size_x() as isize, - window_end_x - intersection_start_x, - ); - - let height = std::cmp::min( - tile_info.tile_size_in_pixels.axis_size_y() as isize, - window_end_y - intersection_start_y, - ); - - let mut output_grid = - MaskedGrid2D::from(EmptyGrid2D::new([height as usize, width as usize].into())); - - let shift_offset = intersection_start - tile_start; - let shifted_source = tile - .grid_array - .shift_by_offset(GridIdx([-1, -1]) * shift_offset); - - output_grid.grid_blit_from(&shifted_source); - - (intersection_start - self.window_start, output_grid) - }; - - let window = (output_ul_x, output_ul_y); - let [shape_y, shape_x] = grid_array.axis_size(); - let window_size = (shape_x, shape_y); + let tile_grid_bounds = tile_info.global_pixel_bounds(); + + let out_data_bounds = self + .output_pixel_grid_bounds + .intersection(&tile_grid_bounds); + + if out_data_bounds.is_none() { + return Ok(()); + } + + let out_data_bounds = out_data_bounds.expect("was checked before"); + + let mut write_buffer_grid = GridOrEmpty::<_, P>::new_empty_shape(out_data_bounds); + + write_buffer_grid.grid_blit_from(&tile.into_inner_positioned_grid()); + + let window_start = out_data_bounds.min_index() - self.output_pixel_grid_bounds.min_index(); + let window = (window_start.x(), window_start.y()); + + let window_size = ( + write_buffer_grid.shape_ref().axis_size_x(), + write_buffer_grid.shape_ref().axis_size_y(), + ); + + let grid_array = write_buffer_grid + .into_materialized_masked_grid() + .unbounded(); // Check if the gdal_tiff_metadata no-data value is set. // If it is set write a geotiff with no-data values. @@ -1014,7 +953,6 @@ fn create_gdal_tiff_options( ) -> Result { let mut options = RasterCreationOptions::new(); options.add_name_value("COMPRESS", COMPRESSION_FORMAT)?; - options.add_name_value("TILED", "YES")?; options.add_name_value("ZLEVEL", COMPRESSION_LEVEL)?; options.add_name_value("NUM_THREADS", compression_num_threads)?; options.add_name_value("INTERLEAVE", "BAND")?; @@ -1023,6 +961,8 @@ fn create_gdal_tiff_options( // COGs require a block size of 512x512, so we enforce it now so that we do the work only once. options.add_name_value("BLOCKXSIZE", COG_BLOCK_SIZE)?; options.add_name_value("BLOCKYSIZE", COG_BLOCK_SIZE)?; + } else { + options.add_name_value("TILED", "YES")?; } if as_big_tiff { @@ -1119,58 +1059,47 @@ fn geotiff_to_cog( #[cfg(test)] mod tests { - use super::*; - use crate::engine::RasterResultDescriptor; + use std::marker::PhantomData; + use std::ops::Add; + + use crate::engine::{MockExecutionContext, RasterResultDescriptor, TimeDescriptor}; use crate::mock::MockRasterSourceProcessor; use crate::util::gdal::gdal_open_dataset; - use crate::{ - engine::MockQueryContext, source::GdalSourceProcessor, util::gdal::create_ndvi_meta_data, + use crate::{source::GdalSourceProcessor, util::gdal::create_ndvi_meta_data}; + use geoengine_datatypes::primitives::{ + BandSelection, CacheHint, DateTime, Duration, SpatialPartition2D, TimeInterval, }; - use geoengine_datatypes::primitives::CacheHint; - use geoengine_datatypes::primitives::{DateTime, Duration}; - use geoengine_datatypes::raster::{Grid, RasterDataType}; + use geoengine_datatypes::raster::{Grid, GridBoundingBox2D, RasterDataType}; use geoengine_datatypes::test_data; + use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::util::{ImageFormat, assert_image_equals_with_format}; - use geoengine_datatypes::{ - primitives::{Coordinate2D, SpatialPartition2D, SpatialResolution, TimeInterval}, - raster::TilingSpecification, - util::test::TestDefault, - }; - use std::marker::PhantomData; - use std::ops::Add; + + use super::*; #[tokio::test] async fn geotiff_with_no_data_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1183,7 +1112,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1194,45 +1122,37 @@ mod tests { // "../test_data/raster/geotiff_from_stream_compressed.tiff", // ); - assert_image_equals_with_format( - test_data!("raster/geotiff_from_stream_compressed.tiff"), - &bytes, - ImageFormat::Tiff, + assert_eq!( + include_bytes!("../../../test_data/raster/geotiff_from_stream_compressed.tiff") + as &[u8], + bytes.as_slice() ); } #[tokio::test] async fn geotiff_with_mask_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: None, @@ -1245,7 +1165,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1260,36 +1179,28 @@ mod tests { #[tokio::test] async fn geotiff_big_tiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1302,7 +1213,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1322,36 +1232,28 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_big_tiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1364,7 +1266,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1386,36 +1287,28 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1428,7 +1321,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1450,39 +1342,28 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_multiple_timesteps_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let mut bytes = raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new( - 1_388_534_400_000, - 1_388_534_400_000 + 7_776_000_000, - ) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 7_776_000_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1495,7 +1376,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1537,39 +1417,28 @@ mod tests { #[tokio::test] async fn cloud_optimized_geotiff_multiple_timesteps_from_stream_wrong_request() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new( - 1_388_534_400_000, - 1_388_534_400_000 + 7_776_000_000, - ) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 7_776_000_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1582,7 +1451,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await; @@ -1592,36 +1460,28 @@ mod tests { #[tokio::test] async fn geotiff_from_stream_limit() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - query_bbox.size_x() / 600., - query_bbox.size_y() / 600., - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-201, 499]).unwrap(), + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1634,7 +1494,6 @@ mod tests { }, Some(1), Box::pin(futures::future::pending()), - tiling_specification, (), ) .await; @@ -1644,38 +1503,38 @@ mod tests { #[tokio::test] async fn geotiff_from_stream_in_range_of_window() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); + let query_bbox = + SpatialPartition2D::new((-180., -66.227_224_576_271_84).into(), (180., -90.).into()) + .unwrap(); + + let query_grid_bounds = metadata + .result_descriptor + .tiling_grid_definition(ctx.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds(&query_bbox); + let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), - tiling_specification, + produced_result_descriptor: metadata.result_descriptor.clone(), + tiling_specification: ctx.tiling_specification(), + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = - SpatialPartition2D::new((-180., -66.227_224_576_271_84).into(), (180., -90.).into()) - .unwrap(); - let bytes = single_timestep_raster_stream_to_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.228_716_645_489_199_48, - 0.226_407_384_987_887_26, - ), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + query_grid_bounds, + TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: Some(0.), @@ -1688,7 +1547,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await; @@ -1743,27 +1601,42 @@ mod tests { }, ]; + let time_bounds = + TimeInterval::new(data[0].time.start(), data.last().unwrap().time.end()).unwrap(); + + let ecx = + MockExecutionContext::new_with_tiling_spec(TilingSpecification::new([600, 600].into())); + let ctx = ecx.mock_query_context_test_default(); + + let result_descriptor = RasterResultDescriptor::with_datatype_and_num_bands( + RasterDataType::U8, + 1, + GridBoundingBox2D::new([-4, -4], [4, 4]).unwrap(), + GeoTransform::test_default(), + TimeDescriptor::new_regular( + Some(time_bounds), + time_bounds.start(), + time_step.try_into().unwrap(), + ), + ); + let query_time = TimeInterval::new(data[0].time.start(), data[1].time.end()).unwrap(); - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); let processor = MockRasterSourceProcessor { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + result_descriptor, data, - tiling_specification, + tiling_specification: ctx.tiling_specification(), } .boxed(); - let query_rectangle = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 2.).into(), (2., 0.).into()).unwrap(), - time_interval: query_time, - spatial_resolution: GeoTransform::test_default().spatial_resolution(), - attributes: BandSelection::first(), - }; + let query_rectangle: geoengine_datatypes::primitives::QueryRectangle< + geoengine_datatypes::raster::GridBoundingBox<[isize; 2]>, + BandSelection, + > = RasterQueryRectangle::new( + GridBoundingBox2D::new([-2, -1], [0, 1]).unwrap(), + query_time, + BandSelection::first(), + ); let file_path = PathBuf::from(format!("/vsimem/{}/", uuid::Uuid::new_v4())); let expected_paths = file_suffixes @@ -1786,7 +1659,6 @@ mod tests { }, None, Box::pin(futures::future::pending()), - tiling_specification, (), ) .await @@ -1861,33 +1733,30 @@ mod tests { #[tokio::test] async fn multi_band_geotriff() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [512, 512].into()); + let exe_ctx = MockExecutionContext::test_default(); + let ctx = exe_ctx.mock_query_context_test_default(); let metadata = create_ndvi_meta_data(); + let tiling_specification = TilingSpecification::new([512, 512].into()); + let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: metadata.result_descriptor.clone(), tiling_specification, + overview_level: 0, meta_data: Box::new(metadata), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_bbox = SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); - let (mut bytes, _) = raster_stream_to_multiband_geotiff_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_bbox, + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(0, 1799, 0, 3599).unwrap(), // 1.1.2014 - 1.4.2014 - time_interval: TimeInterval::new(1_388_534_400_000, 1_396_306_800_000).unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1), - attributes: BandSelection::first(), - }, + TimeInterval::new(1_388_534_400_000, 1_396_306_800_000).unwrap(), + BandSelection::first(), + ), ctx, GdalGeoTiffDatasetMetadata { no_data_value: None, diff --git a/operators/src/util/raster_stream_to_png.rs b/operators/src/util/raster_stream_to_png.rs index 1614af499..0bbf08ffc 100644 --- a/operators/src/util/raster_stream_to_png.rs +++ b/operators/src/util/raster_stream_to_png.rs @@ -1,15 +1,15 @@ use super::abortable_query_execution; -use crate::engine::{QueryAbortTrigger, QueryContext, QueryProcessor, RasterQueryProcessor}; +use crate::engine::{BoxRasterQueryProcessor, QueryAbortTrigger, QueryContext, QueryProcessor}; use crate::util::Result; use futures::TryStreamExt; use futures::{StreamExt, future::BoxFuture}; use geoengine_datatypes::error::{BoxedResultExt, ErrorSource}; -use geoengine_datatypes::operations::image::{ColorMapper, RgbParams}; -use geoengine_datatypes::raster::{FromIndexFn, GridIndexAccess, GridShapeAccess}; +use geoengine_datatypes::operations::image::{ColorMapper, RasterColorizer, RgbParams}; +use geoengine_datatypes::raster::{FromIndexFn, GridIndexAccess, GridShapeAccess, RasterTile2D}; use geoengine_datatypes::{ - operations::image::{Colorizer, RasterColorizer, RgbaColor, ToPng}, - primitives::{AxisAlignedRectangle, CacheHint, RasterQueryRectangle, TimeInterval}, - raster::{Blit, ConvertDataType, EmptyGrid2D, GeoTransform, GridOrEmpty, Pixel, RasterTile2D}, + operations::image::{Colorizer, RgbaColor, ToPng}, + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, + raster::{ChangeGridBounds, GridBlit, GridBoundingBox2D, GridOrEmpty, Pixel}, }; use num_traits::AsPrimitive; use snafu::Snafu; @@ -20,19 +20,19 @@ use tracing::{Level, span}; /// Panics if not three bands were queried. #[allow(clippy::too_many_arguments)] pub async fn raster_stream_to_png_bytes( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, mut query_ctx: C, width: u32, height: u32, - time: Option, + _time: Option, raster_colorizer: Option, conn_closed: BoxFuture<'_, ()>, ) -> Result<(Vec, CacheHint)> { debug_assert!( - query_rect.attributes.count() <= 3 + query_rect.attributes().count() <= 3 || query_rect - .attributes + .attributes() .as_slice() .windows(2) .all(|w| w[0] < w[1]), // TODO: replace with `is_sorted` once it is stable @@ -64,7 +64,7 @@ pub async fn raster_stream_to_png_bytes( .iter() .filter_map(|band| { query_rect - .attributes + .attributes() .as_slice() .iter() .position(|b| b == band) @@ -73,38 +73,19 @@ pub async fn raster_stream_to_png_bytes( if band_positions.len() != required_bands.len() { return Err(PngCreationError::ColorizerBandsMustBePresentInQuery { - bands_present: query_rect.attributes.as_vec(), + bands_present: query_rect.attributes().as_vec(), required_bands, })?; } let query_abort_trigger = query_ctx.abort_trigger()?; - let x_query_resolution = query_rect.spatial_bounds.size_x() / f64::from(width); - let y_query_resolution = query_rect.spatial_bounds.size_y() / f64::from(height); - - // build png - let dim = [height as usize, width as usize]; - let query_geo_transform = GeoTransform::new( - query_rect.spatial_bounds.upper_left(), - x_query_resolution, - -y_query_resolution, // TODO: negative, s.t. geo transform fits... - ); - - let tile_template: RasterTile2D = RasterTile2D::new_without_offset( - time.unwrap_or_default(), - query_geo_transform, - GridOrEmpty::from(EmptyGrid2D::new(dim.into())), - CacheHint::max_duration(), - ); - match raster_colorizer { RasterColorizer::SingleBand { band_colorizer, .. } => { single_band_colorizer_to_png_bytes( processor, query_rect, query_ctx, - tile_template, width, height, band_colorizer, @@ -121,7 +102,6 @@ pub async fn raster_stream_to_png_bytes( processor, query_rect, query_ctx, - tile_template, width, height, rgba_params, @@ -136,39 +116,47 @@ pub async fn raster_stream_to_png_bytes( #[allow(clippy::too_many_arguments)] async fn single_band_colorizer_to_png_bytes( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, query_ctx: C, - tile_template: RasterTile2D, width: u32, height: u32, colorizer: Colorizer, conn_closed: BoxFuture<'_, ()>, query_abort_trigger: QueryAbortTrigger, ) -> Result<(Vec, CacheHint)> { - debug_assert_eq!(query_rect.attributes.count(), 1); + debug_assert_eq!(query_rect.attributes().count(), 1); + // the tile stream will allways produce tiles aligned to the tiling origin let tile_stream = processor.query(query_rect.clone(), &query_ctx).await?; + let output_cache_hint = CacheHint::max_duration(); - let output_tile = Box::pin( - tile_stream.fold(Ok(tile_template), |raster, tile| async move { - blit_tile(raster, tile) - }), - ); + let output_grid = + GridOrEmpty::::new_empty_shape(query_rect.spatial_bounds()); - let result = abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; - Ok(( - result.grid_array.to_png(width, height, &colorizer)?, - result.cache_hint, - )) + let accu = Ok((output_grid, output_cache_hint)); + + let output_tile: BoxFuture, CacheHint)>> = + Box::pin(tile_stream.fold(accu, |accu, tile| { + let result: Result<(GridOrEmpty, CacheHint)> = + blit_tile(accu, tile); + + match result { + Ok(updated_raster2d) => futures::future::ok(updated_raster2d), + Err(error) => futures::future::err(error), + } + })); + + let (result, ch) = + abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; + Ok((result.unbounded().to_png(width, height, &colorizer)?, ch)) } #[allow(clippy::too_many_arguments)] async fn multi_band_colorizer_to_png_bytes( - processor: Box>, + processor: BoxRasterQueryProcessor, query_rect: RasterQueryRectangle, query_ctx: C, - tile_template: RasterTile2D, width: u32, height: u32, rgb_params: RgbParams, @@ -176,17 +164,20 @@ async fn multi_band_colorizer_to_png_bytes( conn_closed: BoxFuture<'_, ()>, query_abort_trigger: QueryAbortTrigger, ) -> Result<(Vec, CacheHint)> { - let rgb_channel_count = query_rect.attributes.count() as usize; + let rgb_channel_count = query_rect.attributes().count() as usize; let no_data_color = rgb_params.no_data_color; - let tile_template: RasterTile2D = tile_template.convert_data_type(); + let tile_template: GridOrEmpty = + GridOrEmpty::new_empty_shape(query_rect.spatial_bounds()); + let output_cache_hint = CacheHint::max_duration(); let red_band_index = band_positions[0]; let green_band_index = band_positions[1]; let blue_band_index = band_positions[2]; let tile_stream = processor.query(query_rect.clone(), &query_ctx).await?; + let accu = Ok((tile_template, output_cache_hint)); let output_tile = Box::pin(tile_stream.try_chunks(rgb_channel_count).fold( - Ok(tile_template), + accu, |raster2d, chunk| async move { let chunk = chunk.boxed_context(error::QueryDidNotProduceNextChunk)?; @@ -212,31 +203,30 @@ async fn multi_band_colorizer_to_png_bytes( }, )); - let result = abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; + let (result, ch) = + abortable_query_execution(output_tile, conn_closed, query_abort_trigger).await?; Ok(( result - .grid_array + .unbounded() .to_png_with_mapper(width, height, ColorMapper::Rgba, no_data_color)?, - result.cache_hint, + ch, )) } fn blit_tile( - raster2d: Result>, + accu: Result<(GridOrEmpty, CacheHint)>, tile: Result>, -) -> Result> +) -> Result<(GridOrEmpty, CacheHint)> where T: Pixel, { - let result: Result> = match (raster2d, tile) { - (Ok(mut raster2d), Ok(tile)) if tile.is_empty() => { - raster2d.cache_hint.merge_with(&tile.cache_hint); - Ok(raster2d) + let result: Result<(GridOrEmpty, CacheHint)> = match (accu, tile) { + (Ok((empty_grid, ch)), Ok(tile)) if tile.is_empty() => Ok((empty_grid, ch)), + (Ok((mut grid, mut ch)), Ok(tile)) => { + ch.merge_with(&tile.cache_hint); + grid.grid_blit_from(&tile.into_inner_positioned_grid()); + Ok((grid, ch)) } - (Ok(mut raster2d), Ok(tile)) => match raster2d.blit(tile) { - Ok(()) => Ok(raster2d), - Err(error) => Err(error.into()), - }, (Err(error), _) | (_, Err(error)) => Err(error), }; @@ -351,19 +341,16 @@ pub enum PngCreationError { mod tests { use std::marker::PhantomData; + use crate::engine::{MockExecutionContext, RasterQueryProcessor}; + use crate::{source::GdalSourceProcessor, util::gdal::create_ndvi_meta_data}; + use geoengine_datatypes::primitives::{DateTime, TimeInstance}; use geoengine_datatypes::{ - primitives::{BandSelection, Coordinate2D, SpatialPartition2D, SpatialResolution}, - raster::{RasterDataType, TilingSpecification}, + primitives::BandSelection, + raster::TilingSpecification, test_data, util::{assert_image_equals, test::TestDefault}, }; - use crate::{ - engine::{MockQueryContext, RasterResultDescriptor}, - source::GdalSourceProcessor, - util::gdal::create_ndvi_meta_data, - }; - use super::*; #[test] @@ -383,32 +370,30 @@ mod tests { #[tokio::test] async fn png_from_stream() { - let ctx = MockQueryContext::test_default(); - let tiling_specification = - TilingSpecification::new(Coordinate2D::default(), [600, 600].into()); + let exe_ctx = MockExecutionContext::test_default(); + let ctx = exe_ctx.mock_query_context_test_default(); + let tiling_specification = TilingSpecification::new([600, 600].into()); + + let meta_data = create_ndvi_meta_data(); let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification, - meta_data: Box::new(create_ndvi_meta_data()), + overview_level: 0, + meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_partition = - SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(); + let query = RasterQueryRectangle::new( + GridBoundingBox2D::new([-800, -100], [-199, 499]).unwrap(), + TimeInstance::from(DateTime::new_utc(2014, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + ); let (image_bytes, _) = raster_stream_to_png_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_partition, - time_interval: TimeInterval::new(1_388_534_400_000, 1_388_534_400_000 + 1000) - .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: BandSelection::first(), - }, + query, ctx, 600, 600, diff --git a/operators/src/util/test.rs b/operators/src/util/test.rs new file mode 100644 index 000000000..2bba3cd0c --- /dev/null +++ b/operators/src/util/test.rs @@ -0,0 +1,154 @@ +use std::path::Path; + +use super::Result; +use crate::{ + engine::{ExecutionContext, QueryContext, RasterOperator, WorkflowOperatorPath}, + util::gdal::gdal_open_dataset, +}; +use futures::StreamExt; +use gdal::raster::GdalType; +use geoengine_datatypes::{ + primitives::{CacheHint, RasterQueryRectangle, TimeInterval}, + raster::{ + GeoTransform, Grid, GridOrEmpty, GridShape2D, MapElements, MaskedGrid, Pixel, RasterTile2D, + TilingSpatialGridDefinition, + }, + util::test::assert_eq_two_list_of_tiles_u8, +}; + +pub async fn raster_operator_to_list_of_tiles_u8( + exe_ctx: &E, + query_ctx: &Q, + operator: Box, + query_rectangle: RasterQueryRectangle, +) -> Result>> { + let initialized_operator = operator + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await?; + let query_processor = initialized_operator.query_processor()?.get_u8().ok_or( + crate::error::Error::MustNotHappen { + message: "Operator does not produce u8 while this function requires it".to_owned(), + }, + )?; + + let res = query_processor + .raster_query(query_rectangle, query_ctx) + .await? + .collect::>() + .await; + + let res = res.into_iter().collect::, _>>()?; + + Ok(res) +} + +/// Compares the output of a raster operators and a list of tiles and panics with a message if they are not equal +/// +/// # Panics +/// +/// If there are tiles that are not equal +pub async fn assert_eq_raster_operator_res_and_list_of_tiles_u8< + E: ExecutionContext, + Q: QueryContext, +>( + exe_ctx: &E, + query_ctx: &Q, + operator: Box, + query_rectangle: RasterQueryRectangle, + compare_cache_hint: bool, + list_of_tiles: &[RasterTile2D], +) { + let res_a = raster_operator_to_list_of_tiles_u8(exe_ctx, query_ctx, operator, query_rectangle) + .await + .expect("raster operator to list failed!"); + + assert_eq_two_list_of_tiles_u8(&res_a, list_of_tiles, compare_cache_hint); +} + +/// Compares the output of two raster operators and panics with a message if they are not equal +/// +/// # Panics +/// +/// If there are tiles that are not equal +pub async fn assert_eq_two_raster_operator_res_u8( + exe_ctx: &E, + query_ctx: &Q, + operator_a: Box, + operator_b: Box, + query_rectangle: RasterQueryRectangle, + compare_cache_hint: bool, +) { + let res_a = raster_operator_to_list_of_tiles_u8( + exe_ctx, + query_ctx, + operator_a, + query_rectangle.clone(), + ) + .await + .expect("raster operator to list failed for operator_a!"); + + assert_eq_raster_operator_res_and_list_of_tiles_u8( + exe_ctx, + query_ctx, + operator_b, + query_rectangle, + compare_cache_hint, + &res_a, + ) + .await; +} + +/// Reads a single raster tile from a single file and returns it as a `RasterTile2D`. +/// This assumes that the file actually contains exactly one geoengine tile according to the spatial grid definition. +pub fn raster_tile_from_file( + file_path: &Path, + tiling_spatial_grid: TilingSpatialGridDefinition, + time: TimeInterval, + band: u32, +) -> Result> { + let ds = gdal_open_dataset(file_path)?; + + let gf: GeoTransform = ds.geo_transform()?.into(); + + let tiling_strategy = tiling_spatial_grid.generate_data_tiling_strategy(); + + let tiling_geo_transform = tiling_spatial_grid.tiling_geo_transform(); + let tile_origin_pixel_idx = + tiling_geo_transform.coordinate_to_grid_idx_2d(gf.origin_coordinate); + let tile_position = tiling_strategy.pixel_idx_to_tile_idx(tile_origin_pixel_idx); + + let rasterband = ds.rasterband(1)?; + + let out_shape: GridShape2D = GridShape2D::new([ds.raster_size().0, ds.raster_size().1]); + + let buffer = rasterband.read_as::( + (0, 0), + ds.raster_size(), + ds.raster_size(), // or: tile_size_in_pixels + None, + )?; + + let (_, buffer_data) = buffer.into_shape_and_vec(); + let data_grid = Grid::new(out_shape, buffer_data)?; + + let mask_band = rasterband.open_mask_band()?; + let mask_buffer = mask_band.read_as::( + (0, 0), + ds.raster_size(), + ds.raster_size(), // or: tile_size_in_pixels + None, + )?; + let (_, mask_buffer_data) = mask_buffer.into_shape_and_vec(); + let mask_grid = Grid::new(out_shape, mask_buffer_data)?.map_elements(|p: u8| p > 0); + + let masked_grid = MaskedGrid::new(data_grid, mask_grid)?; + + Ok(RasterTile2D::::new( + time, + tile_position, + band, + tiling_geo_transform, + GridOrEmpty::from(masked_grid), + CacheHint::default(), + )) +} diff --git a/operators/src/util/wrap_with_projection_and_resample.rs b/operators/src/util/wrap_with_projection_and_resample.rs new file mode 100644 index 000000000..d1663b5ef --- /dev/null +++ b/operators/src/util/wrap_with_projection_and_resample.rs @@ -0,0 +1,268 @@ +use crate::engine::{ + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, RasterOperator, + RasterResultDescriptor, ResultDescriptor, SingleRasterOrVectorSource, WorkflowOperatorPath, +}; +use crate::error::{self, Optimization}; +use crate::processing::{ + DeriveOutRasterSpecsSource, Downsampling, DownsamplingMethod, DownsamplingParams, + DownsamplingResolution, InitializedRasterReprojection, Interpolation, InterpolationMethod, + InterpolationParams, InterpolationResolution, Reprojection, ReprojectionParams, +}; +use crate::util::Result; +use crate::util::input::RasterOrVectorOperator; +use geoengine_datatypes::primitives::{ + Coordinate2D, SpatialResolution, find_next_best_overview_level_resolution, +}; +use geoengine_datatypes::raster::TilingSpecification; +use geoengine_datatypes::spatial_reference::SpatialReference; +use snafu::ResultExt; + +pub struct WrapWithProjectionAndResample { + pub operator: Box, + pub initialized_operator: Box, + pub result_descriptor: RasterResultDescriptor, +} + +impl WrapWithProjectionAndResample { + pub fn new_create_result_descriptor( + operator: Box, + initialized: Box, + ) -> Self { + let result_descriptor = initialized.result_descriptor().clone(); + Self::new(operator, initialized, result_descriptor) + } + + pub fn new( + operator: Box, + initialized: Box, + result_descriptor: RasterResultDescriptor, + ) -> Self { + Self { + operator, + initialized_operator: initialized, + result_descriptor, + } + } + + pub fn wrap_with_projection( + self, + target_sref: SpatialReference, + _target_origin_reference: Option, // TODO: add resampling if origin does not match! Could also do that in projection and avoid extra operation? + tiling_spec: TilingSpecification, + ) -> Result { + let result_sref = self + .result_descriptor + .spatial_reference() + .as_option() + .ok_or(error::Error::SpatialReferenceMustNotBeUnreferenced)?; + + // perform reprojection if necessary + let res = if target_sref == result_sref { + self + } else { + tracing::debug!( + "Target srs: {target_sref}, workflow srs: {result_sref} --> injecting reprojection" + ); + + let reprojection_params = ReprojectionParams { + target_spatial_reference: target_sref, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, + }; + + // create the reprojection operator in order to get the canonic operator name + let reprojected_workflow = Reprojection { + params: reprojection_params, + sources: SingleRasterOrVectorSource { + source: RasterOrVectorOperator::Raster(self.operator), + }, + }; + + // create the inititalized operator directly, to avoid re-initializing everything + // TODO: update the workflow operator path in all operators of the graph! + let irp = InitializedRasterReprojection::try_new_with_input( + CanonicOperatorName::from(&reprojected_workflow), + WorkflowOperatorPath::initialize_root(), // FIXME: this is not correct since the root is the child operator + reprojection_params, + self.initialized_operator, + tiling_spec, + )?; + let rd = irp.result_descriptor().clone(); + + Self::new(reprojected_workflow.boxed(), irp.boxed(), rd) + }; + Ok(res) + } + + pub async fn wrap_with_resample( + self, + target_origin_reference: Option, + target_spatial_resolution: Option, + exe_ctx: &dyn ExecutionContext, + ) -> Result { + if target_origin_reference.is_none() && target_spatial_resolution.is_none() { + return Ok(self); + } + + let rd_resolution = self + .result_descriptor + .spatial_grid_descriptor() + .spatial_resolution(); + let target_spatial_grid = if let Some(tsr) = target_spatial_resolution { + self.result_descriptor + .spatial_grid_descriptor() + .with_changed_resolution(tsr) + } else { + *self.result_descriptor.spatial_grid_descriptor() + }; + let target_spatial_grid = if let Some(tor) = target_origin_reference { + // if the request is to move the origin of the query to a different point, we generate a new grid aligned to that point. + target_spatial_grid + .with_moved_origin_to_nearest_grid_edge(tor) + .as_derived() + .replace_origin(tor) + } else { + target_spatial_grid + }; + + if self + .result_descriptor + .spatial_grid_descriptor() + .is_compatible_grid(&target_spatial_grid) + { + // TODO: resample if origin is not allgned to query? (maybe n + return Ok(self); + } + + // Query resolution is smaller than workflow => compute on full resolution and append interpolation to decrease resolution + if target_spatial_grid.spatial_resolution().x <= rd_resolution.x + && target_spatial_grid.spatial_resolution().y <= rd_resolution.y + //TODO: we should allow to use the "interpolation" as long as the fraction is > 0.5. This would require to keep 4 tiles which seems to be fine. The edge case of resampling with same resolution should also use the interpolation since bilieaner woudl make sense here? + { + self.interpolation( + target_spatial_resolution, + exe_ctx, + rd_resolution, + target_spatial_grid, + ) + .await + } else { + // Query resolution is larger than workflow => compute on overview level and append downsampling to increase resolution + self.downsampling( + target_spatial_resolution, + exe_ctx, + rd_resolution, + target_spatial_grid, + ) + .await + } + } + + async fn downsampling( + self, + target_spatial_resolution: Option, + exe_ctx: &dyn ExecutionContext, + rd_resolution: SpatialResolution, + target_spatial_grid: crate::engine::SpatialGridDescriptor, + ) -> std::result::Result { + tracing::debug!( + "Query res: {target_spatial_resolution:?}, workflow res: {rd_resolution:?} --> optimize workflow and push-down downsampling" + ); + + let snapped_resolution = find_next_best_overview_level_resolution( + self.result_descriptor.spatial_grid.spatial_resolution(), + target_spatial_grid.spatial_resolution(), + ); + debug_assert!(snapped_resolution <= target_spatial_grid.spatial_resolution()); + + let optimized_operator = self + .initialized_operator + .optimize(snapped_resolution) + .context(Optimization)?; + + if snapped_resolution == target_spatial_grid.spatial_resolution() { + // target resolution is an overview level, so we can use the optimized operator directly + let initialized_raster_operator = optimized_operator + .clone() + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await?; + + let rd: RasterResultDescriptor = + initialized_raster_operator.result_descriptor().clone(); + return Ok(Self::new( + optimized_operator, + initialized_raster_operator, + rd, + )); + } + + // target resolution is not an overview level, so we need to downsample the optimized operator + let downsample_params = DownsamplingParams { + sampling_method: DownsamplingMethod::NearestNeighbor, + output_resolution: DownsamplingResolution::Resolution( + target_spatial_grid.spatial_resolution(), + ), + output_origin_reference: None, + }; + let dop = Downsampling { + params: downsample_params, + sources: optimized_operator.into(), + } + .boxed(); + + let ido = dop + .clone() + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await?; + + let rd = ido.result_descriptor().clone(); + Ok(Self::new(dop, ido.boxed(), rd)) + } + + async fn interpolation( + self, + target_spatial_resolution: Option, + exe_ctx: &dyn ExecutionContext, + rd_resolution: SpatialResolution, + target_spatial_grid: crate::engine::SpatialGridDescriptor, + ) -> std::result::Result { + tracing::debug!( + "Target res: {target_spatial_resolution:?}, workflow res: {rd_resolution:?} --> injecting interpolation" + ); + + let interpolation_params = InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, // TODO: choose appropriate interpolation method + output_resolution: InterpolationResolution::Resolution( + target_spatial_grid.spatial_resolution(), + ), + output_origin_reference: None, + }; + + let iop = Interpolation { + params: interpolation_params.clone(), + sources: self.operator.into(), + } + .boxed(); + + let iip = iop + .clone() + .initialize(WorkflowOperatorPath::initialize_root(), exe_ctx) + .await?; + + let rd = iip.result_descriptor().clone(); + + Ok(Self::new(iop, iip.boxed(), rd)) + } + + pub async fn wrap_with_projection_and_resample( + self, + target_origin_reference: Option, + target_spatial_resolution: Option, + target_sref: SpatialReference, + tiling_spec: TilingSpecification, + exe_ctx: &dyn ExecutionContext, + ) -> Result { + self.wrap_with_projection(target_sref, target_origin_reference, tiling_spec)? + .wrap_with_resample(target_origin_reference, target_spatial_resolution, exe_ctx) + .await + } +} diff --git a/services/Cargo.toml b/services/Cargo.toml index ad80747d4..112abf8b7 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -64,6 +64,7 @@ proj-sys = { workspace = true } pwhash = { workspace = true } rand = { workspace = true } rayon = { workspace = true } +regex = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } @@ -111,5 +112,9 @@ vergen-gitcl = { workspace = true, features = ["build"] } name = "quota_check" harness = false +[[bench]] +name = "tiled_gdal_source" +harness = false + [lints] workspace = true diff --git a/services/benches/quota_check.rs b/services/benches/quota_check.rs index f83b7f9a0..4aabc7b22 100644 --- a/services/benches/quota_check.rs +++ b/services/benches/quota_check.rs @@ -56,7 +56,7 @@ async fn bench() { }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), }, diff --git a/services/benches/tiled_gdal_source.rs b/services/benches/tiled_gdal_source.rs new file mode 100644 index 000000000..42903f947 --- /dev/null +++ b/services/benches/tiled_gdal_source.rs @@ -0,0 +1,274 @@ +#![allow(clippy::unwrap_used, clippy::print_stdout, clippy::print_stderr)] // okay in benchmarks + +use futures::StreamExt; +use geoengine_datatypes::{ + primitives::{BandSelection, RasterQueryRectangle, TimeInstance, TimeInterval}, + raster::{GridBoundingBox2D, RasterTile2D}, +}; +use geoengine_operators::{ + engine::{RasterOperator, WorkflowOperatorPath}, + source::{ + GdalSource, GdalSourceParameters, MultiBandGdalSource, MultiBandGdalSourceParameters, + }, + util::{gdal::create_ndvi_result_descriptor, number_statistics::NumberStatistics}, +}; +use geoengine_services::{ + api::{ + handlers::datasets::AddDatasetTile, + model::{ + datatypes::{Coordinate2D, SpatialPartition2D}, + operators::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMultiBand, + }, + services::{AddDataset, DatasetDefinition, MetaDataDefinition, Provenance}, + }, + }, + contexts::{ApplicationContext, PostgresContext, SessionContext}, + datasets::{DatasetName, storage::DatasetStore}, + permissions::{Permission, PermissionDb, Role}, + test_data, + users::{UserAuth, UserSession}, + util::tests::{add_ndvi_to_datasets2, with_temp_context}, +}; +use std::env; +use std::str::FromStr; +use tokio_postgres::NoTls; +use uuid::Uuid; + +const RUNS: usize = 10; + +#[tokio::main] +async fn main() { + bench_gdal_source().await; + bench_multi_band_gdal_source().await; +} + +async fn bench_gdal_source() { + with_temp_context(|app_ctx, _| async move { + let session = app_ctx.create_anonymous_session().await.unwrap(); + let ctx = app_ctx.session_context(session.clone()); + + let exe_ctx = ctx.execution_context().unwrap(); + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(); + + let (_, dataset) = add_ndvi_to_datasets2(&app_ctx, true, true).await; + + let operator = GdalSource { + params: GdalSourceParameters::new(dataset), + } + .boxed(); + + let processor = operator + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap() + .query_processor() + .unwrap() + .get_u8() + .unwrap(); + + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-07-01T00:00:00Z").unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let mut times = NumberStatistics::default(); + + for _ in 0..RUNS { + let (time, result) = time_it(|| async { + let native_query = processor + .raster_query(qrect.clone(), &query_ctx) + .await + .unwrap(); + + native_query.map(Result::unwrap).collect::>().await + }) + .await; + + times.add(time); + + std::hint::black_box(result); + } + + println!( + "GdalSource: {} runs, mean: {:.3} s, std_dev: {:.3} s", + RUNS, + times.mean(), + times.std_dev() + ); + }) + .await; +} + +async fn bench_multi_band_gdal_source() { + with_temp_context(|app_ctx, _| async move { + let session = app_ctx.create_anonymous_session().await.unwrap(); + let ctx = app_ctx.session_context(session.clone()); + + let exe_ctx = ctx.execution_context().unwrap(); + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(); + + let dataset = add_ndvi_multi_tile_dataset(&app_ctx).await; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset.into()), + } + .boxed(); + + let processor = operator + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) + .await + .unwrap() + .query_processor() + .unwrap() + .get_u8() + .unwrap(); + + let qrect = RasterQueryRectangle::new( + GridBoundingBox2D::new([-900, -1800], [899, 1799]).unwrap(), + TimeInterval::new( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-07-01T00:00:00Z").unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let mut times = NumberStatistics::default(); + + for _ in 0..RUNS { + let (time, result) = time_it(|| async { + let native_query = processor + .raster_query(qrect.clone(), &query_ctx) + .await + .unwrap(); + + native_query.map(Result::unwrap).collect::>().await + }) + .await; + + times.add(time); + + std::hint::black_box(result); + } + + println!( + "MultiBandGdalSource: {} runs, mean: {:.3} s, std_dev: {:.3} s", + RUNS, + times.mean(), + times.std_dev() + ); + }) + .await; +} + +async fn time_it(f: F) -> (f64, Vec>) +where + F: FnOnce() -> Fut, + Fut: Future>>, +{ + let start = std::time::Instant::now(); + let result = f().await; + let end = start.elapsed(); + let secs = end.as_secs() as f64 + f64::from(end.subsec_nanos()) / 1_000_000_000.0; + + (secs, result) +} + +async fn add_ndvi_multi_tile_dataset(app_ctx: &PostgresContext) -> DatasetName { + let dataset_name = DatasetName { + namespace: None, + name: "NDVI_multi_tile".to_string(), + }; + + let ndvi = DatasetDefinition { + properties: AddDataset { + name: Some(dataset_name.clone()), + display_name: "NDVI multi tile".to_string(), + description: "NDVI data from MODIS".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: Some(vec![Provenance { + citation: "Sample Citation".to_owned(), + license: "Sample License".to_owned(), + uri: "http://example.org/".to_owned(), + }]), + tags: Some(vec!["raster".to_owned(), "test".to_owned()]), + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + result_descriptor: create_ndvi_result_descriptor(true).into(), + r#type: geoengine_services::api::model::operators::GdalMultiBandTypeTag::GdalMultiBandTypeTag, + }), + }; + + let system_session = UserSession::admin_session(); + + let db = app_ctx.session_context(system_session).db(); + + let dataset_id = db + .add_dataset(ndvi.properties.into(), ndvi.meta_data.into(), None) + .await + .expect("dataset db access") + .id; + + db.add_permission(Role::anonymous_role_id(), dataset_id, Permission::Read) + .await + .unwrap(); + + let time_steps = [ + ("2014-01-01T00:00:00Z", "2014-02-01T00:00:00Z"), + ("2014-02-01T00:00:00Z", "2014-03-01T00:00:00Z"), + ("2014-03-01T00:00:00Z", "2014-04-01T00:00:00Z"), + ("2014-04-01T00:00:00Z", "2014-05-01T00:00:00Z"), + ("2014-05-01T00:00:00Z", "2014-06-01T00:00:00Z"), + ("2014-06-01T00:00:00Z", "2014-07-01T00:00:00Z"), + ]; + + let tiles: Vec = time_steps + .iter() + .map(|(start, end)| AddDatasetTile { + time: TimeInterval::new( + TimeInstance::from_str(start).unwrap(), + TimeInstance::from_str(end).unwrap(), + ) + .unwrap() + .into(), + spatial_partition: SpatialPartition2D { + upper_left_coordinate: Coordinate2D { x: -180., y: 90. }, + lower_right_coordinate: Coordinate2D { x: 180., y: -90. }, + }, + band: 0, + z_index: 0, + params: GdalDatasetParameters { + file_path: test_data!(&format!( + "raster/modis_ndvi/MOD13A2_M_NDVI_{}.TIFF", + start.split('T').next().unwrap() + )) + .into(), + rasterband_channel: 1, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D { x: -180., y: 90. }, + x_pixel_size: 0.1, + y_pixel_size: -0.1, + }, + width: 3600, + height: 1800, + file_not_found_handling: FileNotFoundHandling::Error, + no_data_value: Some(0.0), + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + }, + }) + .collect(); + + db.add_dataset_tiles(dataset_id, tiles).await.unwrap(); + + dataset_name +} diff --git a/services/src/api/apidoc.rs b/services/src/api/apidoc.rs index a7cc0ae4a..0f481d61f 100644 --- a/services/src/api/apidoc.rs +++ b/services/src/api/apidoc.rs @@ -1,7 +1,7 @@ #![allow(clippy::needless_for_each)] // TODO: remove when clippy is fixed for utoipa use crate::api::handlers; -use crate::api::handlers::datasets::VolumeFileLayersResponse; +use crate::api::handlers::datasets::{AddDatasetTile, VolumeFileLayersResponse}; use crate::api::handlers::permissions::{ PermissionListOptions, PermissionListing, PermissionRequest, Resource, }; @@ -9,20 +9,16 @@ use crate::api::handlers::plots::WrappedPlotOutput; use crate::api::handlers::spatial_references::{AxisOrder, SpatialReferenceSpecification}; use crate::api::handlers::tasks::{TaskAbortOptions, TaskResponse}; use crate::api::handlers::upload::{UploadFileLayersResponse, UploadFilesResponse}; -use crate::api::handlers::users::AddRole; -use crate::api::handlers::users::{Quota, UpdateQuota, UsageSummaryGranularity}; -use crate::api::handlers::wfs::{CollectionType, GeoJson}; -use crate::api::handlers::workflows::{ProvenanceEntry, RasterStreamWebsocketResultType}; use crate::api::model::datatypes::{ AxisLabels, BandSelection, BoundingBox2D, Breakpoint, CacheTtlSeconds, ClassificationMeasurement, Colorizer, ContinuousMeasurement, Coordinate2D, DataId, DataProviderId, DatasetId, DateTimeParseFormat, DateTimeString, ExternalDataId, FeatureDataType, GdalConfigOption, LayerId, LinearGradient, LogarithmicGradient, Measurement, MlModelName, MlTensorShape3D, MultiLineString, MultiPoint, MultiPolygon, NamedData, NoGeometry, - Palette, PlotOutputFormat, PlotQueryRectangle, RasterColorizer, RasterDataType, - RasterPropertiesEntryType, RasterPropertiesKey, RasterQueryRectangle, RgbaColor, - SpatialPartition2D, SpatialReferenceAuthority, SpatialResolution, StringPair, TimeGranularity, - TimeInstance, TimeInterval, TimeStep, VectorDataType, VectorQueryRectangle, + Palette, PlotOutputFormat, RasterColorizer, RasterDataType, RasterPropertiesEntryType, + RasterPropertiesKey, RasterToDatasetQueryRectangle, RgbaColor, SpatialPartition2D, + SpatialReferenceAuthority, SpatialResolution, StringPair, TimeGranularity, TimeInstance, + TimeInterval, TimeStep, VectorDataType, }; use crate::api::model::operators::{ CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, @@ -45,8 +41,8 @@ use crate::api::model::services::DatabaseConnectionConfig; use crate::api::model::services::EdrVectorSpec; use crate::api::model::services::LayerProviderListing; use crate::api::model::services::{ - AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, MetaDataSuggestion, - MlModel, Provenance, ProvenanceOutput, Provenances, UpdateDataset, Volume, + AddDataset, CreateDataset, DataPath, Dataset, DatasetDefinition, MetaDataDefinition, + MetaDataSuggestion, MlModel, Provenance, ProvenanceOutput, Provenances, UpdateDataset, Volume, }; use crate::api::model::services::{ ArunaDataProviderDefinition, CopernicusDataspaceDataProviderDefinition, @@ -54,13 +50,25 @@ use crate::api::model::services::{ EbvPortalDataProviderDefinition, EdrDataProviderDefinition, GbifDataProviderDefinition, GfbioAbcdDataProviderDefinition, GfbioCollectionsDataProviderDefinition, NetCdfCfDataProviderDefinition, PangaeaDataProviderDefinition, - SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacBand, StacQueryBuffer, StacZone, + SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacQueryBuffer, TypedDataProviderDefinition, }; -use crate::api::ogc::{util::OgcBoundingBox, wcs, wfs, wms}; +use crate::api::ogc::util::OgcBoundingBox; +use crate::api::ogc::{wcs, wfs, wms}; +use crate::api::{ + handlers::{ + users::{AddRole, Quota, UpdateQuota, UsageSummaryGranularity}, + wfs::{CollectionType, GeoJson}, + workflows::{ProvenanceEntry, RasterStreamWebsocketResultType}, + }, + model::{ + datatypes::{GeoTransform, GridBoundingBox2D, GridIdx2D, SpatialGridDefinition}, + operators::{SpatialGridDescriptor, SpatialGridDescriptorState}, + }, +}; use crate::contexts::SessionId; use crate::datasets::listing::{DatasetListing, OrderBy}; -use crate::datasets::storage::{AutoCreateDataset, Dataset, SuggestMetaData}; +use crate::datasets::storage::{AutoCreateDataset, SuggestMetaData}; use crate::datasets::upload::{UploadId, VolumeName}; use crate::datasets::{DatasetName, RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult}; use crate::layers::layer::{ @@ -109,6 +117,7 @@ use utoipa::{Modify, OpenApi}; handlers::datasets::update_dataset_provenance_handler, handlers::datasets::update_dataset_symbology_handler, handlers::datasets::update_loading_info_handler, + handlers::datasets::add_dataset_tiles_handler, handlers::layers::add_collection, handlers::layers::add_existing_collection_to_collection, handlers::layers::add_existing_layer_to_collection, @@ -175,14 +184,9 @@ use utoipa::{Modify, OpenApi}; handlers::users::revoke_role_handler, handlers::users::session_handler, handlers::users::update_user_quota_handler, - handlers::wcs::wcs_capabilities_handler, - handlers::wcs::wcs_describe_coverage_handler, - handlers::wcs::wcs_get_coverage_handler, - handlers::wfs::wfs_capabilities_handler, - handlers::wfs::wfs_feature_handler, - handlers::wms::wms_capabilities_handler, - handlers::wms::wms_legend_graphic_handler, - handlers::wms::wms_map_handler, + handlers::wcs::wcs_handler, + handlers::wfs::wfs_handler, + handlers::wms::wms_handler, handlers::workflows::dataset_from_workflow_handler, handlers::workflows::get_workflow_all_metadata_zip_handler, handlers::workflows::get_workflow_metadata_handler, @@ -281,9 +285,7 @@ use utoipa::{Modify, OpenApi}; VectorColumnInfo, RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult, - RasterQueryRectangle, - VectorQueryRectangle, - PlotQueryRectangle, + RasterToDatasetQueryRectangle, BandSelection, TaskAbortOptions, @@ -329,28 +331,21 @@ use utoipa::{Modify, OpenApi}; OgcBoundingBox, + wcs::request::WcsRequest, wcs::request::WcsService, wcs::request::WcsVersion, - wcs::request::GetCapabilitiesRequest, - wcs::request::DescribeCoverageRequest, - wcs::request::GetCoverageRequest, wcs::request::GetCoverageFormat, wcs::request::WcsBoundingbox, + wms::request::WmsRequest, wms::request::WmsService, wms::request::WmsVersion, - wms::request::GetCapabilitiesFormat, - wms::request::GetCapabilitiesRequest, - wms::request::GetMapRequest, wms::request::GetMapExceptionFormat, - wms::request::GetMapFormat, - wms::request::GetLegendGraphicRequest, + wms::request::WmsResponseFormat, + wfs::request::WfsRequest, wfs::request::WfsService, wfs::request::WfsVersion, - wfs::request::GetCapabilitiesRequest, - wfs::request::WfsResolution, - wfs::request::GetFeatureRequest, wfs::request::TypeNames, GeoJson, @@ -410,6 +405,7 @@ use utoipa::{Modify, OpenApi}; Volume, VolumeName, DataPath, + AddDatasetTile, PlotOutputFormat, WrappedPlotOutput, @@ -428,6 +424,12 @@ use utoipa::{Modify, OpenApi}; RasterStreamWebsocketResultType, CacheTtlSeconds, + SpatialGridDefinition, + SpatialGridDescriptorState, + SpatialGridDescriptor, + GridBoundingBox2D, + GridIdx2D, + GeoTransform, TypedDataProviderDefinition, ArunaDataProviderDefinition, DatasetLayerListingProviderDefinition, @@ -442,8 +444,6 @@ use utoipa::{Modify, OpenApi}; SentinelS2L2ACogsProviderDefinition, DatabaseConnectionConfig, EdrVectorSpec, - StacBand, - StacZone, StacApiRetries, StacQueryBuffer, DatasetLayerListingCollection, diff --git a/services/src/api/handlers/datasets.rs b/services/src/api/handlers/datasets.rs index cfc907b6d..2c007de46 100755 --- a/services/src/api/handlers/datasets.rs +++ b/services/src/api/handlers/datasets.rs @@ -1,21 +1,21 @@ use crate::{ api::model::{ - operators::{GdalLoadingInfoTemporalSlice, GdalMetaDataList}, + datatypes::SpatialPartition2D, + operators::{GdalDatasetParameters, GdalLoadingInfoTemporalSlice, GdalMetaDataList}, responses::{ ErrorResponse, datasets::{DatasetNameResponse, errors::*}, }, services::{ - AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, + AddDataset, CreateDataset, DataPath, Dataset, DatasetDefinition, MetaDataDefinition, MetaDataSuggestion, Provenances, UpdateDataset, Volume, }, }, - config::{Data, get_config_element}, contexts::{ApplicationContext, SessionContext}, datasets::{ DatasetName, listing::{DatasetListOptions, DatasetListing, DatasetProvider}, - storage::{AutoCreateDataset, Dataset, DatasetStore, SuggestMetaData}, + storage::{AutoCreateDataset, DatasetStore, SuggestMetaData}, upload::{AdjustFilePath, Upload, UploadDb, UploadId, UploadRootPath, VolumeName, Volumes}, }, error::{self, Error, Result}, @@ -26,7 +26,10 @@ use crate::{ path_with_base_path, }, }; -use actix_web::{FromRequest, HttpResponse, HttpResponseBuilder, Responder, web}; +use actix_web::{ + FromRequest, HttpResponse, HttpResponseBuilder, Responder, + web::{self, Json}, +}; use gdal::{ DatasetOptions, vector::{Layer, LayerAccess, OGRFieldType}, @@ -40,10 +43,13 @@ use geoengine_datatypes::{ spatial_reference::{SpatialReference, SpatialReferenceOption}, }; use geoengine_operators::{ - engine::{StaticMetaData, VectorColumnInfo, VectorResultDescriptor}, + engine::{ + OperatorName, RasterResultDescriptor, StaticMetaData, TypedResultDescriptor, + VectorColumnInfo, VectorResultDescriptor, + }, source::{ - OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, OgrSourceDurationSpec, - OgrSourceErrorSpec, OgrSourceTimeFormat, + MultiBandGdalSource, OgrSourceColumnSpec, OgrSourceDataset, OgrSourceDatasetTimeType, + OgrSourceDurationSpec, OgrSourceErrorSpec, OgrSourceTimeFormat, }, util::gdal::{ gdal_open_dataset, gdal_open_dataset_ex, gdal_parameters_from_dataset, @@ -51,7 +57,7 @@ use geoengine_operators::{ }, }; use serde::{Deserialize, Serialize}; -use snafu::ResultExt; +use snafu::{ResultExt, ensure}; use std::{ collections::HashMap, convert::{TryFrom, TryInto}, @@ -88,6 +94,10 @@ where web::resource("/{dataset}/provenance") .route(web::put().to(update_dataset_provenance_handler::)), ) + .service( + web::resource("/{dataset}/tiles") + .route(web::post().to(add_dataset_tiles_handler::)), + ) .service( web::resource("/{dataset}") .route(web::get().to(get_dataset_handler::)) @@ -178,6 +188,194 @@ pub async fn list_datasets_handler( Ok(web::Json(list)) } +/// Add a tile to a gdal dataset. +#[utoipa::path( + tag = "Datasets", + post, + path = "/dataset/{dataset}/tiles", + request_body = AutoCreateDataset, + responses( + (status = 200), + ), + params( + ("dataset" = DatasetName, description = "Dataset Name"), + ), + security( + ("session_token" = []) + ) +)] +pub async fn add_dataset_tiles_handler( + session: C::Session, + app_ctx: web::Data, + dataset: web::Path, + tiles: Json>, +) -> Result { + let session_context = app_ctx.session_context(session); + let db = session_context.db(); + + let dataset = dataset.into_inner(); + let dataset_id = db + .resolve_dataset_name_to_id(&dataset) + .await + .context(CannotLoadDatasetForAddingTiles)?; + + // handle the case where the dataset name is not known + let dataset_id = dataset_id + .ok_or(error::Error::UnknownDatasetName { + dataset_name: dataset.to_string(), + }) + .context(CannotLoadDatasetForAddingTiles)?; + + let dataset = db + .load_dataset(&dataset_id) + .await + .context(CannotLoadDatasetForAddingTiles)?; + + ensure!( + dataset.source_operator == MultiBandGdalSource::TYPE_NAME, + DatasetIsNotGdalMultiBand + ); + + let TypedResultDescriptor::Raster(dataset_descriptor) = dataset.result_descriptor else { + return Err(AddDatasetTilesError::DatasetIsNotGdalMultiBand); + }; + + let tiles = tiles.into_inner(); + + let data_path_file_path = file_path_from_data_path( + &dataset + .data_path + .ok_or(AddDatasetTilesError::DatasetIsMissingDataPath)?, + &session_context, + ) + .context(CannotAddTilesToDataset)?; + + for tile in &tiles { + validate_tile(tile, &data_path_file_path, &dataset_descriptor)?; + } + + db.add_dataset_tiles(dataset_id, tiles) + .await + .context(CannotAddTilesToDataset)?; + + Ok(HttpResponse::Ok().finish()) +} + +fn validate_tile( + tile: &AddDatasetTile, + data_path_file_path: &Path, + dataset_descriptor: &RasterResultDescriptor, +) -> Result<(), AddDatasetTilesError> { + ensure!( + tile.params.file_path.is_relative(), + TileFilePathNotRelative { + file_path: tile.params.file_path.to_string_lossy().to_string() + } + ); + + let absolute_path = data_path_file_path.join(&tile.params.file_path); + + ensure!( + absolute_path.exists(), + TileFilePathDoesNotExist { + file_path: tile.params.file_path.to_string_lossy().to_string(), + absolute_path: absolute_path.to_string_lossy().to_string(), + } + ); + + let ds = gdal_open_dataset_ex(&absolute_path, DatasetOptions::default()).context( + CannotOpenTileFile { + file_path: tile.params.file_path.to_string_lossy().to_string(), + }, + )?; + + let rd = raster_descriptor_from_dataset(&ds, tile.params.rasterband_channel).context( + CannotGetRasterDescriptorFromTileFile { + file_path: tile.params.file_path.to_string_lossy().to_string(), + }, + )?; + + // TODO: move this inside the db? we do not want to open datasets while keeping a database transaction, though + ensure!( + rd.data_type == dataset_descriptor.data_type, + TileFileDataTypeMismatch { + expected: dataset_descriptor.data_type, + found: rd.data_type, + file_path: tile.params.file_path.to_string_lossy().to_string(), + } + ); + + ensure!( + rd.spatial_reference == dataset_descriptor.spatial_reference, + TileFileSpatialReferenceMismatch { + expected: dataset_descriptor.spatial_reference, + found: rd.spatial_reference, + file_path: tile.params.file_path.to_string_lossy().to_string(), + } + ); + + ensure!( + tile.band < dataset_descriptor.bands.count(), + TileFileBandDoesNotExist { + band_count: dataset_descriptor.bands.count(), + found: tile.band, + file_path: tile.params.file_path.to_string_lossy().to_string(), + } + ); + + // TODO: also check that the tiles bbox (from the tile definition, and not the actual gdal dataset of the tile's file) fits into the dataset's spatial grid? + let tile_geotransform = geoengine_datatypes::raster::GeoTransform::try_from( + geoengine_operators::source::GdalDatasetGeoTransform::from(tile.params.geo_transform), + ) + .map_err(|_| AddDatasetTilesError::InvalidTileFileGeoTransform { + file_path: tile.params.file_path.to_string_lossy().to_string(), + })?; + ensure!( + dataset_descriptor + .spatial_grid + .geo_transform() + .is_compatible_grid(tile_geotransform), + TileFileGeoTransformMismatch { + expected: dataset_descriptor.spatial_grid.geo_transform(), + found: tile_geotransform, + file_path: tile.params.file_path.to_string_lossy().to_string(), + } + ); + + Ok(()) +} + +fn file_path_from_data_path( + data_path: &DataPath, + session_context: &T, +) -> Result { + Ok(match data_path { + DataPath::Volume(volume_name) => session_context + .volumes()? + .iter() + .find(|v| v.name == volume_name.0) + .ok_or(Error::UnknownVolumeName { + volume_name: volume_name.0.clone(), + })? + .path + .clone() + .ok_or(Error::CannotAccessVolumePath { + volume_name: volume_name.0.clone(), + })? + .into(), + DataPath::Upload(upload_id) => upload_id.root_path()?, + }) +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Debug, ToSchema)] +pub struct AddDatasetTile { + pub time: crate::api::model::datatypes::TimeInterval, + pub spatial_partition: SpatialPartition2D, + pub band: u32, + pub z_index: u32, + pub params: GdalDatasetParameters, +} + /// Retrieves details about a dataset using the internal name. #[utoipa::path( tag = "Datasets", @@ -242,6 +440,8 @@ pub async fn get_dataset_handler( .await .context(CannotLoadDataset)?; + let dataset: Dataset = dataset.into(); + Ok(web::Json(dataset)) } @@ -490,7 +690,11 @@ pub async fn create_upload_dataset( .context(CannotResolveUploadFilePath)?; let result = db - .add_dataset(definition.properties.into(), definition.meta_data.into()) + .add_dataset( + definition.properties.into(), + definition.meta_data.into(), + Some(DataPath::Upload(upload_id)), + ) .await .context(CannotCreateDataset)?; @@ -522,6 +726,9 @@ pub fn adjust_meta_data_path( } } } + MetaDataDefinition::GdalMultiBand(_gdal_multi_band) => { + // do nothing, the file paths are not inside the meta data defintion but inside the dataset's tiles + } } Ok(()) } @@ -609,7 +816,13 @@ pub async fn auto_create_dataset_handler( tags: Some(vec!["upload".to_owned(), "auto".to_owned()]), }; - let result = db.add_dataset(properties.into(), meta_data).await?; + let result = db + .add_dataset( + properties.into(), + meta_data, + Some(DataPath::Upload(upload.id)), + ) + .await?; Ok(web::Json(result.name.into())) } @@ -1326,27 +1539,23 @@ async fn create_system_dataset( volume_name: VolumeName, mut definition: DatasetDefinition, ) -> Result, CreateDatasetError> { - let volumes = get_config_element::() - .context(CannotAccessConfig)? - .volumes; - let volume_path = volumes - .get(&volume_name) + let volumes = Volumes::default().volumes; + let volume = volumes + .iter() + .find(|v| v.name == volume_name) .ok_or(CreateDatasetError::UnknownVolume)?; - let volume = Volume { - name: volume_name.to_string(), - path: Some(volume_path.to_string_lossy().into()), - }; - adjust_meta_data_path( - &mut definition.meta_data, - &crate::datasets::upload::Volume::from(&volume), - ) - .context(CannotResolveUploadFilePath)?; + adjust_meta_data_path(&mut definition.meta_data, volume) + .context(CannotResolveUploadFilePath)?; let db = app_ctx.session_context(session).db(); let dataset = db - .add_dataset(definition.properties.into(), definition.meta_data.into()) + .add_dataset( + definition.properties.into(), + definition.meta_data.into(), + Some(DataPath::Volume(volume_name)), + ) .await .context(CannotCreateDataset)?; @@ -1369,47 +1578,76 @@ async fn create_system_dataset( #[cfg(test)] mod tests { + + use std::str::FromStr; + use super::*; - use crate::api::model::datatypes::NamedData; - use crate::api::model::responses::IdResponse; - use crate::api::model::responses::datasets::DatasetNameResponse; - use crate::api::model::services::{DatasetDefinition, Provenance}; - use crate::contexts::PostgresContext; - use crate::contexts::{Session, SessionId}; - use crate::datasets::DatasetIdAndName; - use crate::datasets::storage::DatasetStore; - use crate::datasets::upload::{UploadId, VolumeName}; - use crate::error::Result; - use crate::ge_context; - use crate::projects::{PointSymbology, RasterSymbology, Symbology}; - use crate::test_data; - use crate::users::{RoleDb, UserAuth}; - use crate::util::tests::admin_login; - use crate::util::tests::{ - MockQueryContext, SetMultipartBody, TestDataUploads, add_file_definition_to_datasets, - read_body_json, read_body_string, send_test_request, + use crate::{ + api::model::{ + datatypes::{ + Coordinate2D, DataId, GeoTransform, GridBoundingBox2D, GridIdx2D, InternalDataId, + NamedData, RasterDataType, SingleBandRasterColorizer, SpatialGridDefinition, + TimeInstance, + }, + operators::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalMultiBand, RasterBandDescriptor, + RasterBandDescriptors, RasterResultDescriptor, SpatialGridDescriptor, + SpatialGridDescriptorState, TimeDescriptor, TimeDimension, + }, + responses::{IdResponse, datasets::DatasetNameResponse}, + services::{DatasetDefinition, Provenance}, + }, + contexts::{PostgresContext, PostgresSessionContext, Session, SessionId}, + datasets::{ + DatasetIdAndName, + storage::DatasetStore, + upload::{UploadId, VolumeName}, + }, + error::Result, + ge_context, + projects::{PointSymbology, RasterSymbology, Symbology}, + test_data, + users::{RoleDb, UserAuth}, + util::tests::{ + MockQueryContext, SetMultipartBody, TestDataUploads, add_file_definition_to_datasets, + admin_login, read_body_json, read_body_string, send_test_request, + }, + workflows::{registry::WorkflowRegistry, workflow::Workflow}, }; use actix_web; use actix_web::http::header; use actix_web_httpauth::headers::authorization::Bearer; use futures::TryStreamExt; - use geoengine_datatypes::collections::{ - GeometryCollection, MultiPointCollection, VectorDataType, - }; - use geoengine_datatypes::operations::image::{RasterColorizer, RgbaColor}; - use geoengine_datatypes::primitives::{BoundingBox2D, ColumnSelection, SpatialResolution}; - use geoengine_datatypes::raster::{GridShape2D, TilingSpecification}; - use geoengine_datatypes::spatial_reference::SpatialReferenceOption; - use geoengine_operators::engine::{ - ExecutionContext, InitializedVectorOperator, QueryProcessor, StaticMetaData, - VectorOperator, VectorResultDescriptor, WorkflowOperatorPath, + use geoengine_datatypes::{ + collections::{GeometryCollection, MultiPointCollection, VectorDataType}, + operations::image::{RasterColorizer, RgbaColor}, + primitives::{ + BandSelection, BoundingBox2D, ColumnSelection, DateTimeParseFormat, + RasterQueryRectangle, SpatialPartition2D, + }, + raster::{GridShape2D, TilingSpecification}, + spatial_reference::SpatialReferenceOption, + util::{assert_image_equals, test::assert_eq_two_list_of_tiles}, }; - use geoengine_operators::source::{ - OgrSource, OgrSourceDataset, OgrSourceErrorSpec, OgrSourceParameters, + use geoengine_operators::{ + engine::{ + ExecutionContext, InitializedVectorOperator, MetaData, MetaDataProvider, + QueryProcessor, RasterOperator, StaticMetaData, VectorOperator, VectorResultDescriptor, + WorkflowOperatorPath, + }, + source::{ + MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, MultiBandGdalSource, + MultiBandGdalSourceParameters, OgrSource, OgrSourceDataset, OgrSourceErrorSpec, + OgrSourceParameters, + }, + util::{ + gdal::{create_ndvi_meta_data, create_ndvi_result_descriptor}, + test::raster_tile_from_file, + }, }; - use geoengine_operators::util::gdal::create_ndvi_meta_data; use serde_json::{Value, json}; use tokio_postgres::NoTls; + use uuid::Uuid; #[ge_context::test] #[allow(clippy::too_many_lines)] @@ -1455,7 +1693,8 @@ mod tests { }); let db = ctx.db(); - let DatasetIdAndName { id: id1, name: _ } = db.add_dataset(ds.into(), meta).await.unwrap(); + let DatasetIdAndName { id: id1, name: _ } = + db.add_dataset(ds.into(), meta, None).await.unwrap(); let ds = AddDataset { name: Some(DatasetName::new(None, "My_Dataset2")), @@ -1486,7 +1725,8 @@ mod tests { phantom: Default::default(), }); - let DatasetIdAndName { id: id2, name: _ } = db.add_dataset(ds.into(), meta).await.unwrap(); + let DatasetIdAndName { id: id2, name: _ } = + db.add_dataset(ds.into(), meta, None).await.unwrap(); let req = actix_web::test::TestRequest::get() .uri(&format!( @@ -1700,6 +1940,74 @@ mod tests { .map_err(Into::into) } + fn ctx_tiling_spec_600x600() -> TilingSpecification { + TilingSpecification { + tile_size_in_pixels: GridShape2D::new([600, 600]), + } + } + + #[ge_context::test(tiling_spec = "ctx_tiling_spec_600x600")] + async fn create_dataset(app_ctx: PostgresContext) -> Result<()> { + let mut test_data = TestDataUploads::default(); // remember created folder and remove them on drop + + let session = app_ctx.create_anonymous_session().await.unwrap(); + let session_id = session.id(); + let session_context = app_ctx.session_context(session); + + let upload_id = upload_ne_10m_ports_files(app_ctx.clone(), session_id).await?; + test_data.uploads.push(upload_id); + + let dataset_name = + construct_dataset_from_upload(app_ctx.clone(), upload_id, session_id).await; + let exe_ctx = session_context.execution_context()?; + + let source = make_ogr_source( + &exe_ctx, + NamedData { + namespace: dataset_name.namespace, + provider: None, + name: dataset_name.name, + }, + ) + .await?; + + let query_processor = source.query_processor()?.multi_point().unwrap(); + let query_ctx = session_context.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let query = query_processor + .query( + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), + &query_ctx, + ) + .await?; + + let result: Vec = query.try_collect().await?; + + let coords = result[0].coordinates(); + assert_eq!(coords.len(), 10); + assert_eq!( + coords, + &[ + [2.933_686_69, 51.23].into(), + [3.204_593_64_f64, 51.336_388_89].into(), + [4.651_413_428, 51.805_833_33].into(), + [4.11, 51.95].into(), + [4.386_160_188, 50.886_111_11].into(), + [3.767_373_38, 51.114_444_44].into(), + [4.293_757_362, 51.297_777_78].into(), + [1.850_176_678, 50.965_833_33].into(), + [2.170_906_949, 51.021_666_67].into(), + [4.292_873_969, 51.927_222_22].into(), + ] + ); + + Ok(()) + } + #[ge_context::test] async fn it_creates_system_dataset(app_ctx: PostgresContext) -> Result<()> { let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -2283,7 +2591,7 @@ mod tests { let DatasetIdAndName { id, name: dataset_name, - } = db.add_dataset(ds.into(), meta).await?; + } = db.add_dataset(ds.into(), meta, None).await?; let req = actix_web::test::TestRequest::get() .uri(&format!("/dataset/{dataset_name}")) @@ -2314,6 +2622,7 @@ mod tests { "symbology": null, "provenance": null, "tags": ["upload", "test"], + "dataPath": null, }) ); @@ -2567,7 +2876,7 @@ mod tests { let DatasetIdAndName { id: _, name: dataset_name, - } = db.add_dataset(ds.into(), meta).await?; + } = db.add_dataset(ds.into(), meta, None).await?; let req = actix_web::test::TestRequest::get() .uri(&format!("/dataset/{dataset_name}/loadingInfo")) @@ -2658,7 +2967,7 @@ mod tests { let DatasetIdAndName { id, name: dataset_name, - } = db.add_dataset(ds.into(), meta).await?; + } = db.add_dataset(ds.into(), meta, None).await?; let update: MetaDataDefinition = crate::datasets::storage::MetaDataDefinition::OgrMetaData(StaticMetaData { @@ -2903,13 +3212,12 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn create_dataset_tiling_specification() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } #[ge_context::test(tiling_spec = "create_dataset_tiling_specification")] - async fn create_dataset(app_ctx: PostgresContext) -> Result<()> { + async fn create_datasets(app_ctx: PostgresContext) -> Result<()> { let mut test_data = TestDataUploads::default(); // remember created folder and remove them on drop let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -2933,12 +3241,11 @@ mod tests { let query = query_processor .query( - VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, - time_interval: Default::default(), - spatial_resolution: SpatialResolution::new(1., 1.)?, - attributes: ColumnSelection::all(), - }, + VectorQueryRectangle::new( + BoundingBox2D::new((1.85, 50.88).into(), (4.82, 52.95).into())?, + Default::default(), + ColumnSelection::all(), + ), &query_ctx, ) .await @@ -3174,4 +3481,1926 @@ mod tests { Ok(()) } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_adds_tiles_to_dataset(app_ctx: PostgresContext) -> Result<()> { + let volume = VolumeName("test_data".to_string()); + + // add data + let create = CreateDataset { + data_path: DataPath::Volume(volume.clone()), + definition: DatasetDefinition { + properties: AddDataset { + name: None, + display_name: "ndvi (tiled)".to_string(), + description: "ndvi".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: Some(vec!["upload".to_owned(), "test".to_owned()]), + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: Default::default(), + result_descriptor: create_ndvi_result_descriptor(true).into(), + }), + }, + }; + + let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let req = actix_web::test::TestRequest::post() + .uri("/dataset") + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&create)?); + let res = send_test_request(req, app_ctx.clone()).await; + + let DatasetNameResponse { dataset_name } = actix_web::test::read_body_json(res).await; + let dataset_id = db + .resolve_dataset_name_to_id(&dataset_name) + .await + .unwrap() + .unwrap(); + + assert!(db.load_dataset(&dataset_id).await.is_ok()); + + // add tiles + let tiles = create_ndvi_tiles(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 200, "response: {res:?}"); + + // create workflow + let workflow = Workflow { + operator: geoengine_operators::engine::TypedOperator::Raster( + MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(), + ), + }; + + let id = ctx.db().register_workflow(workflow.clone()).await.unwrap(); + + let colorizer = geoengine_datatypes::operations::image::Colorizer::linear_gradient( + vec![ + (0.0, RgbaColor::white()).try_into().unwrap(), + (255.0, RgbaColor::black()).try_into().unwrap(), + ], + RgbaColor::transparent(), + RgbaColor::white(), + RgbaColor::black(), + ) + .unwrap(); + + let raster_colorizer = + crate::api::model::datatypes::RasterColorizer::SingleBand(SingleBandRasterColorizer { + r#type: Default::default(), + band: 0, + band_colorizer: colorizer.into(), + }); + + let params = &[ + ("request", "GetMap"), + ("service", "WMS"), + ("version", "1.3.0"), + ("layers", &id.to_string()), + ("bbox", "-90,-180,90,180"), + ("width", "3600"), + ("height", "1800"), + ("crs", "EPSG:4326"), + ( + "styles", + &format!( + "custom:{}", + serde_json::to_string(&raster_colorizer).unwrap() + ), + ), + ("format", "image/png"), + ("time", "2014-01-01T00:00:00.0Z"), + ]; + + let req = actix_web::test::TestRequest::get() + .uri(&format!( + "/wms/{}?{}", + id, + serde_urlencoded::to_string(params).unwrap() + )) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))); + let res = send_test_request(req, app_ctx).await; + + assert_eq!(res.status(), 200); + + let image_bytes = actix_web::test::read_body(res).await; + + // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "wms.png"); + + assert_image_equals(test_data!("raster/multi_tile/wms.png"), &image_bytes); + + Ok(()) + } + + pub fn create_ndvi_tiles() -> Vec { + let no_data_value = Some(0.); // TODO: is it really 0? + + let starts: Vec = vec![ + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-02-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-03-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-04-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-05-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-06-01T00:00:00Z").unwrap(), + ]; + + let ends: Vec = vec![ + TimeInstance::from_str("2014-02-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-03-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-04-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-05-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-06-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-07-01T00:00:00Z").unwrap(), + ]; + + let mut tiles = vec![]; + + for (start, end) in starts.iter().zip(ends.iter()) { + let start_time: geoengine_datatypes::primitives::TimeInstance = (*start).into(); + let time_str = start_time + .as_date_time() + .unwrap() + .format(&DateTimeParseFormat::custom("%Y-%m-%d".to_string())); + + // left + tiles.push(AddDatasetTile { + time: TimeInterval::new_unchecked(*start, *end).into(), + spatial_partition: + geoengine_datatypes::primitives::SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (0.0, -90.).into(), + ) + .into(), + band: 0, + z_index: 0, + params: GdalDatasetParameters { + file_path: format!("raster/modis_ndvi/tiled/MOD13A2_M_NDVI_{time_str}_1_1.tif") + .into(), + rasterband_channel: 1, + geo_transform: geoengine_operators::source::GdalDatasetGeoTransform { + origin_coordinate: (-180., 90.).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + } + .into(), + width: 1800, + height: 1800, + file_not_found_handling: FileNotFoundHandling::Error, + no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + }, + }); + + // right + tiles.push(AddDatasetTile { + time: TimeInterval::new_unchecked(*start, *end).into(), + spatial_partition: + geoengine_datatypes::primitives::SpatialPartition2D::new_unchecked( + (0., 90.).into(), + (180.0, -90.).into(), + ) + .into(), + band: 0, + z_index: 0, + params: GdalDatasetParameters { + file_path: format!("raster/modis_ndvi/tiled/MOD13A2_M_NDVI_{time_str}_1_2.tif") + .into(), + rasterband_channel: 1, + geo_transform: geoengine_operators::source::GdalDatasetGeoTransform { + origin_coordinate: (0., 90.).into(), + x_pixel_size: 0.1, + y_pixel_size: -0.1, + } + .into(), + width: 1800, + height: 1800, + file_not_found_handling: FileNotFoundHandling::Error, + no_data_value, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: true, + }, + }); + } + + tiles + } + + async fn add_multi_tile_dataset( + app_ctx: &PostgresContext, + reverse_z_order: bool, + as_regular_dataset: bool, + ) -> Result<(PostgresSessionContext, DatasetName)> { + // add data + let create: CreateDataset = if as_regular_dataset { + serde_json::from_str(&std::fs::read_to_string(test_data!( + "raster/multi_tile/metadata/dataset_regular.json" + ))?)? + } else { + serde_json::from_str(&std::fs::read_to_string(test_data!( + "raster/multi_tile/metadata/dataset_irregular.json" + ))?)? + }; + + let session = admin_login(app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let req = actix_web::test::TestRequest::post() + .uri("/dataset") + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&create)?); + let res = send_test_request(req, app_ctx.clone()).await; + + let DatasetNameResponse { dataset_name } = actix_web::test::read_body_json(res).await; + let dataset_id = db + .resolve_dataset_name_to_id(&dataset_name) + .await + .unwrap() + .unwrap(); + + assert!(db.load_dataset(&dataset_id).await.is_ok()); + + // add tiles + let tiles: Vec = if reverse_z_order { + serde_json::from_str(&std::fs::read_to_string(test_data!( + "raster/multi_tile/metadata/loading_info_rev.json" + ))?)? + } else { + serde_json::from_str(&std::fs::read_to_string(test_data!( + "raster/multi_tile/metadata/loading_info.json" + ))?)? + }; + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_json(tiles); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 200, "response: {res:?}"); + + Ok((ctx, dataset_name)) + } + + #[ge_context::test] + async fn it_loads_multi_band_multi_file_mosaics(app_ctx: PostgresContext) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, true).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + "2025-01-01_global_b0_tile_0.tif", + "2025-01-01_global_b0_tile_1.tif", + "2025-01-01_global_b0_tile_2.tif", + "2025-01-01_global_b0_tile_3.tif", + "2025-01-01_global_b0_tile_4.tif", + "2025-01-01_global_b0_tile_5.tif", + "2025-01-01_global_b0_tile_6.tif", + "2025-01-01_global_b0_tile_7.tif", + ]; + + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|f| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + expected_time, + 0, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[ge_context::test] + async fn it_loads_multi_band_multi_file_mosaics_2_bands( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32), + ("2025-01-01_global_b1_tile_0.tif", 1), + ("2025-01-01_global_b0_tile_1.tif", 0), + ("2025-01-01_global_b1_tile_1.tif", 1), + ("2025-01-01_global_b0_tile_2.tif", 0), + ("2025-01-01_global_b1_tile_2.tif", 1), + ("2025-01-01_global_b0_tile_3.tif", 0), + ("2025-01-01_global_b1_tile_3.tif", 1), + ("2025-01-01_global_b0_tile_4.tif", 0), + ("2025-01-01_global_b1_tile_4.tif", 1), + ("2025-01-01_global_b0_tile_5.tif", 0), + ("2025-01-01_global_b1_tile_5.tif", 1), + ("2025-01-01_global_b0_tile_6.tif", 0), + ("2025-01-01_global_b1_tile_6.tif", 1), + ("2025-01-01_global_b0_tile_7.tif", 0), + ("2025-01-01_global_b1_tile_7.tif", 1), + ]; + + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + expected_time, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_loads_multi_band_multi_file_mosaics_2_bands_2_timesteps( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_time1 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_time2 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32, expected_time1), + ("2025-01-01_global_b1_tile_0.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_1.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_1.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_2.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_2.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_3.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_3.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_4.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_4.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_5.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_5.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_6.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_6.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_7.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_7.tif", 1, expected_time1), + ("2025-02-01_global_b0_tile_0.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_0.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_1.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_1.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_2.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_2.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_3.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_3.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_4.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_4.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_5.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_5.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_6.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_6.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_7.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_7.tif", 1, expected_time2), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_loads_multi_band_multi_file_mosaics_with_time_gaps_regular( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, true).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + // query a time interval that is greater than the time interval of the tiles and covers a region with a temporal gap + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-12-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-15T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let mut tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + // first 8 (spatial) x 2 (bands) tiles must be no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-12-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data + let expected_time1 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_time2 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32, expected_time1), + ("2025-01-01_global_b1_tile_0.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_1.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_1.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_2.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_2.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_3.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_3.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_4.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_4.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_5.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_5.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_6.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_6.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_7.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_7.tif", 1, expected_time1), + ("2025-02-01_global_b0_tile_0.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_0.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_1.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_1.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_2.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_2.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_3.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_3.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_4.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_4.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_5.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_5.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_6.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_6.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_7.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_7.tif", 1, expected_time2), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // next comes a gap of no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data again + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-04-01_global_b0_tile_0.tif", 0u32, expected_time), + ("2025-04-01_global_b1_tile_0.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_1.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_1.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_2.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_2.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_3.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_3.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_4.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_4.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_5.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_5.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_6.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_6.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_7.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_7.tif", 1, expected_time), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // last 8 (spatial) x 2 (bands) tiles must be no data + for tile in &tiles[..16] { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-06-01T00:00:00Z") + .unwrap() + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_loads_multi_band_multi_file_mosaics_with_time_gaps_irregular( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + // query a time interval that is greater than the time interval of the tiles and covers a region with a temporal gap + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-12-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-15T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first_n(2), + ); + + let mut tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + // first 8 (spatial) x 2 (bands) tiles must be no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::MIN, + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data + let expected_time1 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_time2 = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-01-01_global_b0_tile_0.tif", 0u32, expected_time1), + ("2025-01-01_global_b1_tile_0.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_1.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_1.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_2.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_2.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_3.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_3.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_4.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_4.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_5.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_5.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_6.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_6.tif", 1, expected_time1), + ("2025-01-01_global_b0_tile_7.tif", 0, expected_time1), + ("2025-01-01_global_b1_tile_7.tif", 1, expected_time1), + ("2025-02-01_global_b0_tile_0.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_0.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_1.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_1.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_2.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_2.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_3.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_3.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_4.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_4.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_5.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_5.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_6.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_6.tif", 1, expected_time2), + ("2025-02-01_global_b0_tile_7.tif", 0, expected_time2), + ("2025-02-01_global_b1_tile_7.tif", 1, expected_time2), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // next comes a gap of no data + for tile in tiles.drain(..16) { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + // next comes data again + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles = [ + ("2025-04-01_global_b0_tile_0.tif", 0u32, expected_time), + ("2025-04-01_global_b1_tile_0.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_1.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_1.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_2.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_2.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_3.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_3.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_4.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_4.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_5.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_5.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_6.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_6.tif", 1, expected_time), + ("2025-04-01_global_b0_tile_7.tif", 0, expected_time), + ("2025-04-01_global_b1_tile_7.tif", 1, expected_time), + ]; + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|(f, b, t)| { + raster_tile_from_file::( + test_data!(format!("raster/multi_tile/results/z_index/tiles/{f}")), + tiling_spatial_grid_definition, + *t, + *b, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles( + &tiles.drain(..expected_tiles.len()).collect::>(), + &expected_tiles, + false, + ); + + // last 8 (spatial) x 2 (bands) tiles must be no data + for tile in &tiles[..16] { + assert_eq!( + tile.time, + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::MAX + ) + .unwrap() + ); + assert!(tile.is_empty()); + } + + Ok(()) + } + + #[ge_context::test] + async fn it_loads_multi_band_multi_file_mosaics_reverse_z_index( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, true, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + let expected_tiles = [ + "2025-01-01_global_b0_tile_0.tif", + "2025-01-01_global_b0_tile_1.tif", + "2025-01-01_global_b0_tile_2.tif", + "2025-01-01_global_b0_tile_3.tif", + "2025-01-01_global_b0_tile_4.tif", + "2025-01-01_global_b0_tile_5.tif", + "2025-01-01_global_b0_tile_6.tif", + "2025-01-01_global_b0_tile_7.tif", + ]; + + let expected_time = TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(); + + let expected_tiles: Vec<_> = expected_tiles + .iter() + .map(|f| { + raster_tile_from_file::( + test_data!(format!( + "raster/multi_tile/results/z_index_reversed/tiles/{f}" + )), + tiling_spatial_grid_definition, + expected_time, + 0, + ) + .unwrap() + }) + .collect(); + + assert_eq_two_list_of_tiles(&tiles, &expected_tiles, false); + + Ok(()) + } + + #[ge_context::test] + async fn it_loads_multi_band_nodata_only(app_ctx: PostgresContext) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let tiling_spec = execution_context.tiling_specification(); + + let tiling_spatial_grid_definition = processor + .result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec); + + let query_tiling_pixel_grid = tiling_spatial_grid_definition + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(SpatialPartition2D::new_unchecked( + (-180., 90.).into(), + (180.0, -90.).into(), + )); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + TimeInterval::new_instant( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + BandSelection::first(), + ); + + let tiles = processor + .get_u16() + .unwrap() + .query(query_rect, &query_ctx) + .await + .unwrap() + .try_collect::>() + .await?; + + assert_eq!(tiles.len(), 8); + + for tile in tiles { + assert!(tile.is_empty()); + assert_eq!(tile.time, geoengine_datatypes::primitives::TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::MIN, + geoengine_datatypes::primitives::TimeInstance::from_str( + "2025-01-01T00:00:00Z", + ) + .unwrap(), + ).unwrap()); + } + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_checks_tile_times_regular_before_adding_to_dataset( + app_ctx: PostgresContext, + ) -> Result<()> { + let volume = VolumeName("test_data".to_string()); + + // add data + let create = CreateDataset { + data_path: DataPath::Volume(volume.clone()), + definition: DatasetDefinition { + properties: AddDataset { + name: None, + display_name: "ndvi (tiled)".to_string(), + description: "ndvi".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: Some(vec!["upload".to_owned(), "test".to_owned()]), + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: Default::default(), + result_descriptor: create_ndvi_result_descriptor(true).into(), + }), + }, + }; + + let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let req = actix_web::test::TestRequest::post() + .uri("/dataset") + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&create)?); + let res = send_test_request(req, app_ctx.clone()).await; + + let DatasetNameResponse { dataset_name } = actix_web::test::read_body_json(res).await; + let dataset_id = db + .resolve_dataset_name_to_id(&dataset_name) + .await + .unwrap() + .unwrap(); + + assert!(db.load_dataset(&dataset_id).await.is_ok()); + + // add tiles + let tiles = create_ndvi_tiles(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 200, "response: {res:?}"); + + // try to insert tiles with time that conflicts with existing tiles + let mut tiles = tiles[..1].to_vec(); + tiles[0].time = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-03T00:00:00Z").unwrap(), + ) + .into(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 400, "response: {res:?}"); + + let read_body: ErrorResponse = actix_web::test::read_body_json(res).await; + + assert_eq!(read_body, ErrorResponse { + error: "CannotAddTilesToDataset".to_string(), + message: "Cannot add tiles to dataset: Dataset tile times `[TimeInterval [1388534400000, 1388707200000)]` conflict with dataset regularity RegularTimeDimension { origin: TimeInstance(0), step: TimeStep { granularity: Months, step: 1 } }".to_string(), + }); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_checks_tile_times_irregular_before_adding_to_dataset( + app_ctx: PostgresContext, + ) -> Result<()> { + let volume = VolumeName("test_data".to_string()); + + // add data + let create = CreateDataset { + data_path: DataPath::Volume(volume.clone()), + definition: DatasetDefinition { + properties: AddDataset { + name: None, + display_name: "ndvi (tiled)".to_string(), + description: "ndvi".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: Some(vec!["upload".to_owned(), "test".to_owned()]), + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: Default::default(), + result_descriptor: create_ndvi_result_descriptor(false).into(), + }), + }, + }; + + let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let req = actix_web::test::TestRequest::post() + .uri("/dataset") + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&create)?); + let res = send_test_request(req, app_ctx.clone()).await; + + let DatasetNameResponse { dataset_name } = actix_web::test::read_body_json(res).await; + let dataset_id = db + .resolve_dataset_name_to_id(&dataset_name) + .await + .unwrap() + .unwrap(); + + assert!(db.load_dataset(&dataset_id).await.is_ok()); + + // add tiles + let tiles = create_ndvi_tiles(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 200, "response: {res:?}"); + + // try to insert tiles with time that conflicts with existing tiles + let mut tiles = tiles[..1].to_vec(); + tiles[0].time = TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-03T00:00:00Z").unwrap(), + ) + .into(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 400, "response: {res:?}"); + + let read_body: ErrorResponse = actix_web::test::read_body_json(res).await; + + assert_eq!(read_body, ErrorResponse { + error: "CannotAddTilesToDataset".to_string(), + message: "Cannot add tiles to dataset: Dataset tile time `[TimeInterval [1388534400000, 1388707200000)]` conflict with existing times `[TimeInterval [1388534400000, 1391212800000)]`".to_string(), + }); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_checks_tile_z_indexes_before_adding_to_dataset( + app_ctx: PostgresContext, + ) -> Result<()> { + let volume = VolumeName("test_data".to_string()); + + // add data + let create = CreateDataset { + data_path: DataPath::Volume(volume.clone()), + definition: DatasetDefinition { + properties: AddDataset { + name: None, + display_name: "ndvi (tiled)".to_string(), + description: "ndvi".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: Some(vec!["upload".to_owned(), "test".to_owned()]), + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: Default::default(), + result_descriptor: create_ndvi_result_descriptor(true).into(), + }), + }, + }; + + let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let req = actix_web::test::TestRequest::post() + .uri("/dataset") + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&create)?); + let res = send_test_request(req, app_ctx.clone()).await; + + let DatasetNameResponse { dataset_name } = actix_web::test::read_body_json(res).await; + let dataset_id = db + .resolve_dataset_name_to_id(&dataset_name) + .await + .unwrap() + .unwrap(); + + assert!(db.load_dataset(&dataset_id).await.is_ok()); + + // add tiles + let tiles = create_ndvi_tiles(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 200, "response: {res:?}"); + + // try to insert tiles with z index that conflicts with existing tiles + let tiles = tiles[..1].to_vec(); + + let req = actix_web::test::TestRequest::post() + .uri(&format!("/dataset/{dataset_name}/tiles")) + .append_header((header::CONTENT_LENGTH, 0)) + .append_header((header::AUTHORIZATION, Bearer::new(session.id().to_string()))) + .append_header((header::CONTENT_TYPE, "application/json")) + .set_payload(serde_json::to_string(&tiles)?); + + let res = send_test_request(req, app_ctx.clone()).await; + + assert_eq!(res.status(), 400, "response: {res:?}"); + + let read_body: ErrorResponse = actix_web::test::read_body_json(res).await; + + let conflict_tile = "raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_1.tif"; + + assert_eq!( + read_body, + ErrorResponse { + error: "CannotAddTilesToDataset".to_string(), + message: format!( + "Cannot add tiles to dataset: Dataset tile z-index of files `[\"{conflict_tile}\"]` conflict with existing tiles with the same z-indexes", + ), + } + ); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_extends_dataset_bounds_when_inserting_new_tiles( + app_ctx: PostgresContext, + ) -> Result<()> { + let volume = VolumeName("test_data".to_string()); + + let create = CreateDataset { + data_path: DataPath::Volume(volume.clone()), + definition: DatasetDefinition { + properties: AddDataset { + name: None, + display_name: "fake dataset".to_string(), + description: "fake".to_string(), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: None, + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: Default::default(), + result_descriptor: RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReferenceOption::SpatialReference( + SpatialReference::epsg_4326(), + ) + .into(), + time: TimeDescriptor { + bounds: Some( + TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-02T00:00:00Z").unwrap(), + ) + .into(), + ), + dimension: TimeDimension::Irregular, + }, + spatial_grid: SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition { + geo_transform: GeoTransform { + origin_coordinate: Coordinate2D { x: 0., y: 0. }, + x_pixel_size: 1., + y_pixel_size: -1., + }, + grid_bounds: GridBoundingBox2D { + top_left_idx: GridIdx2D { y_idx: 0, x_idx: 0 }, + bottom_right_idx: GridIdx2D { + y_idx: 100, + x_idx: 100, + }, + }, + }, + descriptor: SpatialGridDescriptorState::Source, + }, + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor { + name: "band_1".to_string(), + measurement: Measurement::Unitless.into(), + }]) + .unwrap(), + }, + }), + }, + }; + + let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); + + let db = ctx.db(); + + let id_and_name = db + .add_dataset( + create.definition.properties.into(), + create.definition.meta_data.into(), + Some(DataPath::Volume(volume.clone())), + ) + .await + .unwrap(); + + let tile = AddDatasetTile { + time: TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-02T00:00:00Z").unwrap(), + ) + .into(), + spatial_partition: SpatialPartition2D::new_unchecked( + (50., -50.).into(), + (150., -150.).into(), + ) + .into(), + band: 0, + z_index: 0, + params: GdalDatasetParameters { + file_path: "fake_path".into(), + rasterband_channel: 0, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D { x: 0., y: 0. }, + x_pixel_size: 1., + y_pixel_size: -1., + }, + width: 100, + height: 100, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: false, + }, + }; + + db.add_dataset_tiles(id_and_name.id, vec![tile]).await?; + + let ds = db.load_dataset(&id_and_name.id).await?; + let TypedResultDescriptor::Raster(dataset_rd) = ds.result_descriptor else { + panic!("expected raster dataset"); + }; + + let meta_data: Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + geoengine_operators::engine::RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + > = db + .meta_data( + &DataId::Internal(InternalDataId { + dataset_id: id_and_name.id.into(), + r#type: + crate::api::model::datatypes::InternalDataIdTypeTag::InternalDataIdTypeTag, + }) + .into(), + ) + .await?; + let metadata_rd = meta_data.result_descriptor().await?; + + assert_eq!(metadata_rd, dataset_rd); + + assert_eq!( + dataset_rd.spatial_grid, + SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition { + geo_transform: GeoTransform { + origin_coordinate: Coordinate2D { x: 0., y: 0. }, + x_pixel_size: 1., + y_pixel_size: -1., + }, + grid_bounds: GridBoundingBox2D { + top_left_idx: GridIdx2D { y_idx: 0, x_idx: 0 }, + bottom_right_idx: GridIdx2D { + y_idx: 100, + x_idx: 100, + }, + }, + }, + descriptor: SpatialGridDescriptorState::Source, + } + .into() + ); + + assert_eq!( + dataset_rd.time.bounds, + TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-02T00:00:00Z").unwrap(), + ) + .into() + ); + + let tile = AddDatasetTile { + time: TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-03T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-04T00:00:00Z").unwrap(), + ) + .into(), + spatial_partition: SpatialPartition2D::new_unchecked( + (50., -50.).into(), + (150., -150.).into(), + ) + .into(), + band: 0, + z_index: 0, + params: GdalDatasetParameters { + file_path: "fake_path".into(), + rasterband_channel: 0, + geo_transform: GdalDatasetGeoTransform { + origin_coordinate: Coordinate2D { x: -50., y: 50. }, + x_pixel_size: 1., + y_pixel_size: -1., + }, + width: 50, + height: 50, + file_not_found_handling: FileNotFoundHandling::NoData, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: false, + }, + }; + + db.add_dataset_tiles(id_and_name.id, vec![tile]).await?; + + let ds = db.load_dataset(&id_and_name.id).await?; + let TypedResultDescriptor::Raster(dataset_rd) = ds.result_descriptor else { + panic!("expected raster dataset"); + }; + + let meta_data: Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + geoengine_operators::engine::RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + > = db + .meta_data( + &DataId::Internal(InternalDataId { + dataset_id: id_and_name.id.into(), + r#type: + crate::api::model::datatypes::InternalDataIdTypeTag::InternalDataIdTypeTag, + }) + .into(), + ) + .await?; + let metadata_rd = meta_data.result_descriptor().await?; + + assert_eq!(metadata_rd, dataset_rd); + + assert_eq!( + dataset_rd.spatial_grid, + SpatialGridDescriptor { + spatial_grid: SpatialGridDefinition { + geo_transform: GeoTransform { + origin_coordinate: Coordinate2D { x: 0., y: 0. }, + x_pixel_size: 1., + y_pixel_size: -1., + }, + grid_bounds: GridBoundingBox2D { + top_left_idx: GridIdx2D { + y_idx: -50, + x_idx: -50 + }, + bottom_right_idx: GridIdx2D { + y_idx: 100, + x_idx: 100, + }, + }, + }, + descriptor: SpatialGridDescriptorState::Source, + } + .into() + ); + + assert_eq!( + dataset_rd.time.bounds, + TimeInterval::new_unchecked( + TimeInstance::from_str("2014-01-01T00:00:00Z").unwrap(), + TimeInstance::from_str("2014-01-04T00:00:00Z").unwrap(), + ) + .into() + ); + + Ok(()) + } + + #[ge_context::test] + #[allow(clippy::too_many_lines)] + async fn it_answers_multiband_time_queries(app_ctx: PostgresContext) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let processor = processor.get_u16().unwrap(); + + let time_stream = processor + .time_query( + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2024-12-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-15T00:00:00Z") + .unwrap(), + ) + .unwrap(), + &query_ctx, + ) + .await?; + + let times: Vec = time_stream.try_collect().await?; + + assert_eq!( + times, + vec![ + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::MIN, + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-01-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-02-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-05-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::MAX + ) + .unwrap() + ] + ); + + Ok(()) + } + + #[ge_context::test] + async fn it_loads_time_gap_in_multi_band_multi_file_mosaics( + app_ctx: PostgresContext, + ) -> Result<()> { + let (ctx, dataset_name) = add_multi_tile_dataset(&app_ctx, false, false).await?; + + let operator = MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(dataset_name.into()), + } + .boxed(); + + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized = operator + .clone() + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let processor = initialized.query_processor()?; + + let query_ctx = ctx.query_context(Uuid::new_v4(), Uuid::new_v4())?; + + let times = processor + .get_u16() + .unwrap() + .time_query( + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + &query_ctx, + ) + .await + .unwrap() + .try_collect::>() + .await?; + + assert_eq!( + times, + vec![ + TimeInterval::new( + geoengine_datatypes::primitives::TimeInstance::from_str("2025-03-01T00:00:00Z") + .unwrap(), + geoengine_datatypes::primitives::TimeInstance::from_str("2025-04-01T00:00:00Z") + .unwrap(), + ) + .unwrap(), + ] + ); + + Ok(()) + } } diff --git a/services/src/api/handlers/layers.rs b/services/src/api/handlers/layers.rs index 1e5e7f66b..2bcc90346 100644 --- a/services/src/api/handlers/layers.rs +++ b/services/src/api/handlers/layers.rs @@ -1,13 +1,18 @@ +use std::sync::Arc; + use super::tasks::TaskResponse; + use crate::api::model::datatypes::LayerId; use crate::api::model::responses::IdResponse; use crate::api::model::services::LayerProviderListing; use crate::api::model::services::TypedDataProviderDefinition; use crate::config::get_config_element; use crate::contexts::ApplicationContext; -use crate::datasets::{RasterDatasetFromWorkflow, schedule_raster_dataset_from_workflow_task}; +use crate::datasets::{ + RasterDatasetFromWorkflowParams, schedule_raster_dataset_from_workflow_task, +}; use crate::error::Error::NotImplemented; -use crate::error::{Error, Result}; +use crate::error::Result; use crate::layers::layer::{ AddLayer, AddLayerCollection, CollectionItem, Layer, LayerCollection, LayerCollectionListing, ProviderLayerCollectionId, UpdateLayer, UpdateLayerCollection, @@ -23,10 +28,8 @@ use crate::workflows::workflow::WorkflowId; use crate::{contexts::SessionContext, layers::layer::LayerCollectionListOptions}; use actix_web::{FromRequest, HttpResponse, Responder, web}; use geoengine_datatypes::dataset::DataProviderId; -use geoengine_datatypes::primitives::{BandSelection, QueryRectangle}; -use geoengine_operators::engine::WorkflowOperatorPath; + use serde::{Deserialize, Serialize}; -use std::sync::Arc; use utoipa::IntoParams; pub const ROOT_PROVIDER_ID: DataProviderId = @@ -798,47 +801,11 @@ async fn layer_to_dataset( let workflow_id = db.register_workflow(layer.workflow.clone()).await?; - let execution_context = ctx.execution_context()?; - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - let raster_operator = layer - .workflow - .operator - .clone() - .get_raster()? - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - let result_descriptor = raster_operator.result_descriptor(); - - let qr = QueryRectangle { - spatial_bounds: result_descriptor.bbox.ok_or( - Error::LayerResultDescriptorMissingFields { - field: "bbox".to_string(), - cause: "is None".to_string(), - }, - )?, - time_interval: result_descriptor - .time - .ok_or(Error::LayerResultDescriptorMissingFields { - field: "time".to_string(), - cause: "is None".to_string(), - })?, - spatial_resolution: result_descriptor.resolution.ok_or( - Error::LayerResultDescriptorMissingFields { - field: "spatial_resolution".to_string(), - cause: "is None".to_string(), - }, - )?, - attributes: BandSelection::first(), // TODO: add to API - }; - - let from_workflow = RasterDatasetFromWorkflow { + let from_workflow = RasterDatasetFromWorkflowParams { name: None, display_name: layer.name, description: Some(layer.description), - query: qr.into(), + query: None, as_cog: true, }; @@ -848,7 +815,6 @@ async fn layer_to_dataset( let task_id = schedule_raster_dataset_from_workflow_task( format!("layer {item}"), workflow_id, - layer.workflow, ctx, from_workflow, compression_num_threads, @@ -1357,52 +1323,54 @@ async fn delete_provider( mod tests { use super::*; - use crate::api::model::responses::ErrorResponse; - use crate::config::get_config_element; - use crate::contexts::PostgresContext; - use crate::contexts::SessionId; - use crate::datasets::RasterDatasetFromWorkflowResult; - use crate::datasets::dataset_listing_provider::{ - DatasetLayerListingCollection, DatasetLayerListingProviderDefinition, + use crate::{ + api::model::responses::ErrorResponse, + contexts::{PostgresContext, Session, SessionId}, + datasets::{ + RasterDatasetFromWorkflowResult, + dataset_listing_provider::{ + DatasetLayerListingCollection, DatasetLayerListingProviderDefinition, + }, + external::aruna::ArunaDataProviderDefinition, + }, + ge_context, + layers::{layer::Layer, storage::INTERNAL_PROVIDER_ID}, + tasks::{TaskManager, TaskStatus, util::test::wait_for_task_to_finish}, + users::{UserAuth, UserSession}, + util::tests::{TestDataUploads, admin_login, read_body_string, send_test_request}, + workflows::workflow::Workflow, }; - use crate::datasets::external::aruna::ArunaDataProviderDefinition; - use crate::ge_context; - use crate::layers::layer::Layer; - use crate::layers::storage::INTERNAL_PROVIDER_ID; - use crate::tasks::util::test::wait_for_task_to_finish; - use crate::tasks::{TaskManager, TaskStatus}; - use crate::users::{UserAuth, UserSession}; - use crate::util::tests::admin_login; - use crate::util::tests::{ - MockQueryContext, TestDataUploads, read_body_string, send_test_request, + use actix_web::{ + dev::ServiceResponse, + http::header, + test::{self, TestRequest, read_body_json}, }; - use crate::{contexts::Session, workflows::workflow::Workflow}; - use actix_web::dev::ServiceResponse; - use actix_web::{http::header, test}; use actix_web_httpauth::headers::authorization::Bearer; - use geoengine_datatypes::primitives::{CacheHint, CacheTtlSeconds, Coordinate2D}; - use geoengine_datatypes::primitives::{ - RasterQueryRectangle, SpatialPartition2D, TimeGranularity, TimeInterval, - }; - use geoengine_datatypes::raster::{ - GeoTransform, Grid, GridShape, RasterDataType, RasterTile2D, TilingSpecification, - }; - use geoengine_datatypes::spatial_reference::SpatialReference; - use geoengine_datatypes::util::test::TestDefault; - use geoengine_operators::engine::{ - ExecutionContext, InitializedRasterOperator, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, SingleRasterOrVectorSource, TypedOperator, - }; - use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; - use geoengine_operators::processing::{TimeShift, TimeShiftParams}; - use geoengine_operators::source::{GdalSource, GdalSourceParameters}; - use geoengine_operators::util::raster_stream_to_geotiff::{ - GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, raster_stream_to_geotiff_bytes, + use geoengine_datatypes::{ + primitives::{ + BandSelection, CacheHint, CacheTtlSeconds, Coordinate2D, RasterQueryRectangle, + TimeGranularity, TimeInterval, + }, + raster::{ + GeoTransform, Grid, GridBoundingBox2D, GridShape, RasterDataType, RasterTile2D, + TilingSpecification, + }, + spatial_reference::SpatialReference, + util::test::TestDefault, }; use geoengine_operators::{ - engine::VectorOperator, - mock::{MockPointSource, MockPointSourceParams}, + engine::{ + RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SingleRasterOrVectorSource, SpatialGridDescriptor, TimeDescriptor, TypedOperator, + VectorOperator, + }, + mock::{MockPointSource, MockPointSourceParams, MockRasterSource, MockRasterSourceParams}, + processing::{TimeShift, TimeShiftParams}, + source::{GdalSource, GdalSourceParameters}, + util::test::assert_eq_two_raster_operator_res_u8, }; + use uuid::Uuid; + use std::sync::Arc; use tokio_postgres::NoTls; @@ -1415,7 +1383,7 @@ mod tests { let collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/layers")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1439,7 +1407,7 @@ mod tests { assert!(response.status().is_success(), "{response:?}"); - let result: IdResponse = test::read_body_json(response).await; + let result: IdResponse = read_body_json(response).await; ctx.db() .load_layer(&result.id.clone().into()) @@ -1475,9 +1443,10 @@ mod tests { description: "Layer Description".to_string(), workflow: Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![ + (0.0, 0.1).into(), + (1.0, 1.1).into(), + ]), } .boxed() .into(), @@ -1504,7 +1473,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!( "/layerDb/collections/{collection_id}/layers/{layer_id}" )) @@ -1530,7 +1499,7 @@ mod tests { let collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/collections")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1541,7 +1510,7 @@ mod tests { assert!(response.status().is_success(), "{response:?}"); - let result: IdResponse = test::read_body_json(response).await; + let result: IdResponse = read_body_json(response).await; ctx.db() .load_layer_collection(&result.id, LayerCollectionListOptions::default()) @@ -1569,7 +1538,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/collections/{collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!({ @@ -1606,6 +1575,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1632,6 +1602,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(4., 5.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1642,7 +1613,7 @@ mod tests { properties: Default::default(), }; - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(serde_json::json!(update_layer.clone())); @@ -1682,7 +1653,7 @@ mod tests { } } }); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layerDb/collections/{collection_id}/layers")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(invalid_workflow_layer.clone()); @@ -1706,6 +1677,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1724,7 +1696,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::put() + let req = TestRequest::put() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .set_json(invalid_workflow_layer); @@ -1755,6 +1727,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -1773,7 +1746,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/layers/{layer_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); @@ -1821,7 +1794,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!( "/layerDb/collections/{collection_a_id}/collections/{collection_b_id}" )) @@ -1869,9 +1842,10 @@ mod tests { description: "Layer Description".to_string(), workflow: Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![ + (0.0, 0.1).into(), + (1.0, 1.1).into(), + ]), } .boxed() .into(), @@ -1885,7 +1859,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!( "/layerDb/collections/{collection_id}/layers/{layer_id}" )) @@ -1925,7 +1899,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/collections/{collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -1939,7 +1913,7 @@ mod tests { // try removing root collection id --> should fail - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!("/layerDb/collections/{root_collection_id}")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -1969,7 +1943,7 @@ mod tests { .await .unwrap(); - let req = test::TestRequest::delete() + let req = TestRequest::delete() .uri(&format!( "/layerDb/collections/{root_collection_id}/collections/{collection_id}" )) @@ -1999,7 +1973,7 @@ mod tests { let session_id = session.id(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!("/layers/{INTERNAL_PROVIDER_ID}/capabilities",)) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx.clone()).await; @@ -2016,7 +1990,7 @@ mod tests { let root_collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!( "/layers/collections/search/{INTERNAL_PROVIDER_ID}/{root_collection_id}?limit=5&offset=0&searchType=fulltext&searchString=x" )) @@ -2035,7 +2009,7 @@ mod tests { let root_collection_id = ctx.db().get_root_layer_collection_id().await.unwrap(); - let req = test::TestRequest::get() + let req = TestRequest::get() .uri(&format!( "/layers/collections/search/autocomplete/{INTERNAL_PROVIDER_ID}/{root_collection_id}?limit=5&offset=0&searchType=fulltext&searchString=x" )) @@ -2595,12 +2569,7 @@ mod tests { } impl MockRasterWorkflowLayerDescription { - fn new( - has_time: bool, - has_bbox: bool, - has_resolution: bool, - time_shift_millis: i32, - ) -> Self { + fn new(has_time: bool, time_shift_millis: i32) -> Self { let data: Vec> = vec![ RasterTile2D { time: TimeInterval::new_unchecked(1_671_868_800_000, 1_671_955_200_000), @@ -2622,35 +2591,28 @@ mod tests { }, ]; + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + time: TimeDescriptor::new_irregular(if has_time { + Some(TimeInterval::new_unchecked( + 1_671_868_800_000, + 1_672_041_600_000, + )) + } else { + None + }), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(-2, 0, 0, 2).unwrap(), + ), + bands: RasterBandDescriptors::new_single_band(), + }; + let raster_source = MockRasterSource { params: MockRasterSourceParams { data, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: if has_time { - Some(TimeInterval::new_unchecked( - 1_671_868_800_000, - 1_672_041_600_000, - )) - } else { - None - }, - bbox: if has_bbox { - Some(SpatialPartition2D::new_unchecked( - (0., 2.).into(), - (2., 0.).into(), - )) - } else { - None - }, - resolution: if has_resolution { - Some(GeoTransform::test_default().spatial_resolution()) - } else { - None - }, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed(); @@ -2674,19 +2636,17 @@ mod tests { }; let tiling_specification = TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape::new([2, 2]), }; - let query_rectangle = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((0., 2.).into(), (2., 0.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked( - 1_671_868_800_000 + i64::from(time_shift_millis), - 1_672_041_600_000 + i64::from(time_shift_millis), + let query_rectangle = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-2, -1, 0, 1).unwrap(), + TimeInterval::new_unchecked( + 1_671_868_800_000 - i64::from(time_shift_millis), + 1_672_041_600_000 - i64::from(time_shift_millis), ), - spatial_resolution: GeoTransform::test_default().spatial_resolution(), - attributes: BandSelection::first(), - }; + BandSelection::first(), + ); MockRasterWorkflowLayerDescription { workflow, @@ -2747,7 +2707,7 @@ mod tests { let provider_id = layer.id.provider_id; // create dataset from workflow - let req = test::TestRequest::post() + let req = TestRequest::post() .uri(&format!("/layers/{provider_id}/{layer_id}/dataset")) .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))) .append_header((header::CONTENT_TYPE, mime::APPLICATION_JSON)); @@ -2784,47 +2744,6 @@ mod tests { } } - async fn raster_operator_to_geotiff_bytes( - ctx: &C, - operator: Box, - query_rectangle: RasterQueryRectangle, - ) -> geoengine_operators::util::Result>> { - let exe_ctx = ctx.execution_context().unwrap(); - let query_ctx = ctx.mock_query_context().unwrap(); - - let initialized_operator = operator - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - let query_processor = initialized_operator - .query_processor() - .unwrap() - .get_u8() - .unwrap(); - - raster_stream_to_geotiff_bytes( - query_processor, - query_rectangle, - query_ctx, - GdalGeoTiffDatasetMetadata { - no_data_value: Some(0.), - spatial_reference: SpatialReference::epsg_4326(), - }, - GdalGeoTiffOptions { - compression_num_threads: get_config_element::() - .unwrap() - .compression_num_threads, - as_cog: true, - force_big_tiff: false, - }, - None, - Box::pin(futures::future::pending()), - exe_ctx.tiling_specification(), - (), - ) - .await - } - async fn raster_layer_to_dataset_success( app_ctx: PostgresContext, mock_source: MockRasterWorkflowLayerDescription, @@ -2843,45 +2762,37 @@ mod tests { // query the layer let workflow_operator = mock_source.workflow.operator.get_raster().unwrap(); - let workflow_result = raster_operator_to_geotiff_bytes( - &ctx, - workflow_operator, - mock_source.query_rectangle.clone(), - ) - .await - .unwrap(); // query the newly created dataset let dataset_operator = GdalSource { - params: GdalSourceParameters { - data: response.dataset.into(), - }, + params: GdalSourceParameters::new(response.dataset.into()), } .boxed(); - let dataset_result = raster_operator_to_geotiff_bytes( - &ctx, + + assert_eq_two_raster_operator_res_u8( + &ctx.execution_context().unwrap(), + &ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + workflow_operator, dataset_operator, - mock_source.query_rectangle.clone(), + mock_source.query_rectangle, + false, ) - .await - .unwrap(); - - assert_eq!(workflow_result.as_slice(), dataset_result.as_slice()); + .await; } fn test_raster_layer_to_dataset_success_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); mock_source.tiling_specification } #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_success_tiling_spec")] async fn test_raster_layer_to_dataset_success(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); raster_layer_to_dataset_success(app_ctx, mock_source).await; } fn test_raster_layer_with_timeshift_to_dataset_success_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 1_000); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 1_000); mock_source.tiling_specification } @@ -2889,46 +2800,22 @@ mod tests { tiling_spec = "test_raster_layer_with_timeshift_to_dataset_success_tiling_spec" )] async fn test_raster_layer_with_timeshift_to_dataset_success(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, true, 1_000); + let mock_source: MockRasterWorkflowLayerDescription = + MockRasterWorkflowLayerDescription::new(true, 1_000); raster_layer_to_dataset_success(app_ctx, mock_source).await; } fn test_raster_layer_to_dataset_no_time_interval_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(false, true, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(true, 0); mock_source.tiling_specification } #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_no_time_interval_tiling_spec")] async fn test_raster_layer_to_dataset_no_time_interval(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(false, true, true, 0); - - let session = admin_login(&app_ctx).await; - - let session_id = session.id(); - - let layer = mock_source.create_layer_in_context(&app_ctx).await; - - let res = send_dataset_creation_test_request(app_ctx, layer, session_id).await; - - ErrorResponse::assert( - res, - 400, - "LayerResultDescriptorMissingFields", - "Result Descriptor field 'time' is None", - ) - .await; - } - - fn test_raster_layer_to_dataset_no_bounding_box_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, false, true, 0); - mock_source.tiling_specification - } - - #[ge_context::test(tiling_spec = "test_raster_layer_to_dataset_no_bounding_box_tiling_spec")] - async fn test_raster_layer_to_dataset_no_bounding_box(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, false, true, 0); + let mock_source = MockRasterWorkflowLayerDescription::new(false, 0); let session = admin_login(&app_ctx).await; + let ctx = app_ctx.session_context(session.clone()); let session_id = session.id(); @@ -2936,40 +2823,36 @@ mod tests { let res = send_dataset_creation_test_request(app_ctx, layer, session_id).await; - ErrorResponse::assert( - res, - 400, - "LayerResultDescriptorMissingFields", - "Result Descriptor field 'bbox' is None", - ) - .await; - } - - fn test_raster_layer_to_dataset_no_spatial_resolution_tiling_spec() -> TilingSpecification { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, false, 0); - mock_source.tiling_specification - } + assert_eq!(res.status(), 200, "{:?}", res.response()); - #[ge_context::test( - tiling_spec = "test_raster_layer_to_dataset_no_spatial_resolution_tiling_spec" - )] - async fn test_raster_layer_to_dataset_no_spatial_resolution(app_ctx: PostgresContext) { - let mock_source = MockRasterWorkflowLayerDescription::new(true, true, false, 0); + let task_response = + serde_json::from_str::(&read_body_string(res).await).unwrap(); - let session = admin_login(&app_ctx).await; + let task_manager = Arc::new(ctx.tasks()); + wait_for_task_to_finish(task_manager.clone(), task_response.task_id).await; - let session_id = session.id(); + let status = task_manager + .get_task_status(task_response.task_id) + .await + .unwrap(); - let layer = mock_source.create_layer_in_context(&app_ctx).await; + let error_res = if let TaskStatus::Failed { error, .. } = status { + error + .clone() + .into_any_arc() + .downcast::() + .unwrap() + } else { + panic!("Task must fail"); + }; - let res = send_dataset_creation_test_request(app_ctx, layer, session_id).await; + let crate::error::Error::LayerResultDescriptorMissingFields { field: f, cause: c } = + error_res.as_ref() + else { + panic!("Error must be LayerResultDescriptorMissingFields") + }; - ErrorResponse::assert( - res, - 400, - "LayerResultDescriptorMissingFields", - "Result Descriptor field 'spatial_resolution' is None", - ) - .await; + assert_eq!(f, "time"); + assert_eq!(c, "is None"); } } diff --git a/services/src/api/handlers/permissions.rs b/services/src/api/handlers/permissions.rs index 9e0d597bb..b152222b2 100644 --- a/services/src/api/handlers/permissions.rs +++ b/services/src/api/handlers/permissions.rs @@ -1,14 +1,16 @@ -use crate::api::model::datatypes::{DataProviderId, LayerId}; -use crate::contexts::{ApplicationContext, GeoEngineDb, SessionContext}; -use crate::datasets::DatasetName; -use crate::datasets::storage::DatasetDb; -use crate::error::{self, Error, Result}; -use crate::layers::listing::LayerCollectionId; -use crate::machine_learning::MlModelDb; -use crate::permissions::{ - Permission, PermissionDb, PermissionListing as DbPermissionListing, ResourceId, Role, RoleId, +use crate::{ + api::model::datatypes::{DataProviderId, LayerId}, + contexts::{ApplicationContext, GeoEngineDb, SessionContext}, + datasets::{DatasetName, storage::DatasetDb}, + error::{self, Error, Result}, + layers::listing::LayerCollectionId, + machine_learning::MlModelDb, + permissions::{ + Permission, PermissionDb, PermissionListing as DbPermissionListing, ResourceId, Role, + RoleId, + }, + projects::ProjectId, }; -use crate::projects::ProjectId; use actix_web::{FromRequest, HttpResponse, web}; use geoengine_datatypes::error::BoxedResultExt; use geoengine_datatypes::machine_learning::MlModelName; @@ -39,12 +41,12 @@ where } /// Request for adding a new permission to the given role on the given resource -#[derive(Debug, PartialEq, Eq, Deserialize, Clone, ToSchema)] +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, ToSchema)] #[serde(rename_all = "camelCase")] pub struct PermissionRequest { - resource: Resource, - role_id: RoleId, - permission: Permission, + pub resource: Resource, + pub role_id: RoleId, + pub permission: Permission, } #[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone, ToSchema)] @@ -400,6 +402,7 @@ mod tests { let gdal = GdalSource { params: GdalSourceParameters { data: gdal_dataset_name, + overview_level: None, }, } .boxed(); @@ -673,6 +676,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), diff --git a/services/src/api/handlers/plots.rs b/services/src/api/handlers/plots.rs index d580b9d4d..a118bedb4 100644 --- a/services/src/api/handlers/plots.rs +++ b/services/src/api/handlers/plots.rs @@ -10,11 +10,10 @@ use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, Responder, web}; use base64::Engine; -use geoengine_datatypes::operations::reproject::reproject_query; +use geoengine_datatypes::operations::reproject::reproject_spatial_query; use geoengine_datatypes::plots::PlotOutputFormat; -use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, SpatialResolution, VectorQueryRectangle, -}; +use geoengine_datatypes::primitives::{BoundingBox2D, SpatialResolution}; +use geoengine_datatypes::primitives::{PlotQueryRectangle, PlotSeriesSelection}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ QueryContext, ResultDescriptor, TypedPlotQueryProcessor, WorkflowOperatorPath, @@ -131,17 +130,21 @@ async fn get_plot_handler( let request_spatial_ref: SpatialReference = params.crs.ok_or(error::Error::MissingSpatialReference)?; - let query_rect = VectorQueryRectangle { - spatial_bounds: params.bbox, - time_interval: params.time.into(), - spatial_resolution: params.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let query_rect = + PlotQueryRectangle::new(params.bbox, params.time.into(), PlotSeriesSelection::all()); let query_rect = if request_spatial_ref == workflow_spatial_ref { Some(query_rect) } else { - reproject_query(query_rect, workflow_spatial_ref, request_spatial_ref, true)? + let repr_spatial_query = reproject_spatial_query( + query_rect.spatial_bounds(), + workflow_spatial_ref, + request_spatial_ref, + true, + )?; + repr_spatial_query.map(|r| { + PlotQueryRectangle::new(r, query_rect.time_interval(), *query_rect.attributes()) + }) }; let Some(query_rect) = query_rect else { @@ -162,18 +165,18 @@ async fn get_plot_handler( let data = match processor { TypedPlotQueryProcessor::JsonPlain(processor) => { - let json = processor.plot_query(query_rect.into(), &query_ctx); + let json = processor.plot_query(query_rect, &query_ctx); abortable_query_execution(json, conn_closed, query_abort_trigger).await? } TypedPlotQueryProcessor::JsonVega(processor) => { - let chart = processor.plot_query(query_rect.into(), &query_ctx); + let chart = processor.plot_query(query_rect, &query_ctx); let chart = abortable_query_execution(chart, conn_closed, query_abort_trigger).await; let chart = chart?; serde_json::to_value(chart).context(error::SerdeJson)? } TypedPlotQueryProcessor::ImagePng(processor) => { - let png_bytes = processor.plot_query(query_rect.into(), &query_ctx); + let png_bytes = processor.plot_query(query_rect, &query_ctx); let png_bytes = abortable_query_execution(png_bytes, conn_closed, query_abort_trigger).await; let png_bytes = png_bytes?; @@ -224,12 +227,15 @@ mod tests { use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::DateTime; use geoengine_datatypes::raster::{ - Grid2D, RasterDataType, RasterTile2D, TileInformation, TilingSpecification, + GeoTransform, Grid2D, GridBoundingBox2D, RasterDataType, RasterTile2D, TileInformation, + TilingSpecification, }; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::test::TestDefault; + use geoengine_operators::engine::TimeDescriptor; use geoengine_operators::engine::{ PlotOperator, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, + SpatialGridDescriptor, }; use geoengine_operators::mock::{MockRasterSource, MockRasterSourceParams}; use geoengine_operators::plot::{ @@ -239,6 +245,17 @@ mod tests { use tokio_postgres::NoTls; fn example_raster_source() -> Box { + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::U8, + spatial_reference: SpatialReference::epsg_4326().into(), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + GridBoundingBox2D::new_min_max(-3, 0, 0, 2).unwrap(), + ), + time: TimeDescriptor::new_irregular(None), + bands: RasterBandDescriptors::new_single_band(), + }; + MockRasterSource { params: MockRasterSourceParams { data: vec![RasterTile2D::new_with_tile_info( @@ -254,21 +271,14 @@ mod tests { .into(), CacheHint::default(), )], - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U8, - spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, }, } .boxed() } fn json_tiling_spec() -> TilingSpecification { - TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()) + TilingSpecification::new([3, 2].into()) } #[ge_context::test(tiling_spec = "json_tiling_spec")] @@ -320,7 +330,7 @@ mod tests { "plotType": "Statistics", "data": { "Raster-1": { - "valueCount": 24, // Note: this is caused by the query being a BoundingBox where the right and lower bounds are inclusive. This requires that the tiles that inculde the right and lower bounds are also produced. + "valueCount": 6, // TODO: investigate why the bbox is satisfied with 6 pixels while the borders should in theory also be included ... "validCount": 6, "min": 1.0, "max": 6.0, @@ -334,7 +344,7 @@ mod tests { } fn json_vega_tiling_spec() -> TilingSpecification { - TilingSpecification::new([0.0, 0.0].into(), [3, 2].into()) + TilingSpecification::new([3, 2].into()) } #[ge_context::test(tiling_spec = "json_vega_tiling_spec")] diff --git a/services/src/api/handlers/spatial_references.rs b/services/src/api/handlers/spatial_references.rs index 82a5617ec..e1b93c591 100755 --- a/services/src/api/handlers/spatial_references.rs +++ b/services/src/api/handlers/spatial_references.rs @@ -99,6 +99,15 @@ pub enum AxisOrder { EastNorth, } +impl AxisOrder { + pub fn xy_to_native_order(&self, input: [T; 2]) -> [T; 2] { + match &self { + AxisOrder::EastNorth => input, + AxisOrder::NorthEast => [input[1], input[0]], + } + } +} + /// Get the proj json information for the given `srs_string` if it is known. // TODO: expose method in proj crate instead fn proj_json(srs_string: &str) -> Option { @@ -203,15 +212,16 @@ pub(crate) async fn get_spatial_reference_specification_handler, _session: C::Session, ) -> Result { - spatial_reference_specification(&srs_string).map(web::Json) + let spatial_ref = SpatialReference::from_str(&srs_string)?; + spatial_reference_specification(spatial_ref).map(web::Json) } /// custom spatial references not known by proj or that shall be overriden fn custom_spatial_reference_specification( - srs_string: &str, + spatial_ref: SpatialReference, ) -> Option { // TODO: provide a generic storage for custom spatial reference specifications - match srs_string.to_uppercase().as_str() { + match spatial_ref.srs_string().to_uppercase().as_str() { "SR-ORG:81" => Some(SpatialReferenceSpecification { name: "GEOS - GEOstationary Satellite".to_owned(), spatial_reference: SpatialReference::new(SpatialReferenceAuthority::SrOrg, 81), @@ -234,18 +244,22 @@ fn custom_spatial_reference_specification( } } -pub fn spatial_reference_specification(srs_string: &str) -> Result { - if let Some(sref) = custom_spatial_reference_specification(srs_string) { +pub fn spatial_reference_specification( + spatial_reference: SpatialReference, +) -> Result { + if let Some(sref) = custom_spatial_reference_specification(spatial_reference) { return Ok(sref); } - let spatial_reference = - geoengine_datatypes::spatial_reference::SpatialReference::from_str(srs_string)?; - let json = proj_json(srs_string).ok_or_else(|| Error::UnknownSrsString { - srs_string: srs_string.to_owned(), + let spatial_reference: geoengine_datatypes::spatial_reference::SpatialReference = + spatial_reference.into(); + let srs_string = spatial_reference.srs_string(); + + let json = proj_json(&srs_string).ok_or_else(|| Error::UnknownSrsString { + srs_string: srs_string.clone(), })?; - let proj_string = proj_proj_string(srs_string).ok_or_else(|| Error::UnknownSrsString { - srs_string: srs_string.to_owned(), + let proj_string = proj_proj_string(&srs_string).ok_or_else(|| Error::UnknownSrsString { + srs_string: srs_string.clone(), })?; let extent: geoengine_datatypes::primitives::BoundingBox2D = @@ -323,7 +337,10 @@ mod tests { #[test] fn spec_webmercator() { - let spec = spatial_reference_specification("EPSG:3857").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:3857").unwrap().into(), + ) + .unwrap(); assert_eq!(spec.name, "WGS 84 / Pseudo-Mercator"); assert_eq!( spec.spatial_reference, @@ -351,7 +368,10 @@ mod tests { #[test] fn spec_wgs84() { - let spec = spatial_reference_specification("EPSG:4326").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:4326").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "WGS 84".to_owned(), @@ -374,7 +394,10 @@ mod tests { #[test] fn spec_utm32n() { - let spec = spatial_reference_specification("EPSG:32632").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("EPSG:32632").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "WGS 84 / UTM zone 32N".to_owned(), @@ -395,7 +418,10 @@ mod tests { #[test] fn spec_geos() { - let spec = spatial_reference_specification("SR-ORG:81").unwrap(); + let spec = spatial_reference_specification( + SpatialReference::from_str("SR-ORG:81").unwrap().into(), + ) + .unwrap(); assert_eq!( SpatialReferenceSpecification { name: "GEOS - GEOstationary Satellite".to_owned(), diff --git a/services/src/api/handlers/wcs.rs b/services/src/api/handlers/wcs.rs index 99104d1a1..1bf1da862 100644 --- a/services/src/api/handlers/wcs.rs +++ b/services/src/api/handlers/wcs.rs @@ -1,34 +1,35 @@ -use crate::api::handlers::spatial_references::{AxisOrder, spatial_reference_specification}; -use crate::api::model::datatypes::TimeInterval; -use crate::api::ogc::util::{OgcProtocol, OgcRequestGuard, ogc_endpoint_url}; +use crate::api::handlers::spatial_references::spatial_reference_specification; +use crate::api::ogc::util::{OgcProtocol, OgcQueryExtractor, ogc_endpoint_url}; use crate::api::ogc::wcs::request::{DescribeCoverage, GetCapabilities, GetCoverage, WcsVersion}; use crate::config; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; use crate::error::Result; use crate::error::{self, Error}; -use crate::util::server::{CacheControlHeader, connection_closed, not_implemented_handler}; +use crate::util::server::{CacheControlHeader, connection_closed}; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BandSelection, RasterQueryRectangle, SpatialPartition2D, + AxisAlignedRectangle, BandSelection, RasterQueryRectangle, SpatialResolution, TimeInterval, }; -use geoengine_datatypes::raster::GeoTransform; -use geoengine_datatypes::{primitives::SpatialResolution, spatial_reference::SpatialReference}; +use geoengine_datatypes::raster::GridShape2D; +use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::call_on_generic_raster_processor_gdal_types; -use geoengine_operators::engine::{ExecutionContext, RasterOperator, WorkflowOperatorPath}; -use geoengine_operators::engine::{ResultDescriptor, SingleRasterOrVectorSource}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; -use geoengine_operators::util::input::RasterOrVectorOperator; +use geoengine_operators::engine::{ + ExecutionContext, InitializedRasterOperator, WorkflowOperatorPath, +}; use geoengine_operators::util::raster_stream_to_geotiff::{ GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, raster_stream_to_multiband_geotiff_bytes, }; +use serde::Deserialize; use snafu::ensure; use std::str::FromStr; use std::time::Duration; use tracing::info; use url::Url; +use utoipa::IntoParams; +use utoipa::openapi::{Ref, Required}; use uuid::Uuid; pub(crate) fn init_wcs_routes(cfg: &mut web::ServiceConfig) @@ -36,25 +37,56 @@ where C: ApplicationContext, C::Session: FromRequest, { - cfg.service( - web::resource("/wcs/{workflow}") - .route( - web::get() - .guard(OgcRequestGuard::new("GetCapabilities")) - .to(wcs_capabilities_handler::), - ) - .route( - web::get() - .guard(OgcRequestGuard::new("DescribeCoverage")) - .to(wcs_describe_coverage_handler::), - ) - .route( - web::get() - .guard(OgcRequestGuard::new("GetCoverage")) - .to(wcs_get_coverage_handler::), - ) - .route(web::get().to(not_implemented_handler)), - ); + cfg.service(web::resource("/wcs/{workflow}").route(web::get().to(wcs_handler::))); +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "request", rename_all = "PascalCase")] +pub enum WcsQueryParams { + GetCapabilities(GetCapabilities), + DescribeCoverage(DescribeCoverage), + GetCoverage(GetCoverage), +} + +/// manual implementation because derive macro does not support enums +/// Note, that WCS is not really OpenAPI compatible, so this is just an approximation to enable client generation +impl IntoParams for WcsQueryParams { + fn into_params( + _parameter_in_provider: impl Fn() -> Option, + ) -> Vec { + let pip = || Some(utoipa::openapi::path::ParameterIn::Query); + + let mut params = Vec::new(); + + params.push( + utoipa::openapi::path::ParameterBuilder::new() + .name("request") + .required(utoipa::openapi::Required::True) + .parameter_in(pip().unwrap_or_default()) + .description(Some("type of WCS request")) + .schema(Some(Ref::from_schema_name("WcsRequest"))) + .build(), + ); + + for mut p in GetCapabilities::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in DescribeCoverage::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetCoverage::into_params(pip) { + p.required = Required::False; + params.push(p); + } + + // remove duplicate parameters + params.sort_by(|a, b| a.name.cmp(&b.name)); + params.dedup_by(|a, b| a.name == b.name); + + params + } } fn wcs_url(workflow: WorkflowId) -> Result { @@ -64,11 +96,11 @@ fn wcs_url(workflow: WorkflowId) -> Result { ogc_endpoint_url(&base, OgcProtocol::Wcs, workflow) } -/// Get WCS Capabilities +/// OGC WCS endpoint #[utoipa::path( tag = "OGC WCS", get, - path = "/wcs/{workflow}?request=GetCapabilities", + path = "/wcs/{workflow}", responses( (status = 200, description = "OK", content_type = "text/xml", body = String, // TODO: add example when utoipa supports more than just json examples @@ -76,29 +108,48 @@ fn wcs_url(workflow: WorkflowId) -> Result { ), params( ("workflow" = WorkflowId, description = "Workflow id"), - GetCapabilities + WcsQueryParams ), security( ("session_token" = []) ) )] + +async fn wcs_handler( + req: HttpRequest, + workflow: web::Path, + request: OgcQueryExtractor, + app_ctx: web::Data, + session: C::Session, +) -> Result { + match request.into_inner() { + WcsQueryParams::GetCapabilities(r) => wcs_get_capabilities::(workflow, r, session).await, + WcsQueryParams::DescribeCoverage(r) => { + wcs_describe_coverage::(workflow.into_inner(), r, app_ctx, session).await + } + WcsQueryParams::GetCoverage(r) => { + wcs_get_coverage::(req, workflow.into_inner(), r, app_ctx, session).await + } + } +} + #[allow( clippy::unused_async, // the function signature of request handlers requires it clippy::no_effect_underscore_binding // need `_session` to quire authentication )] -async fn wcs_capabilities_handler( +async fn wcs_get_capabilities( workflow: web::Path, - request: web::Query, + request: GetCapabilities, _session: C::Session, ) -> Result { - let workflow = workflow.into_inner(); - info!("{request:?}"); // TODO: workflow bounding box // TODO: host schema file(?) // TODO: load ServiceIdentification and ServiceProvider from config + let workflow = workflow.into_inner(); + let wcs_url = wcs_url(workflow)?; let mock = format!( r#" @@ -158,32 +209,14 @@ async fn wcs_capabilities_handler( Ok(HttpResponse::Ok().content_type(mime::TEXT_XML).body(mock)) } -/// Get WCS Coverage Description -#[utoipa::path( - tag = "OGC WCS", - get, - path = "/wcs/{workflow}?request=DescribeCoverage", - responses( - (status = 200, description = "OK", content_type = "text/xml", body = String, - // TODO: add example when utoipa supports more than just json examples - ) - ), - params( - ("workflow" = WorkflowId, description = "Workflow id"), - DescribeCoverage - ), - security( - ("session_token" = []) - ) -)] #[allow(clippy::too_many_lines)] -async fn wcs_describe_coverage_handler( - workflow: web::Path, - request: web::Query, +async fn wcs_describe_coverage( + workflow: WorkflowId, + request: DescribeCoverage, app_ctx: web::Data, session: C::Session, ) -> Result { - let endpoint = workflow.into_inner(); + let endpoint = workflow; info!("{request:?}"); @@ -217,46 +250,34 @@ async fn wcs_describe_coverage_handler( let result_descriptor = operator.result_descriptor(); + let spatial_grid_descriptor = result_descriptor.spatial_grid_descriptor(); + let spatial_reference: Option = result_descriptor.spatial_reference.into(); let spatial_reference = spatial_reference.ok_or(error::Error::MissingSpatialReference)?; + let spatial_reference_spec = spatial_reference_specification(spatial_reference.into())?; + let spatial_ref_axis_order = + spatial_reference_spec + .axis_order + .ok_or(Error::AxisOrderingNotKnownForSrs { + srs_string: spatial_reference.srs_string(), + })?; + let bounds = spatial_grid_descriptor.spatial_partition(); + + let [bbox_ll_0, bbox_ll_1] = + spatial_ref_axis_order.xy_to_native_order([bounds.lower_left().x, bounds.lower_left().y]); + let [bbox_ur_0, bbox_ur_1] = + spatial_ref_axis_order.xy_to_native_order([bounds.upper_right().x, bounds.upper_right().y]); + + let GridShape2D { shape_array } = spatial_grid_descriptor.grid_shape(); + + let [raster_size_0, raster_size_1] = spatial_ref_axis_order.xy_to_native_order(shape_array); - let resolution = result_descriptor - .resolution - .unwrap_or(SpatialResolution::zero_point_one()); - - let pixel_size_x = resolution.x; - let pixel_size_y = -resolution.y; - - let bbox = if let Some(bbox) = result_descriptor.bbox { - bbox - } else { - spatial_reference.area_of_use_projected()? - }; - - let axis_order = spatial_reference_specification(&spatial_reference.proj_string()?)? - .axis_order - .ok_or(Error::AxisOrderingNotKnownForSrs { - srs_string: spatial_reference.srs_string(), - })?; - let (bbox_ll_0, bbox_ll_1, bbox_ur_0, bbox_ur_1) = match axis_order { - AxisOrder::EastNorth => ( - bbox.lower_left().x, - bbox.lower_left().y, - bbox.upper_right().x, - bbox.upper_right().y, - ), - AxisOrder::NorthEast => ( - bbox.lower_left().y, - bbox.lower_left().x, - bbox.upper_right().y, - bbox.upper_right().x, - ), - }; - - let geo_transform = GeoTransform::new(bbox.upper_left(), pixel_size_x, pixel_size_y); - - let [raster_size_x, raster_size_y] = - *(geo_transform.lower_right_pixel_idx(&bbox) + [1, 1]).inner(); + let SpatialResolution { + x: pixel_size_x, + y: pixel_size_y, + } = spatial_grid_descriptor.spatial_resolution(); + + let band_0 = &result_descriptor.bands[0]; let mock = format!( r#" @@ -277,7 +298,7 @@ async fn wcs_describe_coverage_handler( 0 0 - {raster_size_y} {raster_size_x} + {raster_size_0} {raster_size_1} urn:ogc:def:crs:{srs_authority}::{srs_code} @@ -306,39 +327,29 @@ async fn wcs_describe_coverage_handler( workflow_id = identifiers, srs_authority = spatial_reference.authority(), srs_code = spatial_reference.code(), - origin_x = bbox.upper_left().x, - origin_y = bbox.upper_left().y, - band_name = result_descriptor.bands[0].name, + origin_x = bounds.upper_left().x, + origin_y = bounds.upper_left().y, + bbox_ll_0 = bbox_ll_0, + bbox_ll_1 = bbox_ll_1, + bbox_ur_0 = bbox_ur_0, + bbox_ur_1 = bbox_ur_1, + band_name = band_0.name, + pixel_size_y = -pixel_size_y, // TODO: use the "real" sign in the resolution? + pixel_size_x = pixel_size_x ); Ok(HttpResponse::Ok().content_type(mime::TEXT_XML).body(mock)) } -/// Get WCS Coverage -#[utoipa::path( - tag = "OGC WCS", - get, - path = "/wcs/{workflow}?request=GetCoverage", - responses( - (status = 200, response = crate::api::model::responses::PngResponse), - ), - params( - ("workflow" = WorkflowId, description = "Workflow id"), - GetCoverage - ), - security( - ("session_token" = []) - ) -)] #[allow(clippy::too_many_lines)] -async fn wcs_get_coverage_handler( +async fn wcs_get_coverage( req: HttpRequest, - workflow: web::Path, - request: web::Query, + workflow: WorkflowId, + request: GetCoverage, app_ctx: web::Data, session: C::Session, ) -> Result { - let endpoint = workflow.into_inner(); + let endpoint = workflow; info!("{request:?}"); @@ -364,21 +375,13 @@ async fn wcs_get_coverage_handler( .map(Duration::from_secs), ); + let request_spatial_ref: SpatialReference = request.spatial_ref().map(Into::into)?; + let request_resolution = request.spatial_resolution().transpose()?; let request_partition = request.spatial_partition()?; - - if let Some(gridorigin) = request.gridorigin { - ensure!( - gridorigin.coordinate(request.gridbasecrs)? == request_partition.upper_left(), - error::WcsGridOriginMustEqualBoundingboxUpperLeft - ); - } - - if let Some(bbox_spatial_reference) = request.boundingbox.spatial_reference { - ensure!( - request.gridbasecrs == bbox_spatial_reference, - error::WcsBoundingboxCrsMustEqualGridBaseCrs - ); - } + let request_time: TimeInterval = request + .time + .map_or_else(default_time_from_config, Into::into); + let request_no_data_value = request.nodatavalue; let ctx = app_ctx.session_context(session); @@ -395,84 +398,36 @@ async fn wcs_get_coverage_handler( .initialize(workflow_operator_path_root, &execution_context) .await?; - // handle request and workflow crs matching - let workflow_spatial_ref: Option = - initialized.result_descriptor().spatial_reference().into(); - let workflow_spatial_ref = workflow_spatial_ref.ok_or(error::Error::InvalidSpatialReference)?; + let tiling_spec = execution_context.tiling_specification(); - let request_spatial_ref: SpatialReference = request.gridbasecrs.into(); - let request_no_data_value = request.nodatavalue; - - // perform reprojection if necessary - let initialized = if request_spatial_ref == workflow_spatial_ref { - initialized - } else { - tracing::debug!( - "WCS query srs: {request_spatial_ref}, workflow srs: {workflow_spatial_ref} --> injecting reprojection" - ); - - let reprojection_params = ReprojectionParams { - target_spatial_reference: request_spatial_ref, - }; + let wrapped = + geoengine_operators::util::WrapWithProjectionAndResample::new_create_result_descriptor( + operator, + initialized, + ) + .wrap_with_projection_and_resample( + Some(request_partition.upper_left()), // TODO: set none if not changed? But how to handle mapping to grid? + request_resolution, + request_spatial_ref, + tiling_spec, + &execution_context, + ) + .await?; - // create the reprojection operator in order to get the canonic operator name - let reprojected_workflow = Reprojection { - params: reprojection_params, - sources: SingleRasterOrVectorSource { - source: RasterOrVectorOperator::Raster(operator), - }, - } - .boxed(); - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - // TODO: avoid re-initialization and re-use unprojected workflow. However, this requires updating all operator paths - - // In order to check whether we need to inject a reprojection, we first need to initialize the - // original workflow. Then we can check the result projection. Previously, we then just wrapped - // the initialized workflow with an initialized reprojection. IMHO this is wrong because - // initialization propagates the workflow path down the children and appends a new segment for - // each level. So we can't re-use an already initialized workflow, because all the workflow path/ - // operator names will be wrong. That's why I now build a new workflow with a reprojection and - // perform a full initialization. I only added the TODO because we did some optimization here - // which broke at some point when the workflow operator paths were introduced but no one noticed. - - let irp = reprojected_workflow - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - Box::new(irp) - }; - - let processor = initialized.query_processor()?; - - let spatial_resolution: SpatialResolution = - if let Some(spatial_resolution) = request.spatial_resolution() { - spatial_resolution? - } else { - // TODO: proper default resolution - SpatialResolution { - x: request_partition.size_x() / 256., - y: request_partition.size_y() / 256., - } - }; - - // snap bbox to grid - let geo_transform = GeoTransform::new( - request_partition.upper_left(), - spatial_resolution.x, - -spatial_resolution.y, + let query_tiling_pixel_grid = wrapped + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(request_partition); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + request_time, + BandSelection::first(), // TODO: support multi bands in API and set the selection here ); - let idx = geo_transform.lower_right_pixel_idx(&request_partition) + [1, 1]; - let lower_right = geo_transform.grid_idx_to_pixel_upper_left_coordinate_2d(idx); - let snapped_partition = SpatialPartition2D::new(request_partition.upper_left(), lower_right)?; - let query_rect = RasterQueryRectangle { - spatial_bounds: snapped_partition, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - spatial_resolution, - attributes: BandSelection::first(), // TODO: support multi bands in API and set the selection here - }; + let processor = wrapped.initialized_operator.query_processor()?; let query_ctx = ctx.query_context(identifier.0, Uuid::new_v4())?; @@ -514,7 +469,7 @@ fn default_time_from_config() -> TimeInterval { .and_then(|ogc| ogc.default_time) .map_or_else( || { - geoengine_datatypes::primitives::TimeInterval::new_instant( + TimeInterval::new_instant( geoengine_datatypes::primitives::TimeInstance::now(), ) .expect("is a valid time interval") @@ -524,7 +479,6 @@ fn default_time_from_config() -> TimeInterval { }, |time| time.time_interval(), ) - .into() } #[cfg(test)] @@ -538,12 +492,17 @@ mod tests { use actix_web::http::header; use actix_web::test; use actix_web_httpauth::headers::authorization::Bearer; - use geoengine_datatypes::raster::{GridShape2D, TilingSpecification}; + use geoengine_datatypes::raster::GridShape2D; + use geoengine_datatypes::raster::TilingSpecification; use geoengine_datatypes::test_data; use geoengine_datatypes::util::ImageFormat; use geoengine_datatypes::util::assert_image_equals_with_format; use tokio_postgres::NoTls; + fn tiling_spec() -> TilingSpecification { + TilingSpecification::new(GridShape2D::new([600, 600])) + } + #[ge_context::test] async fn get_capabilities(app_ctx: PostgresContext) { let session = app_ctx.create_anonymous_session().await.unwrap(); @@ -662,7 +621,7 @@ mod tests { xmlns:ogc="http://www.opengis.net/ogc" xmlns:ows="http://www.opengis.net/ows/1.1" xmlns:gml="http://www.opengis.net/gml" - xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wcs/1.1.1 http://127.0.0.1:3030/api/wcs/1625f9b9-7408-58ab-9482-4d5fb0e3c6e4/schemas/wcs/1.1.1/wcsDescribeCoverage.xsd"> + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.opengis.net/wcs/1.1.1 http://127.0.0.1:3030/api/wcs/{workflow_id}/schemas/wcs/1.1.1/wcsDescribeCoverage.xsd"> Workflow {workflow_id} {workflow_id} @@ -706,13 +665,6 @@ mod tests { // TODO: add get_coverage with masked band - fn tiling_spec() -> TilingSpecification { - TilingSpecification { - origin_coordinate: (0., 0.).into(), - tile_size_in_pixels: GridShape2D::new([600, 600]), - } - } - #[ge_context::test(tiling_spec = "tiling_spec")] async fn get_coverage_with_nodatavalue(app_ctx: PostgresContext) { // override the pixel size since this test was designed for 600 x 600 pixel tiles diff --git a/services/src/api/handlers/wfs.rs b/services/src/api/handlers/wfs.rs index 5bde73e6a..53cff55f7 100644 --- a/services/src/api/handlers/wfs.rs +++ b/services/src/api/handlers/wfs.rs @@ -1,24 +1,21 @@ use crate::api::model::datatypes::TimeInterval; -use crate::api::ogc::util::{OgcProtocol, OgcRequestGuard, ogc_endpoint_url}; +use crate::api::ogc::util::{OgcProtocol, OgcQueryExtractor, ogc_endpoint_url}; use crate::api::ogc::wfs::request::{GetCapabilities, GetFeature}; use crate::config; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; use crate::error; use crate::error::Result; -use crate::util::server::{CacheControlHeader, connection_closed, not_implemented_handler}; +use crate::util::server::{CacheControlHeader, connection_closed}; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::{Workflow, WorkflowId}; use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; use futures::future::BoxFuture; use futures_util::TryStreamExt; use geoengine_datatypes::collections::ToGeoJson; +use geoengine_datatypes::collections::{FeatureCollection, MultiPointCollection}; use geoengine_datatypes::primitives::VectorQueryRectangle; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; -use geoengine_datatypes::{ - collections::{FeatureCollection, MultiPointCollection}, - primitives::SpatialResolution, -}; use geoengine_datatypes::{ primitives::{FeatureData, Geometry, MultiPoint}, spatial_reference::SpatialReference, @@ -28,7 +25,9 @@ use geoengine_operators::engine::{ VectorOperator, VectorQueryProcessor, }; use geoengine_operators::engine::{QueryProcessor, WorkflowOperatorPath}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; +use geoengine_operators::processing::{ + DeriveOutRasterSpecsSource, Reprojection, ReprojectionParams, +}; use geoengine_operators::util::abortable_query_execution; use geoengine_operators::util::input::RasterOrVectorOperator; use reqwest::Url; @@ -37,7 +36,8 @@ use serde_json::json; use snafu::ensure; use std::str::FromStr; use std::time::Duration; -use utoipa::ToSchema; +use utoipa::openapi::{Ref, Required}; +use utoipa::{IntoParams, ToSchema}; use uuid::Uuid; pub(crate) fn init_wfs_routes(cfg: &mut web::ServiceConfig) @@ -45,27 +45,59 @@ where C: ApplicationContext, C::Session: FromRequest, { - cfg.service( - web::resource("/wfs/{workflow}") - .route( - web::get() - .guard(OgcRequestGuard::new("GetCapabilities")) - .to(wfs_capabilities_handler::), - ) - .route( - web::get() - .guard(OgcRequestGuard::new("GetFeature")) - .to(wfs_feature_handler::), - ) - .route(web::get().to(not_implemented_handler)), - ); + cfg.service(web::resource("/wfs/{workflow}").route(web::get().to(wfs_handler::))); +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "request", rename_all = "PascalCase")] +#[allow(clippy::large_enum_variant)] // variants may have many fields +pub enum WfsQueryParams { + GetCapabilities(GetCapabilities), + GetFeature(GetFeature), +} + +/// manual implementation because derive macro does not support enums +/// Note, that WFS is not really OpenAPI compatible, so this is just an approximation to enable client generation +impl IntoParams for WfsQueryParams { + fn into_params( + _parameter_in_provider: impl Fn() -> Option, + ) -> Vec { + let pip = || Some(utoipa::openapi::path::ParameterIn::Query); + + let mut params = Vec::new(); + + params.push( + utoipa::openapi::path::ParameterBuilder::new() + .name("request") + .required(utoipa::openapi::Required::True) + .parameter_in(pip().unwrap_or_default()) + .description(Some("type of WFS request")) + .schema(Some(Ref::from_schema_name("WfsRequest"))) + .build(), + ); + + for mut p in GetCapabilities::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetFeature::into_params(pip) { + p.required = Required::False; + params.push(p); + } + + // remove duplicate parameters + params.sort_by(|a, b| a.name.cmp(&b.name)); + params.dedup_by(|a, b| a.name == b.name); + + params + } } -/// Get WFS Capabilities +/// OGC WFS endpoint #[utoipa::path( tag = "OGC WFS", get, - path = "/wfs/{workflow}?request=GetCapabilities", + path = "/wfs/{workflow}", responses( (status = 200, description = "OK", content_type = "text/xml", body = String, // TODO: add example when utoipa supports more than just json examples @@ -156,28 +188,148 @@ where // // // "# - ) + ), + (status = 200, description = "OK", body = GeoJson, + example = json!( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 0.0, + 0.1 + ] + }, + "properties": { + "foo": 0 + }, + "when": { + "start": "1970-01-01T00:00:00+00:00", + "end": "1970-01-01T00:00:00.001+00:00", + "type": "Interval" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 1.0, + 1.1 + ] + }, + "properties": { + "foo": null + }, + "when": { + "start": "1970-01-01T00:00:00+00:00", + "end": "1970-01-01T00:00:00.001+00:00", + "type": "Interval" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 2.0, + 3.1 + ] + }, + "properties": { + "foo": 2 + }, + "when": { + "start": "1970-01-01T00:00:00+00:00", + "end": "1970-01-01T00:00:00.001+00:00", + "type": "Interval" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 3.0, + 3.1 + ] + }, + "properties": { + "foo": 3 + }, + "when": { + "start": "1970-01-01T00:00:00+00:00", + "end": "1970-01-01T00:00:00.001+00:00", + "type": "Interval" + } + }, + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [ + 4.0, + 4.1 + ] + }, + "properties": { + "foo": 4 + }, + "when": { + "start": "1970-01-01T00:00:00+00:00", + "end": "1970-01-01T00:00:00.001+00:00", + "type": "Interval" + } + } + ] + } + )), ), params( ("workflow" = WorkflowId, description = "Workflow id"), - GetCapabilities + WfsQueryParams ), security( ("session_token" = []) ) )] -#[allow(clippy::too_many_lines)] // long xml body -async fn wfs_capabilities_handler( + +async fn wfs_handler( + req: HttpRequest, workflow_id: web::Path, + request: OgcQueryExtractor, // case insensitive query params + app_ctx: web::Data, + session: C::Session, +) -> Result +where + C: ApplicationContext, +{ + let request = request.into_inner(); + + match request { + WfsQueryParams::GetCapabilities(r) => { + wfs_get_capabilities(workflow_id.into_inner(), r, app_ctx, session).await + } + WfsQueryParams::GetFeature(r) => { + wfs_get_feature(req, workflow_id.into_inner(), r, app_ctx, session).await + } + } +} + +#[allow(clippy::too_many_lines)] // long xml body +async fn wfs_get_capabilities( + workflow_id: WorkflowId, // TODO: react on capabilities - // _request: web::Query, + _request: GetCapabilities, app_ctx: web::Data, session: C::Session, ) -> Result where C: ApplicationContext, { - let workflow_id = workflow_id.into_inner(); let wfs_url = wfs_url(workflow_id)?; let ctx = app_ctx.session_context(session); @@ -303,129 +455,13 @@ fn wfs_url(workflow: WorkflowId) -> Result { ogc_endpoint_url(&base, OgcProtocol::Wfs, workflow) } -/// Get WCS Features -#[utoipa::path( - tag = "OGC WFS", - get, - path = "/wfs/{workflow}?request=GetFeature", - responses( - (status = 200, description = "OK", body = GeoJson, - example = json!( - { - "type": "FeatureCollection", - "features": [ - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 0.0, - 0.1 - ] - }, - "properties": { - "foo": 0 - }, - "when": { - "start": "1970-01-01T00:00:00+00:00", - "end": "1970-01-01T00:00:00.001+00:00", - "type": "Interval" - } - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 1.0, - 1.1 - ] - }, - "properties": { - "foo": null - }, - "when": { - "start": "1970-01-01T00:00:00+00:00", - "end": "1970-01-01T00:00:00.001+00:00", - "type": "Interval" - } - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 2.0, - 3.1 - ] - }, - "properties": { - "foo": 2 - }, - "when": { - "start": "1970-01-01T00:00:00+00:00", - "end": "1970-01-01T00:00:00.001+00:00", - "type": "Interval" - } - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 3.0, - 3.1 - ] - }, - "properties": { - "foo": 3 - }, - "when": { - "start": "1970-01-01T00:00:00+00:00", - "end": "1970-01-01T00:00:00.001+00:00", - "type": "Interval" - } - }, - { - "type": "Feature", - "geometry": { - "type": "Point", - "coordinates": [ - 4.0, - 4.1 - ] - }, - "properties": { - "foo": 4 - }, - "when": { - "start": "1970-01-01T00:00:00+00:00", - "end": "1970-01-01T00:00:00.001+00:00", - "type": "Interval" - } - } - ] - } - )), - ), - params( - ("workflow" = WorkflowId, description = "Workflow id"), - GetFeature - ), - security( - ("session_token" = []) - ) -)] -async fn wfs_feature_handler( +async fn wfs_get_feature( req: HttpRequest, - endpoint: web::Path, - request: web::Query, + endpoint: WorkflowId, + request: GetFeature, app_ctx: web::Data, session: C::Session, ) -> Result { - let endpoint = endpoint.into_inner(); - let request = request.into_inner(); - let type_names = match request.type_names.namespace.as_deref() { None => WorkflowId::from_str(&request.type_names.feature_type)?, Some(_) => { @@ -488,6 +524,7 @@ async fn wfs_feature_handler( let reprojection_params = ReprojectionParams { target_spatial_reference: request_spatial_ref, + derive_out_spec: DeriveOutRasterSpecsSource::ProjectionBounds, }; // create the reprojection operator in order to get the canonic operator name @@ -519,15 +556,11 @@ async fn wfs_feature_handler( let processor = initialized.query_processor()?; - let query_rect = VectorQueryRectangle { - spatial_bounds: request.bbox.bounds_naive()?, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - // TODO: find reasonable default - spatial_resolution: request - .query_resolution - .map_or_else(SpatialResolution::zero_point_one, |r| r.0), - attributes: ColumnSelection::all(), - }; + let query_rect = VectorQueryRectangle::new( + request.bbox.bounds_naive()?, + request.time.unwrap_or_else(default_time_from_config).into(), + ColumnSelection::all(), + ); let query_ctx = ctx.query_context(type_names.0, Uuid::new_v4())?; let (json, cache_hint) = match processor { @@ -1126,7 +1159,6 @@ x;y /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn raster_vector_join_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } @@ -1179,7 +1211,10 @@ x;y "rasters": [{ "type": "GdalSource", "params": { + "data": ndvi_name, + "overviewLevel:": null + } }], } diff --git a/services/src/api/handlers/wms.rs b/services/src/api/handlers/wms.rs index e6ae2e51e..e0a00da2e 100644 --- a/services/src/api/handlers/wms.rs +++ b/services/src/api/handlers/wms.rs @@ -2,36 +2,37 @@ use crate::api::model::datatypes::{ RasterColorizer, SpatialReference, SpatialReferenceOption, TimeInterval, }; use crate::api::model::responses::ErrorResponse; -use crate::api::ogc::util::{OgcProtocol, OgcRequestGuard, ogc_endpoint_url}; +use crate::api::ogc::util::{OgcProtocol, OgcQueryExtractor, ogc_endpoint_url}; use crate::api::ogc::wms::request::{ - GetCapabilities, GetLegendGraphic, GetMap, GetMapExceptionFormat, + GetCapabilities, GetFeatureInfo, GetLegendGraphic, GetMap, GetMapExceptionFormat, GetStyles, }; use crate::config; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; -use crate::error::Result; -use crate::error::{self, Error}; +use crate::error::{self, Error, Result}; use crate::util::server::{CacheControlHeader, connection_closed, not_implemented_handler}; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::WorkflowId; use actix_web::{FromRequest, HttpRequest, HttpResponse, web}; -use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, RasterQueryRectangle, SpatialPartition2D, + AxisAlignedRectangle, BandSelection, CacheHint, SpatialResolution, }; -use geoengine_datatypes::primitives::{BandSelection, CacheHint}; -use geoengine_operators::engine::{ - RasterOperator, ResultDescriptor, SingleRasterOrVectorSource, WorkflowOperatorPath, -}; -use geoengine_operators::processing::{Reprojection, ReprojectionParams}; -use geoengine_operators::util::input::RasterOrVectorOperator; +use geoengine_datatypes::primitives::{RasterQueryRectangle, SpatialPartition2D}; +use geoengine_datatypes::raster::GridIntersection; +use geoengine_operators::util::raster_stream_to_png::default_colorizer_gradient; use geoengine_operators::{ - call_on_generic_raster_processor, util::raster_stream_to_png::raster_stream_to_png_bytes, + call_on_generic_raster_processor, + engine::{ExecutionContext, WorkflowOperatorPath}, + util::raster_stream_to_png::raster_stream_to_png_bytes, }; use reqwest::Url; +use serde::Deserialize; use snafu::ensure; use std::str::FromStr; use std::time::Duration; +use tracing::debug; +use utoipa::IntoParams; +use utoipa::openapi::{Ref, Required}; use uuid::Uuid; pub(crate) fn init_wms_routes(cfg: &mut web::ServiceConfig) @@ -39,32 +40,74 @@ where C: ApplicationContext, C::Session: FromRequest, { - cfg.service( - web::resource("/wms/{workflow}") - .route( - web::get() - .guard(OgcRequestGuard::new("GetCapabilities")) - .to(wms_capabilities_handler::), - ) - .route( - web::get() - .guard(OgcRequestGuard::new("GetMap")) - .to(wms_map_handler::), - ) - .route( - web::get() - .guard(OgcRequestGuard::new("GetLegendGraphic")) - .to(wms_legend_graphic_handler::), - ) - .route(web::get().to(not_implemented_handler)), - ); + cfg.service(web::resource("/wms/{workflow}").route(web::get().to(wms_handler::))); +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "request", rename_all = "PascalCase")] +#[allow(clippy::large_enum_variant, clippy::enum_variant_names)] +pub enum WmsQueryParams { + GetCapabilities(GetCapabilities), + GetMap(GetMap), + GetFeatureInfo(GetFeatureInfo), + GetStyles(GetStyles), + GetLegendGraphic(GetLegendGraphic), +} + +/// manual implementation because derive macro does not support enums +/// Note, that WMS is not really OpenAPI compatible, so this is just an approximation to enable client generation +impl IntoParams for WmsQueryParams { + fn into_params( + _parameter_in_provider: impl Fn() -> Option, + ) -> Vec { + let pip = || Some(utoipa::openapi::path::ParameterIn::Query); + + let mut params = Vec::new(); + + params.push( + utoipa::openapi::path::ParameterBuilder::new() + .name("request") + .required(utoipa::openapi::Required::True) + .parameter_in(pip().unwrap_or_default()) + .description(Some("type of WMS request")) + .schema(Some(Ref::from_schema_name("WmsRequest"))) + .build(), + ); + + for mut p in GetCapabilities::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetMap::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetFeatureInfo::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetStyles::into_params(pip) { + p.required = Required::False; + params.push(p); + } + for mut p in GetLegendGraphic::into_params(pip) { + p.required = Required::False; + params.push(p); + } + + // remove duplicate parameters + params.sort_by(|a, b| a.name.cmp(&b.name)); + params.dedup_by(|a, b| a.name == b.name); + + params + } } -/// Get WMS Capabilities +/// OGC WMS endpoint #[utoipa::path( tag = "OGC WMS", get, - path = "/wms/{workflow}?request=GetCapabilities", + path = "/wms/{workflow}", responses( (status = 200, description = "OK", content_type = "text/xml", body = String, // TODO: add example when utoipa supports more than just json examples @@ -119,27 +162,48 @@ where // // // "# - ) + ), + (status = 200, response = crate::api::model::responses::PngResponse), ), params( ("workflow" = WorkflowId, description = "Workflow id"), - GetCapabilities + WmsQueryParams ), security( ("session_token" = []) ) )] -async fn wms_capabilities_handler( +async fn wms_handler( + req: HttpRequest, workflow: web::Path, - // TODO: incorporate `GetCapabilities` request - // _request: web::Query, + request: OgcQueryExtractor, + app_ctx: web::Data, + session: C::Session, +) -> Result +where + C: ApplicationContext, +{ + match request.into_inner() { + WmsQueryParams::GetCapabilities(_) => { + wms_get_capabilities(workflow.into_inner(), app_ctx, session).await + } + WmsQueryParams::GetMap(get_map) => { + wms_get_map(req, workflow.into_inner(), get_map, app_ctx, session).await + } + WmsQueryParams::GetFeatureInfo(_) + | WmsQueryParams::GetStyles(_) + | WmsQueryParams::GetLegendGraphic(_) => Ok(not_implemented_handler().await), + } +} + +async fn wms_get_capabilities( + workflow_id: WorkflowId, app_ctx: web::Data, session: C::Session, ) -> Result where C: ApplicationContext, { - let workflow_id = workflow.into_inner(); let wms_url = wms_url(workflow_id)?; let ctx = app_ctx.session_context(session); @@ -227,38 +291,22 @@ fn wms_url(workflow: WorkflowId) -> Result { ogc_endpoint_url(&base, OgcProtocol::Wms, workflow) } -/// Get WMS Map -#[utoipa::path( - tag = "OGC WMS", - get, - path = "/wms/{workflow}?request=GetMap", - responses( - (status = 200, response = crate::api::model::responses::PngResponse), - ), - params( - ("workflow" = WorkflowId, description = "Workflow id"), - GetMap - ), - security( - ("session_token" = []) - ) -)] #[allow(clippy::too_many_lines)] -async fn wms_map_handler( +async fn wms_get_map( req: HttpRequest, - workflow: web::Path, - request: web::Query, + workflow: WorkflowId, + request: GetMap, app_ctx: web::Data, session: C::Session, ) -> Result { async fn compute_result( req: HttpRequest, - workflow: web::Path, - request: &web::Query, + workflow: WorkflowId, + request: &GetMap, app_ctx: web::Data, session: C::Session, ) -> Result<(Vec, CacheHint)> { - let endpoint = workflow.into_inner(); + let endpoint = workflow; let layer = WorkflowId::from_str(&request.layers)?; ensure!( @@ -275,8 +323,12 @@ async fn wms_map_handler( .map(Duration::from_secs), ); + let raster_colorizer = raster_colorizer_from_style(&request.styles)?; + let ctx = app_ctx.session_context(session); + let tiling_spec = ctx.execution_context()?.tiling_specification(); + let workflow_id = WorkflowId::from_str(&request.layers)?; let workflow = ctx.db().load_workflow(&workflow_id).await?; @@ -291,67 +343,49 @@ async fn wms_map_handler( .initialize(workflow_operator_path_root, &execution_context) .await?; - // handle request and workflow crs matching - let workflow_spatial_ref: SpatialReferenceOption = - initialized.result_descriptor().spatial_reference().into(); - let workflow_spatial_ref: Option = workflow_spatial_ref.into(); - let workflow_spatial_ref = - workflow_spatial_ref.ok_or(error::Error::InvalidSpatialReference)?; - - // TODO: use a default spatial reference if it is not set? - let request_spatial_ref: SpatialReference = - request.crs.ok_or(error::Error::MissingSpatialReference)?; - - // perform reprojection if necessary - let initialized = if request_spatial_ref == workflow_spatial_ref { - initialized - } else { - tracing::debug!( - "WMS query srs: {request_spatial_ref}, workflow srs: {workflow_spatial_ref} --> injecting reprojection" - ); + // Use the datasets crs as default spatial reference if none is requested + let result_desc_crs_option = initialized + .result_descriptor() + .spatial_reference + .as_option(); + let request_spatial_ref: SpatialReference = request + .crs + .or(result_desc_crs_option.map(Into::into)) + .ok_or(error::Error::MissingSpatialReference)?; + + let request_bounds: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; + let x_request_res = request_bounds.size_x() / f64::from(request.width); + let y_request_res = request_bounds.size_y() / f64::from(request.height); + let request_resolution = SpatialResolution::new(x_request_res.abs(), y_request_res.abs())?; + + let wrapped = + geoengine_operators::util::WrapWithProjectionAndResample::new_create_result_descriptor( + operator, + initialized, + ) + .wrap_with_projection_and_resample( + None, // this needs to be `None`` to avoid moving the data origin to a png pixel origin. Since the WMS requests are not consistent with their origins using them distortes the results more then it helps. + Some(request_resolution), + request_spatial_ref.into(), + tiling_spec, + &execution_context, + ) + .await?; + // TODO: add a resammple operator for downsampling AND resample push down! - let reprojection_params = ReprojectionParams { - target_spatial_reference: request_spatial_ref.into(), - }; - - // create the reprojection operator in order to get the canonic operator name - let reprojected_workflow = Reprojection { - params: reprojection_params, - sources: SingleRasterOrVectorSource { - source: RasterOrVectorOperator::Raster(operator), - }, - } - .boxed(); - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - // TODO: avoid re-initialization and re-use unprojected workflow. However, this requires updating all operator paths - - // In order to check whether we need to inject a reprojection, we first need to initialize the - // original workflow. Then we can check the result projection. Previously, we then just wrapped - // the initialized workflow with an initialized reprojection. IMHO this is wrong because - // initialization propagates the workflow path down the children and appends a new segment for - // each level. So we can't re-use an already initialized workflow, because all the workflow path/ - // operator names will be wrong. That's why I now build a new workflow with a reprojection and - // perform a full initialization. I only added the TODO because we did some optimization here - // which broke at some point when the workflow operator paths were introduced but no one noticed. - - let irp = reprojected_workflow - .initialize(workflow_operator_path_root, &execution_context) - .await?; - - Box::new(irp) - }; + let initialized = wrapped.initialized_operator; let processor = initialized.query_processor()?; - let query_bbox: SpatialPartition2D = request.bbox.bounds(request_spatial_ref)?; - let x_query_resolution = query_bbox.size_x() / f64::from(request.width); - let y_query_resolution = query_bbox.size_y() / f64::from(request.height); - - let raster_colorizer = raster_colorizer_from_style(&request.styles)?; + let query_tiling_pixel_grid = wrapped + .result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_spatial_grid_definition() + .spatial_bounds_to_compatible_spatial_grid(request_bounds); let attributes = raster_colorizer.as_ref().map_or_else( + // TODO: move this to a method of RasterColorizer || BandSelection::new_single(0), |colorizer: &RasterColorizer| { RasterColorizer::band_selection(colorizer) @@ -360,23 +394,66 @@ async fn wms_map_handler( }, ); - let query_rect = RasterQueryRectangle { - spatial_bounds: query_bbox, - time_interval: request.time.unwrap_or_else(default_time_from_config).into(), - spatial_resolution: SpatialResolution::new_unchecked( - x_query_resolution, - y_query_resolution, - ), - attributes, + let raster_colorizer: Option = + raster_colorizer.map(Into::into); + + let raster_colorizer = match raster_colorizer { + Some(colorizer) => colorizer, + None => geoengine_datatypes::operations::image::RasterColorizer::SingleBand { + band: 0, + band_colorizer: default_colorizer_gradient::(), + }, }; + let query_time = request.time.unwrap_or_else(default_time_from_config).into(); + let result_descriptor_intersects_query_time = wrapped + .result_descriptor + .time + .bounds + .is_none_or(|b| b.intersects(&query_time)); + + let result_descriptor_intersects_query_space = wrapped + .result_descriptor + .spatial_grid + .tiling_grid_definition(tiling_spec) + .tiling_grid_bounds() + .intersects(&query_tiling_pixel_grid.grid_bounds()); + + if !result_descriptor_intersects_query_time || !result_descriptor_intersects_query_space { + debug!( + "WMS request does not intersect data bounds (time: {}, space: {}), returning empty image", + result_descriptor_intersects_query_time, result_descriptor_intersects_query_space + ); + + let empty_image = + geoengine_datatypes::operations::image::create_empty_no_data_color_png_bytes( + request.width, + request.height, + raster_colorizer.no_data_color(), + )?; + + return Ok((empty_image, CacheHint::max_duration())); + } + + debug!("WMS re-scale-project: {:?}", query_tiling_pixel_grid); + + let query_rect = RasterQueryRectangle::new( + query_tiling_pixel_grid.grid_bounds(), + query_time, + attributes, + ); + + debug!("WMS query rect: {:?}", query_rect); + let query_ctx = ctx.query_context(workflow_id.0, Uuid::new_v4())?; + // The raster to png code already resamples when the tiles are filled. We should add resample for lower resolutions call_on_generic_raster_processor!( processor, p => - raster_stream_to_png_bytes(p, query_rect, query_ctx, request.width, request.height, request.time.map(Into::into), raster_colorizer.map(Into::into), conn_closed).await - ).map_err(error::Error::from) + raster_stream_to_png_bytes(p, query_rect, query_ctx, request.width, request.height, request.time.map(Into::into), Some(raster_colorizer), conn_closed).await // TODO: pass raster colorizer here + ) + .map_err(error::Error::from) } match compute_result(req, workflow, &request, app_ctx, session).await { @@ -421,36 +498,6 @@ fn raster_colorizer_from_style(styles: &str) -> Result> } } -/// Get WMS Legend Graphic -#[utoipa::path( - tag = "OGC WMS", - get, - path = "/wms/{workflow}?request=GetLegendGraphic", - responses( - (status = 501, description = "Not implemented") - ), - params( - ("workflow" = WorkflowId, description = "Workflow id"), - GetLegendGraphic - ), - security( - ("session_token" = []) - ) -)] -#[allow( - clippy::unused_async, // the function signature of request handlers requires it - clippy::no_effect_underscore_binding // need `_session` to quire authentication -)] -async fn wms_legend_graphic_handler( - // TODO: incorporate workflow and `GetLegendGraphic` query - // _workflow: web::Path, - // _request: web::Query, - // _app_ctx: web::Data, - _session: C::Session, -) -> HttpResponse { - HttpResponse::NotImplemented().finish() -} - fn default_time_from_config() -> TimeInterval { get_config_element::() .ok() @@ -490,23 +537,20 @@ mod tests { use crate::ge_context; use crate::users::UserAuth; use crate::util::tests::{ - MockQueryContext, check_allowed_http_methods, read_body_string, + admin_login, check_allowed_http_methods, read_body_string, register_ndvi_workflow_helper, register_ndvi_workflow_helper_with_cache_ttl, register_ne2_multiband_workflow, send_test_request, }; - use crate::util::tests::{admin_login, register_ndvi_workflow_helper}; use actix_http::header::{self, CONTENT_TYPE}; use actix_web::dev::ServiceResponse; use actix_web::http::Method; use actix_web_httpauth::headers::authorization::Bearer; use geoengine_datatypes::operations::image::{Colorizer, RgbaColor}; use geoengine_datatypes::primitives::CacheTtlSeconds; - use geoengine_datatypes::raster::{GridShape2D, RasterDataType, TilingSpecification}; + use geoengine_datatypes::raster::{GridBoundingBox2D, GridShape2D, TilingSpecification}; use geoengine_datatypes::test_data; use geoengine_datatypes::util::assert_image_equals; - use geoengine_operators::engine::{ - ExecutionContext, RasterQueryProcessor, RasterResultDescriptor, - }; + use geoengine_operators::engine::{ExecutionContext, RasterQueryProcessor}; use geoengine_operators::source::GdalSourceProcessor; use geoengine_operators::util::gdal::create_ndvi_meta_data; use std::convert::TryInto; @@ -638,32 +682,29 @@ mod tests { let ctx = app_ctx.session_context(session.clone()); let exe_ctx = ctx.execution_context().unwrap(); + let meta_data = create_ndvi_meta_data(); + let gdal_source = GdalSourceProcessor:: { - result_descriptor: RasterResultDescriptor::with_datatype_and_num_bands( - RasterDataType::U8, - 1, - ), + produced_result_descriptor: meta_data.result_descriptor.clone(), tiling_specification: exe_ctx.tiling_specification(), - meta_data: Box::new(create_ndvi_meta_data()), + overview_level: 0, + meta_data: Box::new(meta_data), + original_resolution_spatial_grid: None, _phantom_data: PhantomData, }; - let query_partition = - SpatialPartition2D::new((-180., 90.).into(), (180., -90.).into()).unwrap(); - let (image_bytes, _) = raster_stream_to_png_bytes( gdal_source.boxed(), - RasterQueryRectangle { - spatial_bounds: query_partition, - time_interval: geoengine_datatypes::primitives::TimeInterval::new( + RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-900, 899, -1800, 1799).unwrap(), + geoengine_datatypes::primitives::TimeInterval::new( 1_388_534_400_000, 1_388_534_400_000 + 1000, ) .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(1.0, 1.0), - attributes: BandSelection::first(), - }, - ctx.mock_query_context().unwrap(), + BandSelection::first(), + ), + ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), 360, 180, None, @@ -673,7 +714,7 @@ mod tests { .await .unwrap(); - // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, test_data!("wms/raster_small.png")); + // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "raster_small_22.png"); assert_image_equals(test_data!("wms/raster_small.png"), &image_bytes); } @@ -681,7 +722,6 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn get_map_test_helper_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape2D::new([600, 600]), } } @@ -751,7 +791,7 @@ mod tests { let image_bytes = actix_web::test::read_body(response).await; - // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "get_map_ndvi.png"); + // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "get_map_ndvi_2.png"); assert_image_equals(test_data!("wms/get_map_ndvi.png"), &image_bytes); } @@ -927,7 +967,10 @@ mod tests { let image_bytes = actix_web::test::read_body(res).await; - // geoengine_datatypes::util::test::save_test_bytes(&image_bytes, "ne2_rgb_colorizer.png"); + // geoengine_datatypes::util::test::save_test_bytes( + // &image_bytes, + // test_data!("wms/ne2_rgb_colorizer.png").to_str().unwrap(), + // ); assert_image_equals(test_data!("wms/ne2_rgb_colorizer.png"), &image_bytes); } @@ -1130,7 +1173,7 @@ mod tests { - No CoordinateProjector available for: SpatialReference { authority: Epsg, code: 4326 } --> SpatialReference { authority: Epsg, code: 432 } + Spatial reference system 'EPSG:432' is unknown "# ); @@ -1194,7 +1237,13 @@ mod tests { .append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let res = send_test_request(req, app_ctx).await; - ErrorResponse::assert(res, 200, "NoCoordinateProjector", "No CoordinateProjector available for: SpatialReference { authority: Epsg, code: 4326 } --> SpatialReference { authority: Epsg, code: 432 }").await; + ErrorResponse::assert( + res, + 200, + "UnknownSrsString", + "Spatial reference system 'EPSG:432' is unknown", + ) + .await; } #[ge_context::test] @@ -1205,7 +1254,7 @@ mod tests { let (_, id) = register_ndvi_workflow_helper(&app_ctx).await; - let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=335&height=168&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); + let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=3600&height=1800&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx).await; assert_eq!( @@ -1230,7 +1279,7 @@ mod tests { let (_, id) = register_ndvi_workflow_helper_with_cache_ttl(&app_ctx, CacheTtlSeconds::new(60)).await; - let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=335&height=168&crs=EPSG:4326&bbox=-90.0,-180.0,90.0,180.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); + let req = actix_web::test::TestRequest::get().uri(&format!("/wms/{id}?service=WMS&version=1.3.0&request=GetMap&layers={id}&styles=&width=360&height=180&crs=EPSG:4326&bbox=-9.0,-18.0,9.0,18.0&format=image/png&transparent=FALSE&bgcolor=0xFFFFFF&exceptions=application/json&time=2014-04-01T12%3A00%3A00.000%2B00%3A00", id = id.to_string())).append_header((header::AUTHORIZATION, Bearer::new(session_id.to_string()))); let response = send_test_request(req, app_ctx).await; assert_eq!( @@ -1247,6 +1296,15 @@ mod tests { cache_header == "private, max-age=60" || cache_header == "private, max-age=59" || cache_header == "private, max-age=58" + || cache_header == "private, max-age=57" + || cache_header == "private, max-age=56" + || cache_header == "private, max-age=55" + || cache_header == "private, max-age=54" + || cache_header == "private, max-age=53" + || cache_header == "private, max-age=52" + || cache_header == "private, max-age=51" // TODO: find out what takes so long. Keep in mind we use a lot of tiles here... + || cache_header == "private, max-age=50", + "Cache header is {cache_header:?} and not one of the exprected 60, 59, 58" ); } } diff --git a/services/src/api/handlers/workflows.rs b/services/src/api/handlers/workflows.rs index 451e79ff7..418e9459c 100755 --- a/services/src/api/handlers/workflows.rs +++ b/services/src/api/handlers/workflows.rs @@ -5,12 +5,13 @@ use crate::api::ogc::util::{parse_bbox, parse_time}; use crate::config::get_config_element; use crate::contexts::{ApplicationContext, SessionContext}; use crate::datasets::listing::{DatasetProvider, Provenance, ProvenanceOutput}; -use crate::datasets::{RasterDatasetFromWorkflow, schedule_raster_dataset_from_workflow_task}; +use crate::datasets::{ + RasterDatasetFromWorkflow, RasterDatasetFromWorkflowParams, + schedule_raster_dataset_from_workflow_task, +}; use crate::error::Result; use crate::layers::storage::LayerProviderDb; -use crate::util::parsing::{ - parse_band_selection, parse_spatial_partition, parse_spatial_resolution, -}; +use crate::util::parsing::{parse_band_selection, parse_spatial_partition}; use crate::util::workflows::validate_workflow; use crate::workflows::registry::WorkflowRegistry; use crate::workflows::workflow::{Workflow, WorkflowId}; @@ -20,13 +21,10 @@ use futures::StreamExt; use futures::future::join_all; use geoengine_datatypes::error::{BoxedResultExt, ErrorSource}; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, RasterQueryRectangle, SpatialPartition2D, SpatialResolution, - VectorQueryRectangle, + BoundingBox2D, ColumnSelection, RasterQueryRectangle, SpatialPartition2D, VectorQueryRectangle, }; use geoengine_operators::call_on_typed_operator; -use geoengine_operators::engine::{ - ExecutionContext, OperatorData, TypedResultDescriptor, WorkflowOperatorPath, -}; +use geoengine_operators::engine::{ExecutionContext, OperatorData, WorkflowOperatorPath}; use serde::{Deserialize, Serialize}; use snafu::Snafu; use std::collections::HashMap; @@ -201,11 +199,11 @@ async fn get_workflow_metadata_handler( async fn workflow_metadata( workflow: Workflow, execution_context: C::ExecutionContext, -) -> Result { +) -> Result { // TODO: use cache here let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - let result_descriptor: TypedResultDescriptor = call_on_typed_operator!( + let result_descriptor: geoengine_operators::engine::TypedResultDescriptor = call_on_typed_operator!( workflow.operator, operator => { let operator = operator @@ -217,7 +215,7 @@ async fn workflow_metadata( } ); - Ok(result_descriptor) + Ok(result_descriptor.into()) } /// Gets the provenance of all datasets used in a workflow. @@ -440,16 +438,18 @@ async fn dataset_from_workflow_handler( let ctx = Arc::new(app_ctx.session_context(session)); let id = id.into_inner(); - let workflow = ctx.db().load_workflow(&id).await?; + let compression_num_threads = get_config_element::()?.compression_num_threads; + let info_inner = + RasterDatasetFromWorkflowParams::from_request_and_result_descriptor(info.into_inner()); + let task_id = schedule_raster_dataset_from_workflow_task( format!("workflow {id}"), id, - workflow, ctx, - info.into_inner(), + info_inner, compression_num_threads, ) .await?; @@ -467,9 +467,6 @@ pub struct RasterStreamWebsocketQuery { #[serde(deserialize_with = "parse_time")] #[param(value_type = String)] pub time_interval: TimeInterval, - #[serde(deserialize_with = "parse_spatial_resolution")] - #[param(value_type = crate::api::model::datatypes::SpatialResolution)] - pub spatial_resolution: SpatialResolution, #[serde(deserialize_with = "parse_band_selection")] #[param(value_type = String)] pub attributes: BandSelection, @@ -526,12 +523,26 @@ async fn raster_stream_websocket( .get_raster() .boxed_context(error::WorkflowMustBeOfTypeRaster)?; - let query_rectangle = RasterQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: query.time_interval.into(), - spatial_resolution: query.spatial_resolution, - attributes: query.attributes.clone().try_into()?, - }; + let execution_context = ctx.execution_context()?; + + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized_operator = operator + .initialize(workflow_operator_path_root, &execution_context) + .await?; + + let query = query.into_inner(); + + let query_bounds = initialized_operator + .result_descriptor() + .tiling_grid_definition(execution_context.tiling_specification()) + .tiling_geo_transform() + .spatial_to_grid_bounds(&query.spatial_bounds); + let query_rectangle = RasterQueryRectangle::new( + query_bounds, + query.time_interval.into(), + query.attributes.clone().try_into()?, + ); // this is the only result type for now debug_assert!(matches!( @@ -539,11 +550,12 @@ async fn raster_stream_websocket( RasterStreamWebsocketResultType::Arrow )); - let mut stream_task = WebsocketStreamTask::new_raster::( - operator, + let query_ctx = ctx.query_context(workflow_id.0, Uuid::new_v4())?; + + let mut stream_task = WebsocketStreamTask::new_raster_initialized::<_>( + initialized_operator, query_rectangle, - ctx.execution_context()?, - ctx.query_context(workflow_id.0, Uuid::new_v4())?, + query_ctx, ) .await?; @@ -591,9 +603,6 @@ pub struct VectorStreamWebsocketQuery { #[serde(deserialize_with = "parse_time")] #[param(value_type = String)] pub time_interval: TimeInterval, - #[serde(deserialize_with = "parse_spatial_resolution")] - #[param(value_type = crate::api::model::datatypes::SpatialResolution)] - pub spatial_resolution: SpatialResolution, pub result_type: RasterStreamWebsocketResultType, } @@ -640,12 +649,11 @@ async fn vector_stream_websocket( .get_vector() .boxed_context(error::WorkflowMustBeOfTypeVector)?; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: query.spatial_bounds, - time_interval: query.time_interval.into(), - spatial_resolution: query.spatial_resolution, - attributes: ColumnSelection::all(), - }; + let query_rectangle = VectorQueryRectangle::new( + query.spatial_bounds, + query.time_interval.into(), + ColumnSelection::all(), + ); // this is the only result type for now debug_assert!(matches!( @@ -717,7 +725,6 @@ mod tests { use super::*; use crate::api::model::responses::ErrorResponse; - use crate::config::get_config_element; use crate::contexts::PostgresContext; use crate::contexts::Session; use crate::datasets::storage::DatasetStore; @@ -744,17 +751,18 @@ mod tests { use geoengine_datatypes::primitives::DateTime; use geoengine_datatypes::primitives::{ ContinuousMeasurement, FeatureData, Measurement, MultiPoint, RasterQueryRectangle, - SpatialPartition2D, SpatialResolution, TimeInterval, + TimeInterval, + }; + use geoengine_datatypes::raster::{ + GeoTransform, GridBoundingBox2D, GridShape, RasterDataType, TilingSpecification, }; - use geoengine_datatypes::raster::{GridShape, RasterDataType, TilingSpecification}; use geoengine_datatypes::spatial_reference::SpatialReference; - use geoengine_datatypes::test_data; - use geoengine_datatypes::util::ImageFormat; use geoengine_datatypes::util::arrow::arrow_ipc_file_to_record_batches; - use geoengine_datatypes::util::assert_image_equals_with_format; + use geoengine_datatypes::util::test::TestDefault; + use geoengine_operators::engine::TimeDescriptor; use geoengine_operators::engine::{ - ExecutionContext, MultipleRasterOrSingleVectorSource, PlotOperator, RasterBandDescriptor, - RasterBandDescriptors, TypedOperator, + MultipleRasterOrSingleVectorSource, PlotOperator, RasterBandDescriptor, + RasterBandDescriptors, SpatialGridDescriptor, TypedOperator, }; use geoengine_operators::engine::{RasterOperator, RasterResultDescriptor, VectorOperator}; use geoengine_operators::mock::{ @@ -766,10 +774,7 @@ mod tests { use geoengine_operators::source::OgrSourceParameters; use geoengine_operators::source::{GdalSource, GdalSourceParameters}; use geoengine_operators::util::input::MultiRasterOrVectorOperator::Raster; - use geoengine_operators::util::raster_stream_to_geotiff::{ - GdalGeoTiffDatasetMetadata, GdalGeoTiffOptions, - single_timestep_raster_stream_to_geotiff_bytes, - }; + use geoengine_operators::util::test::assert_eq_two_raster_operator_res_u8; use serde_json::json; use std::io::Read; use std::sync::Arc; @@ -787,9 +792,7 @@ mod tests { let workflow = Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } .boxed() .into(), @@ -827,9 +830,7 @@ mod tests { async fn register_missing_header(app_ctx: PostgresContext) { let workflow = Workflow { operator: MockPointSource { - params: MockPointSourceParams { - points: vec![(0.0, 0.1).into(), (1.0, 1.1).into()], - }, + params: MockPointSourceParams::new(vec![(0.0, 0.1).into(), (1.0, 1.1).into()]), } .boxed() .into(), @@ -1069,9 +1070,12 @@ mod tests { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::test_default(), + geoengine_datatypes::raster::GridBoundingBox2D::new([0, 0], [199, 199]) + .unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), Measurement::Continuous(ContinuousMeasurement { @@ -1104,9 +1108,28 @@ mod tests { "type": "raster", "dataType": "U8", "spatialReference": "EPSG:4326", - "time": null, - "bbox": null, - "resolution": null, + "time": { + "bounds": null, + "dimension": { + "type": "irregular", + } + }, + "spatialGrid": { + "descriptor": "source", + "spatialGrid": { + "geoTransform": {"originCoordinate":{"x":0.0,"y":0.0}, "xPixelSize": 1., "yPixelSize": -1.}, + "gridBounds": { + "bottomRightIdx": { + "xIdx": 199, + "yIdx": 199 + }, + "topLeftIdx": { + "xIdx": 0, + "yIdx": 0 + } + } + } + }, "bands": [{ "name": "band", "measurement": { @@ -1221,7 +1244,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), ), @@ -1300,7 +1323,6 @@ mod tests { fn test_download_all_metadata_zip_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), tile_size_in_pixels: GridShape::new([600, 600]), } } @@ -1325,9 +1347,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: dataset_name.clone(), - }, + params: GdalSourceParameters::new(dataset_name.clone()), } .boxed(), ), @@ -1356,7 +1376,8 @@ mod tests { "operator": { "type": "GdalSource", "params": { - "data": dataset_name + "data": dataset_name, + "overviewLevel": null } } }) @@ -1369,22 +1390,35 @@ mod tests { "dataType": "U8", "spatialReference": "EPSG:4326", "time": { - "start": 1_388_534_400_000_i64, - "end": 1_404_172_800_000_i64, - }, - "bbox": { - "upperLeftCoordinate": { - "x": -180.0, - "y": 90.0, + "dimension": { + "type": "regular", + "origin": 0, + "step": { + "granularity": "months", + "step": 1 + } }, - "lowerRightCoordinate": { - "x": 180.0, - "y": -90.0 + "bounds": { + "start": 1_388_534_400_000_i64, + "end": 1_404_172_800_000_i64, } }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "descriptor": "source", + "spatialGrid" : { + "geoTransform": {"originCoordinate":{"x":-180.0,"y":90.0}, "xPixelSize": 0.1, "yPixelSize": -0.1}, + "gridBounds": { + "bottomRightIdx": { + "xIdx": 3599, + "yIdx": 1799 + }, + "topLeftIdx": { + "xIdx": 0, + "yIdx": 0 + } + } + } + }, "bands": [{ "name": "ndvi", @@ -1420,8 +1454,7 @@ mod tests { /// override the pixel size since this test was designed for 600 x 600 pixel tiles fn dataset_from_workflow_task_success_tiling_spec() -> TilingSpecification { TilingSpecification { - origin_coordinate: (0., 0.).into(), - tile_size_in_pixels: GridShape::new([600, 600]), + tile_size_in_pixels: GridShape::new([512, 512]), } } @@ -1435,13 +1468,13 @@ mod tests { let (_, dataset) = add_ndvi_to_datasets(&app_ctx).await; + let operator_a = GdalSource { + params: GdalSourceParameters::new(dataset), + } + .boxed(); + let workflow = Workflow { - operator: TypedOperator::Raster( - GdalSource { - params: GdalSourceParameters { data: dataset }, - } - .boxed(), - ), + operator: TypedOperator::Raster(operator_a.clone()), }; let workflow_id = ctx.db().register_workflow(workflow).await.unwrap(); @@ -1458,12 +1491,12 @@ mod tests { "query": { "spatialBounds": { "upperLeftCoordinate": { - "x": -10.0, - "y": 80.0 + "x": 0.0, + "y": 52.0 }, "lowerRightCoordinate": { - "x": 50.0, - "y": 20.0 + "x": 52.0, + "y": 0.0 } }, "timeInterval": { @@ -1506,58 +1539,26 @@ mod tests { }; // query the newly created dataset - let op = GdalSource { - params: GdalSourceParameters { - data: response.dataset.into(), - }, + let operator_b = GdalSource { + params: GdalSourceParameters::new(response.dataset.into()), } .boxed(); - let exe_ctx = ctx.execution_context().unwrap(); - - let o = op - .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) - .await - .unwrap(); - - let query_ctx = ctx.query_context(workflow_id.0, Uuid::new_v4()).unwrap(); - let query_rect = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new((-10., 80.).into(), (50., 20.).into()).unwrap(), - time_interval: TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_000 + 1000), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: geoengine_datatypes::primitives::BandSelection::first(), - }; - - let processor = o.query_processor().unwrap().get_u8().unwrap(); + let query_rectangle = RasterQueryRectangle::new( + GridBoundingBox2D::new_min_max(-512, -1, 0, 511).unwrap(), + TimeInterval::new_unchecked(1_388_534_400_000, 1_388_534_400_000 + 1000), + geoengine_datatypes::primitives::BandSelection::first(), + ); - let result = single_timestep_raster_stream_to_geotiff_bytes( - processor, - query_rect, - query_ctx, - GdalGeoTiffDatasetMetadata { - no_data_value: Some(0.), - spatial_reference: SpatialReference::epsg_4326(), - }, - GdalGeoTiffOptions { - compression_num_threads: get_config_element::() - .unwrap() - .compression_num_threads, - as_cog: false, - force_big_tiff: false, - }, - None, - Box::pin(futures::future::pending()), - exe_ctx.tiling_specification(), - (), + assert_eq_two_raster_operator_res_u8( + &ctx.execution_context().unwrap(), + &ctx.query_context(Uuid::new_v4(), Uuid::new_v4()).unwrap(), + operator_a, + operator_b, + query_rectangle, + false, ) - .await - .unwrap(); - - assert_image_equals_with_format( - test_data!("raster/geotiff_from_stream_compressed.tiff"), - result.as_slice(), - ImageFormat::Tiff, - ); + .await; } #[ge_context::test] @@ -1571,7 +1572,10 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters { + data: dataset, + overview_level: None, + }, } .boxed(), ), @@ -1607,7 +1611,6 @@ mod tests { )) .unwrap() .into(), - spatial_resolution: SpatialResolution::one(), attributes: geoengine_datatypes::primitives::BandSelection::first().into(), result_type: RasterStreamWebsocketResultType::Arrow, }), @@ -1688,7 +1691,6 @@ mod tests { )) .unwrap() .into(), - spatial_resolution: SpatialResolution::one(), result_type: RasterStreamWebsocketResultType::Arrow, }), req, diff --git a/services/src/api/model/datatypes.rs b/services/src/api/model/datatypes.rs index 7df43e2c7..0cd3e7f14 100644 --- a/services/src/api/model/datatypes.rs +++ b/services/src/api/model/datatypes.rs @@ -4,6 +4,7 @@ use geoengine_datatypes::operations::image::RgbParams; use geoengine_datatypes::primitives::{ AxisAlignedRectangle, MultiLineStringAccess, MultiPointAccess, MultiPolygonAccess, }; +use geoengine_datatypes::raster::GridBounds; use geoengine_macros::type_tag; use ordered_float::NotNan; use postgres_types::{FromSql, ToSql}; @@ -11,12 +12,13 @@ use serde::de::Error as SerdeError; use serde::ser::SerializeMap; use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Visitor}; use snafu::ResultExt; +use std::borrow::Cow; use std::{ collections::{BTreeMap, HashMap}, fmt::{Debug, Formatter}, str::FromStr, }; -use utoipa::{PartialSchema, ToSchema}; +use utoipa::{PartialSchema, ToSchema, openapi}; identifier!(DataProviderId); @@ -746,6 +748,109 @@ impl From for geoengine_datatypes::primitives::BoundingBox2D { } } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDefinition { + pub geo_transform: GeoTransform, + pub grid_bounds: GridBoundingBox2D, +} + +impl From for SpatialGridDefinition { + fn from(value: geoengine_datatypes::raster::SpatialGridDefinition) -> Self { + Self { + geo_transform: value.geo_transform().into(), + grid_bounds: value.grid_bounds().into(), + } + } +} + +impl From for geoengine_datatypes::raster::SpatialGridDefinition { + fn from(value: SpatialGridDefinition) -> Self { + geoengine_datatypes::raster::SpatialGridDefinition::new( + value.geo_transform.into(), + value.grid_bounds.into(), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GeoTransform { + pub origin_coordinate: Coordinate2D, + pub x_pixel_size: f64, + pub y_pixel_size: f64, +} + +impl From for GeoTransform { + fn from(value: geoengine_datatypes::raster::GeoTransform) -> Self { + GeoTransform { + origin_coordinate: value.origin_coordinate().into(), + x_pixel_size: value.x_pixel_size(), + y_pixel_size: value.y_pixel_size(), + } + } +} + +impl From for geoengine_datatypes::raster::GeoTransform { + fn from(value: GeoTransform) -> Self { + geoengine_datatypes::raster::GeoTransform::new( + value.origin_coordinate.into(), + value.x_pixel_size, + value.y_pixel_size, + ) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GridIdx2D { + pub y_idx: isize, + pub x_idx: isize, +} + +impl From for GridIdx2D { + fn from(value: geoengine_datatypes::raster::GridIdx2D) -> Self { + Self { + y_idx: value.y(), + x_idx: value.x(), + } + } +} + +impl From for geoengine_datatypes::raster::GridIdx2D { + fn from(value: GridIdx2D) -> Self { + geoengine_datatypes::raster::GridIdx::new_y_x(value.y_idx, value.x_idx) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GridBoundingBox2D { + pub top_left_idx: GridIdx2D, + pub bottom_right_idx: GridIdx2D, +} + +impl From for GridBoundingBox2D { + fn from(value: geoengine_datatypes::raster::GridBoundingBox2D) -> Self { + Self { + top_left_idx: value.min_index().into(), + bottom_right_idx: value.max_index().into(), + } + } +} + +impl From for geoengine_datatypes::raster::GridBoundingBox2D { + fn from(value: GridBoundingBox2D) -> Self { + geoengine_datatypes::raster::GridBoundingBox2D::new_min_max( + value.top_left_idx.y_idx, + value.bottom_right_idx.y_idx, + value.top_left_idx.x_idx, + value.bottom_right_idx.x_idx, + ) + .expect("Bounds were correct before") // TODO: maybe try from? + } +} + /// An object that composes the date and a timestamp with time zone. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub struct DateTimeString { @@ -898,6 +1003,48 @@ impl From for geoengine_datatypes::primitives::Continuous } } +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct SerializableClasses(BTreeMap); + +impl PartialSchema for SerializableClasses { + fn schema() -> openapi::RefOr { + BTreeMap::::schema() + } +} + +impl ToSchema for SerializableClasses { + fn name() -> Cow<'static, str> { + as ToSchema>::name() // TODO: is this needed? + } +} + +impl Serialize for SerializableClasses { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + let classes: BTreeMap = + self.0.iter().map(|(k, v)| (k.to_string(), v)).collect(); + classes.serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for SerializableClasses { + fn deserialize(deserializer: D) -> std::result::Result + where + D: Deserializer<'de>, + { + let tree: BTreeMap = Deserialize::deserialize(deserializer)?; + let classes: Result, _> = tree + .into_iter() + .map(|(k, v)| k.parse::().map(|x| (x, v))) + .collect(); + Ok(SerializableClasses( + classes.map_err(serde::de::Error::custom)?, + )) + } +} + #[type_tag(value = "classification")] #[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] pub struct ClassificationMeasurement { @@ -939,15 +1086,10 @@ impl From for ClassificationMeasurement { fn from(value: geoengine_datatypes::primitives::ClassificationMeasurement) -> Self { - let mut classes = BTreeMap::new(); - for (k, v) in value.classes { - classes.insert(k, v); - } - Self { r#type: Default::default(), measurement: value.measurement, - classes, + classes: value.classes, } } } @@ -956,14 +1098,9 @@ impl From for geoengine_datatypes::primitives::ClassificationMeasurement { fn from(measurement: ClassificationMeasurement) -> Self { - let mut classes = HashMap::with_capacity(measurement.classes.len()); - for (k, v) in measurement.classes { - classes.insert(k, v); - } - Self { measurement: measurement.measurement, - classes, + classes: measurement.classes, } } } @@ -997,10 +1134,9 @@ impl From for geoengine_datatypes::primitives::SpatialPartit /// A spatio-temporal rectangle with a specified resolution #[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] -pub struct RasterQueryRectangle { +pub struct RasterToDatasetQueryRectangle { pub spatial_bounds: SpatialPartition2D, pub time_interval: TimeInterval, - pub spatial_resolution: SpatialResolution, } /// A spatio-temporal rectangle with a specified resolution @@ -1011,6 +1147,7 @@ pub struct VectorQueryRectangle { pub time_interval: TimeInterval, pub spatial_resolution: SpatialResolution, } + /// A spatio-temporal rectangle with a specified resolution #[derive(Copy, Clone, Debug, PartialEq, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "camelCase")] @@ -1020,39 +1157,6 @@ pub struct PlotQueryRectangle { pub spatial_resolution: SpatialResolution, } -impl - From< - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::SpatialPartition2D, - geoengine_datatypes::primitives::BandSelection, - >, - > for RasterQueryRectangle -{ - fn from( - value: geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::SpatialPartition2D, - geoengine_datatypes::primitives::BandSelection, - >, - ) -> Self { - Self { - spatial_bounds: value.spatial_bounds.into(), - time_interval: value.time_interval.into(), - spatial_resolution: value.spatial_resolution.into(), - } - } -} - -impl From for geoengine_datatypes::primitives::RasterQueryRectangle { - fn from(value: RasterQueryRectangle) -> Self { - Self { - spatial_bounds: value.spatial_bounds.into(), - time_interval: value.time_interval.into(), - spatial_resolution: value.spatial_resolution.into(), - attributes: geoengine_datatypes::primitives::BandSelection::first(), // TODO: adjust once API supports attribute selection - } - } -} - #[derive(Clone, Debug, PartialEq, Deserialize, Serialize, ToSchema)] pub struct BandSelection(pub Vec); @@ -1257,8 +1361,8 @@ impl From for geoengine_datatypes::primitives::TimeStep { /// Stores time intervals in ms in close-open semantic [start, end) #[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq, ToSql, FromSql, ToSchema)] pub struct TimeInterval { - start: TimeInstance, - end: TimeInstance, + pub start: TimeInstance, + pub end: TimeInstance, } impl From for geoengine_datatypes::primitives::TimeInterval { diff --git a/services/src/api/model/operators.rs b/services/src/api/model/operators.rs index 63233a895..e8e16a99c 100644 --- a/services/src/api/model/operators.rs +++ b/services/src/api/model/operators.rs @@ -1,14 +1,15 @@ +use super::datatypes::{ + BoundingBox2D, FeatureDataType, Measurement, RasterDataType, SpatialGridDefinition, + SpatialReferenceOption, TimeInterval, VectorDataType, +}; use crate::api::model::datatypes::{ - BoundingBox2D, CacheTtlSeconds, Coordinate2D, DateTimeParseFormat, FeatureDataType, - GdalConfigOption, Measurement, MlTensorShape3D, MultiLineString, MultiPoint, MultiPolygon, - NoGeometry, RasterDataType, RasterPropertiesEntryType, RasterPropertiesKey, SpatialPartition2D, - SpatialReferenceOption, SpatialResolution, TimeInstance, TimeInterval, TimeStep, - VectorDataType, + CacheTtlSeconds, Coordinate2D, DateTimeParseFormat, GdalConfigOption, MlTensorShape3D, + MultiLineString, MultiPoint, MultiPolygon, NoGeometry, RasterPropertiesEntryType, + RasterPropertiesKey, TimeInstance, TimeStep, }; use crate::error::{ RasterBandNameMustNotBeEmpty, RasterBandNameTooLong, RasterBandNamesMustBeUnique, Result, }; -use geoengine_datatypes::primitives::ColumnSelection; use geoengine_datatypes::util::ByteSize; use geoengine_macros::type_tag; use geoengine_operators::util::input::float_option_with_nan; @@ -26,12 +27,127 @@ pub struct RasterResultDescriptor { pub data_type: RasterDataType, #[schema(value_type = String)] pub spatial_reference: SpatialReferenceOption, - pub time: Option, - pub bbox: Option, - pub resolution: Option, + pub time: TimeDescriptor, + pub spatial_grid: SpatialGridDescriptor, pub bands: RasterBandDescriptors, } +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub enum SpatialGridDescriptorState { + Source, + Derived, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct SpatialGridDescriptor { + pub spatial_grid: SpatialGridDefinition, + pub descriptor: SpatialGridDescriptorState, +} + +impl From for geoengine_operators::engine::SpatialGridDescriptor { + fn from(value: SpatialGridDescriptor) -> Self { + let sp = geoengine_operators::engine::SpatialGridDescriptor::new_source( + value.spatial_grid.into(), + ); + match value.descriptor { + SpatialGridDescriptorState::Source => sp, + SpatialGridDescriptorState::Derived => sp.as_derived(), + } + } +} + +impl From for SpatialGridDescriptor { + fn from(value: geoengine_operators::engine::SpatialGridDescriptor) -> Self { + if value.is_source() { + let sp = value.source_spatial_grid_definition().expect("is source"); + return SpatialGridDescriptor { + spatial_grid: sp.into(), + descriptor: SpatialGridDescriptorState::Source, + }; + } + let sp = value + .derived_spatial_grid_definition() + .expect("if not source it must be derived"); + SpatialGridDescriptor { + spatial_grid: sp.into(), + descriptor: SpatialGridDescriptorState::Derived, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase", tag = "type")] +pub enum TimeDimension { + Regular(RegularTimeDimension), + Irregular, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct RegularTimeDimension { + pub origin: TimeInstance, + pub step: TimeStep, +} + +impl From for geoengine_datatypes::primitives::RegularTimeDimension { + fn from(value: RegularTimeDimension) -> Self { + geoengine_datatypes::primitives::RegularTimeDimension { + origin: value.origin.into(), + step: value.step.into(), + } + } +} + +impl From for RegularTimeDimension { + fn from(value: geoengine_datatypes::primitives::RegularTimeDimension) -> Self { + Self { + origin: value.origin.into(), + step: value.step.into(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct TimeDescriptor { + pub bounds: Option, + pub dimension: TimeDimension, +} + +impl From for geoengine_operators::engine::TimeDescriptor { + fn from(value: TimeDescriptor) -> Self { + geoengine_operators::engine::TimeDescriptor::new( + value.bounds.map(Into::into), + match value.dimension { + TimeDimension::Regular(d) => { + geoengine_datatypes::primitives::TimeDimension::Regular(d.into()) + } + TimeDimension::Irregular => { + geoengine_datatypes::primitives::TimeDimension::Irregular + } + }, + ) + } +} + +impl From for TimeDescriptor { + fn from(value: geoengine_operators::engine::TimeDescriptor) -> Self { + Self { + bounds: value.bounds.map(Into::into), + dimension: match value.dimension { + geoengine_datatypes::primitives::TimeDimension::Regular(d) => { + TimeDimension::Regular(d.into()) + } + geoengine_datatypes::primitives::TimeDimension::Irregular => { + TimeDimension::Irregular + } + }, + } + } +} + #[derive(Debug, Clone, PartialEq, Serialize, ToSchema)] pub struct RasterBandDescriptors(Vec); @@ -107,9 +223,8 @@ impl From for RasterResultD Self { data_type: value.data_type.into(), spatial_reference: value.spatial_reference.into(), - time: value.time.map(Into::into), - bbox: value.bbox.map(Into::into), - resolution: value.resolution.map(Into::into), + time: value.time.into(), + spatial_grid: value.spatial_grid.into(), bands: value.bands.into(), } } @@ -120,9 +235,8 @@ impl From for geoengine_operators::engine::RasterResultD Self { data_type: value.data_type.into(), spatial_reference: value.spatial_reference.into(), - time: value.time.map(Into::into), - bbox: value.bbox.map(Into::into), - resolution: value.resolution.map(Into::into), + time: value.time.into(), + spatial_grid: value.spatial_grid.into(), bands: value.bands.into(), } } @@ -332,6 +446,26 @@ impl From for TypedResultDes } } +impl From for geoengine_operators::engine::TypedResultDescriptor { + fn from(value: TypedResultDescriptor) -> Self { + match value { + TypedResultDescriptor::Plot(p) => { + geoengine_operators::engine::TypedResultDescriptor::Plot(p.result_descriptor.into()) + } + TypedResultDescriptor::Raster(r) => { + geoengine_operators::engine::TypedResultDescriptor::Raster( + r.result_descriptor.into(), + ) + } + TypedResultDescriptor::Vector(v) => { + geoengine_operators::engine::TypedResultDescriptor::Vector( + v.result_descriptor.into(), + ) + } + } + } +} + impl From for TypedResultDescriptor { fn from(value: geoengine_operators::engine::PlotResultDescriptor) -> Self { Self::Plot(TypedPlotResultDescriptor { @@ -431,10 +565,7 @@ impl geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, > for MockMetaData { @@ -442,10 +573,7 @@ impl value: geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, ) -> Self { Self { @@ -461,10 +589,7 @@ impl geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, > for OgrMetaData { @@ -472,10 +597,7 @@ impl value: geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, >, ) -> Self { Self { @@ -490,10 +612,7 @@ impl From for geoengine_operators::engine::StaticMetaData< geoengine_operators::mock::MockDatasetDataSourceLoadingInfo, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, > { fn from(value: MockMetaData) -> Self { @@ -509,10 +628,7 @@ impl From for geoengine_operators::engine::StaticMetaData< geoengine_operators::source::OgrSourceDataset, geoengine_operators::engine::VectorResultDescriptor, - geoengine_datatypes::primitives::QueryRectangle< - geoengine_datatypes::primitives::BoundingBox2D, - ColumnSelection, - >, + geoengine_datatypes::primitives::VectorQueryRectangle, > { fn from(value: OgrMetaData) -> Self { @@ -1382,6 +1498,30 @@ impl From } } +#[type_tag(value = "GdalMultiBand")] +#[derive(PartialEq, Serialize, Deserialize, Debug, Clone, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct GdalMultiBand { + pub result_descriptor: RasterResultDescriptor, +} + +impl From for GdalMultiBand { + fn from(value: geoengine_operators::source::GdalMultiBand) -> Self { + Self { + r#type: Default::default(), + result_descriptor: value.result_descriptor.into(), + } + } +} + +impl From for geoengine_operators::source::GdalMultiBand { + fn from(value: GdalMultiBand) -> Self { + Self { + result_descriptor: value.result_descriptor.into(), + } + } +} + #[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, ToSchema)] #[serde(rename_all = "lowercase")] pub enum CsvHeader { diff --git a/services/src/api/model/responses/datasets/errors.rs b/services/src/api/model/responses/datasets/errors.rs index 8ecdb2c30..94ccb5ba4 100644 --- a/services/src/api/model/responses/datasets/errors.rs +++ b/services/src/api/model/responses/datasets/errors.rs @@ -104,3 +104,75 @@ impl fmt::Debug for UpdateDatasetError { write!(f, "{}", ge_report(self)) } } + +#[derive(Snafu, IntoStaticStr)] +#[snafu(visibility(pub(crate)))] +#[snafu(context(suffix(false)))] // disables default `Snafu` suffix +pub enum AddDatasetTilesError { + #[snafu(display("Cannot load dataset for adding tiles"))] + CannotLoadDatasetForAddingTiles { + source: error::Error, + }, + #[snafu(display("Cannot add tiles to dataset: {source}"))] + CannotAddTilesToDataset { + source: error::Error, + }, + #[snafu(display("Cannot add tiles to dataset that is not a GdalMultiBand"))] + DatasetIsNotGdalMultiBand, + #[snafu(display("Tile file path does not exist: {file_path}"))] + TileFilePathDoesNotExist { + file_path: String, + absolute_path: String, + }, + DatasetIsMissingDataPath, + TileFilePathNotRelative { + file_path: String, + }, + CannotOpenTileFile { + source: geoengine_operators::error::Error, + file_path: String, + }, + CannotGetRasterDescriptorFromTileFile { + source: geoengine_operators::error::Error, + file_path: String, + }, + TileFileDataTypeMismatch { + expected: geoengine_datatypes::raster::RasterDataType, + found: geoengine_datatypes::raster::RasterDataType, + file_path: String, + }, + TileFileSpatialReferenceMismatch { + expected: geoengine_datatypes::spatial_reference::SpatialReferenceOption, + found: geoengine_datatypes::spatial_reference::SpatialReferenceOption, + file_path: String, + }, + TileFileBandDoesNotExist { + band_count: u32, + found: u32, + file_path: String, + }, + TileFileGeoTransformMismatch { + expected: geoengine_datatypes::raster::GeoTransform, + found: geoengine_datatypes::raster::GeoTransform, + file_path: String, + }, + InvalidTileFileGeoTransform { + file_path: String, + }, +} + +impl ResponseError for AddDatasetTilesError { + fn error_response(&self) -> HttpResponse { + HttpResponse::build(self.status_code()).json(ErrorResponse::from(self)) + } + + fn status_code(&self) -> StatusCode { + StatusCode::BAD_REQUEST + } +} + +impl fmt::Debug for AddDatasetTilesError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", ge_report(self)) + } +} diff --git a/services/src/api/model/services.rs b/services/src/api/model/services.rs index c386e0199..a4a54de3a 100644 --- a/services/src/api/model/services.rs +++ b/services/src/api/model/services.rs @@ -1,4 +1,5 @@ -use super::datatypes::{CacheTtlSeconds, DataId, DataProviderId, GdalConfigOption, RasterDataType}; +use super::datatypes::{CacheTtlSeconds, DataId, DataProviderId, DatasetId, GdalConfigOption}; +use super::operators::TypedResultDescriptor; use crate::api::model::datatypes::MlModelName; use crate::api::model::operators::{ GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf, @@ -13,6 +14,7 @@ use crate::util::Secret; use crate::util::oidc::RefreshToken; use crate::util::parsing::deserialize_base_url; use geoengine_datatypes::primitives::DateTime; +use geoengine_datatypes::util::test::TestDefault; use geoengine_macros::type_tag; use serde::{Deserialize, Serialize}; use std::path::PathBuf; @@ -20,6 +22,7 @@ use url::Url; use utoipa::ToSchema; use validator::{Validate, ValidationErrors}; +use super::operators::GdalMultiBand; pub const SECRET_REPLACEMENT: &str = "*****"; #[allow(clippy::large_enum_variant)] @@ -33,6 +36,7 @@ pub enum MetaDataDefinition { GdalStatic(GdalMetaDataStatic), GdalMetadataNetCdfCf(GdalMetadataNetCdfCf), GdalMetaDataList(GdalMetaDataList), + GdalMultiBand(GdalMultiBand), } impl From for MetaDataDefinition { @@ -64,6 +68,9 @@ impl From for MetaDataDefinition { crate::datasets::storage::MetaDataDefinition::GdalMetaDataList(x) => { Self::GdalMetaDataList(x.into()) } + crate::datasets::storage::MetaDataDefinition::GdalMultiBand(x) => { + Self::GdalMultiBand(x.into()) + } } } } @@ -77,6 +84,7 @@ impl From for crate::datasets::storage::MetaDataDefinition { MetaDataDefinition::GdalStatic(x) => Self::GdalStatic(x.into()), MetaDataDefinition::GdalMetadataNetCdfCf(x) => Self::GdalMetadataNetCdfCf(x.into()), MetaDataDefinition::GdalMetaDataList(x) => Self::GdalMetaDataList(x.into()), + MetaDataDefinition::GdalMultiBand(x) => Self::GdalMultiBand(x.into()), } } } @@ -144,7 +152,7 @@ pub struct DatasetDefinition { #[derive(Deserialize, Serialize, Debug, Clone, ToSchema)] #[serde(rename_all = "camelCase")] pub struct CreateDataset { - pub data_path: DataPath, + pub data_path: DataPath, // TODO: move into `AddDataset`? pub definition: DatasetDefinition, } @@ -155,6 +163,12 @@ pub enum DataPath { Upload(UploadId), } +impl TestDefault for DataPath { + fn test_default() -> Self { + DataPath::Volume(VolumeName("test_data".to_string())) + } +} + #[derive(Deserialize, Serialize, Debug, Clone, ToSchema, Validate)] pub struct UpdateDataset { pub name: DatasetName, @@ -753,8 +767,6 @@ pub struct SentinelS2L2ACogsProviderDefinition { pub description: String, pub priority: Option, pub api_url: String, - pub bands: Vec, - pub zones: Vec, #[serde(default)] pub stac_api_retries: StacApiRetries, #[serde(default)] @@ -765,58 +777,6 @@ pub struct SentinelS2L2ACogsProviderDefinition { pub query_buffer: StacQueryBuffer, } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ToSchema)] -#[serde(rename_all = "camelCase")] -pub struct StacBand { - pub name: String, - pub no_data_value: Option, - pub data_type: RasterDataType, -} - -impl From for crate::datasets::external::sentinel_s2_l2a_cogs::StacBand { - fn from(value: StacBand) -> Self { - crate::datasets::external::sentinel_s2_l2a_cogs::StacBand { - name: value.name, - no_data_value: value.no_data_value, - data_type: value.data_type.into(), - } - } -} - -impl From for StacBand { - fn from(value: crate::datasets::external::sentinel_s2_l2a_cogs::StacBand) -> Self { - StacBand { - name: value.name, - no_data_value: value.no_data_value, - data_type: value.data_type.into(), - } - } -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, ToSchema)] -pub struct StacZone { - pub name: String, - pub epsg: u32, -} - -impl From for crate::datasets::external::sentinel_s2_l2a_cogs::StacZone { - fn from(value: StacZone) -> Self { - crate::datasets::external::sentinel_s2_l2a_cogs::StacZone { - name: value.name, - epsg: value.epsg, - } - } -} - -impl From for StacZone { - fn from(value: crate::datasets::external::sentinel_s2_l2a_cogs::StacZone) -> Self { - StacZone { - name: value.name, - epsg: value.epsg, - } - } -} - #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, ToSchema, Default)] #[serde(rename_all = "camelCase")] pub struct StacApiRetries { @@ -896,8 +856,6 @@ impl From description: value.description, priority: value.priority, api_url: value.api_url, - bands: value.bands.into_iter().map(Into::into).collect(), - zones: value.zones.into_iter().map(Into::into).collect(), stac_api_retries: value.stac_api_retries.into(), gdal_retries: value.gdal_retries.into(), cache_ttl: value.cache_ttl.into(), @@ -919,8 +877,6 @@ impl From for crate::datasets::upload::Volume { } } +#[derive(Debug, Serialize, Deserialize, Clone, ToSchema, Validate)] +#[serde(rename_all = "camelCase")] +pub struct Dataset { + pub id: DatasetId, + pub name: DatasetName, + pub display_name: String, + pub description: String, + pub result_descriptor: TypedResultDescriptor, + pub source_operator: String, + pub symbology: Option, + pub provenance: Option>, + pub tags: Option>, + pub data_path: Option, +} + +impl From for crate::datasets::storage::Dataset { + fn from(value: Dataset) -> Self { + crate::datasets::storage::Dataset { + id: value.id.into(), + name: value.name, + display_name: value.display_name, + description: value.description, + result_descriptor: value.result_descriptor.into(), + source_operator: value.source_operator, + symbology: value.symbology, + provenance: value + .provenance + .map(|v| v.into_iter().map(Into::into).collect::>()), + tags: value.tags, + data_path: value.data_path, + } + } +} + +impl From for Dataset { + fn from(value: crate::datasets::storage::Dataset) -> Self { + Dataset { + id: value.id.into(), + name: value.name, + display_name: value.display_name, + description: value.description, + result_descriptor: value.result_descriptor.into(), + source_operator: value.source_operator, + symbology: value.symbology, + provenance: value + .provenance + .map(|v| v.into_iter().map(Into::into).collect::>()), + tags: value.tags, + data_path: value.data_path, + } + } +} + #[derive(Debug, Serialize, Deserialize, Clone, PartialEq, ToSchema)] #[serde(rename_all = "camelCase")] pub struct MlModel { diff --git a/services/src/api/ogc/util.rs b/services/src/api/ogc/util.rs index ecef3dcd8..1ee2b4c94 100644 --- a/services/src/api/ogc/util.rs +++ b/services/src/api/ogc/util.rs @@ -1,20 +1,22 @@ use crate::api::model::datatypes::SpatialReference; use actix_web::guard::{Guard, GuardContext}; +use actix_web::{FromRequest, HttpRequest, dev::Payload}; +use futures_util::future::{Ready, ready}; +use geoengine_datatypes::primitives::Coordinate2D; use geoengine_datatypes::primitives::{AxisAlignedRectangle, BoundingBox2D, DateTime}; -use geoengine_datatypes::primitives::{Coordinate2D, SpatialResolution}; use reqwest::Url; +use serde::de::DeserializeOwned; use serde::de::Error; use serde::{Deserialize, Serialize}; -use snafu::ensure; +use snafu::{ResultExt, ensure}; use std::str::FromStr; use utoipa::openapi::schema::{ObjectBuilder, SchemaType}; -use utoipa::{PartialSchema, ToSchema}; +use utoipa::{IntoParams, PartialSchema, ToSchema}; use super::wcs::request::WcsBoundingbox; -use super::wfs::request::WfsResolution; use crate::api::handlers::spatial_references::{AxisOrder, spatial_reference_specification}; use crate::api::model::datatypes::TimeInterval; -use crate::error::{self, Result}; +use crate::error::{self, Result, UnableToParseQueryString, UnableToSerializeQueryString}; use crate::workflows::workflow::WorkflowId; #[derive(PartialEq, Debug, Deserialize, Serialize, Clone, Copy)] @@ -139,34 +141,6 @@ where } } -/// Parse a spatial resolution, format is: "resolution" or "xResolution,yResolution" -pub fn parse_wfs_resolution_option<'de, D>( - deserializer: D, -) -> Result, D::Error> -where - D: serde::Deserializer<'de>, -{ - let s = String::deserialize(deserializer)?; - - if s.is_empty() { - return Ok(None); - } - - let split: Vec> = s.split(',').map(str::parse).collect(); - - let spatial_resolution = match *split.as_slice() { - [Ok(resolution)] => { - SpatialResolution::new(resolution, resolution).map_err(D::Error::custom)? - } - [Ok(x_resolution), Ok(y_resolution)] => { - SpatialResolution::new(x_resolution, y_resolution).map_err(D::Error::custom)? - } - _ => return Err(D::Error::custom("Invalid spatial resolution")), - }; - - Ok(Some(WfsResolution(spatial_resolution))) -} - /// Parse wcs 1.1.1 bbox, format is: "x1,y1,x2,y2,crs", crs format is like `urn:ogc:def:crs:EPSG::4326` #[allow(clippy::missing_panics_doc)] pub fn parse_wcs_bbox<'de, D>(deserializer: D) -> Result @@ -249,11 +223,12 @@ pub fn rectangle_from_ogc_params( spatial_reference: SpatialReference, ) -> Result { let [a, b, c, d] = values; - match spatial_reference_specification(&spatial_reference.proj_string()?)? + let axis_order = spatial_reference_specification(spatial_reference)? .axis_order .ok_or(error::Error::AxisOrderingNotKnownForSrs { srs_string: spatial_reference.srs_string(), - })? { + })?; + match axis_order { AxisOrder::EastNorth => A::from_min_max((a, b).into(), (c, d).into()).map_err(Into::into), AxisOrder::NorthEast => A::from_min_max((b, a).into(), (d, c).into()).map_err(Into::into), } @@ -265,7 +240,7 @@ pub fn tuple_from_ogc_params( b: f64, spatial_reference: SpatialReference, ) -> Result<(f64, f64)> { - match spatial_reference_specification(&spatial_reference.proj_string()?)? + match spatial_reference_specification(spatial_reference)? .axis_order .ok_or(error::Error::AxisOrderingNotKnownForSrs { srs_string: spatial_reference.srs_string(), @@ -300,6 +275,7 @@ pub fn ogc_endpoint_url(base: &Url, protocol: OgcProtocol, workflow: WorkflowId) .map_err(Into::into) } +// TODO: remove when all OGC handlers are using only one handler pub struct OgcRequestGuard<'a> { request: &'a str, } @@ -319,6 +295,78 @@ impl Guard for OgcRequestGuard<'_> { } } +/// a special query extractor for actix-web because some OGC clients use varying casing for query parameter keys +pub struct OgcQueryExtractor(pub T); + +impl OgcQueryExtractor { + #[inline] + pub fn into_inner(self) -> T { + self.0 + } +} + +impl std::ops::Deref for OgcQueryExtractor { + type Target = T; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl std::ops::DerefMut for OgcQueryExtractor { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl FromRequest for OgcQueryExtractor +where + T: DeserializeOwned + 'static, +{ + type Error = crate::error::Error; + type Future = Ready>; + + fn from_request(req: &HttpRequest, _: &mut Payload) -> Self::Future { + // get raw query string + let qs = req.query_string(); + + // parse into key-value pairs + let params: Result> = + serde_urlencoded::from_str(qs).context(UnableToParseQueryString); + + let res = params + .and_then(|pairs| { + // normalize keys to lowercase + let normalized: Vec<(String, String)> = pairs + .into_iter() + .map(|(k, v)| (k.to_lowercase(), v)) + .collect(); + + // re-serialize then deserialize into target type T + let new_qs = serde_urlencoded::to_string(&normalized) + .context(UnableToSerializeQueryString)?; + let value = + serde_urlencoded::from_str::(&new_qs).context(UnableToParseQueryString)?; + Ok(value) + }) + .map(OgcQueryExtractor); + + ready(res) + } +} + +// Tell `utoipa` that our wrapper is a query-parameter extractor by forwarding +// `IntoParams` to the inner type and specifying `ParameterIn::Query`. +impl IntoParams for OgcQueryExtractor +where + T: IntoParams, +{ + fn into_params( + _parameter_in_provider: impl Fn() -> Option, + ) -> Vec { + T::into_params(|| Some(utoipa::openapi::path::ParameterIn::Query)) + } +} + #[cfg(test)] mod tests { use crate::api::model::datatypes::SpatialReferenceAuthority; @@ -453,31 +501,6 @@ mod tests { s.to_owned().into_deserializer() } - #[test] - fn it_parses_spatial_resolution_options() { - assert_eq!( - parse_wfs_resolution_option(to_deserializer("")).unwrap(), - None - ); - - assert_eq!( - parse_wfs_resolution_option(to_deserializer("0.1")).unwrap(), - Some(WfsResolution(SpatialResolution::zero_point_one())) - ); - assert_eq!( - parse_wfs_resolution_option(to_deserializer("1")).unwrap(), - Some(WfsResolution(SpatialResolution::one())) - ); - - assert_eq!( - parse_wfs_resolution_option(to_deserializer("0.1,0.2")).unwrap(), - Some(WfsResolution(SpatialResolution::new(0.1, 0.2).unwrap())) - ); - - assert!(parse_wfs_resolution_option(to_deserializer(",")).is_err()); - assert!(parse_wfs_resolution_option(to_deserializer("0.1,0.2,0.3")).is_err()); - } - #[test] fn it_parses_wcs_bbox() { let s = "-162,-81,162,81,urn:ogc:def:crs:EPSG::3857"; @@ -547,4 +570,25 @@ mod tests { .unwrap() ); } + + #[test] + fn ogc_query_extractor_normalizes_keys() { + use actix_web::test; + use serde::Deserialize; + + #[derive(Debug, Deserialize, PartialEq)] + struct TestQuery { + foo: String, + bar: String, + } + + let req = test::TestRequest::with_uri("/test?FOO=hello&BaR=world").to_http_request(); + + let mut payload = actix_web::dev::Payload::None; + let fut = OgcQueryExtractor::::from_request(&req, &mut payload); + let result = futures::executor::block_on(fut).unwrap(); + + assert_eq!(result.foo, "hello"); + assert_eq!(result.bar, "world"); + } } diff --git a/services/src/api/ogc/wcs/request.rs b/services/src/api/ogc/wcs/request.rs index 80e935aa3..8b046da41 100644 --- a/services/src/api/ogc/wcs/request.rs +++ b/services/src/api/ogc/wcs/request.rs @@ -11,6 +11,14 @@ use serde::de::Error; use serde::{Deserialize, Serialize}; use utoipa::{IntoParams, ToSchema}; +#[derive(Debug, Clone, Deserialize, ToSchema)] +#[serde(rename_all = "PascalCase")] +pub enum WcsRequest { + GetCapabilities, + DescribeCoverage, + GetCoverage, +} + #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] pub enum WcsService { #[serde(rename = "WCS")] @@ -32,14 +40,8 @@ pub struct GetCapabilities { pub version: Option, #[serde(alias = "SERVICE")] pub service: WcsService, - #[serde(alias = "REQUEST")] - pub request: GetCapabilitiesRequest, } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetCapabilitiesRequest { - GetCapabilities, -} // sample: SERVICE=WCS&request=DescribeCoverage&VERSION=1.1.1&IDENTIFIERS=nurc:Arc_Sample #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, IntoParams)] pub struct DescribeCoverage { @@ -47,18 +49,11 @@ pub struct DescribeCoverage { pub version: WcsVersion, #[serde(alias = "SERVICE")] pub service: WcsService, - #[serde(alias = "REQUEST")] - pub request: DescribeCoverageRequest, #[serde(alias = "IDENTIFIERS")] #[param(example = "")] pub identifiers: String, } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum DescribeCoverageRequest { - DescribeCoverage, -} - // sample: SERVICE=WCS&VERSION=1.1.1&request=GetCoverage&FORMAT=image/tiff&IDENTIFIER=nurc:Arc_Sample&BOUNDINGBOX=-81,-162,81,162,urn:ogc:def:crs:EPSG::4326&GRIDBASECRS=urn:ogc:def:crs:EPSG::4326&GRIDCS=urn:ogc:def:cs:OGC:0.0:Grid2dSquareCS&GRIDTYPE=urn:ogc:def:method:WCS:1.1:2dSimpleGrid&GRIDORIGIN=81,-162&GRIDOFFSETS=-18,36 #[derive(PartialEq, Debug, Deserialize, Serialize, IntoParams)] pub struct GetCoverage { @@ -66,8 +61,6 @@ pub struct GetCoverage { pub version: WcsVersion, #[serde(alias = "SERVICE")] pub service: WcsService, - #[serde(alias = "REQUEST")] - pub request: GetCoverageRequest, #[serde(alias = "FORMAT")] pub format: GetCoverageFormat, #[serde(alias = "IDENTIFIER")] @@ -121,11 +114,6 @@ pub struct GetCoverage { pub nodatavalue: Option, } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetCoverageRequest { - GetCoverage, -} - impl GetCoverage { pub fn spatial_resolution(&self) -> Option> { if let Some(grid_offsets) = self.gridoffsets { @@ -150,6 +138,16 @@ impl GetCoverage { rectangle_from_ogc_params(self.boundingbox.bbox, spatial_reference) } + + pub fn spatial_ref(&self) -> Result { + let spatial_ref = self.gridbasecrs; // TODO: maybe this is something different. Lets investigate that later... + if let Some(bbx_sref) = self.boundingbox.spatial_reference + && bbx_sref != spatial_ref + { + return Err(error::Error::WcsBoundingboxCrsMustEqualGridBaseCrs); + } + Ok(spatial_ref) + } } #[derive(PartialEq, Debug, Deserialize, Serialize, ToSchema)] @@ -266,7 +264,6 @@ mod tests { GetCoverage { version: WcsVersion::V1_1_1, service: WcsService::Wcs, - request: GetCoverageRequest::GetCoverage, format: GetCoverageFormat::ImageTiff, identifier: "nurc:Arc_Sample".to_owned(), boundingbox: WcsBoundingbox { diff --git a/services/src/api/ogc/wfs/request.rs b/services/src/api/ogc/wfs/request.rs index cdd0d3fb5..57e756178 100644 --- a/services/src/api/ogc/wfs/request.rs +++ b/services/src/api/ogc/wfs/request.rs @@ -1,15 +1,19 @@ use crate::api::model::datatypes::TimeInterval; -use crate::api::ogc::util::{ - OgcBoundingBox, parse_ogc_bbox, parse_time_option, parse_wfs_resolution_option, -}; +use crate::api::ogc::util::{OgcBoundingBox, parse_ogc_bbox, parse_time_option}; use crate::util::from_str_option; -use geoengine_datatypes::primitives::SpatialResolution; use geoengine_datatypes::spatial_reference::SpatialReference; use serde::{Deserialize, Serialize}; use utoipa::openapi::Type; use utoipa::openapi::schema::{ObjectBuilder, SchemaType}; use utoipa::{IntoParams, PartialSchema, ToSchema}; +#[derive(Debug, Clone, Deserialize, ToSchema)] +#[serde(rename_all = "PascalCase")] +pub enum WfsRequest { + GetCapabilities, + GetFeature, +} + #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] pub enum WfsService { #[serde(rename = "WFS")] @@ -28,13 +32,6 @@ pub struct GetCapabilities { pub version: Option, #[serde(alias = "SERVICE")] pub service: WfsService, - #[serde(alias = "REQUEST")] - pub request: GetCapabilitiesRequest, -} - -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetCapabilitiesRequest { - GetCapabilities, } #[derive(PartialEq, Eq, Debug, Deserialize, Serialize)] @@ -58,8 +55,7 @@ impl ToSchema for TypeNames {} pub struct GetFeature { pub version: Option, pub service: WfsService, - pub request: GetFeatureRequest, - #[serde(deserialize_with = "parse_type_names")] + #[serde(deserialize_with = "parse_type_names", alias = "typenames")] #[param(example = "")] pub type_names: TypeNames, // TODO: fifths parameter can be CRS @@ -71,38 +67,20 @@ pub struct GetFeature { #[param(value_type = String, example = "2014-04-01T12:00:00.000Z")] pub time: Option, #[param(example = "EPSG:4326", value_type = Option)] + #[serde(alias = "srsname")] pub srs_name: Option, pub namespaces: Option, // TODO e.g. xmlns(dog=http://www.example.com/namespaces/dog) #[serde(default)] #[serde(deserialize_with = "from_str_option")] pub count: Option, - pub sort_by: Option, // TODO: Name[+A|+D] (asc/desc) - pub result_type: Option, // TODO: enum: results/hits? - pub filter: Option, // TODO: parse filters + #[serde(alias = "sortby")] + pub sort_by: Option, // TODO: Name[+A|+D] (asc/desc) + #[serde(alias = "resulttype")] + pub result_type: Option, // TODO: enum: results/hits? + pub filter: Option, // TODO: parse filters + #[serde(alias = "propertyname")] pub property_name: Option, // TODO comma separated list - // TODO: feature_id, ... - /// Vendor parameter for specifying a spatial query resolution - #[serde(default)] - #[serde(deserialize_with = "parse_wfs_resolution_option")] - pub query_resolution: Option, -} - -#[derive(Copy, Clone, PartialEq, Debug)] -pub struct WfsResolution(pub SpatialResolution); - -impl PartialSchema for WfsResolution { - fn schema() -> utoipa::openapi::RefOr { - ObjectBuilder::new() - .schema_type(SchemaType::Type(Type::String)) - .into() - } -} - -impl ToSchema for WfsResolution {} - -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetFeatureRequest { - GetFeature, + // TODO: feature_id, ... } #[allow(clippy::option_if_let_else)] @@ -140,7 +118,6 @@ mod tests { let request = GetFeature { service: WfsService::Wfs, - request: GetFeatureRequest::GetFeature, version: Some(WfsVersion::V2_0_0), time: None, srs_name: None, @@ -155,7 +132,6 @@ mod tests { feature_type: "test".into(), }, property_name: None, - query_resolution: None, }; assert_eq!(parsed, request); @@ -183,14 +159,12 @@ mod tests { "), ("propertyName","P1,P2"), - ("queryResolution","0.1,0.1"), ]; let query = serde_urlencoded::to_string(params).unwrap(); let parsed: GetFeature = serde_urlencoded::from_str(&query).unwrap(); let request =GetFeature { service: WfsService::Wfs, - request: GetFeatureRequest::GetFeature, version: Some(WfsVersion::V2_0_0), time: Some(geoengine_datatypes::primitives::TimeInterval::new(946_684_800_000, 946_771_200_000).unwrap().into()), srs_name: Some(SpatialReference::new(SpatialReferenceAuthority::Epsg, 4326)), @@ -210,7 +184,6 @@ mod tests { feature_type: "test".into(), }, property_name: Some("P1,P2".into()), - query_resolution: Some(WfsResolution(SpatialResolution::zero_point_one())), }; assert_eq!(parsed, request); @@ -234,7 +207,6 @@ mod tests { let request = GetFeature { service: WfsService::Wfs, - request: GetFeatureRequest::GetFeature, version: Some(WfsVersion::V2_0_0), time: None, srs_name: None, @@ -249,7 +221,6 @@ mod tests { feature_type: op, }, property_name: None, - query_resolution: None, }; assert_eq!(parsed, request); diff --git a/services/src/api/ogc/wms/request.rs b/services/src/api/ogc/wms/request.rs index 91bcced77..a798c8c1e 100644 --- a/services/src/api/ogc/wms/request.rs +++ b/services/src/api/ogc/wms/request.rs @@ -4,6 +4,17 @@ use crate::util::{bool_option_case_insensitive, from_str}; use serde::{Deserialize, Serialize}; use utoipa::{IntoParams, ToSchema}; +#[derive(Debug, Clone, Deserialize, ToSchema)] +#[serde(rename_all = "PascalCase")] +#[allow(clippy::enum_variant_names)] +pub enum WmsRequest { + GetCapabilities, + GetMap, + GetFeatureInfo, + GetStyles, + GetLegendGraphic, +} + #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] pub enum WmsService { #[serde(rename = "WMS")] @@ -22,21 +33,16 @@ pub struct GetCapabilities { pub version: Option, #[serde(alias = "SERVICE")] pub service: WmsService, - #[serde(alias = "REQUEST")] - pub request: GetCapabilitiesRequest, #[serde(alias = "FORMAT")] - pub format: Option, -} - -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetCapabilitiesRequest { - GetCapabilities, + pub format: Option, } #[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetCapabilitiesFormat { +pub enum WmsResponseFormat { #[serde(rename = "text/xml")] - TextXml, // TODO: remaining formats + TextXml, + #[serde(rename = "image/png")] + ImagePng, // TODO: remaining formats } // TODO: remove serde aliases and use serde-aux and case insensitive keys @@ -46,8 +52,6 @@ pub struct GetMap { pub version: WmsVersion, #[serde(alias = "SERVICE")] pub service: WmsService, - #[serde(alias = "REQUEST")] - pub request: GetMapRequest, #[serde(alias = "WIDTH")] #[serde(deserialize_with = "from_str")] #[param(example = 512)] @@ -61,7 +65,7 @@ pub struct GetMap { #[param(example = "-90,-180,90,180")] pub bbox: OgcBoundingBox, #[serde(alias = "FORMAT")] - pub format: GetMapFormat, + pub format: WmsResponseFormat, #[serde(alias = "LAYERS")] #[param(example = "")] pub layers: String, @@ -95,11 +99,6 @@ pub struct GetMap { // TODO: DIM_ } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetMapRequest { - GetMap, -} - #[derive(Clone, Copy, PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] pub enum GetMapExceptionFormat { #[serde(rename = "XML", alias = "application/vnd.ogc.se_xml")] @@ -108,13 +107,7 @@ pub enum GetMapExceptionFormat { Json, // UNSUPPORTED: INIMAGE, BLANK } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetMapFormat { - #[serde(rename = "image/png")] - ImagePng, // TODO: remaining formats -} - -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize)] +#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, IntoParams)] pub struct GetFeatureInfo { pub version: String, pub query_layers: String, @@ -122,12 +115,12 @@ pub struct GetFeatureInfo { // TODO: remaining fields } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize)] +#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] pub enum GetFeatureInfoFormat { TextXml, // TODO: remaining formats } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize)] +#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, IntoParams)] pub struct GetStyles { pub version: String, pub layer: String, @@ -139,18 +132,11 @@ pub struct GetLegendGraphic { pub version: WmsVersion, #[serde(alias = "SERVICE")] pub service: WmsService, - #[serde(alias = "REQUEST")] - pub request: GetLegendGraphicRequest, #[param(example = "")] pub layer: String, // TODO: remaining fields } -#[derive(PartialEq, Eq, Debug, Deserialize, Serialize, ToSchema)] -pub enum GetLegendGraphicRequest { - GetLegendGraphic, -} - #[cfg(test)] mod tests { use super::*; @@ -163,7 +149,6 @@ mod tests { let request = GetMap { service: WmsService::Wms, version: WmsVersion::V1_3_0, - request: GetMapRequest::GetMap, width: 2, layers: "modis_ndvi".into(), crs: Some(geoengine_datatypes::spatial_reference::SpatialReference::epsg_4326().into()), @@ -183,7 +168,7 @@ mod tests { elevation: Some("elevation".into()), bbox: OgcBoundingBox::new(1., 2., 3., 4.), height: 2, - format: GetMapFormat::ImagePng, + format: WmsResponseFormat::ImagePng, exceptions: Some(GetMapExceptionFormat::Json), }; @@ -198,7 +183,6 @@ mod tests { let request = GetMap { service: WmsService::Wms, version: WmsVersion::V1_3_0, - request: GetMapRequest::GetMap, width: 2, layers: "modis_ndvi".into(), crs: Some(geoengine_datatypes::spatial_reference::SpatialReference::epsg_4326().into()), @@ -211,7 +195,7 @@ mod tests { elevation: None, bbox: OgcBoundingBox::new(1., 2., 3., 4.), height: 2, - format: GetMapFormat::ImagePng, + format: WmsResponseFormat::ImagePng, exceptions: None, }; diff --git a/services/src/bin/geoengine-cli.rs b/services/src/bin/geoengine-cli.rs index 5e1d20775..5580ab4ca 100644 --- a/services/src/bin/geoengine-cli.rs +++ b/services/src/bin/geoengine-cli.rs @@ -1,7 +1,8 @@ use clap::{Parser, Subcommand}; use geoengine_services::cli::{ - CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, check_heartbeat, - check_successful_startup, output_openapi_json, output_toolchain_file, + CheckSuccessfulStartup, ExpressionToolchainFile, Heartbeat, OpenAPIGenerate, TileImport, + check_heartbeat, check_successful_startup, output_openapi_json, output_toolchain_file, + tile_import, }; /// CLI for Geo Engine Utilities @@ -24,6 +25,8 @@ enum Commands { #[command(name = "openapi")] OpenAPI(OpenAPIGenerate), + // Imports a tiled dataset + TileImport(TileImport), /// Generates an rustup toolchain file for compiling expressions /// and outputs it to STDOUT ExpressionToolchainFile(ExpressionToolchainFile), @@ -35,6 +38,7 @@ impl Commands { Commands::CheckSuccessfulStartup(params) => check_successful_startup(params).await, Commands::Heartbeat(params) => check_heartbeat(params).await, Commands::OpenAPI(params) => output_openapi_json(params).await, + Commands::TileImport(params) => tile_import(params).await, Commands::ExpressionToolchainFile(params) => output_toolchain_file(params).await, } } diff --git a/services/src/cli/mod.rs b/services/src/cli/mod.rs index 8b332978f..79328b483 100644 --- a/services/src/cli/mod.rs +++ b/services/src/cli/mod.rs @@ -2,8 +2,10 @@ mod check_successful_startup; mod expression_toolchain_file; mod heartbeat; mod openapi; +mod tile_import; pub use check_successful_startup::{CheckSuccessfulStartup, check_successful_startup}; pub use expression_toolchain_file::{ExpressionToolchainFile, output_toolchain_file}; pub use heartbeat::{Heartbeat, check_heartbeat}; pub use openapi::{OpenAPIGenerate, output_openapi_json}; +pub use tile_import::{TileImport, tile_import}; diff --git a/services/src/cli/tile_import.rs b/services/src/cli/tile_import.rs new file mode 100644 index 000000000..07a511551 --- /dev/null +++ b/services/src/cli/tile_import.rs @@ -0,0 +1,700 @@ +#![allow(clippy::print_stdout)] + +use crate::api::handlers::datasets::AddDatasetTile; +use crate::api::handlers::permissions::{ + LayerCollectionResource, LayerResource, PermissionRequest, +}; +use crate::api::model::datatypes::LayerId; +use crate::api::model::operators::{GdalDatasetParameters, GdalMultiBand}; +use crate::api::model::responses::IdResponse; +use crate::api::model::services::{ + AddDataset, CreateDataset, DataPath, DatasetDefinition, MetaDataDefinition, +}; +use crate::datasets::DatasetName; +use crate::datasets::upload::VolumeName; +use crate::layers::layer::{AddLayer, AddLayerCollection}; +use crate::layers::listing::LayerCollectionId; +use crate::layers::storage::INTERNAL_LAYER_DB_ROOT_COLLECTION_ID; +use crate::permissions::{Permission, Role}; +use crate::workflows::workflow::Workflow; +use anyhow::Context; +use chrono::{NaiveDate, TimeZone}; +use clap::Parser; +use gdal::{Dataset as GdalDataset, Metadata}; +use geoengine_datatypes::dataset::NamedData; +use geoengine_datatypes::primitives::{ + Coordinate2D, DateTime, Measurement, SpatialPartition2D, TimeInstance, TimeInterval, +}; +use geoengine_datatypes::raster::GdalGeoTransform; +use geoengine_operators::engine::{RasterBandDescriptor, RasterOperator, RasterResultDescriptor}; +use geoengine_operators::source::{ + GdalDatasetGeoTransform, MultiBandGdalSource, MultiBandGdalSourceParameters, +}; +use geoengine_operators::util::gdal::{ + measurement_from_rasterband, raster_descriptor_from_dataset, +}; +use regex::Regex; +use serde::Serialize; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::str::FromStr; +use uuid::Uuid; + +/// Checks if the Geo Engine server is alive +#[derive(Debug, Parser)] +pub struct TileImport { + /// where the files are scanned + #[arg(long, default_value = "/home/michael/geodata/force/marburg")] + local_data_dir: String, + + /// volume on the server that points to the path of the same files + #[arg(long, default_value = "geodata")] + volume_name: String, + + /// directory on the server, relative to the volume, where the files are located + #[arg(long, default_value = "force/marburg")] + remote_data_dir: String, + + /// file extension to look for + #[arg(long, default_value = "tif")] + file_extension: String, + + /// whether to scan directories recursively + #[arg(long, value_enum, default_value_t = FileScanMode::Recursive)] + scan_mode: FileScanMode, + + /// regex to extract time from file names + #[arg(long, default_value = r"(\d{8})_LEVEL2")] + time_regex: String, + + /// strftime format for time + #[arg(long, default_value = "%Y%m%d")] + time_format: String, + + /// duration of each tile in seconds + #[arg(long, default_value_t = 60 * 60 * 24)] + tile_duration_seconds: u32, + + /// regex to extract product name from file names + #[arg(long, default_value = r"LEVEL2_(\w+_\w+)\.tif")] + product_name_regex: String, + + /// Geo Engine API URL + #[arg(long, default_value = "http://localhost:3030/api")] + geo_engine_url: String, + + /// Geo Engine API email + #[arg(long, default_value = "admin@localhost")] + geo_engine_email: String, + + /// Geo Engine API password + #[arg(long, default_value = "adminadmin")] + geo_engine_password: String, + + /// Parent layer collection ID + #[arg(long, default_value_t = INTERNAL_LAYER_DB_ROOT_COLLECTION_ID)] + parent_layer_collection_id: Uuid, + + /// Name of the layer collection to create/use + #[arg(long, default_value = "FORCE")] + layer_collection_name: String, + + /// whether to share the layers with registered and anonymous users + #[arg(long, value_enum, default_value_t = LayerShareMode::Share)] + share_layers: LayerShareMode, +} + +#[derive(clap::ValueEnum, Clone, Default, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum FileScanMode { + NonRecursive, + #[default] + Recursive, +} + +#[derive(clap::ValueEnum, Clone, Default, Debug, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum LayerShareMode { + DoNotShare, + #[default] + Share, +} + +/// A simple importer of tiled datasets that have +/// * multiple time steps +/// * each time step has one or more corresponding files +/// * each file of the dataset has the same bands +/// * the time and product name are encoded in each file name +/// +/// One example is the FORCE dataset where the rasters are stored in multiple tiles for each time step and the files look like this +/// `/geodata/force/marburg/X0059_Y0049/20000124_LEVEL2_LND07_BOA.tif` +/// +/// The datasets are inserted into a Geo Engine instance via its REST API. +/// The files are scanned from a given local directory. +/// On the server, the files are expected to be available via a volume mapping. +pub async fn tile_import(params: TileImport) -> Result<(), anyhow::Error> { + let time_regex = Regex::new(¶ms.time_regex).context("Invalid time regex")?; + let product_regex = Regex::new(¶ms.product_name_regex).context("Invalid product regex")?; + + let (client, session_id) = login( + ¶ms.geo_engine_url, + ¶ms.geo_engine_email, + ¶ms.geo_engine_password, + ) + .await?; + + let layer_collection_id = create_layer_collection_if_not_exists( + &session_id, + &client, + ¶ms.geo_engine_url, + params.parent_layer_collection_id, + ¶ms.layer_collection_name, + params.share_layers == LayerShareMode::Share, + ) + .await?; + + let mut files = Vec::new(); + collect_files( + Path::new(¶ms.local_data_dir), + ¶ms.file_extension, + params.scan_mode == FileScanMode::Recursive, + &mut files, + )?; + + println!("Found {} files", files.len()); + + let products = + extract_products_from_files(&time_regex, ¶ms.time_format, &product_regex, files); + + for product in products { + let num_timesteps = product + .1 + .iter() + .map(|f| f.time) + .collect::>() + .len(); + println!( + "Found Product: {} ({} files, {num_timesteps} time steps)", + product.0, + product.1.len() + ); + + let dataset_name = add_dataset_and_tiles_to_geoengine( + &session_id, + &client, + &product.0, + &product.1, + params.tile_duration_seconds, + ¶ms.geo_engine_url, + ¶ms.local_data_dir, + ¶ms.volume_name, + ¶ms.remote_data_dir, + ) + .await?; + + if let Some(dataset_name) = dataset_name { + add_dataset_to_collection( + &session_id, + &client, + &dataset_name, + &layer_collection_id, + &product.0, + ¶ms.geo_engine_url, + params.share_layers == LayerShareMode::Share, + ) + .await?; + } + } + + Ok(()) +} + +async fn add_dataset_to_collection( + session_id: &str, + client: &reqwest::Client, + dataset_name: &str, + layer_collection_id: &str, + layer_name: &str, + geo_engine_url: &str, + share_layer: bool, +) -> anyhow::Result<()> { + let add_layer = AddLayer { + name: layer_name.to_string(), + description: String::new(), + workflow: Workflow { + operator: geoengine_operators::engine::TypedOperator::Raster( + MultiBandGdalSource { + params: MultiBandGdalSourceParameters { + data: NamedData { + namespace: None, + provider: None, + name: dataset_name.to_string(), + }, + overview_level: None, + }, + } + .boxed(), + ), + }, + symbology: None, // TODO: add symbology + properties: vec![], + metadata: Default::default(), + }; + + let response: IdResponse = client + .post(format!( + "{geo_engine_url}/layerDb/collections/{layer_collection_id}/layers" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_layer) + .send() + .await + .context("Failed to add layer to collection")? + .json() + .await + .context("Failed to parse layer response")?; + + if share_layer { + let permissions = vec![ + PermissionRequest { + resource: crate::api::handlers::permissions::Resource::Layer( + LayerResource { + id:response.id.clone(), + r#type: crate::api::handlers::permissions::LayerResourceTypeTag::LayerResourceTypeTag + }, + ), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, + PermissionRequest { + resource: crate::api::handlers::permissions::Resource::Layer( + LayerResource { + id:response.id.clone(), + r#type: crate::api::handlers::permissions::LayerResourceTypeTag::LayerResourceTypeTag + }, + ), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }, + ]; + + for permission in &permissions { + let response = client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&permission) + .send() + .await + .context("Failed to add permission")?; + + println!( + "Layer '{}' shared with role {}: {}", + layer_name, + permission.role_id, + response.text().await.unwrap_or_default() + ); + } + } + + Ok(()) +} + +async fn create_layer_collection_if_not_exists( + session_id: &str, + client: &reqwest::Client, + geo_engine_url: &str, + parent_layer_collection_id: Uuid, + layer_collection_name: &str, + share_layer: bool, +) -> anyhow::Result { + let add_collection = AddLayerCollection { + name: layer_collection_name.to_string(), + description: format!("Layer collection for {layer_collection_name}",), + properties: vec![], + }; + + let response: IdResponse = client + .post(format!( + "{geo_engine_url}/layerDb/collections/{parent_layer_collection_id}/collections" + )) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&add_collection) + .send() + .await + .context("Failed to add layer collection")? + .json() + .await + .context("Failed to parse layer collection response")?; + + // TODO: handle case where collection already exists, but there is no API to get collection by name + + println!( + "Layer collection '{}' created with id {}", + layer_collection_name, response.id.0 + ); + + if share_layer { + let permissions = vec![PermissionRequest { + resource: crate::api::handlers::permissions::Resource::LayerCollection( + LayerCollectionResource { + id: response.id.clone(), + r#type: crate::api::handlers::permissions::LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }, + ), + role_id: Role::registered_user_role_id(), + permission: Permission::Read, + }, PermissionRequest { + resource: crate::api::handlers::permissions::Resource::LayerCollection( + LayerCollectionResource { + id: response.id.clone(), + r#type: crate::api::handlers::permissions::LayerCollectionResourceTypeTag::LayerCollectionResourceTypeTag, + }, + ), + role_id: Role::anonymous_role_id(), + permission: Permission::Read, + }]; + + for permission in &permissions { + let response = client + .put(format!("{geo_engine_url}/permissions")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&permission) + .send() + .await + .context("Failed to add permission")?; + + println!( + "Layer collection '{}' shared with role {}: {}", + layer_collection_name, + permission.role_id, + response.text().await.unwrap_or_default() + ); + } + } + + Ok(response.id.0) +} + +struct ProductFile { + product_name: String, + path: PathBuf, + time: TimeInstance, + geo_transform: GdalDatasetGeoTransform, + spatial_partition: SpatialPartition2D, + width: usize, + height: usize, + result_descriptor: RasterResultDescriptor, +} + +fn naive_date_to_time_instance(date: NaiveDate) -> anyhow::Result { + let time: chrono::DateTime = chrono::Utc.from_utc_datetime( + &date + .and_hms_opt(0, 0, 0) + .with_context(|| format!("Failed to create datetime from date: {date}"))?, + ); + let time: DateTime = time.into(); + Ok(time.into()) +} + +fn collect_files( + dir: &Path, + extension: &str, + recursive: bool, + files: &mut Vec, +) -> anyhow::Result<()> { + let entries = + fs::read_dir(dir).with_context(|| format!("Failed to read directory {}", dir.display()))?; + + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() && recursive { + collect_files(&path, extension, recursive, files)?; + } else if path.is_file() + && let Some(ext) = path.extension().and_then(|e| e.to_str()) + && ext.eq_ignore_ascii_case(extension) + { + files.push(path); + } + } + + Ok(()) +} + +#[allow(clippy::too_many_lines, clippy::too_many_arguments)] +async fn add_dataset_and_tiles_to_geoengine( + session_id: &str, + client: &reqwest::Client, + product_name: &str, + files: &[ProductFile], + tile_duration_seconds: u32, + geo_engine_url: &str, + local_data_dir: &str, + volume_name: &str, + remote_data_dir: &str, +) -> anyhow::Result> { + let create_dataset = CreateDataset { + data_path: DataPath::Volume(VolumeName(volume_name.to_string())), + definition: DatasetDefinition { + properties: AddDataset { + name: Some( + DatasetName::from_str(product_name).context("Failed to create dataset name")?, + ), + display_name: format!("{product_name} Dataset"), + description: format!("Dataset for {product_name}"), + source_operator: "MultiBandGdalSource".to_string(), + symbology: None, + provenance: None, + tags: None, + }, + meta_data: MetaDataDefinition::GdalMultiBand(GdalMultiBand { + r#type: crate::api::model::operators::GdalMultiBandTypeTag::GdalMultiBandTypeTag, + result_descriptor: files[0].result_descriptor.clone().into(), + }), + }, + }; + + let response = client + .post(format!("{geo_engine_url}/dataset")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&create_dataset) + .send() + .await + .context("Failed to send dataset creation request")?; + + let dataset_name = if let Ok(json) = response.json::().await { + if let Some(id) = json.get("datasetName").and_then(|v| v.as_str()) { + println!("Dataset added successfully for product: {product_name}, id: {id}"); + id.to_string() + } else { + println!("Failed to get dataset id from response: {json:?}"); + return Ok(None); + } + } else { + println!("Failed to parse dataset creation response as JSON"); + return Ok(None); + }; + + let mut tiles = Vec::new(); + + for file in files { + for (band_idx, _band) in file.result_descriptor.bands.iter().enumerate() { + let tile = AddDatasetTile { + time: TimeInterval::new( + file.time, + file.time + i64::from(tile_duration_seconds * 1000), + ) + .context("Failed to create time interval")? + .into(), + spatial_partition: file.spatial_partition.into(), + band: band_idx as u32, + z_index: 0, // TODO: implement z-index calculation + params: GdalDatasetParameters { + file_path: Path::new(remote_data_dir).join( + file.path.strip_prefix(local_data_dir).with_context(|| { + format!( + "Failed to strip local data dir from path {}", + file.path.display() + ) + })?, + ), + rasterband_channel: band_idx + 1, + geo_transform: file.geo_transform.into(), + width: file.width, + height: file.height, + file_not_found_handling: + crate::api::model::operators::FileNotFoundHandling::Error, + no_data_value: None, + properties_mapping: None, + gdal_open_options: None, + gdal_config_options: None, + allow_alphaband_as_mask: false, + }, + }; + + tiles.push(tile); + } + } + + let response = client + .post(format!("{geo_engine_url}/dataset/{dataset_name}/tiles")) + .header("Content-Type", "application/json") + .header("Authorization", format!("Bearer {session_id}")) + .json(&tiles) + .send() + .await + .context("Failed to send add tiles request")?; + + if !response.status().is_success() { + let error_text = response.text().await.unwrap_or_default(); + println!("Failed to add tile to dataset: {error_text}"); + return Ok(Some(dataset_name)); + } + + println!("Dataset tiles added successfully for product: {product_name}"); + + Ok(Some(dataset_name)) +} + +async fn login( + geo_engine_url: &str, + geo_engine_email: &str, + geo_engine_password: &str, +) -> anyhow::Result<(reqwest::Client, String)> { + let client = reqwest::Client::new(); + let response = client + .post(format!("{geo_engine_url}/login")) + .header("Content-Type", "application/json") + .json(&serde_json::json!({ + "email": geo_engine_email, + "password": geo_engine_password, + })) + .send() + .await + .context("Failed to authenticate")?; + + let json = response + .json::() + .await + .context("Failed to parse auth response")?; + + let session_id = json["id"] + .as_str() + .context("No session id in response")? + .to_string(); + + Ok((client, session_id)) +} + +fn extract_products_from_files( + time_regex: &Regex, + time_format: &str, + product_regex: &Regex, + files: Vec, +) -> HashMap> { + let product_files = files + .into_iter() + .filter_map(|f| { + match extract_product_from_file(time_regex, time_format, product_regex, f.clone()) { + Ok(product_file) => Some(product_file), + Err(e) => { + println!("Skipped file {}: {}", f.as_os_str().to_string_lossy(), e); + None + } + } + }) + .collect::>(); + + let mut products = HashMap::new(); + + for file in product_files { + let product_name = file.product_name.clone(); + products + .entry(product_name) + .or_insert_with(Vec::new) + .push(file); + } + + products +} + +fn extract_product_from_file( + time_regex: &Regex, + time_format: &str, + product_regex: &Regex, + file_path: PathBuf, +) -> anyhow::Result { + let filename = file_path + .file_name() + .and_then(|f| f.to_str()) + .with_context(|| format!("Invalid filename: {}", file_path.display()))?; + + let time_match = time_regex + .captures(filename) + .with_context(|| format!("Time regex did not match filename: {filename}"))?; + + let product_match = product_regex + .captures(filename) + .with_context(|| format!("Product regex did not match filename: {filename}"))?; + + let time_str = time_match + .get(1) + .context("No time capture group in regex match")? + .as_str(); + + let product_str = product_match + .get(1) + .context("No product capture group in regex match")? + .as_str(); + + let naive_date = NaiveDate::parse_from_str(time_str, time_format) + .with_context(|| format!("Failed to parse time '{time_str}'"))?; + + let time = naive_date_to_time_instance(naive_date)?; + + let gdal_dataset = GdalDataset::open(&file_path) + .with_context(|| format!("Failed to open dataset {}", file_path.display()))?; + + let geo_transform: GdalGeoTransform = gdal_dataset + .geo_transform() + .context("Failed to get geo-transform")?; + let geo_transform: GdalDatasetGeoTransform = geo_transform.into(); + + let (width, height) = gdal_dataset.raster_size(); + + let spatial_partition = SpatialPartition2D::new( + geo_transform.origin_coordinate, + geo_transform.origin_coordinate + + Coordinate2D::new( + width as f64 * geo_transform.x_pixel_size, + height as f64 * geo_transform.y_pixel_size, + ), + ) + .context("Failed to create spatial partition")?; + + // TODO: collect units for all bands + let mut result_descriptor = raster_descriptor_from_dataset(&gdal_dataset, 1) + .context("Could not get raster descriptor")?; + + let measurements = (1..=gdal_dataset.raster_count()) + .map(|band| measurement_from_rasterband(&gdal_dataset, band)) + .collect::>(); + + result_descriptor.bands = measurements + .into_iter() + .enumerate() + .map(|(idx, measurement)| { + let band = gdal_dataset + .rasterband(idx + 1) + .with_context(|| format!("Failed to get raster band {}", idx + 1))?; + + let description = band + .description() + .unwrap_or_else(|_| format!("band {}", idx + 1)); + + Ok(RasterBandDescriptor::new( + description, + measurement.unwrap_or(Measurement::Unitless), + )) + }) + .collect::>>()? + .try_into() + .context("Failed to convert raster bands")?; + + Ok(ProductFile { + product_name: product_str.to_string(), + path: file_path, + time, + geo_transform, + spatial_partition, + width, + height, + result_descriptor, + }) +} diff --git a/services/src/config.rs b/services/src/config.rs index 78243a2ea..f087248a2 100644 --- a/services/src/config.rs +++ b/services/src/config.rs @@ -169,8 +169,6 @@ impl ConfigElement for ProjectService { #[derive(Debug, Deserialize)] pub struct TilingSpecification { - pub origin_coordinate_x: f64, - pub origin_coordinate_y: f64, pub tile_shape_pixels_x: usize, pub tile_shape_pixels_y: usize, } @@ -178,10 +176,6 @@ pub struct TilingSpecification { impl From for geoengine_datatypes::raster::TilingSpecification { fn from(ts: TilingSpecification) -> geoengine_datatypes::raster::TilingSpecification { geoengine_datatypes::raster::TilingSpecification { - origin_coordinate: geoengine_datatypes::primitives::Coordinate2D::new( - ts.origin_coordinate_x, - ts.origin_coordinate_y, - ), tile_size_in_pixels: geoengine_datatypes::raster::GridShape2D::from([ ts.tile_shape_pixels_y, ts.tile_shape_pixels_x, diff --git a/services/src/contexts/db_types.rs b/services/src/contexts/db_types.rs index 6122fcb87..5acc08ff1 100644 --- a/services/src/contexts/db_types.rs +++ b/services/src/contexts/db_types.rs @@ -1,4 +1,5 @@ use crate::{ + api::model::services::DataPath, datasets::{ dataset_listing_provider::DatasetLayerListingProviderDefinition, external::{ @@ -14,6 +15,7 @@ use crate::{ }, listing::Provenance, storage::MetaDataDefinition, + upload::{UploadId, VolumeName}, }, error::Error, layers::external::TypedDataProviderDefinition, @@ -36,7 +38,7 @@ use geoengine_operators::{ mock::MockDatasetDataSourceLoadingInfo, source::{ GdalMetaDataList, GdalMetaDataRegular, GdalMetaDataStatic, GdalMetadataNetCdfCf, - OgrSourceDataset, + GdalMultiBand, OgrSourceDataset, }, }; use postgres_types::{FromSql, ToSql}; @@ -377,6 +379,7 @@ pub struct MetaDataDefinitionDbType { gdal_static: Option, gdal_metadata_net_cdf_cf: Option, gdal_meta_data_list: Option, + gdal_multi_band: Option, } impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { @@ -389,6 +392,7 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, }, MetaDataDefinition::OgrMetaData(meta_data) => Self { mock_meta_data: None, @@ -397,6 +401,7 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, }, MetaDataDefinition::GdalMetaDataRegular(meta_data) => Self { mock_meta_data: None, @@ -405,6 +410,7 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, }, MetaDataDefinition::GdalStatic(meta_data) => Self { mock_meta_data: None, @@ -413,6 +419,7 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: Some(meta_data.clone()), gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, }, MetaDataDefinition::GdalMetadataNetCdfCf(meta_data) => Self { mock_meta_data: None, @@ -421,6 +428,7 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: None, gdal_metadata_net_cdf_cf: Some(meta_data.clone()), gdal_meta_data_list: None, + gdal_multi_band: None, }, MetaDataDefinition::GdalMetaDataList(meta_data) => Self { mock_meta_data: None, @@ -429,6 +437,16 @@ impl From<&MetaDataDefinition> for MetaDataDefinitionDbType { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: Some(meta_data.clone()), + gdal_multi_band: None, + }, + MetaDataDefinition::GdalMultiBand(gdal_multi_band) => Self { + mock_meta_data: None, + ogr_meta_data: None, + gdal_meta_data_regular: None, + gdal_static: None, + gdal_metadata_net_cdf_cf: None, + gdal_meta_data_list: None, + gdal_multi_band: Some(gdal_multi_band.clone()), }, } } @@ -446,6 +464,7 @@ impl TryFrom for MetaDataDefinition { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, } => Ok(MetaDataDefinition::MockMetaData(meta_data)), MetaDataDefinitionDbType { mock_meta_data: None, @@ -454,6 +473,7 @@ impl TryFrom for MetaDataDefinition { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, } => Ok(MetaDataDefinition::OgrMetaData(meta_data)), MetaDataDefinitionDbType { mock_meta_data: None, @@ -462,6 +482,7 @@ impl TryFrom for MetaDataDefinition { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, } => Ok(MetaDataDefinition::GdalMetaDataRegular(meta_data)), MetaDataDefinitionDbType { mock_meta_data: None, @@ -470,6 +491,7 @@ impl TryFrom for MetaDataDefinition { gdal_static: Some(meta_data), gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: None, + gdal_multi_band: None, } => Ok(MetaDataDefinition::GdalStatic(meta_data)), MetaDataDefinitionDbType { mock_meta_data: None, @@ -478,6 +500,7 @@ impl TryFrom for MetaDataDefinition { gdal_static: None, gdal_metadata_net_cdf_cf: Some(meta_data), gdal_meta_data_list: None, + gdal_multi_band: None, } => Ok(MetaDataDefinition::GdalMetadataNetCdfCf(meta_data)), MetaDataDefinitionDbType { mock_meta_data: None, @@ -486,7 +509,17 @@ impl TryFrom for MetaDataDefinition { gdal_static: None, gdal_metadata_net_cdf_cf: None, gdal_meta_data_list: Some(meta_data), + gdal_multi_band: None, } => Ok(MetaDataDefinition::GdalMetaDataList(meta_data)), + MetaDataDefinitionDbType { + mock_meta_data: None, + ogr_meta_data: None, + gdal_meta_data_regular: None, + gdal_static: None, + gdal_metadata_net_cdf_cf: None, + gdal_meta_data_list: None, + gdal_multi_band: Some(meta_data), + } => Ok(MetaDataDefinition::GdalMultiBand(meta_data)), _ => Err(Error::UnexpectedInvalidDbTypeConversion), } } @@ -1323,6 +1356,39 @@ impl TryFrom for TypedDataProviderDefinition } } +#[derive(Debug, ToSql, FromSql)] +#[postgres(name = "DataPath")] +pub struct DataPathDbType { + pub volume_name: Option, + pub upload_id: Option, +} + +impl From<&DataPath> for DataPathDbType { + fn from(other: &DataPath) -> Self { + match other { + DataPath::Volume(volume_name) => Self { + volume_name: Some(volume_name.0.clone()), + upload_id: None, + }, + DataPath::Upload(upload_id) => Self { + volume_name: None, + upload_id: Some(upload_id.0), + }, + } + } +} + +impl TryFrom for DataPath { + type Error = Error; + fn try_from(other: DataPathDbType) -> Result { + match (other.volume_name, other.upload_id) { + (Some(v), None) => Ok(DataPath::Volume(VolumeName(v))), + (None, Some(u)) => Ok(DataPath::Upload(UploadId(u))), + _ => Err(Error::UnexpectedInvalidDbTypeConversion), + } + } +} + delegate_from_to_sql!(ColorParam, ColorParamDbType); delegate_from_to_sql!(DatabaseConnectionConfig, DatabaseConnectionConfigDbType); delegate_from_to_sql!( @@ -1360,18 +1426,22 @@ delegate_from_to_sql!(RasterSymbology, RasterSymbologyDbType); delegate_from_to_sql!(PointSymbology, PointSymbologyDbType); delegate_from_to_sql!(LineSymbology, LineSymbologyDbType); delegate_from_to_sql!(PolygonSymbology, PolygonSymbologyDbType); +delegate_from_to_sql!(DataPath, DataPathDbType); #[cfg(test)] mod tests { use geoengine_datatypes::{ - dataset::DataProviderId, primitives::CacheTtlSeconds, util::Identifier, + dataset::DataProviderId, + primitives::{CacheTtlSeconds, Coordinate2D, SpatialPartition2D, TimeInterval}, + util::Identifier, }; + use tokio_postgres::NoTls; use super::*; use crate::{ - datasets::external::{ - SentinelS2L2ACogsProviderDefinition, StacBand, StacQueryBuffer, StacZone, - }, + contexts::PostgresContext, + datasets::external::{SentinelS2L2ACogsProviderDefinition, StacQueryBuffer}, + ge_context, layers::external::TypedDataProviderDefinition, util::{postgres::assert_sql_type, tests::with_temp_context}, }; @@ -1402,27 +1472,6 @@ mod tests { ) .await; - assert_sql_type( - &pool, - "StacBand", - [StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - ) - .await; - - assert_sql_type( - &pool, - "StacZone", - [StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], - ) - .await; - assert_sql_type( &pool, "SentinelS2L2ACogsProviderDefinition", @@ -1432,15 +1481,6 @@ mod tests { description: "A provider".to_owned(), priority: Some(1), api_url: "http://api.url".to_owned(), - bands: vec![StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - zones: vec![StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], stac_api_retries: StacApiRetries { number_of_retries: 3, initial_delay_ms: 4, @@ -1499,15 +1539,6 @@ mod tests { priority: Some(3), id: DataProviderId::new(), api_url: "http://api.url".to_owned(), - bands: vec![StacBand { - name: "band".to_owned(), - no_data_value: Some(133.7), - data_type: geoengine_datatypes::raster::RasterDataType::F32, - }], - zones: vec![StacZone { - name: "zone".to_owned(), - epsg: 4326, - }], stac_api_retries: StacApiRetries { number_of_retries: 3, initial_delay_ms: 4, @@ -1529,4 +1560,87 @@ mod tests { }) .await; } + + #[ge_context::test] + async fn it_checks_spatial_partition_overlaps( + app_ctx: PostgresContext, + ) -> crate::error::Result<()> { + let conn = app_ctx.pool.get().await?; + + assert!( + conn.query_one( + "SELECT spatial_partition2d_intersects($1, $2)", + &[ + &SpatialPartition2D::new(Coordinate2D::new(0., 1.), Coordinate2D::new(1., 0.))?, + &SpatialPartition2D::new( + Coordinate2D::new(0.5, 1.5), + Coordinate2D::new(1.5, 0.5), + )?, + ], + ) + .await? + .get::<_, bool>(0) + ); + + assert!( + !conn + .query_one( + "SELECT spatial_partition2d_intersects($1, $2)", + &[ + &SpatialPartition2D::new( + Coordinate2D::new(0., 1.), + Coordinate2D::new(1., 0.) + )?, + &SpatialPartition2D::new( + Coordinate2D::new(1.0, 2.0), + Coordinate2D::new(2.0, 1.0), + )?, + ], + ) + .await? + .get::<_, bool>(0) + ); + + Ok(()) + } + + #[ge_context::test] + async fn it_checks_time_interval_overlaps( + app_ctx: PostgresContext, + ) -> crate::error::Result<()> { + let conn = app_ctx.pool.get().await?; + + assert!( + conn.query_one( + "SELECT time_interval_intersects($1, $2)", + &[&TimeInterval::new(0, 10)?, &TimeInterval::new(5, 15)?,], + ) + .await? + .get::<_, bool>(0) + ); + + assert!( + !conn + .query_one( + "SELECT time_interval_intersects($1, $2)", + &[&TimeInterval::new(0, 10)?, &TimeInterval::new(10, 20)?,], + ) + .await? + .get::<_, bool>(0) + ); + + assert!( + conn.query_one( + "SELECT time_interval_intersects($1, $2)", + &[ + &TimeInterval::new(1_388_534_400_000, 1_388_534_400_000)?, + &TimeInterval::new(1_388_534_400_000, 1_391_212_800_000)?, + ], + ) + .await? + .get::<_, bool>(0) + ); + + Ok(()) + } } diff --git a/services/src/contexts/migrations/current_schema.sql b/services/src/contexts/migrations/current_schema.sql index 673f84d74..efe04ddb3 100644 --- a/services/src/contexts/migrations/current_schema.sql +++ b/services/src/contexts/migrations/current_schema.sql @@ -51,6 +51,31 @@ CREATE TYPE "TimeStep" AS ( step OID ); +CREATE TYPE "GridBoundingBox2D" AS ( + y_min bigint, + y_max bigint, + x_min bigint, + x_max bigint +); + +CREATE TYPE "GeoTransform" AS ( + origin_coordinate "Coordinate2D", + x_pixel_size double precision, + y_pixel_size double precision +); + +CREATE TYPE "SpatialGridDefinition" AS ( + geo_transform "GeoTransform", + grid_bounds "GridBoundingBox2D" +); + +CREATE TYPE "SpatialGridDescriptorState" AS ENUM ('Source', 'Merged'); + +CREATE TYPE "SpatialGridDescriptor" AS ( + "state" "SpatialGridDescriptorState", + spatial_grid "SpatialGridDefinition" +); + CREATE TYPE "DatasetName" AS (namespace text, name text); CREATE TYPE "Provenance" AS ( @@ -249,14 +274,33 @@ CREATE TYPE "RasterBandDescriptor" AS ( measurement "Measurement" ); +CREATE TYPE "RegularTimeDimension" AS ( + origin bigint, + step "TimeStep" +); + +CREATE TYPE "TimeDimensionDiscriminator" AS ENUM ( + 'Regular', + 'Irregular' +); + +CREATE TYPE "TimeDimension" AS ( + regular_dimension "RegularTimeDimension", + discriminant "TimeDimensionDiscriminator" +); + +CREATE TYPE "TimeDescriptor" AS ( + bounds "TimeInterval", + dimension "TimeDimension" +); + CREATE TYPE "RasterResultDescriptor" AS ( data_type "RasterDataType", -- SpatialReferenceOption spatial_reference "SpatialReference", - "time" "TimeInterval", - bbox "SpatialPartition2D", - resolution "SpatialResolution", - bands "RasterBandDescriptor" [] + bands "RasterBandDescriptor" [], + spatial_grid "SpatialGridDescriptor", + "time" "TimeDescriptor" ); CREATE TYPE "VectorResultDescriptor" AS ( @@ -517,6 +561,10 @@ CREATE TYPE "GdalMetaDataList" AS ( params "GdalLoadingInfoTemporalSlice" [] ); +CREATE TYPE "GdalMultiBand" AS ( + result_descriptor "RasterResultDescriptor" +); + CREATE TYPE "MetaDataDefinition" AS ( -- oneOf mock_meta_data "MockMetaData", @@ -524,7 +572,8 @@ CREATE TYPE "MetaDataDefinition" AS ( gdal_meta_data_regular "GdalMetaDataRegular", gdal_static "GdalMetaDataStatic", gdal_metadata_net_cdf_cf "GdalMetadataNetCdfCf", - gdal_meta_data_list "GdalMetaDataList" + gdal_meta_data_list "GdalMetaDataList", + gdal_multi_band "GdalMultiBand" ); -- seperate table for projects used in foreign key constraints @@ -590,6 +639,12 @@ CREATE TABLE workflows ( -- TODO: add length constraints +CREATE TYPE "DataPath" AS ( + -- oneOf + volume_name text, + upload_id uuid +); + CREATE TABLE datasets ( id uuid PRIMARY KEY, name "DatasetName" UNIQUE NOT NULL, @@ -600,7 +655,8 @@ CREATE TABLE datasets ( result_descriptor "ResultDescriptor" NOT NULL, meta_data "MetaDataDefinition" NOT NULL, symbology "Symbology", - provenance "Provenance" [] + provenance "Provenance" [], + data_path "DataPath" -- TODO: NOT NULL for GdalMultiBand? ); -- TODO: add constraint not null @@ -765,17 +821,6 @@ CREATE TYPE "DatasetLayerListingProviderDefinition" AS ( priority smallint ); -CREATE TYPE "StacBand" AS ( - "name" text, - no_data_value double precision, - data_type "RasterDataType" -); - -CREATE TYPE "StacZone" AS ( - "name" text, - epsg oid -); - CREATE TYPE "StacApiRetries" AS ( number_of_retries bigint, initial_delay_ms bigint, @@ -795,8 +840,6 @@ CREATE TYPE "SentinelS2L2ACogsProviderDefinition" AS ( "name" text, id uuid, api_url text, - bands "StacBand" [], - zones "StacZone" [], stac_api_retries "StacApiRetries", gdal_retries "GdalRetries", cache_ttl int, @@ -1281,3 +1324,69 @@ CREATE TABLE quota_log ( ); CREATE INDEX ON quota_log (user_id, timestamp, computation_id); + +CREATE TABLE dataset_tiles ( + id uuid NOT NULL PRIMARY KEY, + dataset_id uuid NOT NULL, + time "TimeInterval" NOT NULL, -- noqa: references.keywords + bbox "SpatialPartition2D" NOT NULL, + band oid NOT NULL, + z_index oid NOT NULL, + gdal_params "GdalDatasetParameters" NOT NULL +); + +CREATE UNIQUE INDEX dataset_tiles_unique_idx ON dataset_tiles ( + dataset_id, + time, + bbox, + band, + z_index +); + +-- helper type for batch checking tile validity +CREATE TYPE "TileKey" AS ( + time "TimeInterval", + bbox "SpatialPartition2D", + band oid, + z_index oid +); + +-- helper type for batch inserting tiles +CREATE TYPE "TileEntry" AS ( + id uuid, + dataset_id uuid, + time "TimeInterval", + bbox "SpatialPartition2D", + band oid, + z_index oid, + gdal_params "GdalDatasetParameters" +); + +-- Returns true if the partitions have any space in common +CREATE OR REPLACE FUNCTION SPATIAL_PARTITION2D_INTERSECTS( + a "SpatialPartition2D", b "SpatialPartition2D" +) RETURNS boolean AS $$ +SELECT NOT ( + ( (a).lower_right_coordinate.x <= (b).upper_left_coordinate.x ) OR + ( (a).upper_left_coordinate.x >= (b).lower_right_coordinate.x ) OR + ( (a).lower_right_coordinate.y >= (b).upper_left_coordinate.y ) OR + ( (a).upper_left_coordinate.y <= (b).lower_right_coordinate.y ) +); +$$ LANGUAGE sql IMMUTABLE; + + +-- Returns true if two TimeIntervals overlap +CREATE OR REPLACE FUNCTION TIME_INTERVAL_INTERSECTS( + a "TimeInterval", b "TimeInterval" +) RETURNS boolean AS $$ +SELECT + -- If a is an instant + ((a).start = (a)."end" AND (a).start >= (b).start AND (a).start < (b)."end") + OR + -- If b is an instant + ((b).start = (b)."end" AND (b).start >= (a).start AND (b).start < (a)."end") + OR + -- Regular overlap for intervals + ((a).start < (b)."end" AND (b).start < (a)."end") +; +$$ LANGUAGE sql IMMUTABLE; diff --git a/services/src/contexts/migrations/migration_0030_raster_result_desc.rs b/services/src/contexts/migrations/migration_0030_raster_result_desc.rs new file mode 100644 index 000000000..85dab39e5 --- /dev/null +++ b/services/src/contexts/migrations/migration_0030_raster_result_desc.rs @@ -0,0 +1,27 @@ +use super::database_migration::{DatabaseVersion, Migration}; +use crate::{contexts::migrations::Migration0023WildliveOidc, error::Result}; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +/// This migration reworks the raster result descritptor and some other small changes from the rewrite branch +pub struct Migration0030RasterResultDesc; + +#[async_trait] +impl Migration for Migration0030RasterResultDesc { + fn prev_version(&self) -> Option { + Some(Migration0023WildliveOidc.version()) + } + + fn version(&self) -> DatabaseVersion { + "0030_raster_result_desc".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0030_remove_stack_zone_band.sql")) + .await?; + + tx.batch_execute(include_str!("migration_0030_raster_result_desc.sql")) + .await?; + Ok(()) + } +} diff --git a/services/src/contexts/migrations/migration_0030_raster_result_desc.sql b/services/src/contexts/migrations/migration_0030_raster_result_desc.sql new file mode 100644 index 000000000..4c7ec51f9 --- /dev/null +++ b/services/src/contexts/migrations/migration_0030_raster_result_desc.sql @@ -0,0 +1,183 @@ +-- add the new types +CREATE TYPE "GridBoundingBox2D" AS ( + y_min bigint, + y_max bigint, + x_min bigint, + x_max bigint +); +CREATE TYPE "GeoTransform" AS ( + origin_coordinate "Coordinate2D", + x_pixel_size double precision, + y_pixel_size double precision +); +CREATE TYPE "SpatialGridDefinition" AS ( + geo_transform "GeoTransform", + grid_bounds "GridBoundingBox2D" +); +CREATE TYPE "SpatialGridDescriptorState" AS ENUM ('Source', 'Merged'); +CREATE TYPE "SpatialGridDescriptor" AS ( + "state" "SpatialGridDescriptorState", + spatial_grid "SpatialGridDefinition" +); +-- adapt the RasterResultDescriptor --> add the new attribute +ALTER TYPE "RasterResultDescriptor" +ADD ATTRIBUTE spatial_grid "SpatialGridDescriptor"; +-- migrate gdal_static metadata +WITH cte AS ( + SELECT + id, + (meta_data).gdal_static AS meta + FROM datasets + WHERE (meta_data).gdal_static IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_regular metadata +WITH cte AS ( + SELECT + id, + (meta_data).gdal_meta_data_regular AS meta + FROM datasets + WHERE (meta_data).gdal_meta_data_regular IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_metadata_net_cdf_cf +WITH cte AS ( + SELECT + id, + (meta_data).gdal_metadata_net_cdf_cf AS meta + FROM datasets + WHERE (meta_data).gdal_metadata_net_cdf_cf IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + ( + ( + (cte).meta.params.geo_transform.origin_coordinate, + (cte).meta.params.geo_transform.x_pixel_size, + (cte).meta.params.geo_transform.x_pixel_size + )::"GeoTransform", + ( + 0, + (cte).meta.params.height - 1, + 0, + (cte).meta.params.width - 1 + )::"GridBoundingBox2D" + )::"SpatialGridDefinition" + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- migrate gdal_metadata_lsit +CREATE FUNCTION pg_temp.spatial_grid_def_from_params_array( + t_slices "GdalLoadingInfoTemporalSlice" [] +) RETURNS "SpatialGridDefinition" AS $$ +DECLARE b_size_x double precision; +b_size_y double precision; +b_ul_x double precision; +b_ul_y double precision; +b_lr_x double precision; +b_lr_y double precision; +t_x double precision; +t_y double precision; +n "SpatialGridDefinition"; +t "GdalLoadingInfoTemporalSlice"; +BEGIN FOREACH t IN ARRAY t_slices LOOP IF t.params IS NULL THEN CONTINUE; +END IF; +t_x := (t).params.geo_transform.origin_coordinate.x + (t).params.geo_transform.x_pixel_size * (t).params.width; +t_y := (t).params.geo_transform.origin_coordinate.y + (t).params.geo_transform.y_pixel_size * (t).params.height; +IF b_size_x IS NULL THEN b_size_x := (t).params.geo_transform.x_pixel_size; +b_size_y := (t).params.geo_transform.y_pixel_size; +b_ul_x := (t).params.geo_transform.origin_coordinate.x; +b_ul_y := (t).params.geo_transform.origin_coordinate.y; +b_lr_x := t_x; +b_lr_y := t_y; +END IF; +b_ul_x := LEAST( + b_ul_x, + (t).params.geo_transform.origin_coordinate.x +); +b_ul_y := GREATEST( + b_ul_y, + (t).params.geo_transform.origin_coordinate.y +); +b_lr_x := GREATEST(b_lr_x, t_x); +b_lr_y := LEAST(b_lr_y, t_y); +END LOOP; +RETURN ( + ( + (b_ul_x, b_ul_y)::"Coordinate2D", + b_size_x, + b_size_y + )::"GeoTransform", + ( + 0, + ((b_ul_y - b_lr_y) / b_size_y) -1, + 0, + ((b_lr_x - b_ul_x) / b_size_x) -1 + )::"GridBoundingBox2D" +)::"SpatialGridDefinition"; +END; +$$ LANGUAGE plpgsql; +WITH cte AS ( + SELECT + id, + (meta_data).gdal_meta_data_list AS meta + FROM datasets + WHERE (meta_data).gdal_meta_data_list IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.spatial_grid = ( + 'Source', + pg_temp.spatial_grid_def_from_params_array(((cte).meta.params)) + )::"SpatialGridDescriptor" +FROM cte +WHERE datasets.id = cte.id; +-- remove the old attributes +ALTER TYPE "RasterResultDescriptor" DROP ATTRIBUTE bbox; +ALTER TYPE "RasterResultDescriptor" DROP ATTRIBUTE resolution; +-- mark the spatial_grid as NOT NULL +DROP FUNCTION IF EXISTS pg_temp.spatial_grid_def_from_params_array; diff --git a/services/src/contexts/migrations/migration_0030_remove_stack_zone_band.sql b/services/src/contexts/migrations/migration_0030_remove_stack_zone_band.sql new file mode 100644 index 000000000..097a85f67 --- /dev/null +++ b/services/src/contexts/migrations/migration_0030_remove_stack_zone_band.sql @@ -0,0 +1,4 @@ +ALTER TYPE "SentinelS2L2ACogsProviderDefinition" DROP ATTRIBUTE bands; +ALTER TYPE "SentinelS2L2ACogsProviderDefinition" DROP ATTRIBUTE zones; +DROP TYPE "StacBand"; +DROP TYPE "StacZone"; diff --git a/services/src/contexts/migrations/migration_0031_time_descriptor.rs b/services/src/contexts/migrations/migration_0031_time_descriptor.rs new file mode 100644 index 000000000..0d147849c --- /dev/null +++ b/services/src/contexts/migrations/migration_0031_time_descriptor.rs @@ -0,0 +1,24 @@ +use super::database_migration::{DatabaseVersion, Migration}; +use crate::{contexts::migrations::Migration0030RasterResultDesc, error::Result}; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +pub struct Migration0031TimeDescriptor; + +#[async_trait] +impl Migration for Migration0031TimeDescriptor { + fn prev_version(&self) -> Option { + Some(Migration0030RasterResultDesc.version()) + } + + fn version(&self) -> DatabaseVersion { + "0031_time_descriptor".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0031_time_descriptor.sql")) + .await?; + + Ok(()) + } +} diff --git a/services/src/contexts/migrations/migration_0031_time_descriptor.sql b/services/src/contexts/migrations/migration_0031_time_descriptor.sql new file mode 100644 index 000000000..c83754b24 --- /dev/null +++ b/services/src/contexts/migrations/migration_0031_time_descriptor.sql @@ -0,0 +1,65 @@ +CREATE TYPE "RegularTimeDimension" AS ( + origin bigint, + step "TimeStep" +); + +CREATE TYPE "TimeDimensionDiscriminator" AS ENUM ( + 'Regular', + 'Irregular' +); + +CREATE TYPE "TimeDimension" AS ( + regular_dimension "RegularTimeDimension", + discriminant "TimeDimensionDiscriminator" +); + +CREATE TYPE "TimeDescriptor" AS ( + bounds "TimeInterval", + dimension "TimeDimension" +); + +ALTER TYPE "RasterResultDescriptor" RENAME ATTRIBUTE time TO old_time; +ALTER TYPE "RasterResultDescriptor" ADD ATTRIBUTE time "TimeDescriptor"; + +WITH cte AS ( + SELECT + id, + (result_descriptor).raster.old_time AS old_time + FROM datasets + WHERE (result_descriptor).raster IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.time = ( + cte.old_time, + (NULL, 'Irregular')::"TimeDimension" + )::"TimeDescriptor" +FROM cte +WHERE datasets.id = cte.id; + +WITH cte AS ( + SELECT + id, + (result_descriptor).raster.old_time AS old_time, + (meta_data).gdal_meta_data_regular AS meta + FROM datasets + WHERE (meta_data).gdal_meta_data_regular IS NOT NULL +) + +UPDATE datasets +SET + result_descriptor.raster.time = ( + cte.old_time, + ( + ( + (cte).meta.data_time.start, + (cte).meta.step + )::"RegularTimeDimension", + 'Regular' + )::"TimeDimension" + )::"TimeDescriptor" +FROM cte +WHERE datasets.id = cte.id; + +ALTER TYPE "RasterResultDescriptor" DROP ATTRIBUTE old_time; diff --git a/services/src/contexts/migrations/migration_0032_gdal_tiles.rs b/services/src/contexts/migrations/migration_0032_gdal_tiles.rs new file mode 100644 index 000000000..31ff04a0c --- /dev/null +++ b/services/src/contexts/migrations/migration_0032_gdal_tiles.rs @@ -0,0 +1,25 @@ +use super::database_migration::{DatabaseVersion, Migration}; +use crate::{contexts::migrations::Migration0031TimeDescriptor, error::Result}; +use async_trait::async_trait; +use tokio_postgres::Transaction; + +/// This migration adds support for GDAL tiles in the database schema. +pub struct Migration0032GdalTiles; + +#[async_trait] +impl Migration for Migration0032GdalTiles { + fn prev_version(&self) -> Option { + Some(Migration0031TimeDescriptor.version()) + } + + fn version(&self) -> DatabaseVersion { + "0032_gdal_tiles".into() + } + + async fn migrate(&self, tx: &Transaction<'_>) -> Result<()> { + tx.batch_execute(include_str!("migration_0032_gdal_tiles.sql")) + .await?; + + Ok(()) + } +} diff --git a/services/src/contexts/migrations/migration_0032_gdal_tiles.sql b/services/src/contexts/migrations/migration_0032_gdal_tiles.sql new file mode 100644 index 000000000..1564168b9 --- /dev/null +++ b/services/src/contexts/migrations/migration_0032_gdal_tiles.sql @@ -0,0 +1,80 @@ +CREATE TYPE "GdalMultiBand" AS ( + result_descriptor "RasterResultDescriptor" +); + +ALTER TYPE "MetaDataDefinition" ADD ATTRIBUTE gdal_multi_band "GdalMultiBand"; + +CREATE TYPE "DataPath" AS ( + -- oneOf + volume_name text, + upload_id uuid +); + +-- TODO: NOT NULL for GdalMultiBand? +ALTER TABLE datasets ADD COLUMN data_path "DataPath"; + +CREATE TABLE dataset_tiles ( + id uuid NOT NULL PRIMARY KEY, + dataset_id uuid NOT NULL, + time "TimeInterval" NOT NULL, -- noqa: references.keywords + bbox "SpatialPartition2D" NOT NULL, + band oid NOT NULL, + z_index oid NOT NULL, + gdal_params "GdalDatasetParameters" NOT NULL +); + +CREATE UNIQUE INDEX dataset_tiles_unique_idx ON dataset_tiles ( + dataset_id, + time, + bbox, + band, + z_index +); + +-- helper type for batch checking tile validity +CREATE TYPE "TileKey" AS ( + time "TimeInterval", + bbox "SpatialPartition2D", + band oid, + z_index oid +); + +-- helper type for batch inserting tiles +CREATE TYPE "TileEntry" AS ( + id uuid, + dataset_id uuid, + time "TimeInterval", + bbox "SpatialPartition2D", + band oid, + z_index oid, + gdal_params "GdalDatasetParameters" +); + +-- Returns true if the partitions have any space in common +CREATE OR REPLACE FUNCTION SPATIAL_PARTITION2D_INTERSECTS( + a "SpatialPartition2D", b "SpatialPartition2D" +) RETURNS boolean AS $$ +SELECT NOT ( + ( (a).lower_right_coordinate.x <= (b).upper_left_coordinate.x ) OR + ( (a).upper_left_coordinate.x >= (b).lower_right_coordinate.x ) OR + ( (a).lower_right_coordinate.y >= (b).upper_left_coordinate.y ) OR + ( (a).upper_left_coordinate.y <= (b).lower_right_coordinate.y ) +); +$$ LANGUAGE sql IMMUTABLE; + + +-- Returns true if two TimeIntervals overlap +CREATE OR REPLACE FUNCTION TIME_INTERVAL_INTERSECTS( + a "TimeInterval", b "TimeInterval" +) RETURNS boolean AS $$ +SELECT + -- If a is an instant + ((a).start = (a)."end" AND (a).start >= (b).start AND (a).start < (b)."end") + OR + -- If b is an instant + ((b).start = (b)."end" AND (b).start >= (a).start AND (b).start < (a)."end") + OR + -- Regular overlap for intervals + ((a).start < (b)."end" AND (b).start < (a)."end") +; +$$ LANGUAGE sql IMMUTABLE; diff --git a/services/src/contexts/migrations/mod.rs b/services/src/contexts/migrations/mod.rs index ba576c802..c6f371500 100644 --- a/services/src/contexts/migrations/mod.rs +++ b/services/src/contexts/migrations/mod.rs @@ -1,15 +1,16 @@ +use crate::contexts::migrations::migration_0032_gdal_tiles::Migration0032GdalTiles; pub use crate::contexts::migrations::{ current_schema::CurrentSchemaMigration, migration_0016_merge_providers::Migration0016MergeProviders, migration_0017_ml_model_tensor_shape::Migration0017MlModelTensorShape, migration_0018_wildlive_connector::Migration0018WildliveConnector, + migration_0019_ml_model_no_data::Migration0019MlModelNoData, migration_0020_provider_permissions::Migration0020ProviderPermissions, migration_0021_default_permissions_for_existing_providers::Migration0021DefaultPermissionsForExistingProviders, migration_0022_permission_queries::Migration0022PermissionQueries, -}; -use crate::contexts::migrations::{ - migration_0019_ml_model_no_data::Migration0019MlModelNoData, migration_0023_wildlive_oidc::Migration0023WildliveOidc, + migration_0030_raster_result_desc::Migration0030RasterResultDesc, + migration_0031_time_descriptor::Migration0031TimeDescriptor, }; pub use database_migration::{ DatabaseVersion, Migration, MigrationResult, initialize_database, migrate_database, @@ -25,6 +26,9 @@ mod migration_0020_provider_permissions; mod migration_0021_default_permissions_for_existing_providers; mod migration_0022_permission_queries; mod migration_0023_wildlive_oidc; +mod migration_0030_raster_result_desc; +mod migration_0031_time_descriptor; +mod migration_0032_gdal_tiles; #[cfg(test)] mod schema_info; @@ -54,6 +58,9 @@ pub fn all_migrations() -> Vec> { Box::new(Migration0021DefaultPermissionsForExistingProviders), Box::new(Migration0022PermissionQueries), Box::new(Migration0023WildliveOidc), + Box::new(Migration0030RasterResultDesc), + Box::new(Migration0031TimeDescriptor), + Box::new(Migration0032GdalTiles), ] } diff --git a/services/src/contexts/migrations/schema_info.rs b/services/src/contexts/migrations/schema_info.rs index d1e58dedb..05788a363 100644 --- a/services/src/contexts/migrations/schema_info.rs +++ b/services/src/contexts/migrations/schema_info.rs @@ -343,7 +343,8 @@ schema_info_table!( parameters, "specific_schema", "specific_name, ordinal_position ASC", - (specific_name, String), + // the specific name contains the object ID which is different when migrating vs. when creating from scratch + // (specific_name, String), (ordinal_position, i32), (parameter_mode, String), (parameter_name, String), diff --git a/services/src/contexts/mod.rs b/services/src/contexts/mod.rs index 7baa4b381..ae88530a8 100644 --- a/services/src/contexts/mod.rs +++ b/services/src/contexts/mod.rs @@ -27,7 +27,10 @@ use geoengine_operators::machine_learning::MlModelLoadingInfo; use geoengine_operators::meta::quota::{QuotaCheck, QuotaChecker, QuotaTracking}; use geoengine_operators::meta::wrapper::InitializedOperatorWrapper; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; -use geoengine_operators::source::{GdalLoadingInfo, OgrSourceDataset}; +use geoengine_operators::source::{ + GdalLoadingInfo, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, + OgrSourceDataset, +}; use rayon::ThreadPool; use std::str::FromStr; use std::sync::Arc; @@ -115,6 +118,7 @@ pub trait GeoEngineDb: pub struct QueryContextImpl { chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, thread_pool: Arc, cache: Option>, quota_tracking: Option, @@ -124,10 +128,15 @@ pub struct QueryContextImpl { } impl QueryContextImpl { - pub fn new(chunk_byte_size: ChunkByteSize, thread_pool: Arc) -> Self { + pub fn new( + chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, + thread_pool: Arc, + ) -> Self { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); QueryContextImpl { chunk_byte_size, + tiling_specification, thread_pool, cache: None, quota_tracking: None, @@ -139,6 +148,7 @@ impl QueryContextImpl { pub fn new_with_extensions( chunk_byte_size: ChunkByteSize, + tiling_specification: TilingSpecification, thread_pool: Arc, cache: Option>, quota_tracking: Option, @@ -147,6 +157,7 @@ impl QueryContextImpl { let (abort_registration, abort_trigger) = QueryAbortRegistration::new(); QueryContextImpl { chunk_byte_size, + tiling_specification, thread_pool, cache, quota_checker, @@ -157,6 +168,7 @@ impl QueryContextImpl { } } +#[async_trait::async_trait] impl QueryContext for QueryContextImpl { fn chunk_byte_size(&self) -> ChunkByteSize { self.chunk_byte_size @@ -176,6 +188,10 @@ impl QueryContext for QueryContextImpl { .ok_or(geoengine_operators::error::Error::AbortTriggerAlreadyUsed) } + fn tiling_specification(&self) -> TilingSpecification { + self.tiling_specification + } + fn quota_tracking(&self) -> Option<&geoengine_operators::meta::quota::QuotaTracking> { self.quota_tracking.as_ref() } @@ -225,7 +241,11 @@ where VectorQueryRectangle, > + MetaDataProvider + MetaDataProvider - + LayerProviderDb + + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > + LayerProviderDb + MlModelDb, { fn thread_pool(&self) -> &Arc { @@ -466,6 +486,49 @@ where } } +#[async_trait] +impl + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > for ExecutionContextImpl +where + D: DatasetDb + + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > + LayerProviderDb, +{ + async fn meta_data( + &self, + data_id: &DataId, + ) -> Result< + Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + >, + geoengine_operators::error::Error, + > { + match data_id { + DataId::Internal { dataset_id: _ } => { + self.db.meta_data(&data_id.clone()).await.map_err(|e| { + geoengine_operators::error::Error::LoadingInfo { + source: Box::new(e), + } + }) + } + DataId::External(_external) => { + Err(geoengine_operators::error::Error::NotYetImplemented) + } + } + } +} + pub struct QuotaCheckerImpl { pub(crate) user_db: U, } diff --git a/services/src/contexts/postgres.rs b/services/src/contexts/postgres.rs index f9c854121..92696a094 100644 --- a/services/src/contexts/postgres.rs +++ b/services/src/contexts/postgres.rs @@ -354,6 +354,7 @@ where Ok(QueryContextImpl::new_with_extensions( self.context.query_ctx_chunk_size, + self.context.exe_ctx_tiling_spec, self.context.thread_pool.clone(), Some(self.context.tile_cache.clone()), Some( @@ -395,7 +396,7 @@ where } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct PostgresDb where Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, @@ -462,70 +463,84 @@ where #[cfg(test)] mod tests { use super::*; - use crate::config::QuotaTrackingMode; - use crate::datasets::external::netcdfcf::NetCdfCfDataProviderDefinition; - use crate::datasets::listing::{DatasetListOptions, DatasetListing, ProvenanceOutput}; - use crate::datasets::listing::{DatasetProvider, Provenance}; - use crate::datasets::storage::{DatasetStore, MetaDataDefinition}; - use crate::datasets::upload::{FileId, UploadId}; - use crate::datasets::upload::{FileUpload, Upload, UploadDb}; - use crate::datasets::{AddDataset, DatasetIdAndName}; - use crate::ge_context; - use crate::layers::add_from_directory::UNSORTED_COLLECTION_ID; - use crate::layers::layer::{ - AddLayer, AddLayerCollection, CollectionItem, LayerCollection, LayerCollectionListOptions, - LayerCollectionListing, LayerListing, ProviderLayerCollectionId, ProviderLayerId, - }; - use crate::layers::listing::{ - LayerCollectionId, LayerCollectionProvider, SearchParameters, SearchType, - }; - use crate::layers::storage::{ - INTERNAL_PROVIDER_ID, LayerDb, LayerProviderDb, LayerProviderListing, - LayerProviderListingOptions, - }; - use crate::machine_learning::{MlModel, MlModelDb, MlModelIdAndName}; - use crate::permissions::{Permission, PermissionDb, Role, RoleDescription, RoleId}; - use crate::projects::{ - CreateProject, LayerUpdate, LoadVersion, OrderBy, Plot, PlotUpdate, PointSymbology, - ProjectDb, ProjectId, ProjectLayer, ProjectListOptions, ProjectListing, STRectangle, - UpdateProject, + use crate::{ + config::QuotaTrackingMode, + datasets::{ + AddDataset, DatasetIdAndName, + external::netcdfcf::NetCdfCfDataProviderDefinition, + listing::{ + DatasetListOptions, DatasetListing, DatasetProvider, Provenance, ProvenanceOutput, + }, + storage::{DatasetStore, MetaDataDefinition}, + upload::{FileId, FileUpload, Upload, UploadDb, UploadId}, + }, + ge_context, + layers::{ + add_from_directory::UNSORTED_COLLECTION_ID, + layer::{ + AddLayer, AddLayerCollection, CollectionItem, LayerCollection, + LayerCollectionListOptions, LayerCollectionListing, LayerListing, + ProviderLayerCollectionId, ProviderLayerId, + }, + listing::{LayerCollectionId, LayerCollectionProvider, SearchParameters, SearchType}, + storage::{ + INTERNAL_PROVIDER_ID, LayerDb, LayerProviderDb, LayerProviderListing, + LayerProviderListingOptions, + }, + }, + machine_learning::{MlModel, MlModelDb, MlModelIdAndName}, + permissions::{Permission, PermissionDb, Role, RoleDescription, RoleId}, + projects::{ + CreateProject, LayerUpdate, LoadVersion, OrderBy, Plot, PlotUpdate, PointSymbology, + ProjectDb, ProjectId, ProjectLayer, ProjectListOptions, ProjectListing, STRectangle, + UpdateProject, + }, + users::{ + OidcTokens, RoleDb, SessionTokenStore, UserClaims, UserCredentials, UserDb, UserId, + UserRegistration, + }, + util::tests::{ + MockQuotaTracking, admin_login, + mock_oidc::{MockRefreshServerConfig, mock_refresh_server}, + register_ndvi_workflow_helper, + }, + workflows::{registry::WorkflowRegistry, workflow::Workflow}, }; - use crate::users::{OidcTokens, SessionTokenStore}; - use crate::users::{RoleDb, UserClaims, UserCredentials, UserDb, UserId, UserRegistration}; - use crate::util::tests::mock_oidc::{MockRefreshServerConfig, mock_refresh_server}; - use crate::util::tests::{MockQuotaTracking, admin_login, register_ndvi_workflow_helper}; - use crate::workflows::registry::WorkflowRegistry; - use crate::workflows::workflow::Workflow; use bb8_postgres::tokio_postgres::NoTls; use futures::join; - use geoengine_datatypes::collections::VectorDataType; - use geoengine_datatypes::dataset::{DataProviderId, LayerId}; - use geoengine_datatypes::machine_learning::MlTensorShape3D; - use geoengine_datatypes::primitives::{ - BoundingBox2D, Coordinate2D, DateTime, Duration, FeatureDataType, Measurement, - RasterQueryRectangle, SpatialResolution, TimeGranularity, TimeInstance, TimeInterval, - TimeStep, VectorQueryRectangle, - }; - use geoengine_datatypes::primitives::{CacheTtlSeconds, ColumnSelection}; - use geoengine_datatypes::raster::RasterDataType; - use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceOption}; - use geoengine_datatypes::test_data; - use geoengine_datatypes::util::Identifier; - use geoengine_operators::engine::{ - MetaData, MetaDataProvider, MultipleRasterOrSingleVectorSource, PlotOperator, - RasterBandDescriptors, RasterResultDescriptor, StaticMetaData, TypedOperator, - TypedResultDescriptor, VectorColumnInfo, VectorOperator, VectorResultDescriptor, + use geoengine_datatypes::{ + collections::VectorDataType, + dataset::{DataProviderId, LayerId}, + machine_learning::MlTensorShape3D, + primitives::{ + BoundingBox2D, CacheTtlSeconds, ColumnSelection, Coordinate2D, DateTime, Duration, + FeatureDataType, Measurement, RasterQueryRectangle, TimeGranularity, TimeInstance, + TimeInterval, TimeStep, VectorQueryRectangle, + }, + raster::{GeoTransform, GridBoundingBox2D, RasterDataType}, + spatial_reference::{SpatialReference, SpatialReferenceOption}, + test_data, + util::Identifier, }; - use geoengine_operators::machine_learning::MlModelMetadata; - use geoengine_operators::mock::{MockPointSource, MockPointSourceParams}; - use geoengine_operators::plot::{Statistics, StatisticsParams}; - use geoengine_operators::source::{ - CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, - GdalDatasetParameters, GdalLoadingInfo, GdalMetaDataList, GdalMetaDataRegular, - GdalMetaDataStatic, GdalMetadataNetCdfCf, OgrSourceColumnSpec, OgrSourceDataset, - OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, OgrSourceTimeFormat, + use geoengine_operators::{ + engine::{ + MetaData, MetaDataProvider, MultipleRasterOrSingleVectorSource, PlotOperator, + RasterBandDescriptors, RasterResultDescriptor, StaticMetaData, TimeDescriptor, + TypedOperator, TypedResultDescriptor, VectorColumnInfo, VectorOperator, + VectorResultDescriptor, + }, + machine_learning::MlModelMetadata, + mock::{MockPointSource, MockPointSourceParams}, + plot::{Statistics, StatisticsParams}, + source::{ + CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalDatasetGeoTransform, + GdalDatasetParameters, GdalLoadingInfo, GdalMetaDataList, GdalMetaDataRegular, + GdalMetaDataStatic, GdalMetadataNetCdfCf, OgrSourceColumnSpec, OgrSourceDataset, + OgrSourceDatasetTimeType, OgrSourceDurationSpec, OgrSourceErrorSpec, + OgrSourceTimeFormat, + }, + util::input::MultiRasterOrVectorOperator::Raster, }; - use geoengine_operators::util::input::MultiRasterOrVectorOperator::Raster; use httptest::Server; use oauth2::{AccessToken, RefreshToken}; use openidconnect::SubjectIdentifier; @@ -718,9 +733,7 @@ mod tests { .register_workflow(Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -964,9 +977,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -985,7 +996,7 @@ mod tests { let json = serde_json::to_string(&workflow).unwrap(); assert_eq!( json, - r#"{"type":"Vector","operator":{"type":"MockPointSource","params":{"points":[{"x":1.0,"y":2.0},{"x":1.0,"y":2.0},{"x":1.0,"y":2.0}]}}}"# + r#"{"type":"Vector","operator":{"type":"MockPointSource","params":{"points":[{"x":1.0,"y":2.0},{"x":1.0,"y":2.0},{"x":1.0,"y":2.0}],"spatialBounds":{"type":"none"}}}}"# ); } @@ -1071,6 +1082,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data, + None, ) .await .unwrap(); @@ -1098,6 +1110,7 @@ mod tests { source_operator: "OgrSource".to_owned(), symbology: None, tags: vec!["upload".to_owned(), "test".to_owned()], + // create a TypedResultDescriptor object then concert it to the API model result_descriptor: TypedResultDescriptor::Vector(VectorResultDescriptor { data_type: VectorDataType::MultiPoint, spatial_reference: SpatialReference::epsg_4326().into(), @@ -1113,6 +1126,7 @@ mod tests { time: None, bbox: None, }) + .into() }, ); @@ -1135,15 +1149,11 @@ mod tests { assert_eq!( meta_data - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into() - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all() - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all() + )) .await .unwrap(), loading_info @@ -1269,7 +1279,7 @@ mod tests { phantom: Default::default(), }; - let _id = db1.add_dataset(ds, meta.into()).await.unwrap(); + let _id = db1.add_dataset(ds, meta.into(), None).await.unwrap(); let list1 = db1 .list_datasets(DatasetListOptions { @@ -1343,7 +1353,7 @@ mod tests { phantom: Default::default(), }; - let id = db1.add_dataset(ds, meta.into()).await.unwrap().id; + let id = db1.add_dataset(ds, meta.into(), None).await.unwrap().id; assert!(db1.load_provenance(&id).await.is_ok()); @@ -1395,7 +1405,7 @@ mod tests { phantom: Default::default(), }; - let id = db1.add_dataset(ds, meta.into()).await.unwrap().id; + let id = db1.add_dataset(ds, meta.into(), None).await.unwrap().id; assert!(db1.load_dataset(&id).await.is_ok()); @@ -1453,7 +1463,7 @@ mod tests { phantom: Default::default(), }; - let id = db1.add_dataset(ds, meta.into()).await.unwrap().id; + let id = db1.add_dataset(ds, meta.into(), None).await.unwrap().id; assert!(db1.load_dataset(&id).await.is_ok()); @@ -1511,7 +1521,7 @@ mod tests { phantom: Default::default(), }; - let id = db1.add_dataset(ds, meta.into()).await.unwrap().id; + let id = db1.add_dataset(ds, meta.into(), None).await.unwrap().id; let meta: geoengine_operators::util::Result< Box>, @@ -1554,9 +1564,11 @@ mod tests { let raster_descriptor = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::Unreferenced, - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: geoengine_operators::engine::SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(0., 0.), 1., -1.), + GridBoundingBox2D::new([0, 0], [1, 1]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }; @@ -1618,7 +1630,11 @@ mod tests { phantom: Default::default(), }; - let id = db.add_dataset(vector_ds, meta.into()).await.unwrap().id; + let id = db + .add_dataset(vector_ds, meta.into(), None) + .await + .unwrap() + .id; let meta: geoengine_operators::util::Result< Box>, @@ -1639,7 +1655,7 @@ mod tests { }; let id = db - .add_dataset(raster_ds.clone(), meta.into()) + .add_dataset(raster_ds.clone(), meta.into(), None) .await .unwrap() .id; @@ -1658,7 +1674,7 @@ mod tests { }; let id = db - .add_dataset(raster_ds.clone(), meta.into()) + .add_dataset(raster_ds.clone(), meta.into(), None) .await .unwrap() .id; @@ -1675,7 +1691,7 @@ mod tests { }; let id = db - .add_dataset(raster_ds.clone(), meta.into()) + .add_dataset(raster_ds.clone(), meta.into(), None) .await .unwrap() .id; @@ -1700,7 +1716,7 @@ mod tests { }; let id = db - .add_dataset(raster_ds.clone(), meta.into()) + .add_dataset(raster_ds.clone(), meta.into(), None) .await .unwrap() .id; @@ -1748,9 +1764,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -1948,9 +1962,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -2308,6 +2320,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -2828,9 +2841,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -3012,6 +3023,7 @@ mod tests { MockPointSource { params: MockPointSourceParams { points: vec![Coordinate2D::new(1., 2.); 3], + spatial_bounds: geoengine_operators::mock::SpatialBoundsDerive::Derive, }, } .boxed(), @@ -3546,9 +3558,7 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -3744,9 +3754,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), @@ -3813,9 +3824,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), @@ -3837,9 +3849,10 @@ mod tests { workflow: Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![ + Coordinate2D::new(1., 2.); + 3 + ]), } .boxed(), ), @@ -3989,6 +4002,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data, + None, ) .await .unwrap() @@ -4080,6 +4094,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data, + None, ) .await .unwrap() @@ -4769,6 +4784,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data.clone(), + None, ) .await .unwrap(); @@ -4795,6 +4811,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data, + None, ) .await .unwrap(); diff --git a/services/src/datasets/create_from_workflow.rs b/services/src/datasets/create_from_workflow.rs index 48b2f35b1..b4b6f2bca 100644 --- a/services/src/datasets/create_from_workflow.rs +++ b/services/src/datasets/create_from_workflow.rs @@ -1,21 +1,25 @@ -use super::{DatasetIdAndName, DatasetName}; -use crate::api::model::datatypes::RasterQueryRectangle; +use crate::api::model::datatypes::RasterToDatasetQueryRectangle; +use crate::api::model::services::{AddDataset, DataPath}; use crate::contexts::SessionContext; -use crate::datasets::AddDataset; use crate::datasets::listing::DatasetProvider; use crate::datasets::storage::{DatasetDefinition, DatasetStore, MetaDataDefinition}; use crate::datasets::upload::{UploadId, UploadRootPath}; -use crate::error; -use crate::tasks::{Task, TaskContext, TaskId, TaskManager, TaskStatusInfo}; -use crate::workflows::workflow::{Workflow, WorkflowId}; -use async_trait::async_trait; +use crate::datasets::{DatasetIdAndName, DatasetName}; +use crate::tasks::TaskContext; +use crate::workflows::registry::WorkflowRegistry; +use crate::workflows::workflow::WorkflowId; +use crate::{ + error, + tasks::{Task, TaskId, TaskManager, TaskStatusInfo}, +}; use geoengine_datatypes::error::ErrorSource; -use geoengine_datatypes::primitives::TimeInterval; +use geoengine_datatypes::primitives::{BandSelection, TimeInterval}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::Identifier; use geoengine_operators::call_on_generic_raster_processor_gdal_types; use geoengine_operators::engine::{ - ExecutionContext, InitializedRasterOperator, RasterResultDescriptor, WorkflowOperatorPath, + ExecutionContext, InitializedRasterOperator, RasterResultDescriptor, TimeDescriptor, + WorkflowOperatorPath, }; use geoengine_operators::source::{ GdalLoadingInfoTemporalSlice, GdalMetaDataList, GdalMetaDataStatic, @@ -29,23 +33,44 @@ use snafu::{ResultExt, ensure}; use std::path::PathBuf; use std::sync::Arc; use tokio::fs; +use tonic::async_trait; use utoipa::ToSchema; use uuid::Uuid; /// parameter for the dataset from workflow handler (body) #[derive(Clone, Debug, Deserialize, Serialize, ToSchema)] -#[schema(example = json!({"name": "foo", "displayName": "a new dataset", "description": null, "query": {"spatialBounds": {"upperLeftCoordinate": {"x": -10.0, "y": 80.0}, "lowerRightCoordinate": {"x": 50.0, "y": 20.0}}, "timeInterval": {"start": 1_388_534_400_000_i64, "end": 1_388_534_401_000_i64}, "spatialResolution": {"x": 0.1, "y": 0.1}}}))] +#[schema(example = json!({"name": "foo", "displayName": "a new dataset", "description": null, "query": {"spatialBounds": {"upperLeftCoordinate": {"x": -10.0, "y": 80.0}, "lowerRightCoordinate": {"x": 50.0, "y": 20.0}}, "timeInterval": {"start": 1_388_534_400_000_i64, "end": 1_388_534_401_000_i64}}}))] #[serde(rename_all = "camelCase")] pub struct RasterDatasetFromWorkflow { pub name: Option, pub display_name: String, pub description: Option, - pub query: RasterQueryRectangle, + pub query: RasterToDatasetQueryRectangle, #[schema(default = default_as_cog)] #[serde(default = "default_as_cog")] pub as_cog: bool, } +pub struct RasterDatasetFromWorkflowParams { + pub name: Option, + pub display_name: String, + pub description: Option, + pub query: Option, + pub as_cog: bool, +} + +impl RasterDatasetFromWorkflowParams { + pub fn from_request_and_result_descriptor(request: RasterDatasetFromWorkflow) -> Self { + Self { + name: request.name, + display_name: request.display_name, + description: request.description, + query: Some(request.query), + as_cog: request.as_cog, + } + } +} + /// By default, we set [`RasterDatasetFromWorkflow::as_cog`] to true to produce cloud-optmized `GeoTiff`s. #[inline] const fn default_as_cog() -> bool { @@ -76,10 +101,10 @@ impl ToGeoTiffProgressConsumer for ToGeoTiffTaskContext { pub struct RasterDatasetFromWorkflowTask { pub source_name: String, + pub workflow_id: WorkflowId, - pub workflow: Workflow, pub ctx: Arc, - pub info: RasterDatasetFromWorkflow, + pub info: RasterDatasetFromWorkflowParams, pub upload: UploadId, pub file_path: PathBuf, pub compression_num_threads: GdalCompressionNumThreads, @@ -88,37 +113,67 @@ pub struct RasterDatasetFromWorkflowTask { impl RasterDatasetFromWorkflowTask { async fn process( &self, - task_ctx: ToGeoTiffTaskContext, + to_geo_tiff_task_context: ToGeoTiffTaskContext, ) -> error::Result { - let operator = self.workflow.operator.clone(); - - let operator = operator.get_raster()?; - - let execution_context = self.ctx.execution_context()?; - - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - let initialized = operator - .initialize(workflow_operator_path_root, &execution_context) + let workflow = self.ctx.db().load_workflow(&self.workflow_id).await?; + let exe_ctx = self.ctx.execution_context()?; + + let initialized_operator = workflow + .clone() + .operator + .get_raster() + .expect("must be raster here") + .initialize(WorkflowOperatorPath::initialize_root(), &exe_ctx) .await?; - let result_descriptor = initialized.result_descriptor(); + let tiling_spec = exe_ctx.tiling_specification(); + let result_descriptor = initialized_operator.result_descriptor(); + + let query_rect = if let Some(sq) = self.info.query { + let grid_bounds = result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(tiling_spec) + .tiling_geo_transform() + .spatial_to_grid_bounds(&sq.spatial_bounds.into()); + + geoengine_datatypes::primitives::RasterQueryRectangle::new( + grid_bounds, + sq.time_interval.into(), + BandSelection::first_n(result_descriptor.bands.len() as u32), + ) + } else { + let grid_bounds = result_descriptor + .tiling_grid_definition(tiling_spec) + .tiling_grid_bounds(); + + let qt = result_descriptor.time.bounds.ok_or( + crate::error::Error::LayerResultDescriptorMissingFields { + field: "time".to_string(), + cause: "is None".to_string(), + }, + )?; + + geoengine_datatypes::primitives::RasterQueryRectangle::new( + grid_bounds, + qt, // TODO: is this a good default? + BandSelection::first_n(result_descriptor.bands.len() as u32), + ) + }; - let processor = initialized.query_processor()?; - - let query_rect = self.info.query; let query_ctx = self.ctx.query_context(self.workflow_id.0, Uuid::new_v4())?; let request_spatial_ref = Option::::from(result_descriptor.spatial_reference) .ok_or(crate::error::Error::MissingSpatialReference)?; let tile_limit = None; // TODO: set a reasonable limit or make configurable? + let processor = initialized_operator.query_processor()?; + // build the geotiff let res = call_on_generic_raster_processor_gdal_types!(processor, p => raster_stream_to_geotiff( &self.file_path, p, - query_rect.into(), + query_rect.clone(), query_ctx, GdalGeoTiffDatasetMetadata { no_data_value: Default::default(), // TODO: decide how to handle the no data here @@ -131,18 +186,17 @@ impl RasterDatasetFromWorkflowTask { }, tile_limit, Box::pin(futures::future::pending()), // datasets shall continue to be built in the background and not cancelled - execution_context.tiling_specification(), - task_ctx + to_geo_tiff_task_context, ).await)? .map_err(crate::error::Error::from)?; - // create the dataset let dataset = create_dataset( - self.info.clone(), + &self.info, res, result_descriptor, - query_rect, + &query_rect, self.ctx.as_ref(), + DataPath::Upload(self.upload), ) .await?; @@ -199,9 +253,8 @@ impl Task for RasterDatasetFromWorkflowTask( source_name: String, workflow_id: WorkflowId, - workflow: Workflow, ctx: Arc, - info: RasterDatasetFromWorkflow, + info: RasterDatasetFromWorkflowParams, compression_num_threads: GdalCompressionNumThreads, ) -> error::Result { if let Some(dataset_name) = &info.name { @@ -229,7 +282,6 @@ pub async fn schedule_raster_dataset_from_workflow_task( let task = RasterDatasetFromWorkflowTask { source_name, workflow_id, - workflow, ctx: ctx.clone(), info, upload, @@ -244,11 +296,12 @@ pub async fn schedule_raster_dataset_from_workflow_task( } async fn create_dataset( - info: RasterDatasetFromWorkflow, + info: &RasterDatasetFromWorkflowParams, mut slice_info: Vec, origin_result_descriptor: &RasterResultDescriptor, - query_rectangle: RasterQueryRectangle, + query_rectangle: &geoengine_datatypes::primitives::RasterQueryRectangle, ctx: &C, + data_path: DataPath, ) -> error::Result { ensure!(!slice_info.is_empty(), error::EmptyDatasetCannotBeImported); @@ -264,12 +317,32 @@ async fn create_dataset( .end(); let result_time_interval = TimeInterval::new(first_start, last_end)?; + let exe_ctx = ctx.execution_context()?; + + let source_tiling_spatial_grid = + origin_result_descriptor.tiling_grid_definition(exe_ctx.tiling_specification()); + let query_tiling_spatial_grid = + source_tiling_spatial_grid.with_other_bounds(query_rectangle.spatial_bounds()); + let result_descriptor_bounds = origin_result_descriptor + .spatial_grid_descriptor() + .intersection_with_tiling_grid(&query_tiling_spatial_grid) + .ok_or(error::Error::EmptyDatasetCannotBeImported)?; // TODO: maybe allow empty datasets? + + // TODO: this is not how it is intended to work with the spatial grid descriptor. The source should propably not need that defined in its params since it can be derived from the dataset! + let (_state, dataset_source_descriptor_spatial_grid) = result_descriptor_bounds.as_parts(); + + let dataset_spatial_grid = geoengine_operators::engine::SpatialGridDescriptor::new_source( + dataset_source_descriptor_spatial_grid, + ); + let result_descriptor = RasterResultDescriptor { data_type: origin_result_descriptor.data_type, spatial_reference: origin_result_descriptor.spatial_reference, - time: Some(result_time_interval), - bbox: Some(query_rectangle.spatial_bounds.into()), - resolution: Some(query_rectangle.spatial_resolution.into()), + time: TimeDescriptor::new( + Some(result_time_interval), + origin_result_descriptor.time.dimension, + ), + spatial_grid: dataset_spatial_grid, bands: origin_result_descriptor.bands.clone(), }; //TODO: Recognize MetaDataDefinition::GdalMetaDataRegular @@ -295,20 +368,25 @@ async fn create_dataset( let dataset_definition = DatasetDefinition { properties: AddDataset { - name: info.name, - display_name: info.display_name, - description: info.description.unwrap_or_default(), + name: info.name.clone(), + display_name: info.display_name.clone(), + description: info.description.clone().unwrap_or_default(), source_operator: "GdalSource".to_owned(), symbology: None, // TODO add symbology? provenance: None, // TODO add provenance that references the workflow tags: Some(vec!["workflow".to_owned()]), - }, + } + .into(), meta_data, }; let db = ctx.db(); let result = db - .add_dataset(dataset_definition.properties, dataset_definition.meta_data) + .add_dataset( + dataset_definition.properties, + dataset_definition.meta_data, + Some(data_path), + ) .await?; Ok(result) diff --git a/services/src/datasets/dataset_listing_provider.rs b/services/src/datasets/dataset_listing_provider.rs index 267c25f26..eebb176b9 100644 --- a/services/src/datasets/dataset_listing_provider.rs +++ b/services/src/datasets/dataset_listing_provider.rs @@ -1,5 +1,21 @@ use std::{borrow::Cow, collections::HashMap, str::FromStr}; +use async_trait::async_trait; +use geoengine_datatypes::{ + dataset::{DataId, LayerId}, + primitives::{RasterQueryRectangle, VectorQueryRectangle}, +}; +use geoengine_operators::{ + engine::{MetaData, MetaDataProvider, RasterResultDescriptor, VectorResultDescriptor}, + mock::MockDatasetDataSourceLoadingInfo, + source::{ + GdalLoadingInfo, MultiBandGdalLoadingInfo, MultiBandGdalLoadingInfoQueryRectangle, + OgrSourceDataset, + }, +}; +use postgres_types::{FromSql, ToSql}; +use serde::{Deserialize, Serialize}; + use crate::{ contexts::GeoEngineDb, datasets::listing::DatasetProvider, @@ -18,18 +34,6 @@ use crate::{ util::operators::source_operator_from_dataset, workflows::workflow::Workflow, }; -use async_trait::async_trait; -use geoengine_datatypes::{ - dataset::{DataId, LayerId}, - primitives::{RasterQueryRectangle, VectorQueryRectangle}, -}; -use geoengine_operators::{ - engine::{MetaData, MetaDataProvider, RasterResultDescriptor, VectorResultDescriptor}, - mock::MockDatasetDataSourceLoadingInfo, - source::{GdalLoadingInfo, OgrSourceDataset}, -}; -use postgres_types::{FromSql, ToSql}; -use serde::{Deserialize, Serialize}; use geoengine_datatypes::dataset::{DataProviderId, DatasetId}; @@ -420,6 +424,34 @@ where } } +#[async_trait] +impl + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > for DatasetLayerListingProvider +where + D: DatasetProvider + Send + Sync + 'static, +{ + async fn meta_data( + &self, + _id: &geoengine_datatypes::dataset::DataId, + ) -> Result< + Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + >, + geoengine_operators::error::Error, + > { + // never called but handled by the dataset provider + Err(geoengine_operators::error::Error::NotImplemented) + } +} + fn tags_from_collection_id(collection_id: &LayerCollectionId) -> Result>> { let collection_str = &collection_id.0; @@ -462,11 +494,11 @@ mod tests { use geoengine_datatypes::{ collections::VectorDataType, primitives::{CacheTtlSeconds, TimeGranularity, TimeStep}, - raster::RasterDataType, + raster::{GeoTransform, GridBoundingBox, RasterDataType}, spatial_reference::SpatialReferenceOption, }; use geoengine_operators::{ - engine::{RasterBandDescriptors, StaticMetaData}, + engine::{RasterBandDescriptors, SpatialGridDescriptor, StaticMetaData, TimeDescriptor}, source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, GdalMetaDataRegular, OgrSourceErrorSpec, @@ -709,9 +741,11 @@ mod tests { let raster_descriptor = RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::Unreferenced, - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((0., 0.).into(), 1., -1.), + GridBoundingBox::new([0, 0], [0, 0]).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }; @@ -743,8 +777,8 @@ mod tests { x_pixel_size: 0.0, y_pixel_size: 0.0, }, - width: 0, - height: 0, + width: 1, + height: 1, file_not_found_handling: FileNotFoundHandling::NoData, no_data_value: None, properties_mapping: None, @@ -785,10 +819,13 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }; - let _ = db.add_dataset(vector_ds, vector_meta.into()).await.unwrap(); + let _ = db + .add_dataset(vector_ds, vector_meta.into(), None) + .await + .unwrap(); let _ = db - .add_dataset(raster_ds.clone(), raster_meta.into()) + .add_dataset(raster_ds.clone(), raster_meta.into(), None) .await .unwrap(); } diff --git a/services/src/datasets/external/aruna/mod.rs b/services/src/datasets/external/aruna/mod.rs index 0aa02773d..60e903ef5 100644 --- a/services/src/datasets/external/aruna/mod.rs +++ b/services/src/datasets/external/aruna/mod.rs @@ -35,13 +35,14 @@ use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId}; use geoengine_datatypes::primitives::CacheTtlSeconds; use geoengine_datatypes::primitives::{ - FeatureDataType, Measurement, RasterQueryRectangle, SpatialResolution, VectorQueryRectangle, + FeatureDataType, Measurement, RasterQueryRectangle, VectorQueryRectangle, }; +use geoengine_datatypes::raster::{BoundedGrid, GeoTransform, GridShape2D}; use geoengine_datatypes::spatial_reference::SpatialReferenceOption; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, ResultDescriptor, TypedOperator, VectorColumnInfo, VectorOperator, - VectorResultDescriptor, + RasterResultDescriptor, ResultDescriptor, SpatialGridDescriptor, TimeDescriptor, TypedOperator, + VectorColumnInfo, VectorOperator, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -63,6 +64,7 @@ pub use self::error::ArunaProviderError; pub mod error; pub mod metadata; + #[cfg(test)] #[macro_use] mod mock_grpc_server; @@ -526,19 +528,14 @@ impl ArunaDataProvider { crs: SpatialReferenceOption, info: &RasterInfo, ) -> geoengine_operators::util::Result { + let shape = GridShape2D::new_2d(info.width, info.height).bounding_box(); + let geo_transform = GeoTransform::try_from(info.geo_transform)?; + Ok(RasterResultDescriptor { data_type: info.data_type, spatial_reference: crs, - - time: Some(info.time_interval), - bbox: Some( - info.geo_transform - .spatial_partition(info.width, info.height), - ), - resolution: Some(SpatialResolution::try_from(( - info.geo_transform.x_pixel_size, - info.geo_transform.y_pixel_size, - ))?), + time: TimeDescriptor::new_irregular(Some(info.time_interval)), + spatial_grid: SpatialGridDescriptor::source_from_parts(geo_transform, shape), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( "band".into(), info.measurement @@ -903,12 +900,12 @@ impl LayerCollectionProvider for ArunaDataProvider { ), DataType::SingleRasterFile(_) => TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_provider( self.id.to_string(), id.to_string(), ), - }, + ), } .boxed(), ), @@ -1102,12 +1099,11 @@ mod tests { use geoengine_datatypes::collections::{FeatureCollectionInfos, MultiPointCollection}; use geoengine_datatypes::dataset::{DataId, DataProviderId, ExternalDataId, LayerId}; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheTtlSeconds, ColumnSelection, SpatialResolution, TimeInterval, - VectorQueryRectangle, + BoundingBox2D, CacheTtlSeconds, ColumnSelection, TimeInterval, VectorQueryRectangle, }; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ - MetaData, MetaDataProvider, MockExecutionContext, MockQueryContext, QueryProcessor, + MetaData, MetaDataProvider, MockExecutionContext, QueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorResultDescriptor, WorkflowOperatorPath, }; use geoengine_operators::source::{OgrSource, OgrSourceDataset, OgrSourceParameters}; @@ -2040,7 +2036,8 @@ mod tests { "operator": { "type": "GdalSource", "params": { - "data": "_:86a7f7ce-1bab-4ce9-a32b-172c0f958ee0:DATASET_ID" + "data": "_:86a7f7ce-1bab-4ce9-a32b-172c0f958ee0:DATASET_ID", + "overviewLevel": null } } }), @@ -2450,14 +2447,13 @@ mod tests { panic!("Expected MultiPoint QueryProcessor"); }; - let ctx = MockQueryContext::test_default(); + let ctx = context.mock_query_context_test_default(); - let qr = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; + let qr = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); let result: Vec = proc .query(qr, &ctx) diff --git a/services/src/datasets/external/copernicus_dataspace/ids.rs b/services/src/datasets/external/copernicus_dataspace/ids.rs index 5ac53c5e2..d9d41a558 100644 --- a/services/src/datasets/external/copernicus_dataspace/ids.rs +++ b/services/src/datasets/external/copernicus_dataspace/ids.rs @@ -1,8 +1,6 @@ use geoengine_datatypes::{ dataset::{DataId, DataProviderId, ExternalDataId, LayerId, NamedData}, - primitives::SpatialPartition2D, raster::RasterDataType, - spatial_reference::{SpatialReference, SpatialReferenceAuthority}, }; use std::str::FromStr; use strum::IntoEnumIterator; @@ -11,6 +9,7 @@ use strum_macros::{EnumIter, EnumString}; use crate::{ error::{Error, Result}, layers::listing::LayerCollectionId, + util::sentinel_2_utm_zones::UtmZone, }; #[derive(Debug, Clone)] @@ -91,6 +90,13 @@ impl Sentinel2ProductBand { Sentinel2ProductBand::L2A(band) => format!("{band}"), } } + + pub fn resolution_meters(self) -> usize { + match self { + Sentinel2ProductBand::L1C(l1c) => l1c.resolution_meters(), + Sentinel2ProductBand::L2A(l2a) => l2a.resolution_meters(), + } + } } #[derive(Debug, Clone, Copy, EnumString, strum::Display, EnumIter)] @@ -138,18 +144,6 @@ pub enum L2ABand { WVP_60M, } -#[derive(Debug, Clone, Copy)] -pub struct UtmZone { - pub zone: u8, - pub direction: UtmZoneDirection, -} - -#[derive(Debug, Clone, Copy)] -pub enum UtmZoneDirection { - North, - South, -} - impl Sentinel2ProductBand { // TODO: move to sentinel2 to separate concerns pub fn product_type(&self) -> &str { @@ -327,24 +321,6 @@ impl Sentinel2Band for Sentinel2ProductBand { } } -impl UtmZone { - pub fn epsg_code(self) -> u32 { - match self.direction { - UtmZoneDirection::North => 32600 + u32::from(self.zone), - UtmZoneDirection::South => 32700 + u32::from(self.zone), - } - } - - pub fn spatial_reference(self) -> SpatialReference { - SpatialReference::new(SpatialReferenceAuthority::Epsg, self.epsg_code()) - } - - pub fn extent(self) -> Option { - // TODO: as Sentinel uses enlarged grids, we could return a larger extent - self.spatial_reference().area_of_use().ok() - } -} - impl FromStr for CopernicusDataspaceLayerCollectionId { type Err = crate::error::Error; @@ -473,7 +449,7 @@ impl FromStr for Sentinel2LayerCollectionId { [product, zone] => Self::ProductZone { product: Sentinel2Product::from_str(product) .map_err(|_| Error::InvalidLayerCollectionId)?, - zone: UtmZone::from_str(zone)?, + zone: UtmZone::from_str(zone).map_err(|_| Error::InvalidLayerCollectionId)?, }, _ => return Err(Error::InvalidLayerCollectionId), }) @@ -522,7 +498,7 @@ impl FromStr for Sentinel2LayerId { [product, zone, band] => Self { product_band: Sentinel2ProductBand::with_product_and_band_as_str(product, band) .map_err(|_| Error::InvalidLayerId)?, - zone: UtmZone::from_str(zone)?, + zone: UtmZone::from_str(zone).map_err(|_| Error::InvalidLayerId)?, }, _ => return Err(Error::InvalidLayerId), }) @@ -555,62 +531,3 @@ impl From for DataId { }) } } - -impl FromStr for UtmZone { - type Err = crate::error::Error; - - fn from_str(s: &str) -> Result { - if s.len() < 5 || &s[..3] != "UTM" { - return Err(Error::InvalidLayerCollectionId); - } - - let (zone_str, dir_char) = s[3..].split_at(s.len() - 4); - let zone = zone_str - .parse::() - .map_err(|_| Error::InvalidLayerCollectionId)?; - - // TODO: check if zone is in valid range - - let north = match dir_char { - "N" => UtmZoneDirection::North, - "S" => UtmZoneDirection::South, - _ => return Err(Error::InvalidLayerCollectionId), - }; - - Ok(Self { - zone, - direction: north, - }) - } -} - -impl std::fmt::Display for UtmZone { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!( - f, - "UTM{}{}", - self.zone, - match self.direction { - UtmZoneDirection::North => "N", - UtmZoneDirection::South => "S", - } - ) - } -} - -impl UtmZone { - pub fn zones() -> impl Iterator { - (1..=60).flat_map(|zone| { - vec![ - UtmZone { - zone, - direction: UtmZoneDirection::North, - }, - UtmZone { - zone, - direction: UtmZoneDirection::South, - }, - ] - }) - } -} diff --git a/services/src/datasets/external/copernicus_dataspace/provider.rs b/services/src/datasets/external/copernicus_dataspace/provider.rs index f4adc3570..d59f03c4d 100644 --- a/services/src/datasets/external/copernicus_dataspace/provider.rs +++ b/services/src/datasets/external/copernicus_dataspace/provider.rs @@ -12,6 +12,7 @@ use crate::{ listing::LayerCollectionId, }, projects::RasterSymbology, + util::sentinel_2_utm_zones::UtmZone, workflows::workflow::Workflow, }; use async_trait::async_trait; @@ -36,7 +37,6 @@ use super::{ ids::{ CopernicusDataId, CopernicusDataspaceLayerCollectionId, CopernicusDataspaceLayerId, Sentinel2LayerCollectionId, Sentinel2LayerId, Sentinel2Product, Sentinel2ProductBand, - UtmZone, }, sentinel2::Sentinel2Metadata, }; @@ -309,6 +309,7 @@ impl CopernicusDataspaceDataProvider { self.id, ) .into(), + overview_level: None, }, } .boxed(), diff --git a/services/src/datasets/external/copernicus_dataspace/sentinel2.rs b/services/src/datasets/external/copernicus_dataspace/sentinel2.rs index c0120018c..635388941 100644 --- a/services/src/datasets/external/copernicus_dataspace/sentinel2.rs +++ b/services/src/datasets/external/copernicus_dataspace/sentinel2.rs @@ -1,18 +1,25 @@ use std::path::PathBuf; -use crate::datasets::external::copernicus_dataspace::stac::{ - load_stac_items, resolve_datetime_duplicates, +use crate::{ + datasets::external::copernicus_dataspace::stac::{ + StacQueryRectangle, load_stac_items, resolve_datetime_duplicates, + }, + util::sentinel_2_utm_zones::UtmZone, }; use gdal::{DatasetOptions, GdalOpenFlags}; use geoengine_datatypes::{ primitives::{ - CacheTtlSeconds, DateTime, RasterQueryRectangle, SpatialResolution, TimeInstance, + AxisAlignedRectangle, CacheTtlSeconds, DateTime, RasterQueryRectangle, TimeInstance, TimeInterval, }, + raster::{GeoTransform, SpatialGridDefinition}, spatial_reference::{SpatialReference, SpatialReferenceAuthority}, }; use geoengine_operators::{ - engine::{MetaData, RasterBandDescriptor, RasterResultDescriptor}, + engine::{ + MetaData, RasterBandDescriptor, RasterResultDescriptor, SpatialGridDescriptor, + TimeDescriptor, + }, source::{ GdalDatasetParameters, GdalLoadingInfo, GdalLoadingInfoTemporalSlice, GdalLoadingInfoTemporalSliceIterator, @@ -28,7 +35,7 @@ use snafu::{ResultExt, Snafu}; use url::Url; use super::{ - ids::{Sentinel2Band, Sentinel2ProductBand, UtmZone}, + ids::{Sentinel2Band, Sentinel2ProductBand}, stac::{CopernicusStacError, StacItemExt}, }; @@ -84,7 +91,7 @@ pub struct Sentinel2Metadata { impl Sentinel2Metadata { async fn crate_loading_info( &self, - query: RasterQueryRectangle, + query: StacQueryRectangle, // TODO: here the name is misleading :( ) -> Result { let mut stac_items = load_stac_items( Url::parse(&self.stac_url).context(CannotParseStacUrl)?, @@ -241,14 +248,44 @@ impl MetaData for &self, query: RasterQueryRectangle, ) -> geoengine_operators::util::Result { - self.crate_loading_info(query).await.map_err(|e| { - geoengine_operators::error::Error::LoadingInfo { + let utm_extent = self.zone.native_extent(); + let px_size = self.product_band.resolution_meters() as f64; + let geo_transform = GeoTransform::new(utm_extent.upper_left(), px_size, -px_size); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&utm_extent); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + // TODO: maybe get tiling_specification from self/ctx? + let tiling_specification = crate::config::get_config_element::< + crate::config::TilingSpecification, + >() + .map_err(|e| geoengine_operators::error::Error::LoadingInfo { + source: Box::new(e), + })?; + + let spatial_bounds = SpatialGridDescriptor::new_source(spatial_grid) + .tiling_grid_definition(tiling_specification.into()) + .tiling_geo_transform() + .grid_to_spatial_bounds(&query.spatial_bounds()); + + let spatial_bounds_query = + StacQueryRectangle::new(spatial_bounds.as_bbox(), query.time_interval(), ()); + + self.crate_loading_info(spatial_bounds_query) + .await + .map_err(|e| geoengine_operators::error::Error::LoadingInfo { source: Box::new(e), - } - }) + }) } async fn result_descriptor(&self) -> geoengine_operators::util::Result { + let utm_extent = self.zone.native_extent(); + let px_size = self.product_band.resolution_meters() as f64; + let geo_transform = GeoTransform::new(utm_extent.upper_left(), px_size, -px_size); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&utm_extent); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + + let spatial_grid_desc = SpatialGridDescriptor::new_source(spatial_grid); + Ok(RasterResultDescriptor { data_type: self.product_band.data_type(), spatial_reference: SpatialReference::new( @@ -260,15 +297,11 @@ impl MetaData for // first image: // $ s3cmd -c .s3cfg ls s3://eodata/Sentinel-2/MSI/L1C/2015/07/04/ // DIR s3://eodata/Sentinel-2/MSI/L1C/2015/07/04/S2A_MSIL1C_20150704T101006_N0204_R022_T32SKB_20150704T101337.SAFE/ - time: Some(TimeInterval::new_unchecked( + time: TimeDescriptor::new_irregular(Some(TimeInterval::new_unchecked( DateTime::new_utc(2015, 7, 4, 10, 10, 6), DateTime::now(), - )), - bbox: self.zone.extent(), - resolution: Some(SpatialResolution::new( - self.product_band.resolution_meters() as f64, - self.product_band.resolution_meters() as f64, - )?), + ))), + spatial_grid: spatial_grid_desc, bands: vec![RasterBandDescriptor::new_unitless( self.product_band.band_name(), )] @@ -285,10 +318,13 @@ impl MetaData for #[cfg(test)] mod tests { - use std::env; - + use super::*; + use crate::{ + datasets::external::copernicus_dataspace::ids::L2ABand, + util::sentinel_2_utm_zones::UtmZoneDirection, + }; use geoengine_datatypes::{ - primitives::{BandSelection, Coordinate2D, DateTime, SpatialPartition2D}, + primitives::{Coordinate2D, DateTime, SpatialPartition2D}, test_data, }; use geoengine_operators::source::{FileNotFoundHandling, GdalDatasetGeoTransform}; @@ -297,10 +333,7 @@ mod tests { matchers::{contains, request, url_decoded}, responders::status_code, }; - - use crate::datasets::external::copernicus_dataspace::ids::{L2ABand, UtmZoneDirection}; - - use super::*; + use std::env; fn add_partial_responses( server: &Server, @@ -479,18 +512,18 @@ mod tests { // time=2020-07-01T12%3A00%3A00.000Z/2020-07-03T12%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=256&HEIGHT=256&CRS=EPSG%3A32632&BBOX=482500%2C5627500%2C483500%2C5628500 let loading_info = metadata - .crate_loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( + .crate_loading_info(StacQueryRectangle::new( + SpatialPartition2D::new_unchecked( (482_500., 5_627_500.).into(), (483_500., 5_628_500.).into(), - ), - time_interval: TimeInterval::new_unchecked( + ) + .as_bbox(), + TimeInterval::new_unchecked( DateTime::parse_from_rfc3339("2020-07-01T12:00:00.000Z").unwrap(), DateTime::parse_from_rfc3339("2020-07-03T12:00:00.000Z").unwrap(), ), - spatial_resolution: SpatialResolution::new(10., 10.).unwrap(), - attributes: BandSelection::first(), - }) + (), + )) .await .unwrap(); diff --git a/services/src/datasets/external/copernicus_dataspace/stac.rs b/services/src/datasets/external/copernicus_dataspace/stac.rs index fa65eb39d..b8c2d8ea6 100644 --- a/services/src/datasets/external/copernicus_dataspace/stac.rs +++ b/services/src/datasets/external/copernicus_dataspace/stac.rs @@ -1,9 +1,6 @@ use geoengine_datatypes::{ operations::reproject::{CoordinateProjection, CoordinateProjector, ReprojectClipped}, - primitives::{ - AxisAlignedRectangle, BoundingBox2D, DateTime, Duration, RasterQueryRectangle, - SpatialPartitioned, - }, + primitives::{AxisAlignedRectangle, BoundingBox2D, DateTime, Duration, QueryRectangle}, spatial_reference::SpatialReference, }; use snafu::{ResultExt, Snafu}; @@ -15,6 +12,8 @@ use crate::util::join_base_url_and_path; const MAX_NUM_PAGES: usize = 100; const MAX_PAGE_SIZE: usize = 1000; +pub type StacQueryRectangle = QueryRectangle; + #[derive(Debug, Snafu)] #[snafu(visibility(pub(crate)))] #[snafu(context(suffix(false)))] // disables default `Snafu` suffix @@ -41,22 +40,18 @@ pub enum CopernicusStacError { } fn bbox_time_query( - query: &RasterQueryRectangle, + query: &StacQueryRectangle, query_projection: SpatialReference, ) -> Result<[(&'static str, String); 2], CopernicusStacError> { // TODO: add query buffer like in Element84 provider? - let time_start = query.time_interval.start(); - let time_end = query.time_interval.end(); + let time_start = query.time_interval().start(); + let time_end = query.time_interval().end(); let projector = CoordinateProjector::from_known_srs(query_projection, SpatialReference::epsg_4326()) .context(CannotReprojectBbox)?; - let spatial_partition = query.spatial_partition(); // TODO: use SpatialPartition2D directly - let bbox = BoundingBox2D::new_upper_left_lower_right_unchecked( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ); + let bbox = query.spatial_bounds(); // TODO: use SpatialPartition2D directly // TODO: query the whole zone instead? (for Sentinel-2) let bbox = bbox @@ -97,7 +92,7 @@ fn bbox_time_query( pub async fn load_stac_items( stac_url: Url, collection: &str, - query: RasterQueryRectangle, + query: StacQueryRectangle, query_projection: SpatialReference, product_type: &str, ) -> Result, CopernicusStacError> { diff --git a/services/src/datasets/external/edr.rs b/services/src/datasets/external/edr.rs index 1f7e28f02..25ff48ce8 100644 --- a/services/src/datasets/external/edr.rs +++ b/services/src/datasets/external/edr.rs @@ -16,15 +16,18 @@ use gdal::Dataset; use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, Coordinate2D, - FeatureDataType, Measurement, RasterQueryRectangle, SpatialPartition2D, TimeInstance, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, CacheTtlSeconds, ContinuousMeasurement, Coordinate2D, FeatureDataType, + Measurement, RasterQueryRectangle, TimeInstance, TimeInterval, VectorQueryRectangle, +}; +use geoengine_datatypes::raster::{ + BoundedGrid, GeoTransform, GridIdx2D, GridShape2D, GridSize, RasterDataType, + SpatialGridDefinition, }; -use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterBandDescriptors, RasterOperator, RasterResultDescriptor, - StaticMetaData, TypedOperator, VectorColumnInfo, VectorOperator, VectorResultDescriptor, + SpatialGridDescriptor, StaticMetaData, TypedOperator, VectorColumnInfo, VectorOperator, + VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -681,16 +684,32 @@ impl EdrCollectionMetaData { fn get_raster_result_descriptor( &self, + geo_transform: GeoTransform, + grid_shape: GridShape2D, ) -> Result { - let bbox = self.get_bounding_box()?; - let bbox = SpatialPartition2D::new_unchecked(bbox.upper_left(), bbox.lower_right()); + // IF the dataset has a fliped y-axis and we want to use it up-up we need to flip the grid! + + let spatial_grid = if geo_transform.y_axis_is_neg() { + SpatialGridDefinition::new(geo_transform, grid_shape.bounding_box()) + } else { + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_shape.bounding_box()); + spatial_grid + .flip_axis_y() + .shift_bounds_relative_by_pixel_offset(GridIdx2D::new_y_x( + spatial_grid.grid_bounds.axis_size_y() as isize, + 0, + )) + }; + + let spatial_grid_def = SpatialGridDescriptor::new_source(spatial_grid); Ok(RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: Some(self.get_time_interval()?), - bbox: Some(bbox), - resolution: None, + time: geoengine_operators::engine::TimeDescriptor::new_irregular(Some( + self.get_time_interval()?, + )), // TODO: can we find out if it is regular? + spatial_grid: spatial_grid_def, bands: RasterBandDescriptors::new_single_band(), }) } @@ -985,12 +1004,12 @@ impl LayerCollectionProvider for EdrDataProvider { let operator = if collection.is_raster_file()? { TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_provider( self.id.to_string(), id.to_string(), ), - }, + ), } .boxed(), ) @@ -1188,8 +1207,23 @@ impl MetaDataProvider = + GridShape2D::new_2d(first_params.height, first_params.width); + Ok(Box::new(GdalMetaDataList { - result_descriptor: collection.get_raster_result_descriptor()?, + result_descriptor: collection + .get_raster_result_descriptor(geo_transform, grid_shape)?, params, })) } @@ -1228,7 +1262,8 @@ mod tests { use geoengine_datatypes::{ dataset::ExternalDataId, hashmap, - primitives::{BandSelection, ColumnSelection, SpatialResolution}, + primitives::{BandSelection, ColumnSelection}, + raster::GridBoundingBox2D, util::gdal::hide_gdal_errors, }; use geoengine_operators::{engine::ResultDescriptor, source::GdalDatasetGeoTransform}; @@ -1630,15 +1665,11 @@ mod tests { .await .unwrap(); let loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .unwrap(); assert_eq!( @@ -1672,7 +1703,6 @@ mod tests { cache_ttl: Default::default(), } ); - let result_descriptor = meta.result_descriptor().await.unwrap(); assert_eq!( result_descriptor, @@ -1759,23 +1789,15 @@ mod tests { if meta_result.is_err() { server.verify_and_clear(); } - meta_result.unwrap() }; let loading_info_parts = meta - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (0., 90.).into(), - (360., -90.).into(), - ), - time_interval: TimeInterval::new_unchecked( - 1_692_144_000_000, - 1_692_500_400_000, - ), - spatial_resolution: SpatialResolution::new_unchecked(1., 1.), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [361, 720]).unwrap(), + TimeInterval::new_unchecked(1_692_144_000_000, 1_692_500_400_000), + BandSelection::first(), + )) .await .unwrap() .info @@ -1853,15 +1875,17 @@ mod tests { RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReference::epsg_4326().into(), - time: Some(TimeInterval::new_unchecked( - 1_692_144_000_000, - 1_692_500_400_000 + time: geoengine_operators::engine::TimeDescriptor::new_irregular(Some( + TimeInterval::new_unchecked(1_692_144_000_000, 1_692_500_400_000) )), - bbox: Some(SpatialPartition2D::new_unchecked( - (0., 90.).into(), - (359.500_000_000_000_06, -90.).into() - )), - resolution: None, + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new( + (0., 90.).into(), + 0.499_305_555_555_555_6, + -0.498_614_958_448_753_5 + ), + GridBoundingBox2D::new_min_max(0, 360, 0, 719).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); diff --git a/services/src/datasets/external/gbif.rs b/services/src/datasets/external/gbif.rs index bb18ee152..fdeee2203 100644 --- a/services/src/datasets/external/gbif.rs +++ b/services/src/datasets/external/gbif.rs @@ -1570,17 +1570,17 @@ mod tests { use crate::layers::layer::Layer; use crate::layers::layer::ProviderLayerCollectionId; use crate::test_data; + use crate::util::tests::MockQueryContext; use bb8_postgres::bb8::ManageConnection; use futures::StreamExt; use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; use geoengine_datatypes::dataset::ExternalDataId; use geoengine_datatypes::primitives::{ - BoundingBox2D, CacheHint, FeatureData, MultiPoint, SpatialResolution, TimeInterval, + BoundingBox2D, CacheHint, FeatureData, MultiPoint, TimeInterval, }; use geoengine_datatypes::primitives::{ColumnSelection, TimeInstance}; - use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::QueryProcessor; - use geoengine_operators::{engine::MockQueryContext, source::OgrSourceProcessor}; + use geoengine_operators::source::OgrSourceProcessor; use std::collections::HashMap; use std::{fs::File, io::Read, path::PathBuf}; @@ -2239,15 +2239,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2438,15 +2434,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2576,15 +2568,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -2683,17 +2671,17 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + TimeInterval::default(), + ColumnSelection::all(), + ); + + let ctx = ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3052,17 +3040,17 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + TimeInterval::default(), + ColumnSelection::all(), + ); + + let ctx = ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3171,17 +3159,17 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new( (-61.065_22, 14.775_33).into(), (-61.065_22, 14.775_33).into(), ) .unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + TimeInterval::default(), + ColumnSelection::all(), + ); + + let ctx = ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3282,14 +3270,13 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(1_517_011_200_000).unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(1_517_011_200_000).unwrap(), + ColumnSelection::all(), + ); + + let ctx = ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3326,14 +3313,11 @@ mod tests { return Err(format!("{result:?} != {expected:?}")); } - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new_instant(1_517_443_200_000).unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new_instant(1_517_443_200_000).unwrap(), + ColumnSelection::all(), + ); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3376,7 +3360,6 @@ mod tests { add_test_data(&db_config).await; let result = test(ctx, db_config).await; - assert!(result.is_ok()); } @@ -3384,10 +3367,10 @@ mod tests { #[allow(clippy::too_many_lines)] async fn it_loads_for_lite_subset_selected_columns_with_time_range_filter( db_config: DatabaseConnectionConfig, - ctx: PostgresSessionContext, + ps_ctx: PostgresSessionContext, ) { async fn test( - ctx: PostgresSessionContext, + ps_ctx: PostgresSessionContext, db_config: DatabaseConnectionConfig, ) -> Result<(), String> { let provider = Box::new(GbifDataProviderDefinition { @@ -3399,7 +3382,7 @@ mod tests { autocomplete_timeout: 5, columns: vec!["gbifid".to_string()], }) - .initialize(ctx.db()) + .initialize(ps_ctx.db()) .await .map_err(|e| e.to_string())?; @@ -3425,18 +3408,17 @@ mod tests { vec![], ); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( TimeInstance::from_millis_unchecked(1_517_011_200_000), TimeInstance::from_millis_unchecked(1_517_443_200_000), ) .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + ColumnSelection::all(), + ); + + let ctx = ps_ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3473,18 +3455,17 @@ mod tests { return Err(format!("{result:?} != {expected:?}")); } - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()) - .unwrap(), - time_interval: TimeInterval::new( + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::new( TimeInstance::from_millis_unchecked(1_517_011_200_000), TimeInstance::from_millis_unchecked(1_517_443_200_001), ) .unwrap(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + ColumnSelection::all(), + ); + + let ctx = ps_ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) @@ -3539,7 +3520,7 @@ mod tests { add_test_data(&db_config).await; - let result = test(ctx, db_config).await; + let result = test(ps_ctx, db_config).await; assert!(result.is_ok()); } diff --git a/services/src/datasets/external/gfbio_abcd.rs b/services/src/datasets/external/gfbio_abcd.rs index 4bf911c2e..014b6c58a 100644 --- a/services/src/datasets/external/gfbio_abcd.rs +++ b/services/src/datasets/external/gfbio_abcd.rs @@ -661,21 +661,18 @@ impl mod tests { use super::*; use crate::config; - use crate::contexts::SessionContext; - use crate::contexts::{PostgresContext, PostgresSessionContext}; + use crate::contexts::{PostgresContext, PostgresSessionContext, SessionContext}; use crate::layers::layer::ProviderLayerCollectionId; + use crate::util::tests::MockQueryContext; use crate::{ge_context, test_data}; use bb8_postgres::bb8::ManageConnection; use futures::StreamExt; use geoengine_datatypes::collections::{ChunksEqualIgnoringCacheHint, MultiPointCollection}; use geoengine_datatypes::dataset::ExternalDataId; - use geoengine_datatypes::primitives::{ - BoundingBox2D, FeatureData, MultiPoint, SpatialResolution, TimeInterval, - }; + use geoengine_datatypes::primitives::{BoundingBox2D, FeatureData, MultiPoint, TimeInterval}; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; - use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::QueryProcessor; - use geoengine_operators::{engine::MockQueryContext, source::OgrSourceProcessor}; + use geoengine_operators::source::OgrSourceProcessor; use rand::RngCore; use std::{fs::File, io::Read, path::PathBuf}; use tokio_postgres::Config; @@ -1297,15 +1294,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; @@ -1477,13 +1470,12 @@ mod tests { bbox: None, },meta, vec![]); - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((0., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((0., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = ctx.mock_query_context().unwrap(); let result: Vec<_> = processor .query(query_rectangle, &ctx) diff --git a/services/src/datasets/external/gfbio_collections.rs b/services/src/datasets/external/gfbio_collections.rs index 4f6dcb8ed..ae5a77ebe 100644 --- a/services/src/datasets/external/gfbio_collections.rs +++ b/services/src/datasets/external/gfbio_collections.rs @@ -843,7 +843,7 @@ mod tests { use bb8_postgres::bb8::ManageConnection; use geoengine_datatypes::{ dataset::ExternalDataId, - primitives::{BoundingBox2D, ColumnSelection, SpatialResolution, TimeInterval}, + primitives::{BoundingBox2D, ColumnSelection, TimeInterval}, test_data, }; use httptest::{ @@ -1142,15 +1142,11 @@ mod tests { } let mut loading_info = meta - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new_unchecked( - (-180., -90.).into(), - (180., 90.).into(), - ), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }) + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new_unchecked((-180., -90.).into(), (180., 90.).into()), + TimeInterval::default(), + ColumnSelection::all(), + )) .await .map_err(|e| e.to_string())?; diff --git a/services/src/datasets/external/mod.rs b/services/src/datasets/external/mod.rs index 6ac1affb7..19a216bce 100644 --- a/services/src/datasets/external/mod.rs +++ b/services/src/datasets/external/mod.rs @@ -11,8 +11,7 @@ mod wildlive; pub use copernicus_dataspace::CopernicusDataspaceDataProviderDefinition; pub use sentinel_s2_l2a_cogs::{ - GdalRetries, SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacBand, StacQueryBuffer, - StacZone, + GdalRetries, SentinelS2L2ACogsProviderDefinition, StacApiRetries, StacQueryBuffer, }; pub use wildlive::{ WildliveDataConnectorAuth, WildliveDataConnectorDefinition, WildliveDbCache, WildliveError, diff --git a/services/src/datasets/external/netcdfcf/loading.rs b/services/src/datasets/external/netcdfcf/loading.rs index 9d5cfe78d..3af2de915 100644 --- a/services/src/datasets/external/netcdfcf/loading.rs +++ b/services/src/datasets/external/netcdfcf/loading.rs @@ -16,7 +16,7 @@ use crate::{ use geoengine_datatypes::{ dataset::{DataProviderId, LayerId, NamedData}, operations::image::{Colorizer, RasterColorizer}, - primitives::{CacheTtlSeconds, TimeInstance}, + primitives::{CacheTtlSeconds, Duration, TimeInstance, TimeInterval}, }; use geoengine_operators::{ engine::{RasterOperator, RasterResultDescriptor, TypedOperator}, @@ -148,7 +148,7 @@ pub fn create_layer( workflow: Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: data_id }, + params: GdalSourceParameters::new(data_id), } .boxed(), ), @@ -217,7 +217,9 @@ fn create_loading_info_part( params.file_path = file_path.with_file_name(time_instance.as_datetime_string_with_millis() + ".tiff"); - time_instance.into() + // Note: was a TimeInstance before which is not valid so we add 1 millisecond just to get an interval. + TimeInterval::new(time_instance, time_instance + Duration::milliseconds(1)) + .expect("increasing one millisecond must work") } ParamModification::Channel { channel, @@ -225,7 +227,8 @@ fn create_loading_info_part( } => { params.rasterband_channel = channel; - time_instance.into() + TimeInterval::new(time_instance, time_instance + Duration::milliseconds(1)) + .expect("increasing one millisecond must work") } }; diff --git a/services/src/datasets/external/netcdfcf/mod.rs b/services/src/datasets/external/netcdfcf/mod.rs index 5caf4f6bd..4f4e45daf 100644 --- a/services/src/datasets/external/netcdfcf/mod.rs +++ b/services/src/datasets/external/netcdfcf/mod.rs @@ -29,12 +29,14 @@ use geoengine_datatypes::primitives::{ CacheTtlSeconds, DateTime, Measurement, RasterQueryRectangle, TimeInstance, VectorQueryRectangle, }; -use geoengine_datatypes::raster::{GdalGeoTransform, RasterDataType}; +use geoengine_datatypes::raster::{ + BoundedGrid, GdalGeoTransform, GeoTransform, GridShape2D, RasterDataType, +}; use geoengine_datatypes::spatial_reference::SpatialReference; use geoengine_datatypes::util::canonicalize_subpath; use geoengine_datatypes::util::gdal::ResamplingMethod; -use geoengine_operators::engine::RasterBandDescriptor; -use geoengine_operators::engine::RasterBandDescriptors; +use geoengine_operators::engine::{RasterBandDescriptor, SpatialGridDescriptor}; +use geoengine_operators::engine::{RasterBandDescriptors, TimeDescriptor}; use geoengine_operators::source::{ FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, }; @@ -453,6 +455,7 @@ impl NetCdfCfDataProvider { .boxed_context(error::UnexpectedExecution)? } + #[allow(clippy::too_many_lines)] fn meta_data_from_netcdf( base_path: &Path, dataset_id: &NetCdfCf4DDatasetId, @@ -464,9 +467,7 @@ impl NetCdfCfDataProvider { const TIME_DIMENSION_INDEX: usize = 1; let dataset = gdal_netcdf_open(Some(base_path), Path::new(&dataset_id.file_name))?; - let root_group = dataset.root_group().context(error::GdalMd)?; - let time_coverage = TimeCoverage::from_dimension(&root_group)?; let geo_transform = { @@ -502,30 +503,6 @@ impl NetCdfCfDataProvider { let dimensions = data_array.dimensions().context(error::GdalMd)?; - let result_descriptor = RasterResultDescriptor { - data_type: RasterDataType::from_gdal_data_type( - data_array - .datatype() - .numeric_datatype() - .try_into() - .unwrap_or(GdalDataType::Float64), - ) - .unwrap_or(RasterDataType::F64), - spatial_reference: SpatialReference::try_from( - data_array.spatial_reference().context(error::GdalMd)?, - ) - .context(error::CannotParseCrs)? - .into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( - "band".into(), - derive_measurement(data_array.unit()), - )]) - .context(error::GeneratingResultDescriptorFromDataset)?, - }; - let params = GdalDatasetParameters { file_path: netcfg_gdal_path( Some(base_path), @@ -551,6 +528,36 @@ impl NetCdfCfDataProvider { retry: None, }; + let pixel_shape = GridShape2D::new_2d(params.height as usize, params.width as usize); + let geo_transform = + GeoTransform::try_from(params.geo_transform).expect("GeoTransform must be valid"); // TODO: check how the axis in netcfd are stored; + + let result_descriptor = RasterResultDescriptor { + data_type: RasterDataType::from_gdal_data_type( + data_array + .datatype() + .numeric_datatype() + .try_into() + .unwrap_or(GdalDataType::Float64), + ) + .unwrap_or(RasterDataType::F64), + spatial_reference: SpatialReference::try_from( + data_array.spatial_reference().context(error::GdalMd)?, + ) + .context(error::CannotParseCrs)? + .into(), + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + geo_transform, + pixel_shape.bounding_box(), + ), + bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new( + "band".into(), + derive_measurement(data_array.unit()), + )]) + .expect("must work since derive_measurement can't fail"), + }; + let dimensions_time = dimensions .get(TIME_DIMENSION_INDEX) .map(Dimension::size) @@ -1578,24 +1585,31 @@ mod tests { use crate::ge_context; use crate::layers::layer::LayerListing; use crate::layers::storage::LayerProviderDb; - use crate::{tasks::util::NopTaskContext, util::tests::add_land_cover_to_datasets}; + use crate::tasks::util::NopTaskContext; + use crate::util::tests::add_land_cover_to_datasets; use geoengine_datatypes::dataset::ExternalDataId; use geoengine_datatypes::plots::{PlotData, PlotMetaData}; - use geoengine_datatypes::primitives::{BandSelection, PlotSeriesSelection}; + use geoengine_datatypes::primitives::{ + BandSelection, BoundingBox2D, Coordinate2D, PlotQueryRectangle, PlotSeriesSelection, + SpatialResolution, + }; use geoengine_datatypes::raster::RenameBands; + use geoengine_datatypes::raster::{GeoTransform, GridBoundingBox2D}; use geoengine_datatypes::{ - primitives::{ - BoundingBox2D, PlotQueryRectangle, SpatialPartition2D, SpatialResolution, TimeInterval, - }, - spatial_reference::SpatialReferenceAuthority, - test_data, + primitives::TimeInterval, spatial_reference::SpatialReferenceAuthority, test_data, util::gdal::hide_gdal_errors, }; use geoengine_operators::engine::{ MultipleRasterSources, RasterBandDescriptors, RasterOperator, SingleRasterSource, + TimeDescriptor, }; use geoengine_operators::processing::{ - RasterStacker, RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams, + Interpolation, InterpolationMethod, InterpolationParams, RasterStacker, + RasterStackerParams, RasterTypeConversion, RasterTypeConversionParams, + }; + use geoengine_operators::source::{ + FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, + GdalLoadingInfoTemporalSlice, }; use geoengine_operators::source::{GdalSource, GdalSourceParameters}; use geoengine_operators::{ @@ -1605,10 +1619,6 @@ mod tests { MeanRasterPixelValuesOverTimePosition, }, processing::{Expression, ExpressionParams}, - source::{ - FileNotFoundHandling, GdalDatasetGeoTransform, GdalDatasetParameters, - GdalLoadingInfoTemporalSlice, - }, }; use tokio_postgres::NoTls; @@ -1866,7 +1876,7 @@ mod tests { .await .unwrap(); - pretty_assertions::assert_eq!( + assert_eq!( collection, LayerCollection { id: ProviderLayerCollectionId { @@ -1896,7 +1906,7 @@ mod tests { provider_id: NETCDF_CF_PROVIDER_ID, collection_id: LayerCollectionId("dataset_sm.nc/scenario_3".to_string()) }, - name: "Regional Rivalry".to_string(), + name: "Regional Rivalry".to_string(), description: "SSP3-RCP6.0".to_string(), properties: Default::default(), }), CollectionItem::Collection(LayerCollectionListing { r#type: Default::default(), @@ -1955,27 +1965,25 @@ mod tests { data_type: RasterDataType::I16, spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035) .into(), - time: None, - bbox: None, - resolution: None, + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((3_580_000.0, 2_370_000.0).into(), 1000.0, -1000.0), // FIXME: move to tiling bounds + GridBoundingBox2D::new( + [0, 0], // 0 + [9, 9] // 10 + ) + .unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); let loading_info = metadata - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (43.945_312_5, 0.791_015_625_25).into(), - (44.033_203_125, 0.703_125_25).into(), - ) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.000_343_322_7, // 256 pixel - 0.000_343_322_7, // 256 pixel - ), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [9, 9]).unwrap(), + TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + )) .await .unwrap(); @@ -1995,10 +2003,11 @@ mod tests { ) .into(); + let expected_time = TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)); assert_eq!( loading_info_parts[0], GdalLoadingInfoTemporalSlice { - time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + time: TimeInterval::new_unchecked(expected_time, expected_time + 1), params: Some(GdalDatasetParameters { file_path, rasterband_channel: 4, @@ -2091,27 +2100,25 @@ mod tests { data_type: RasterDataType::I16, spatial_reference: SpatialReference::new(SpatialReferenceAuthority::Epsg, 3035) .into(), - time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1000.0, 1000.0)), + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((3_580_000.0, 2_370_000.0).into(), 1000.0, -1000.0), + GridBoundingBox2D::new( + [0, 0], // 0 + [9, 9] // 10 + ) + .unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), } ); let loading_info = metadata - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (43.945_312_5, 0.791_015_625_25).into(), - (44.033_203_125, 0.703_125_25).into(), - ) - .unwrap(), - time_interval: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), - spatial_resolution: SpatialResolution::new_unchecked( - 0.000_343_322_7, // 256 pixel - 0.000_343_322_7, // 256 pixel - ), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new( + GridBoundingBox2D::new([0, 0], [9, 9]).unwrap(), // Fixme: adapt to tiling bounds + TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + BandSelection::first(), + )) .await .unwrap(); @@ -2126,10 +2133,11 @@ mod tests { .path() .join("dataset_sm.nc/scenario_5/metric_2/1/2000-01-01T00:00:00.000Z.tiff"); + let expected_time = TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)); assert_eq!( loading_info_parts[0], GdalLoadingInfoTemporalSlice { - time: TimeInstance::from(DateTime::new_utc(2000, 1, 1, 0, 0, 0)).into(), + time: TimeInterval::new_unchecked(expected_time, expected_time + 1), params: Some(GdalDatasetParameters { file_path, rasterband_channel: 1, @@ -2275,10 +2283,10 @@ mod tests { }, sources: Expression { params: ExpressionParams { - expression: "A".to_string(), + expression: "if A is NODATA {NODATA} else {A}".to_string(), // FIXME: was "A" because nodata pixels would be skipped. --> The landcover pixels overlapping are NODATA, but why? output_type: RasterDataType::F64, output_band: None, - map_no_data: false, + map_no_data: true, }, sources: SingleRasterSource { raster: RasterStacker { @@ -2287,29 +2295,41 @@ mod tests { }, sources: MultipleRasterSources { rasters: vec![ - GdalSource { - params: GdalSourceParameters { - data: geoengine_datatypes::dataset::NamedData::with_system_provider( - EBV_PROVIDER_ID.to_string(), - serde_json::json!({ - "fileName": "dataset_irr_ts.nc", - "groupNames": ["metric_1"], - "entity": 0 - }) - .to_string(), - ), + Interpolation{ + params: InterpolationParams { + interpolation: InterpolationMethod::NearestNeighbor, + output_resolution: geoengine_operators::processing::InterpolationResolution::Resolution(SpatialResolution::new_unchecked(0.1, 0.1)), // The test data has a resolution of 1.0! + output_origin_reference: Some(Coordinate2D::new(0.0, 0.0)), }, - } - .boxed(), + sources: SingleRasterSource { + raster: GdalSource { + params: GdalSourceParameters { + data: geoengine_datatypes::dataset::NamedData::with_system_provider( + EBV_PROVIDER_ID.to_string(), + serde_json::json!({ + "fileName": "dataset_irr_ts.nc", + "groupNames": ["metric_1"], + "entity": 0 + }) + .to_string(), + ), + overview_level: None, + }, + } + .boxed(), + } + }.boxed(), RasterTypeConversion { params: RasterTypeConversionParams { output_data_type: RasterDataType::I16, }, sources: SingleRasterSource { raster: GdalSource { - params: GdalSourceParameters { - data: land_cover_dataset_name.into(), - }, + params: GdalSourceParameters::new( + geoengine_datatypes::dataset::NamedData::with_system_name( + land_cover_dataset_name.to_string(), + ) + ), }.boxed(), } }.boxed(), @@ -2317,9 +2337,7 @@ mod tests { } }.boxed() } - } - .boxed() - .into(), + }.boxed().into(), } .boxed(); @@ -2343,27 +2361,26 @@ mod tests { let result = processor .plot_query( - PlotQueryRectangle { - spatial_bounds: BoundingBox2D::new( + PlotQueryRectangle::new( + BoundingBox2D::new( (46.478_278_849, 40.584_655_660_000_1).into(), (87.323_796_021_000_1, 55.434_550_273).into(), ) .unwrap(), - time_interval: TimeInterval::new( + TimeInterval::new( DateTime::new_utc(1900, 4, 1, 0, 0, 0), DateTime::new_utc_with_millis(2055, 4, 1, 0, 0, 0, 1), ) .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(0.1, 0.1), - attributes: PlotSeriesSelection::all(), - }, + PlotSeriesSelection::all(), + ), &query_context, ) .await .unwrap(); assert_eq!(result, PlotData { - vega_string: "{\"$schema\":\"https://vega.github.io/schema/vega-lite/v4.17.0.json\",\"data\":{\"values\":[{\"x\":\"2015-01-01T00:00:00+00:00\",\"y\":46.342800000000004},{\"x\":\"2055-01-01T00:00:00+00:00\",\"y\":43.54399999999997}]},\"description\":\"Area Plot\",\"encoding\":{\"x\":{\"field\":\"x\",\"title\":\"Time\",\"type\":\"temporal\"},\"y\":{\"field\":\"y\",\"title\":\"\",\"type\":\"quantitative\"}},\"mark\":{\"line\":true,\"point\":true,\"type\":\"line\"}}".to_string(), + vega_string: "{\"$schema\":\"https://vega.github.io/schema/vega-lite/v4.17.0.json\",\"data\":{\"values\":[{\"x\":\"2015-01-01T00:00:00+00:00\",\"y\":46.68000000000007},{\"x\":\"2055-01-01T00:00:00+00:00\",\"y\":43.72000000000009}]},\"description\":\"Area Plot\",\"encoding\":{\"x\":{\"field\":\"x\",\"title\":\"Time\",\"type\":\"temporal\"},\"y\":{\"field\":\"y\",\"title\":\"\",\"type\":\"quantitative\"}},\"mark\":{\"line\":true,\"point\":true,\"type\":\"line\"}}".to_string(), metadata: PlotMetaData::None, }); } diff --git a/services/src/datasets/external/netcdfcf/overviews.rs b/services/src/datasets/external/netcdfcf/overviews.rs index 2f4cf106e..88675aa18 100644 --- a/services/src/datasets/external/netcdfcf/overviews.rs +++ b/services/src/datasets/external/netcdfcf/overviews.rs @@ -664,7 +664,6 @@ impl CogRasterCreationOptionss { fn inner(this: &CogRasterCreationOptionss) -> Result { let mut options = RasterCreationOptions::new(); options.add_name_value("COMPRESS", &this.compression_format)?; - options.add_name_value("TILED", "YES")?; options.add_name_value("LEVEL", &this.compression_level)?; options.add_name_value("NUM_THREADS", &this.num_threads)?; options.add_name_value("BLOCKSIZE", COG_BLOCK_SIZE)?; @@ -769,12 +768,13 @@ mod tests { use crate::{contexts::SessionContext, ge_context, tasks::util::NopTaskContext}; use gdal::{DatasetOptions, GdalOpenFlags}; use geoengine_datatypes::{ - primitives::{DateTime, SpatialResolution, TimeInterval}, - raster::RasterDataType, + primitives::{DateTime, TimeInterval}, + raster::{GeoTransform, GridBoundingBox2D, RasterDataType}, spatial_reference::SpatialReference, test_data, util::gdal::hide_gdal_errors, }; + use geoengine_operators::engine::{SpatialGridDescriptor, TimeDescriptor}; use geoengine_operators::{ engine::{RasterBandDescriptors, RasterResultDescriptor}, source::{ @@ -819,24 +819,24 @@ mod tests { ) .unwrap(); + let expected_time_1: TimeInstance = DateTime::new_utc(2020, 1, 1, 0, 0, 0).into(); + let expected_time_2: TimeInstance = DateTime::new_utc(2020, 2, 1, 0, 0, 0).into(); assert_eq!( loading_info, GdalMetaDataList { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::I16, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1.0, 1.0)), + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((50., 55.).into(), 1., -1.), + GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2020, 1, 1, 0, 0, 0), - DateTime::new_utc(2020, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_1, expected_time_1 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: Path::new("foo/2020-01-01T00:00:00.000Z.tiff").into(), rasterband_channel: 1, @@ -858,11 +858,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2020, 2, 1, 0, 0, 0), - DateTime::new_utc(2020, 2, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_2, expected_time_2 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: Path::new("foo/2020-02-01T00:00:00.000Z.tiff").into(), rasterband_channel: 1, @@ -994,24 +990,27 @@ mod tests { .await .unwrap() .unwrap(); + + let expected_time_1: TimeInstance = DateTime::new_utc(1900, 1, 1, 0, 0, 0).into(); + let expected_time_2: TimeInstance = DateTime::new_utc(2015, 1, 1, 0, 0, 0).into(); + let expected_time_3: TimeInstance = DateTime::new_utc(2055, 1, 1, 0, 0, 0).into(); + pretty_assertions::assert_eq!( sample_loading_info, GdalMetaDataList { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::I16, spatial_reference: SpatialReference::epsg_4326().into(), - time: None, - bbox: None, - resolution: Some(SpatialResolution::new_unchecked(1.0, 1.0)), + time: TimeDescriptor::new_irregular(None), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new((50., 55.).into(), 1., -1.), + GridBoundingBox2D::new_min_max(0, 4, 0, 4).unwrap(), + ), bands: RasterBandDescriptors::new_single_band(), }, params: vec![ GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(1900, 1, 1, 0, 0, 0), - DateTime::new_utc(1900, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_1, expected_time_1 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/1900-01-01T00:00:00.000Z.tiff"), @@ -1034,11 +1033,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2015, 1, 1, 0, 0, 0), - DateTime::new_utc(2015, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_2, expected_time_2 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/2015-01-01T00:00:00.000Z.tiff"), @@ -1061,11 +1056,7 @@ mod tests { cache_ttl: CacheTtlSeconds::default(), }, GdalLoadingInfoTemporalSlice { - time: TimeInterval::new( - DateTime::new_utc(2055, 1, 1, 0, 0, 0), - DateTime::new_utc(2055, 1, 1, 0, 0, 0) - ) - .unwrap(), + time: TimeInterval::new(expected_time_3, expected_time_3 + 1).unwrap(), params: Some(GdalDatasetParameters { file_path: dataset_folder .join("metric_2/0/2055-01-01T00:00:00.000Z.tiff"), diff --git a/services/src/datasets/external/pangaea/mod.rs b/services/src/datasets/external/pangaea/mod.rs index 27e2dc624..870a5bc61 100644 --- a/services/src/datasets/external/pangaea/mod.rs +++ b/services/src/datasets/external/pangaea/mod.rs @@ -273,14 +273,13 @@ mod tests { }; use geoengine_datatypes::dataset::{DataId, ExternalDataId, LayerId}; use geoengine_datatypes::primitives::{ - BoundingBox2D, ColumnSelection, Coordinate2D, MultiPointAccess, SpatialResolution, - TimeInterval, VectorQueryRectangle, + BoundingBox2D, ColumnSelection, Coordinate2D, MultiPointAccess, TimeInterval, + VectorQueryRectangle, }; use geoengine_datatypes::util::test::TestDefault; use geoengine_operators::engine::{ - InitializedVectorOperator, MetaData, MockExecutionContext, MockQueryContext, - QueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorResultDescriptor, - WorkflowOperatorPath, + InitializedVectorOperator, MetaData, MockExecutionContext, QueryProcessor, + TypedVectorQueryProcessor, VectorOperator, VectorResultDescriptor, WorkflowOperatorPath, }; use geoengine_operators::source::{OgrSource, OgrSourceDataset, OgrSourceParameters}; use httptest::{ @@ -515,13 +514,12 @@ mod tests { panic!("Expected Data QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = context.mock_query_context_test_default(); let result = proc.query(query_rectangle, &ctx).await; @@ -580,13 +578,12 @@ mod tests { panic!("Expected MultiPoint QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = context.mock_query_context_test_default(); let result: Vec = proc .query(query_rectangle, &ctx) @@ -656,13 +653,12 @@ mod tests { panic!("Expected MultiPolygon QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = context.mock_query_context_test_default(); let result: Vec = proc .query(query_rectangle, &ctx) @@ -727,13 +723,12 @@ mod tests { panic!("Expected MultiPoint QueryProcessor"); }; - let query_rectangle = VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), - time_interval: TimeInterval::default(), - spatial_resolution: SpatialResolution::zero_point_one(), - attributes: ColumnSelection::all(), - }; - let ctx = MockQueryContext::test_default(); + let query_rectangle = VectorQueryRectangle::new( + BoundingBox2D::new((-180., -90.).into(), (180., 90.).into()).unwrap(), + TimeInterval::default(), + ColumnSelection::all(), + ); + let ctx = context.mock_query_context_test_default(); let result: Vec = proc .query(query_rectangle, &ctx) diff --git a/services/src/datasets/external/sentinel_s2_l2a_cogs.rs b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs similarity index 85% rename from services/src/datasets/external/sentinel_s2_l2a_cogs.rs rename to services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs index 92fa9981b..f4d1237c4 100644 --- a/services/src/datasets/external/sentinel_s2_l2a_cogs.rs +++ b/services/src/datasets/external/sentinel_s2_l2a_cogs/mod.rs @@ -12,23 +12,23 @@ use crate::layers::listing::{ use crate::projects::{RasterSymbology, Symbology}; use crate::stac::{Feature as StacFeature, FeatureCollection as StacCollection, StacAsset}; use crate::util::operators::source_operator_from_dataset; +use crate::util::sentinel_2_utm_zones::UtmZone; use crate::workflows::workflow::Workflow; use async_trait::async_trait; use geoengine_datatypes::dataset::{DataId, DataProviderId, LayerId, NamedData}; use geoengine_datatypes::operations::image::{RasterColorizer, RgbaColor}; use geoengine_datatypes::operations::reproject::{ - CoordinateProjection, CoordinateProjector, ReprojectClipped, + CoordinateProjection, CoordinateProjector, Reproject, }; -use geoengine_datatypes::primitives::CacheTtlSeconds; +use geoengine_datatypes::primitives::{AxisAlignedRectangle, CacheTtlSeconds}; use geoengine_datatypes::primitives::{ - AxisAlignedRectangle, BoundingBox2D, DateTime, Duration, RasterQueryRectangle, - SpatialPartitioned, TimeInstance, TimeInterval, VectorQueryRectangle, + DateTime, Duration, RasterQueryRectangle, TimeInstance, TimeInterval, VectorQueryRectangle, }; -use geoengine_datatypes::raster::RasterDataType; +use geoengine_datatypes::raster::{GeoTransform, SpatialGridDefinition}; use geoengine_datatypes::spatial_reference::{SpatialReference, SpatialReferenceAuthority}; use geoengine_operators::engine::{ MetaData, MetaDataProvider, OperatorName, RasterBandDescriptors, RasterOperator, - RasterResultDescriptor, TypedOperator, VectorResultDescriptor, + RasterResultDescriptor, SpatialGridDescriptor, TypedOperator, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{ @@ -39,15 +39,37 @@ use geoengine_operators::source::{ use geoengine_operators::util::retry::retry; use postgres_types::{FromSql, ToSql}; use reqwest::Client; +use sentinel_2_l2a_bands::{ImageProduct, ImageProductpec}; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use std::collections::HashMap; use std::convert::TryInto; use std::fmt::Debug; +use std::ops::Neg; use std::path::PathBuf; use tracing::debug; +mod sentinel_2_l2a_bands; + static STAC_RETRY_MAX_BACKOFF_MS: u64 = 60 * 60 * 1000; +static ELEMENT_84_STAC_SENTINEL2_L2A_PRODUCTS: &[ImageProduct] = &[ + ImageProduct::B01, + ImageProduct::B02, + ImageProduct::B03, + ImageProduct::B04, + ImageProduct::B05, + ImageProduct::B06, + ImageProduct::B07, + ImageProduct::B08, + ImageProduct::B8A, + ImageProduct::B09, + ImageProduct::B10, + ImageProduct::B11, + ImageProduct::B12, + ImageProduct::Aot, + ImageProduct::Wvp, + ImageProduct::Scl, +]; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, FromSql, ToSql)] #[serde(rename_all = "camelCase")] @@ -57,8 +79,6 @@ pub struct SentinelS2L2ACogsProviderDefinition { pub description: String, pub priority: Option, pub api_url: String, - pub bands: Vec, - pub zones: Vec, #[serde(default)] pub stac_api_retries: StacApiRetries, #[serde(default)] @@ -129,8 +149,6 @@ impl DataProviderDefinition for SentinelS2L2ACogsProviderDefi self.name, self.description, self.api_url, - &self.bands, - &self.zones, self.stac_api_retries, self.gdal_retries, self.cache_ttl, @@ -155,24 +173,10 @@ impl DataProviderDefinition for SentinelS2L2ACogsProviderDefi } } -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, FromSql, ToSql)] -#[serde(rename_all = "camelCase")] -pub struct StacBand { - pub name: String, - pub no_data_value: Option, - pub data_type: RasterDataType, -} - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, FromSql, ToSql)] -pub struct StacZone { - pub name: String, - pub epsg: u32, -} - #[derive(Debug, Clone, PartialEq)] pub struct SentinelDataset { - band: StacBand, - zone: StacZone, + band: ImageProduct, + zone: UtmZone, listing: Layer, } @@ -202,8 +206,6 @@ impl SentinelS2L2aCogsDataProvider { name: String, description: String, api_url: String, - bands: &[StacBand], - zones: &[StacZone], stac_api_retries: StacApiRetries, gdal_retries: GdalRetries, cache_ttl: CacheTtlSeconds, @@ -214,7 +216,7 @@ impl SentinelS2L2aCogsDataProvider { name, description, api_url, - datasets: Self::create_datasets(&id, bands, zones), + datasets: Self::create_datasets(&id), stac_api_retries, gdal_retries, cache_ttl, @@ -222,22 +224,19 @@ impl SentinelS2L2aCogsDataProvider { } } - fn create_datasets( - id: &DataProviderId, - bands: &[StacBand], - zones: &[StacZone], - ) -> HashMap { - zones - .iter() + fn create_datasets(id: &DataProviderId) -> HashMap { + UtmZone::zones() .flat_map(|zone| { - bands.iter().map(move |band| { - let layer_id = LayerId(format!("{}:{}", zone.name, band.name)); - let listing = Layer { + ELEMENT_84_STAC_SENTINEL2_L2A_PRODUCTS + .iter() + .map(move |band| { + let layer_id = LayerId(format!("{}:{}", zone, band.name())); + let listing = Layer { id: ProviderLayerId { provider_id: *id, layer_id: layer_id.clone(), }, - name: format!("Sentinel S2 L2A COGS {}:{}", zone.name, band.name), + name: format!("Sentinel S2 L2A COGS {}:{} ({})", zone, band.long_name(), band.name()), description: String::new(), workflow: Workflow { operator: source_operator_from_dataset( @@ -274,14 +273,14 @@ impl SentinelS2L2aCogsDataProvider { metadata: HashMap::new(), }; - let dataset = SentinelDataset { - zone: zone.clone(), - band: band.clone(), - listing, - }; + let dataset = SentinelDataset { + zone, + band: *band, + listing, + }; - (layer_id, dataset) - }) + (layer_id, dataset) + }) }) .collect() } @@ -377,13 +376,11 @@ impl LayerCollectionProvider for SentinelS2L2aCogsDataProvider { workflow: Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { - data: NamedData { - namespace: None, - provider: Some(self.id.to_string()), - name: id.to_string(), - }, - }, + params: GdalSourceParameters::new(NamedData { + namespace: None, + provider: Some(self.id.to_string()), + name: id.to_string(), + }), } .boxed(), ), @@ -398,8 +395,8 @@ impl LayerCollectionProvider for SentinelS2L2aCogsDataProvider { #[derive(Debug, Clone)] pub struct SentinelS2L2aCogsMetaData { api_url: String, - zone: StacZone, - band: StacBand, + zone: UtmZone, + band: ImageProduct, stac_api_retries: StacApiRetries, gdal_retries: GdalRetries, cache_ttl: CacheTtlSeconds, @@ -423,8 +420,8 @@ impl SentinelS2L2aCogsMetaData { GdalLoadingInfoTemporalSliceIterator::Static { parts: vec![].into_iter(), }, - query.time_interval.start() - query_start_buffer, - query.time_interval.end() + query_end_buffer, + query.time_interval().start() - query_start_buffer, + query.time_interval().end() + query_end_buffer, )); } @@ -438,7 +435,7 @@ impl SentinelS2L2aCogsMetaData { .filter(|f| { f.properties .proj_epsg - .is_some_and(|epsg| epsg == self.zone.epsg) + .is_some_and(|epsg| epsg == self.zone.epsg_code()) }) .collect(); @@ -465,7 +462,7 @@ impl SentinelS2L2aCogsMetaData { start_times[i + 1] } else { // (or end of query?) - query.time_interval.end() + query_end_buffer + start + query_end_buffer }; /* @@ -512,40 +509,45 @@ impl SentinelS2L2aCogsMetaData { let time_interval = TimeInterval::new(start, end)?; - if time_interval.start() <= query.time_interval.start() { - let t = if time_interval.end() > query.time_interval.start() { - time_interval.start() - } else { - time_interval.end() - }; - known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); - } + if time_interval.contains(&query.time_interval()) { + known_time_start = Some(time_interval.start()); + known_time_end = Some(time_interval.end()); + } else { + if time_interval.start() <= query.time_interval().start() { + let t = if time_interval.end() > query.time_interval().start() { + time_interval.start() + } else { + time_interval.end() + }; + known_time_start = known_time_start.map(|old| old.max(t)).or(Some(t)); + } - if time_interval.end() >= query.time_interval.end() { - let t = if time_interval.start() < query.time_interval.end() { - time_interval.end() - } else { - time_interval.start() - }; - known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + if time_interval.end() >= query.time_interval().end() { + let t = if time_interval.start() < query.time_interval().end() { + time_interval.end() + } else { + time_interval.start() + }; + known_time_end = known_time_end.map(|old| old.min(t)).or(Some(t)); + } } - if time_interval.intersects(&query.time_interval) { + if time_interval.intersects(&query.time_interval()) { debug!( "STAC asset time: {}, url: {}", time_interval, feature .assets - .get(&self.band.name) + .get(self.band.name()) .map_or(&"n/a".to_string(), |a| &a.href) ); let asset = feature .assets - .get(&self.band.name) + .get(self.band.name()) .ok_or(error::Error::StacNoSuchBand { - band_name: self.band.name.clone(), + band_name: self.band.name().to_owned(), })?; parts.push(self.create_loading_info_part(time_interval, asset, self.cache_ttl)?); @@ -554,10 +556,10 @@ impl SentinelS2L2aCogsMetaData { debug!("number of generated loading infos: {}", parts.len()); // if there is no information of time outside the query, we fallback to the only information we know: query -/+ buffer. We also use that information if we did not find a better time - let query_start = query.time_interval.start() - query_start_buffer; + let query_start = query.time_interval().start() - query_start_buffer; let known_time_before = known_time_start.unwrap_or(query_start).min(query_start); - let query_end = query.time_interval.end() + query_end_buffer; + let query_end = query.time_interval().end() + query_end_buffer; let known_time_after = known_time_end.unwrap_or(query_end).max(query_end); Ok(GdalLoadingInfo::new( @@ -619,7 +621,7 @@ impl SentinelS2L2aCogsMetaData { width: stac_shape_x as usize, height: stac_shape_y as usize, file_not_found_handling: geoengine_operators::source::FileNotFoundHandling::NoData, - no_data_value: self.band.no_data_value, + no_data_value: self.band.no_data_value(), properties_mapping: None, gdal_open_options: None, gdal_config_options: Some(vec![ @@ -651,23 +653,26 @@ impl SentinelS2L2aCogsMetaData { &self, query: &RasterQueryRectangle, ) -> Result>> { - let (t_start, t_end) = Self::time_range_request(&query.time_interval)?; + let (t_start, t_end) = Self::time_range_request(&query.time_interval())?; let t_start = t_start - Duration::seconds(self.stac_query_buffer.start_seconds); let t_end = t_end + Duration::seconds(self.stac_query_buffer.end_seconds); + let native_spatial_ref = + SpatialReference::new(SpatialReferenceAuthority::Epsg, self.zone.epsg_code()); + let epsg_4326_ref = SpatialReference::epsg_4326(); + let projector = CoordinateProjector::from_known_srs(native_spatial_ref, epsg_4326_ref)?; + let native_bounds = self.zone.native_extent(); + // request all features in zone in order to be able to determine the temporal validity of individual tile - let projector = CoordinateProjector::from_known_srs( - SpatialReference::new(SpatialReferenceAuthority::Epsg, self.zone.epsg), - SpatialReference::epsg_4326(), - )?; - - let spatial_partition = query.spatial_partition(); // TODO: use SpatialPartition2D directly - let bbox = BoundingBox2D::new_upper_left_lower_right_unchecked( - spatial_partition.upper_left(), - spatial_partition.lower_right(), - ); - let bbox = bbox.reproject_clipped(&projector)?; // TODO: use reproject_clipped on SpatialPartition2D + let bbox = native_bounds + .reproject(&projector) + .inspect_err(|e| { + debug!( + "could not project zone bounds to EPSG:4326. Was: {native_bounds:?}. Source: {e}" + ); + }) + .ok(); Ok(bbox.map(|bbox| { vec![ @@ -795,16 +800,23 @@ impl MetaData } async fn result_descriptor(&self) -> geoengine_operators::util::Result { + let geo_transform = GeoTransform::new( + self.zone.native_extent().upper_left(), + self.band.resolution_m(), + self.band.resolution_m().neg(), + ); + let grid_bounds = geo_transform.spatial_to_grid_bounds(&self.zone.native_extent()); + let spatial_grid = SpatialGridDefinition::new(geo_transform, grid_bounds); + Ok(RasterResultDescriptor { - data_type: self.band.data_type, + data_type: self.band.data_type(), spatial_reference: SpatialReference::new( SpatialReferenceAuthority::Epsg, - self.zone.epsg, + self.zone.epsg_code(), ) .into(), - time: None, - bbox: None, - resolution: None, // TODO: determine from STAC or data or hardcode it + time: geoengine_operators::engine::TimeDescriptor::new_irregular(None), // TODO: can we get the time bounds? + spatial_grid: SpatialGridDescriptor::new_source(spatial_grid), bands: RasterBandDescriptors::new_single_band(), }) } @@ -842,8 +854,8 @@ impl MetaDataProvider> = provider @@ -952,20 +961,29 @@ mod tests { .await .unwrap(); + let data_bounds = + SpatialPartition2D::new((166_021.44, 9_329_005.18).into(), (534_994.66, 0.00).into()) + .unwrap(); + + let raster_result_descriptor = meta.result_descriptor().await.unwrap(); + let tiling_grid_definition = raster_result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(exe_ctx.tiling_specification()); + + let data_bounds_in_pixel_grid = tiling_grid_definition + .tiling_geo_transform() + .spatial_to_grid_bounds(&data_bounds); + let loading_info = meta - .loading_info(RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new( - (166_021.44, 9_329_005.18).into(), - (534_994.66, 0.00).into(), - ) - .unwrap(), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, - spatial_resolution: SpatialResolution::one(), - attributes: BandSelection::first(), - }) + .loading_info(RasterQueryRectangle::new( + data_bounds_in_pixel_grid, + TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, + BandSelection::first(), + )) .await .unwrap(); + // we expect only one tile because there is only this one at the queried time let expected = vec![GdalLoadingInfoTemporalSlice { time: TimeInterval::new_unchecked(1_609_581_746_000, 1_609_581_758_000), params: Some(GdalDatasetParameters { @@ -1055,31 +1073,34 @@ mod tests { ); let op = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &exe) .await .unwrap(); - let processor = op.query_processor()?.get_u16().unwrap(); - - let spatial_bounds = - SpatialPartition2D::new((166_021.44, 9_329_005.18).into(), (534_994.66, 0.00).into()) - .unwrap(); + let sp = SpatialPartition2D::new( + (600_000.000, 5_500_020.000).into(), // 1830 px + (709_800.000, 5_390_220.000).into(), // 1830 px + ) + .unwrap(); - let spatial_resolution = SpatialResolution::new_unchecked( - spatial_bounds.size_x() / 256., - spatial_bounds.size_y() / 256., + let processor = op.query_processor()?.get_u16().unwrap(); + let sp = processor + .raster_result_descriptor() + .spatial_grid_descriptor() + .tiling_grid_definition(exe.tiling_specification) + .tiling_geo_transform() + .spatial_to_grid_bounds(&sp); + + let query = RasterQueryRectangle::new( + sp, + TimeInterval::new_instant(DateTime::new_utc(2018, 10, 19, 13, 23, 25))?, + BandSelection::first(), ); - let query = RasterQueryRectangle { - spatial_bounds, - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 1, 2, 10, 2, 26))?, - spatial_resolution, - attributes: BandSelection::first(), - }; - let ctx = MockQueryContext::new(ChunkByteSize::MAX); + let ctx = exe.mock_query_context(ChunkByteSize::MAX); let result = processor .raster_query(query, &ctx) @@ -1087,8 +1108,8 @@ mod tests { .collect::>() .await; - // TODO: check actual data - assert_eq!(result.len(), 1); + // This are 5 x 5 tiles since the sentinel tile intersects 5 x5 geo engine tiles + assert_eq!(result.len(), 25); // Ok(()) } @@ -1116,7 +1137,7 @@ mod tests { request::query(url_decoded(contains(("limit", "500")))), request::query(url_decoded(contains(( "bbox", - "[33.899332958586406,-2.261536424319933,33.900232774450984,-2.2606312588790414]" + "[9.396566748392315,-83.82852972938498,63.83756656611425,0]" )))), request::query(url_decoded(contains(( "datetime", @@ -1184,7 +1205,8 @@ mod tests { responders::status_code(206) .append_header("Content-Type", "application/json") .body( - include_bytes!("../../../../test_data/stac_responses/cog-header.bin").to_vec(), + include_bytes!("../../../../../test_data/stac_responses/cog-header.bin") + .to_vec(), ) .append_header( "x-amz-id-2", @@ -1252,7 +1274,7 @@ mod tests { .append_header("Content-Type", "application/json") .body( include_bytes!( - "../../../../test_data/stac_responses/cog-tile.bin" + "../../../../../test_data/stac_responses/cog-tile.bin" )[0..2] .to_vec() ).append_header( @@ -1277,7 +1299,7 @@ mod tests { .append_header("Content-Type", "application/json") .body( include_bytes!( - "../../../../test_data/stac_responses/cog-tile.bin" + "../../../../../test_data/stac_responses/cog-tile.bin" ) .to_vec() ).append_header( @@ -1309,15 +1331,6 @@ mod tests { description: "Access to Sentinel 2 L2A COGs on AWS".into(), priority: Some(22), api_url: server.url_str("/v0/collections/sentinel-s2-l2a-cogs/items"), - bands: vec![StacBand { - name: "B04".into(), - no_data_value: Some(0.), - data_type: RasterDataType::U16, - }], - zones: vec![StacZone { - name: "UTM36S".into(), - epsg: 32736, - }], stac_api_retries: Default::default(), gdal_retries: GdalRetries { number_of_retries: 999, @@ -1326,14 +1339,10 @@ mod tests { query_buffer: Default::default(), }); - let provider = provider_def - .initialize( - app_ctx - .session_context(app_ctx.create_anonymous_session().await.unwrap()) - .db(), - ) - .await - .unwrap(); + let session_ctx = + app_ctx.session_context(app_ctx.create_anonymous_session().await.unwrap()); + + let provider = provider_def.initialize(session_ctx.db()).await.unwrap(); let meta: Box> = provider @@ -1347,17 +1356,27 @@ mod tests { .await .unwrap(); - let query = RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (600_000.00, 9_750_100.).into(), - (600_100.0, 9_750_000.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc(2021, 9, 23, 8, 10, 44)) - .unwrap(), - spatial_resolution: SpatialResolution::new_unchecked(10., 10.), - attributes: BandSelection::first(), - }; + let exe_ctx = session_ctx.execution_context().unwrap(); + let result_descriptor = meta.result_descriptor().await.unwrap(); + + let data_bounds = SpatialPartition2D::new_unchecked( + (600_000.00, 9_750_100.).into(), + (600_100.0, 9_750_000.).into(), + ); + + let tiling_geo_transform = result_descriptor + .spatial_grid_descriptor() + .tiling_grid_definition(exe_ctx.tiling_specification()) + .tiling_geo_transform(); + let sp: geoengine_datatypes::raster::GridBoundingBox<[isize; 2]> = + tiling_geo_transform.spatial_to_grid_bounds(&data_bounds); + + let query = RasterQueryRectangle::new( + sp, + TimeInterval::new_instant(DateTime::new_utc(2021, 9, 23, 8, 10, 44)).unwrap(), + BandSelection::first(), + ); let loading_info = meta.loading_info(query).await.unwrap(); let parts = if let GdalLoadingInfoTemporalSliceIterator::Static { parts } = loading_info.info { @@ -1425,21 +1444,14 @@ mod tests { name.clone(), Box::new(GdalMetaDataStatic { time: None, - result_descriptor: RasterResultDescriptor { - data_type: RasterDataType::U16, - spatial_reference: SpatialReference::from_str("EPSG:32736").unwrap().into(), - time: None, - bbox: None, - resolution: None, - bands: RasterBandDescriptors::new_single_band(), - }, + result_descriptor, params, cache_ttl: CacheTtlSeconds::default(), }), ); let gdal_source = GdalSource { - params: GdalSourceParameters { data: name }, + params: GdalSourceParameters::new(name), } .boxed() .initialize(WorkflowOperatorPath::initialize_root(), &execution_context) @@ -1450,22 +1462,23 @@ mod tests { .get_u16() .unwrap(); - let query_context = MockQueryContext::test_default(); + let query_context = execution_context.mock_query_context_test_default(); + + let data_bounds = SpatialPartition2D::new_unchecked( + (499_980., 9_804_800.).into(), + (499_990., 9_804_810.).into(), + ); + + let sp: geoengine_datatypes::raster::GridBoundingBox<[isize; 2]> = + tiling_geo_transform.spatial_to_grid_bounds(&data_bounds); let stream = gdal_source .raster_query( - RasterQueryRectangle { - spatial_bounds: SpatialPartition2D::new_unchecked( - (499_980., 9_804_800.).into(), - (499_990., 9_804_810.).into(), - ), - time_interval: TimeInterval::new_instant(DateTime::new_utc( - 2014, 3, 1, 0, 0, 0, - )) - .unwrap(), - spatial_resolution: SpatialResolution::new(10., 10.).unwrap(), - attributes: BandSelection::first(), - }, + RasterQueryRectangle::new( + sp, + TimeInterval::new_instant(DateTime::new_utc(2014, 3, 1, 0, 0, 0)).unwrap(), + BandSelection::first(), + ), &query_context, ) .await diff --git a/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs b/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs new file mode 100644 index 000000000..20ceba5c0 --- /dev/null +++ b/services/src/datasets/external/sentinel_s2_l2a_cogs/sentinel_2_l2a_bands.rs @@ -0,0 +1,95 @@ +use geoengine_datatypes::raster::RasterDataType; + +#[derive(Debug, Copy, Clone, PartialEq)] +pub enum ImageProduct { + B01, + B02, + B03, + B04, + B05, + B06, + B07, + B08, + B8A, + B09, + B10, + B11, + B12, + Scl, + Wvp, + Aot, + _Tci, +} + +pub trait ImageProductpec { + fn resolution_m(&self) -> f64; + fn name(&self) -> &str; + fn long_name(&self) -> &str; + fn no_data_value(&self) -> Option; + fn data_type(&self) -> RasterDataType; +} + +impl ImageProductpec for ImageProduct { + fn no_data_value(&self) -> Option { + Some(0.) + } + + fn resolution_m(&self) -> f64 { + match self { + ImageProduct::B02 + | ImageProduct::B03 + | ImageProduct::B04 + | ImageProduct::B08 + | ImageProduct::_Tci => 10., + ImageProduct::B05 + | ImageProduct::B06 + | ImageProduct::B07 + | ImageProduct::B8A + | ImageProduct::B11 + | ImageProduct::B12 + | ImageProduct::Scl + | ImageProduct::Wvp + | ImageProduct::Aot => 20., + ImageProduct::B01 | ImageProduct::B09 | ImageProduct::B10 => 60., + } + } + + fn name(&self) -> &str { + match self { + ImageProduct::B01 => "B01", + ImageProduct::B02 => "B02", + ImageProduct::B03 => "B03", + ImageProduct::B04 => "B04", + ImageProduct::B05 => "B05", + ImageProduct::B06 => "B06", + ImageProduct::B07 => "B07", + ImageProduct::B08 => "B08", + ImageProduct::B8A => "B8A", + ImageProduct::B09 => "B09", + ImageProduct::B10 => "B10", + ImageProduct::B11 => "B11", + ImageProduct::B12 => "B12", + ImageProduct::Scl => "SCL", + ImageProduct::Wvp => "WVP", + ImageProduct::Aot => "AOT", + ImageProduct::_Tci => "TCI", + } + } + + fn long_name(&self) -> &str { + match self { + ImageProduct::Scl => "Scene Classification", + ImageProduct::Wvp => "Water Vapour", + ImageProduct::Aot => "Aerosol Optical Thickness", + ImageProduct::_Tci => "True Colour Image", + _ => self.name(), + } + } + + fn data_type(&self) -> RasterDataType { + match self { + ImageProduct::Scl => RasterDataType::U8, + _ => RasterDataType::U16, + } + } +} diff --git a/services/src/datasets/external/wildlive/mod.rs b/services/src/datasets/external/wildlive/mod.rs index 984511037..5b28543f5 100644 --- a/services/src/datasets/external/wildlive/mod.rs +++ b/services/src/datasets/external/wildlive/mod.rs @@ -1017,7 +1017,7 @@ mod tests { dataset::ExternalDataId, primitives::{ BoundingBox2D, CacheTtlSeconds, ColumnSelection, Coordinate2D, FeatureDataType, - Measurement, SpatialResolution, TimeInterval, + Measurement, TimeInterval, }, spatial_reference::SpatialReference, test_data, @@ -1316,16 +1316,15 @@ mod tests { .unwrap(); let loading_info = metadata - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new( Coordinate2D { x: 0.0, y: 0.0 }, Coordinate2D { x: 1.0, y: 1.0 }, ) .unwrap(), - time_interval: TimeInterval::new(0, 1).unwrap(), - spatial_resolution: SpatialResolution::new(1.0, 1.0).unwrap(), - attributes: ColumnSelection::all(), - }) + TimeInterval::new(0, 1).unwrap(), + ColumnSelection::all(), + )) .await .unwrap(); @@ -1611,16 +1610,15 @@ mod tests { .unwrap(); let loading_info = metadata - .loading_info(VectorQueryRectangle { - spatial_bounds: BoundingBox2D::new( + .loading_info(VectorQueryRectangle::new( + BoundingBox2D::new( Coordinate2D { x: 0.0, y: 0.0 }, Coordinate2D { x: 1.0, y: 1.0 }, ) .unwrap(), - time_interval: TimeInterval::new(0, 1).unwrap(), - spatial_resolution: SpatialResolution::new(1.0, 1.0).unwrap(), - attributes: ColumnSelection::all(), - }) + TimeInterval::new(0, 1).unwrap(), + ColumnSelection::all(), + )) .await .unwrap(); diff --git a/services/src/datasets/listing.rs b/services/src/datasets/listing.rs index 4af6e3d3e..f86dbe67e 100644 --- a/services/src/datasets/listing.rs +++ b/services/src/datasets/listing.rs @@ -1,5 +1,6 @@ use super::DatasetName; use super::storage::MetaDataDefinition; +use crate::api::model::operators::TypedResultDescriptor; use crate::config::{DatasetService, get_config_element}; use crate::datasets::storage::{Dataset, validate_tags}; use crate::error::Result; @@ -8,7 +9,7 @@ use async_trait::async_trait; use geoengine_datatypes::dataset::{DataId, DatasetId}; use geoengine_datatypes::primitives::{RasterQueryRectangle, VectorQueryRectangle}; use geoengine_operators::engine::{ - MetaDataProvider, RasterResultDescriptor, TypedResultDescriptor, VectorResultDescriptor, + MetaDataProvider, RasterResultDescriptor, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; use geoengine_operators::source::{GdalLoadingInfo, OgrSourceDataset}; diff --git a/services/src/datasets/mod.rs b/services/src/datasets/mod.rs index 2da973a78..b9c3ee3e4 100644 --- a/services/src/datasets/mod.rs +++ b/services/src/datasets/mod.rs @@ -8,7 +8,7 @@ pub mod storage; pub mod upload; pub(crate) use create_from_workflow::{ - RasterDatasetFromWorkflow, RasterDatasetFromWorkflowResult, + RasterDatasetFromWorkflow, RasterDatasetFromWorkflowParams, RasterDatasetFromWorkflowResult, schedule_raster_dataset_from_workflow_task, }; pub use name::{DatasetIdAndName, DatasetName, DatasetNameError}; diff --git a/services/src/datasets/postgres.rs b/services/src/datasets/postgres.rs index 1eff05acc..9ba77a08f 100644 --- a/services/src/datasets/postgres.rs +++ b/services/src/datasets/postgres.rs @@ -1,13 +1,18 @@ -use crate::api::model::services::UpdateDataset; +use std::path::PathBuf; + +use crate::api::handlers::datasets::AddDatasetTile; +use crate::api::model::datatypes::SpatialPartition2D; +use crate::api::model::services::{DataPath, UpdateDataset}; use crate::contexts::PostgresDb; use crate::datasets::listing::Provenance; use crate::datasets::listing::{DatasetListOptions, DatasetListing, DatasetProvider}; use crate::datasets::listing::{OrderBy, ProvenanceOutput}; use crate::datasets::storage::{Dataset, DatasetDb, DatasetStore, MetaDataDefinition}; -use crate::datasets::upload::FileId; +use crate::datasets::upload::{FileId, UploadRootPath, Volumes}; use crate::datasets::upload::{Upload, UploadDb, UploadId}; use crate::datasets::{AddDataset, DatasetIdAndName, DatasetName}; use crate::error::{self, Error, Result}; +use crate::identifier; use crate::permissions::TxPermissionDb; use crate::permissions::{Permission, RoleId}; use crate::projects::Symbology; @@ -19,16 +24,24 @@ use bb8_postgres::tokio_postgres::Socket; use bb8_postgres::tokio_postgres::tls::{MakeTlsConnect, TlsConnect}; use geoengine_datatypes::dataset::{DataId, DatasetId}; use geoengine_datatypes::error::BoxedResultExt; -use geoengine_datatypes::primitives::RasterQueryRectangle; -use geoengine_datatypes::primitives::VectorQueryRectangle; +use geoengine_datatypes::primitives::{ + CacheHint, RasterQueryRectangle, TimeDimension, TimeInstance, TryIrregularTimeFillIterExt, + TryRegularTimeFillIterExt, +}; +use geoengine_datatypes::primitives::{TimeInterval, VectorQueryRectangle}; +use geoengine_datatypes::raster::{GridBoundingBox2D, SpatialGridDefinition}; use geoengine_datatypes::util::Identifier; use geoengine_operators::engine::TypedResultDescriptor; use geoengine_operators::engine::{ MetaData, MetaDataProvider, RasterResultDescriptor, VectorResultDescriptor, }; use geoengine_operators::mock::MockDatasetDataSourceLoadingInfo; -use geoengine_operators::source::{GdalLoadingInfo, OgrSourceDataset}; +use geoengine_operators::source::{ + GdalDatasetGeoTransform, GdalDatasetParameters, GdalLoadingInfo, MultiBandGdalLoadingInfo, + MultiBandGdalLoadingInfoQueryRectangle, OgrSourceDataset, TileFile, +}; use postgres_types::{FromSql, ToSql}; +use tokio_postgres::Transaction; pub async fn resolve_dataset_name_to_id( conn: &PooledConnection<'_, PostgresConnectionManager>, @@ -228,6 +241,10 @@ where Ok(rows .iter() .map(|row| { + // get the real TypedResultDescriptor and convert it to the API one + let result_desc: TypedResultDescriptor = row.get(6); + let result_desc = result_desc.into(); + Result::::Ok(DatasetListing { id: row.get(0), name: row.get(1), @@ -235,7 +252,7 @@ where description: row.get(3), tags: row.get::<_, Option>>(4).unwrap_or_default(), source_operator: row.get(5), - result_descriptor: row.get(6), + result_descriptor: result_desc, symbology: row.get(7), }) }) @@ -257,7 +274,8 @@ where d.source_operator, d.symbology, d.provenance, - d.tags + d.tags, + d.data_path FROM user_permitted_datasets p JOIN datasets d ON (p.dataset_id = d.id) @@ -284,6 +302,7 @@ where symbology: row.get(6), provenance: row.get(7), tags: row.get(8), + data_path: row.get(9), }) } @@ -588,6 +607,466 @@ where } } +#[async_trait] +impl + MetaDataProvider< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > for PostgresDb +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + async fn meta_data( + &self, + data_id: &DataId, + ) -> geoengine_operators::util::Result< + Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + >, + > { + let id = data_id + .internal() + .ok_or(geoengine_operators::error::Error::DataIdTypeMissMatch)?; + + let mut conn = self.conn_pool.get().await.map_err(|e| { + geoengine_operators::error::Error::MetaData { + source: Box::new(e), + } + })?; + let tx = conn.build_transaction().start().await.map_err(|e| { + geoengine_operators::error::Error::MetaData { + source: Box::new(e), + } + })?; + + if !self + .has_permission_in_tx(id, Permission::Read, &tx) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })? + { + return Err(geoengine_operators::error::Error::PermissionDenied); + } + + let stmt = tx + .prepare( + " + SELECT + d.meta_data, d.data_path + FROM + user_permitted_datasets p JOIN datasets d + ON (p.dataset_id = d.id) + WHERE + d.id = $1 AND p.user_id = $2 + LIMIT + 1", + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + let row = tx + .query_one(&stmt, &[&id, &self.session.user.id]) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + let meta_data: MetaDataDefinition = row.get(0); + + let result_descriptor = match meta_data { + MetaDataDefinition::GdalMultiBand(b) => b.result_descriptor, + _ => return Err(geoengine_operators::error::Error::DataIdTypeMissMatch), + }; + + let data_path: DataPath = row.get(1); + + let data_path = + match data_path { + DataPath::Volume(volume_name) => + // TODO: after Volume management is implemented, this needs to be adapted + { + Volumes::default() + .volumes + .iter() + .find(|v| v.name == volume_name) + .ok_or(Error::UnknownVolumeName { + volume_name: volume_name.0.clone(), + }) + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })? + .path + .clone() + } + DataPath::Upload(upload_id) => upload_id.root_path().map_err(|e| { + geoengine_operators::error::Error::MetaData { + source: Box::new(e), + } + })?, + }; + + Ok(Box::new(MultiBandGdalLoadingInfoProvider { + dataset_id: id, + result_descriptor, + data_path, + db: self.clone(), + })) + } +} + +#[derive(Debug, Clone)] +pub struct MultiBandGdalLoadingInfoProvider +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + dataset_id: DatasetId, + result_descriptor: RasterResultDescriptor, + data_path: PathBuf, + db: PostgresDb, +} + +async fn create_gap_free_time_steps( + dataset_id: DatasetId, + query_time: TimeInterval, + conn: &PooledConnection<'_, PostgresConnectionManager>, +) -> Result> +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + // determine the regularity of the dataset + let time_dim = resolve_time_dim(dataset_id, conn).await?; + + // fill the gaps before, between and after the data tiles to fully cover the query time range + match time_dim { + TimeDimension::Regular(regular_time_dimension) => { + // we do not even need to know the existing tile's time steps here, since the steps are regular + vec![] + .into_iter() + .try_time_regular_range_fill(regular_time_dimension, query_time) + .collect() + } + TimeDimension::Irregular => { + // query all time steps inside the query, across all spatial tiles and bands + let time_steps = collect_timesteps_in_query(dataset_id, query_time, conn) + .await? + .into_iter() + .map(Result::Ok) + .collect::>>(); + + // determine the time range to cover by finding the last tile before, and the first tile after the `time_steps` + let (start, end) = if let Some(Ok(first_time)) = time_steps.first() + && let Some(Ok(last_time)) = time_steps.last() + { + let start = if first_time.start() > query_time.start() { + resolve_first_time_end_before(dataset_id, conn, first_time.start()).await? + } else { + query_time.start() + }; + + let end = if last_time.end() < query_time.end() { + resolve_first_time_start_after(dataset_id, conn, last_time.end()).await? + } else { + query_time.end() + }; + + (start, end) + } else { + let start = + resolve_first_time_end_before(dataset_id, conn, query_time.start()).await?; + let end = + resolve_first_time_start_after(dataset_id, conn, query_time.end()).await?; + + (start, end) + }; + + time_steps + .into_iter() + .try_time_irregular_range_fill(TimeInterval::new(start, end)?) + .collect() + } + } +} + +async fn resolve_first_time_end_before( + dataset_id: DatasetId, + conn: &PooledConnection<'_, PostgresConnectionManager>, + start: TimeInstance, +) -> Result +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + let rows = conn + .query( + " + SELECT DISTINCT + (time).end + FROM + dataset_tiles + WHERE + dataset_id = $1 AND (time).end <= $2 + ORDER BY + (time).end DESC + LIMIT 1", + &[&dataset_id, &start], + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + Ok(if let Some(row) = rows.first() { + row.get(0) + } else { + TimeInstance::MIN + }) +} + +async fn resolve_first_time_start_after( + dataset_id: DatasetId, + conn: &PooledConnection<'_, PostgresConnectionManager>, + end: TimeInstance, +) -> Result +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + let rows = conn + .query( + " + SELECT DISTINCT + (time).start + FROM + dataset_tiles + WHERE + dataset_id = $1 AND (time).start >= $2 + ORDER BY + (time).start ASC + LIMIT 1", + &[&dataset_id, &end], + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + Ok(if let Some(row) = rows.first() { + row.get(0) + } else { + TimeInstance::MAX + }) +} + +async fn resolve_time_dim( + dataset_id: DatasetId, + conn: &PooledConnection<'_, PostgresConnectionManager>, +) -> Result +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + let time_dim: TimeDimension = conn + .query_one( + r#" + SELECT (result_descriptor).raster."time".dimension + FROM datasets + WHERE id = $1; + "#, + &[&dataset_id], + ) + .await? + .get(0); + Ok(time_dim) +} + +async fn collect_timesteps_in_query( + dataset_id: DatasetId, + query_time: TimeInterval, + conn: &PooledConnection<'_, PostgresConnectionManager>, +) -> Result> +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + let rows = conn + .query( + " + SELECT DISTINCT + time, (time).start + FROM + dataset_tiles + WHERE + dataset_id = $1 AND time_interval_intersects(time, $2) + ORDER BY + (time).start", + &[&dataset_id, &query_time], + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + let time_steps: Vec = rows.into_iter().map(|row| row.get(0)).collect(); + + Ok(time_steps) +} + +#[async_trait] +impl + MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + > for MultiBandGdalLoadingInfoProvider +where + Tls: MakeTlsConnect + Clone + Send + Sync + 'static + std::fmt::Debug, + >::Stream: Send + Sync, + >::TlsConnect: Send, + <>::TlsConnect as TlsConnect>::Future: Send, +{ + async fn loading_info( + &self, + query: MultiBandGdalLoadingInfoQueryRectangle, + ) -> Result { + // NOTE: we query only the files that are needed for answering the query, but to retrieve ALL the time steps in the query we have to consider the whole world and not the restrained bbox + + let conn = self.db.conn_pool.get().await.map_err(|e| { + geoengine_operators::error::Error::MetaData { + source: Box::new(e), + } + })?; + + let files = if query.fetch_tiles { + // query the files + let rows = conn + .query( + " + SELECT + bbox, time, band, z_index, gdal_params + FROM + dataset_tiles + WHERE + dataset_id = $1 AND + spatial_partition2d_intersects(bbox, $2) AND + time_interval_intersects(time, $3) AND + band = ANY($4) + ORDER BY + (time).start, band, z_index", + &[ + &self.dataset_id, + &query.query_rectangle.spatial_bounds(), + &query.query_rectangle.time_interval(), + &query.query_rectangle.attributes().as_slice(), + ], + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + let files: Vec = rows + .into_iter() + .map(|row| TileFile { + spatial_partition: row.get(0), + time: row.get(1), + band: row.get(2), + z_index: row.get(3), + params: { + let mut params: GdalDatasetParameters = row.get(4); + // at some point we need to turn the relative file paths of tiles into absolute paths + params.file_path = self.data_path.join(¶ms.file_path); + params + }, + }) + .collect(); + + files + } else { + // TODO: configure the resulting loading info to return an error on accessing the files + vec![] + }; + + let time_steps = create_gap_free_time_steps( + self.dataset_id, + query.query_rectangle.time_interval(), + &conn, + ) + .await + .map_err(|e| geoengine_operators::error::Error::MetaData { + source: Box::new(e), + })?; + + Ok(MultiBandGdalLoadingInfo::new( + time_steps, + files, + CacheHint::default(), // TODO: implement cache hint, should it be one value for the whole dataset? If so, load it once(!) from the database and add it to the loading info. Otherwise add the cache hint as a new attribute to the tiles. + )) + } + + async fn result_descriptor( + &self, + ) -> Result { + Ok(self.result_descriptor.clone()) + } + + fn box_clone( + &self, + ) -> Box< + dyn MetaData< + MultiBandGdalLoadingInfo, + RasterResultDescriptor, + MultiBandGdalLoadingInfoQueryRectangle, + >, + > { + Box::new(self.clone()) + } +} + +#[derive(Debug, Clone, PartialEq, ToSql, FromSql)] +struct TileKey { + time: crate::api::model::datatypes::TimeInterval, + bbox: SpatialPartition2D, + band: u32, + z_index: u32, +} + +identifier!(DatasetTileId); + +#[derive(Debug, Clone, PartialEq, ToSql, FromSql)] +struct TileEntry { + id: DatasetTileId, + dataset_id: DatasetId, + time: crate::api::model::datatypes::TimeInterval, + bbox: SpatialPartition2D, + band: u32, + z_index: u32, + gdal_params: GdalDatasetParameters, +} + #[async_trait] impl DatasetStore for PostgresDb where @@ -600,6 +1079,7 @@ where &self, dataset: AddDataset, meta_data: MetaDataDefinition, + data_path: Option, ) -> Result { let id = DatasetId::new(); let name = dataset.name.unwrap_or_else(|| DatasetName { @@ -615,6 +1095,8 @@ where self.check_dataset_namespace(&name)?; + // TODO: check `data_path` exists? + let typed_meta_data = meta_data.to_typed_metadata(); let mut conn = self.conn_pool.get().await?; @@ -633,9 +1115,10 @@ where meta_data, symbology, provenance, - tags + tags, + data_path ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::text[])", + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10::text[], $11)", &[ &id, &name, @@ -647,6 +1130,7 @@ where &dataset.symbology, &dataset.provenance, &dataset.tags, + &data_path, ], ) .await @@ -812,6 +1296,234 @@ where Ok(()) } + + async fn add_dataset_tiles( + &self, + dataset: DatasetId, + tiles: Vec, + ) -> Result<()> { + let mut conn = self.conn_pool.get().await?; + let tx = conn.build_transaction().start().await?; + + self.ensure_permission_in_tx(dataset.into(), Permission::Owner, &tx) + .await + .boxed_context(crate::error::PermissionDb)?; + + validate_time(&tx, dataset, &tiles).await?; + + validate_z_index(&tx, dataset, &tiles).await?; + + batch_insert_tiles(&tx, dataset, &tiles).await?; + + update_dataset_extents(&tx, dataset, &tiles).await?; + + tx.commit().await?; + + Ok(()) + } +} + +async fn validate_time( + tx: &Transaction<'_>, + dataset: DatasetId, + tiles: &[AddDatasetTile], +) -> Result<()> { + // validate the time of the tiles + let time_dim: TimeDimension = tx + .query_one( + r#" + SELECT (result_descriptor).raster."time".dimension + FROM datasets + WHERE id = $1; + "#, + &[&dataset], + ) + .await? + .get(0); + + match time_dim { + TimeDimension::Regular(regular_time_dimension) => { + // check all tiles if they fit to time step + let invalid_tiles = tiles + .iter() + .filter(|tile| !regular_time_dimension.valid_interval(tile.time.into())) + .collect::>(); + + if !invalid_tiles.is_empty() { + return Err(Error::DatasetTileRegularTimeConflict { + times: invalid_tiles.iter().map(|tile| tile.time).collect(), + time_dim: regular_time_dimension, + }); + } + } + TimeDimension::Irregular => { + // check if any tile conflicts with time of existing tiles. there must be no overlaps. + let times = tiles.iter().map(|tile| tile.time).collect::>(); + + let incompatible_times = tx + .query( + r#" + SELECT DISTINCT + dt.time, + nt + FROM + dataset_tiles dt JOIN + unnest($2::"TimeInterval"[]) as nt + ON (time_interval_intersects(dt.time, nt)) + WHERE + dataset_id = $1 AND ( + (dt.time).start != (nt).start OR + (dt.time)."end" != (nt)."end" + ); + "#, + &[&dataset, ×], + ) + .await?; + + if !incompatible_times.is_empty() { + return Err(Error::DatasetTileTimeConflict { + existing_times: incompatible_times.iter().map(|row| row.get(0)).collect(), + times: incompatible_times.iter().map(|row| row.get(1)).collect(), + }); + } + } + } + + Ok(()) +} + +async fn validate_z_index( + tx: &Transaction<'_>, + dataset: DatasetId, + tiles: &[AddDatasetTile], +) -> Result<()> { + // check, on spatial overlap, if the z-index is different than existing tiles for the same time step and band) + let tile_keys = tiles + .iter() + .map(|tile| TileKey { + time: tile.time, + bbox: tile.spatial_partition, + band: tile.band, + z_index: tile.z_index, + }) + .collect::>(); + let incompatible_z_index = tx + .query( + r#" + SELECT DISTINCT + (gdal_params).file_path + FROM + dataset_tiles dt, unnest($2::"TileKey"[]) as tk + WHERE + dataset_id = $1 AND + dt.time = tk.time AND + SPATIAL_PARTITION2D_INTERSECTS(dt.bbox, tk.bbox) AND + dt.band = tk.band AND + dt.z_index = tk.z_index + ; + "#, + &[&dataset, &tile_keys], + ) + .await?; + + if !incompatible_z_index.is_empty() { + return Err(Error::DatasetTileZIndexConflict { + files: incompatible_z_index.iter().map(|row| row.get(0)).collect(), + }); + } + + Ok(()) +} + +async fn batch_insert_tiles( + tx: &Transaction<'_>, + dataset: DatasetId, + tiles: &[AddDatasetTile], +) -> Result<()> { + // batch insert using array unnesting + let tile_entries = tiles + .iter() + .map(|tile| TileEntry { + id: DatasetTileId::new(), + dataset_id: dataset, + time: tile.time, + bbox: tile.spatial_partition, + band: tile.band, + z_index: tile.z_index, + gdal_params: tile.params.clone().into(), + }) + .collect::>(); + + tx.execute( + r#" + INSERT INTO dataset_tiles (id, dataset_id, time, bbox, band, z_index, gdal_params) + SELECT * FROM unnest($1::"TileEntry"[]); + "#, + &[&tile_entries], + ) + .await?; + + Ok(()) +} + +async fn update_dataset_extents( + tx: &Transaction<'_>, + dataset: DatasetId, + tiles: &[AddDatasetTile], +) -> Result<()> { + // update the dataset extents + let row = tx + .query_one( + r#" + SELECT + (result_descriptor).raster.spatial_grid.spatial_grid, + (result_descriptor).raster."time".bounds + FROM + datasets + WHERE + id = $1; + "#, + &[&dataset], + ) + .await?; + + let (mut dataset_grid, mut time_bounds): (SpatialGridDefinition, Option) = + (row.get(0), row.get(1)); + + for tile in tiles { + // TODO: handle datasets with flipped y axis? + let tile_grid = SpatialGridDefinition::new( + GdalDatasetGeoTransform::from(tile.params.geo_transform).try_into()?, + GridBoundingBox2D::new_unchecked( + [0, 0], + [tile.params.height as isize, tile.params.width as isize], + ), + ); + + dataset_grid = dataset_grid + .merge(&tile_grid) + .expect("grids should be compatible because the compatibility was checked before inserting tiles"); + + if let Some(time_bounds) = &mut time_bounds { + *time_bounds = time_bounds.extend(&tile.time.into()); + } + } + + tx.execute( + r#" + UPDATE datasets + SET + result_descriptor.raster.spatial_grid.spatial_grid = $2, + result_descriptor.raster."time".bounds = $3, + meta_data.gdal_multi_band.result_descriptor.spatial_grid.spatial_grid = $2, + meta_data.gdal_multi_band.result_descriptor."time".bounds = $3 + WHERE id = $1; + "#, + &[&dataset, &dataset_grid, &time_bounds], + ) + .await?; + + Ok(()) } #[async_trait] @@ -888,8 +1600,7 @@ mod tests { use super::*; use crate::{ - contexts::PostgresContext, - contexts::{ApplicationContext, SessionContext}, + contexts::{ApplicationContext, PostgresContext, SessionContext}, ge_context, permissions::PermissionDb, users::{UserAuth, UserSession}, @@ -1047,6 +1758,7 @@ mod tests { tags: Some(vec!["upload".to_owned(), "test".to_owned()]), }, meta_data, + None, ) .await .unwrap() diff --git a/services/src/datasets/storage.rs b/services/src/datasets/storage.rs index a57c539d4..d96404566 100755 --- a/services/src/datasets/storage.rs +++ b/services/src/datasets/storage.rs @@ -1,6 +1,7 @@ use super::listing::Provenance; use super::postgres::DatasetMetaData; use super::{DatasetIdAndName, DatasetName}; +use crate::api::handlers::datasets::AddDatasetTile; use crate::api::model::services::{DataPath, UpdateDataset}; use crate::datasets::listing::{DatasetListing, DatasetProvider}; use crate::datasets::upload::UploadDb; @@ -11,7 +12,7 @@ use async_trait::async_trait; use geoengine_datatypes::dataset::DatasetId; use geoengine_datatypes::primitives::VectorQueryRectangle; use geoengine_operators::engine::{MetaData, TypedResultDescriptor}; -use geoengine_operators::source::{GdalMetaDataList, GdalMetadataNetCdfCf}; +use geoengine_operators::source::{GdalMetaDataList, GdalMetadataNetCdfCf, GdalMultiBand}; use geoengine_operators::{engine::StaticMetaData, source::OgrSourceDataset}; use geoengine_operators::{engine::VectorResultDescriptor, source::GdalMetaDataRegular}; use geoengine_operators::{mock::MockDatasetDataSourceLoadingInfo, source::GdalMetaDataStatic}; @@ -24,20 +25,19 @@ use validator::{Validate, ValidationError}; pub const DATASET_DB_ROOT_COLLECTION_ID: Uuid = Uuid::from_u128(0x5460_73b6_d535_4205_b601_9967_5c9f_6dd7); -#[derive(Debug, Serialize, Deserialize, Clone, ToSchema, Validate)] +#[derive(Debug, Serialize, Deserialize, Clone, Validate)] #[serde(rename_all = "camelCase")] pub struct Dataset { - #[schema(value_type = crate::api::model::datatypes::DatasetId)] pub id: DatasetId, pub name: DatasetName, pub display_name: String, pub description: String, - #[schema(value_type = crate::api::model::operators::TypedResultDescriptor)] pub result_descriptor: TypedResultDescriptor, pub source_operator: String, pub symbology: Option, pub provenance: Option>, pub tags: Option>, + pub data_path: Option, } impl Dataset { @@ -49,7 +49,8 @@ impl Dataset { description: self.description.clone(), tags: self.tags.clone().unwrap_or_default(), // TODO: figure out if we want to use Option> everywhere or if Vec is fine source_operator: self.source_operator.clone(), - result_descriptor: self.result_descriptor.clone(), + // convert the TypedResultDescriptor to the API one + result_descriptor: self.result_descriptor.clone().into(), symbology: self.symbology.clone(), } } @@ -146,6 +147,7 @@ pub enum MetaDataDefinition { GdalMetadataNetCdfCf(GdalMetadataNetCdfCf), GdalMetaDataList(GdalMetaDataList), + GdalMultiBand(GdalMultiBand), } impl From> @@ -182,6 +184,12 @@ impl From for MetaDataDefinition { } } +impl From for MetaDataDefinition { + fn from(meta_data: GdalMultiBand) -> Self { + MetaDataDefinition::GdalMultiBand(meta_data) + } +} + impl MetaDataDefinition { pub fn source_operator_type(&self) -> &str { match self { @@ -191,6 +199,7 @@ impl MetaDataDefinition { | MetaDataDefinition::GdalStatic(_) | MetaDataDefinition::GdalMetadataNetCdfCf(_) | MetaDataDefinition::GdalMetaDataList(_) => "GdalSource", + MetaDataDefinition::GdalMultiBand(_) => "MultiBandGdalSource", } } @@ -202,6 +211,7 @@ impl MetaDataDefinition { MetaDataDefinition::GdalStatic(_) => "GdalStatic", MetaDataDefinition::GdalMetadataNetCdfCf(_) => "GdalMetadataNetCdfCf", MetaDataDefinition::GdalMetaDataList(_) => "GdalMetaDataList", + MetaDataDefinition::GdalMultiBand(_) => "GdalMultiBand", } } @@ -237,6 +247,7 @@ impl MetaDataDefinition { .await .map(Into::into) .map_err(Into::into), + MetaDataDefinition::GdalMultiBand(m) => Ok(m.result_descriptor.clone().into()), } } @@ -266,6 +277,10 @@ impl MetaDataDefinition { meta_data: self, result_descriptor: TypedResultDescriptor::from(d.result_descriptor.clone()), }, + MetaDataDefinition::GdalMultiBand(d) => DatasetMetaData { + meta_data: self, + result_descriptor: TypedResultDescriptor::from(d.result_descriptor.clone()), + }, } } } @@ -281,6 +296,7 @@ pub trait DatasetStore { &self, dataset: AddDataset, meta_data: MetaDataDefinition, + data_path: Option, // TODO: make mandatory when implementing Volumes RFC ) -> Result; async fn update_dataset(&self, dataset: DatasetId, update: UpdateDataset) -> Result<()>; @@ -304,4 +320,7 @@ pub trait DatasetStore { ) -> Result<()>; async fn delete_dataset(&self, dataset: DatasetId) -> Result<()>; + + async fn add_dataset_tiles(&self, dataset: DatasetId, tiles: Vec) + -> Result<()>; } diff --git a/services/src/datasets/upload.rs b/services/src/datasets/upload.rs index 215678ff3..6d5d97966 100644 --- a/services/src/datasets/upload.rs +++ b/services/src/datasets/upload.rs @@ -9,8 +9,6 @@ use crate::{ error, }; use async_trait::async_trait; -use geoengine_datatypes::test_data; -use geoengine_datatypes::util::test::TestDefault; use serde::{Deserialize, Deserializer, Serialize}; use utoipa::ToSchema; @@ -71,28 +69,59 @@ pub struct Volumes { impl Default for Volumes { fn default() -> Self { - Self { - volumes: crate::config::get_config_element::() + // TODO: implement proper volume management in the database + #[cfg(test)] + { + let exe_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .canonicalize() // get a full path + .expect("should be available during testing") + .parent() + .expect("should be available during testing") + .to_path_buf(); + + let volumes = crate::config::get_config_element::() .expect("volumes should be defined, because they are in the default config") .volumes .into_iter() - .map(|(name, path)| Volume { name, path }) - .collect::>(), + .map(|(name, path)| { + // make relative paths absolute with respect to executable for tests (because tests have different working directories depending on how they are run) + let path = if path.is_relative() { + exe_dir.join(path) + } else { + path + }; + + Volume { name, path } + }) + .collect::>(); + + Self { volumes } } - } -} + #[cfg(not(test))] + { + let volumes = crate::config::get_config_element::() + .expect("volumes should be defined, because they are in the default config") + .volumes + .into_iter() + .map(|(name, path)| Volume { name, path }) + .collect::>(); -impl TestDefault for Volumes { - fn test_default() -> Self { - Self { - volumes: vec![Volume { - name: VolumeName("test_data".to_string()), - path: test_data!("").to_path_buf(), - }], + Self { volumes } } } } +// impl TestDefault for Volumes { +// fn test_default() -> Self { +// Self { +// volumes: vec![Volume { +// name: VolumeName("test_data".to_string()), +// path: test_data!("").to_path_buf(), +// }], +// } +// } +// } + pub trait UploadRootPath { fn root_path(&self) -> Result; } diff --git a/services/src/error.rs b/services/src/error.rs index 595668f22..4206addda 100644 --- a/services/src/error.rs +++ b/services/src/error.rs @@ -1,5 +1,5 @@ use crate::api::model::datatypes::{ - DatasetId, SpatialReference, SpatialReferenceOption, TimeInstance, + DatasetId, SpatialReference, SpatialReferenceOption, TimeInstance, TimeInterval, }; use crate::api::model::responses::ErrorResponse; use crate::datasets::external::aruna::error::ArunaProviderError; @@ -9,6 +9,7 @@ use actix_web::HttpResponse; use actix_web::http::StatusCode; use geoengine_datatypes::dataset::{DataProviderId, LayerId}; use geoengine_datatypes::error::ErrorSource; +use geoengine_datatypes::primitives::RegularTimeDimension; use geoengine_datatypes::util::helpers::ge_report; use ordered_float::FloatIsNan; use snafu::prelude::*; @@ -76,6 +77,11 @@ pub enum Error { source: serde_urlencoded::de::Error, }, + #[snafu(display("Unable to serialize query string: {}", source))] + UnableToSerializeQueryString { + source: serde_urlencoded::ser::Error, + }, + ServerStartup, #[snafu(display("Registration failed: {}", reason))] @@ -550,6 +556,26 @@ pub enum Error { Wildlive { source: crate::datasets::external::WildliveError, }, + #[snafu(display( + "Dataset tile time `{times:?}` conflict with existing times `{existing_times:?}`" + ))] + DatasetTileTimeConflict { + times: Vec, + existing_times: Vec, + }, + #[snafu(display( + "Dataset tile times `{times:?}` conflict with dataset regularity {time_dim:?}" + ))] + DatasetTileRegularTimeConflict { + times: Vec, + time_dim: RegularTimeDimension, + }, + #[snafu(display( + "Dataset tile z-index of files `{files:?}` conflict with existing tiles with the same z-indexes" + ))] + DatasetTileZIndexConflict { + files: Vec, + }, } impl actix_web::error::ResponseError for Error { @@ -595,6 +621,18 @@ impl From for Error { } } +impl From for Error { + fn from(e: serde_urlencoded::de::Error) -> Self { + Self::UnableToParseQueryString { source: e } + } +} + +impl From for Error { + fn from(e: serde_urlencoded::ser::Error) -> Self { + Self::UnableToSerializeQueryString { source: e } + } +} + impl From for Error { fn from(e: std::io::Error) -> Self { Self::Io { source: e } diff --git a/services/src/layers/add_from_directory.rs b/services/src/layers/add_from_directory.rs index b68202de7..4e3cb9b6f 100644 --- a/services/src/layers/add_from_directory.rs +++ b/services/src/layers/add_from_directory.rs @@ -274,7 +274,7 @@ pub async fn add_datasets_from_directory( serde_json::from_reader(BufReader::new(File::open(entry.path())?))?; let dataset_id: DatasetId = db - .add_dataset(def.properties.clone(), def.meta_data.clone()) + .add_dataset(def.properties.clone(), def.meta_data.clone(), None) .await? .id; diff --git a/services/src/util/mod.rs b/services/src/util/mod.rs index 3c1183ffd..a05a1bd44 100644 --- a/services/src/util/mod.rs +++ b/services/src/util/mod.rs @@ -24,6 +24,7 @@ pub mod openapi_visitors; pub mod operators; pub mod parsing; pub mod postgres; +pub mod sentinel_2_utm_zones; pub mod server; // TODO: refactor to be gated by `#[cfg(test)]` pub mod tests; diff --git a/services/src/util/operators.rs b/services/src/util/operators.rs index 0f995de0f..510e99d00 100644 --- a/services/src/util/operators.rs +++ b/services/src/util/operators.rs @@ -3,7 +3,10 @@ use geoengine_datatypes::dataset::NamedData; use geoengine_operators::{ engine::{OperatorName, RasterOperator, TypedOperator, VectorOperator}, mock::{MockDatasetDataSource, MockDatasetDataSourceParams}, - source::{GdalSource, GdalSourceParameters, OgrSource, OgrSourceParameters}, + source::{ + GdalSource, GdalSourceParameters, MultiBandGdalSource, MultiBandGdalSourceParameters, + OgrSource, OgrSourceParameters, + }, }; pub fn source_operator_from_dataset( @@ -23,7 +26,7 @@ pub fn source_operator_from_dataset( ), GdalSource::TYPE_NAME => TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: name.clone() }, + params: GdalSourceParameters::new(name.clone()), } .boxed(), ), @@ -33,6 +36,12 @@ pub fn source_operator_from_dataset( } .boxed(), ), + MultiBandGdalSource::TYPE_NAME => TypedOperator::Raster( + MultiBandGdalSource { + params: MultiBandGdalSourceParameters::new(name.clone()), + } + .boxed(), + ), s => { return Err(crate::error::Error::UnknownOperator { operator: s.to_owned(), diff --git a/services/src/util/sentinel_2_utm_zones.rs b/services/src/util/sentinel_2_utm_zones.rs new file mode 100644 index 000000000..26471b9c9 --- /dev/null +++ b/services/src/util/sentinel_2_utm_zones.rs @@ -0,0 +1,133 @@ +use std::str::FromStr; + +use geoengine_datatypes::{ + primitives::{Coordinate2D, SpatialPartition2D}, + spatial_reference::{SpatialReference, SpatialReferenceAuthority}, +}; +use snafu::Snafu; +use strum::IntoStaticStr; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct UtmZone { + pub zone: u8, + pub direction: UtmZoneDirection, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum UtmZoneDirection { + North, + South, +} + +#[derive(Debug, Snafu, IntoStaticStr)] +pub enum UtmZoneError { + IdMustStartWithUtm, + DirectionNotNorthOrSouth, + ZoneSuffixNotANumber, +} + +impl UtmZone { + pub fn epsg_code(self) -> u32 { + match self.direction { + UtmZoneDirection::North => 32600 + u32::from(self.zone), + UtmZoneDirection::South => 32700 + u32::from(self.zone), + } + } + + pub fn spatial_reference(self) -> SpatialReference { + SpatialReference::new(SpatialReferenceAuthority::Epsg, self.epsg_code()) + } + + pub fn extent(self) -> Option { + // TODO: as Sentinel uses enlarged grids, we could return a larger extent + self.spatial_reference().area_of_use().ok() + } + + pub fn native_extent(self) -> SpatialPartition2D { + match (self.zone, self.direction) { + (32 | 34 | 36, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 8_000_040.0), + Coordinate2D::new(909_780.0, -9_780.0), + ), + (60, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 9_100_020.0), + Coordinate2D::new(809_760.0, -9_780.0), + ), + (_, UtmZoneDirection::North) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 9_400_020.0), + Coordinate2D::new(909_780.0, -9_780.0), + ), + (1, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(99_960.0, 10_000_000.0), + Coordinate2D::new(909_780.0, 690_220.0), + ), + (60, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 10_000_000.0), + Coordinate2D::new(809_760.0, 890_200.0), + ), + (_, UtmZoneDirection::South) => SpatialPartition2D::new_unchecked( + Coordinate2D::new(199_980.0, 10_000_000.0), + Coordinate2D::new(909_780.0, 690_220.0), + ), + } + } +} + +impl FromStr for UtmZone { + type Err = UtmZoneError; + + fn from_str(s: &str) -> Result { + if s.len() < 5 || &s[..3] != "UTM" { + return Err(UtmZoneError::IdMustStartWithUtm); + } + + let (zone_str, dir_char) = s[3..].split_at(s.len() - 4); + let zone = zone_str + .parse::() + .map_err(|_| UtmZoneError::ZoneSuffixNotANumber)?; + + // TODO: check if zone is in valid range + + let north = match dir_char { + "N" => UtmZoneDirection::North, + "S" => UtmZoneDirection::South, + _ => return Err(UtmZoneError::DirectionNotNorthOrSouth), + }; + + Ok(Self { + zone, + direction: north, + }) + } +} + +impl std::fmt::Display for UtmZone { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!( + f, + "UTM{}{}", + self.zone, + match self.direction { + UtmZoneDirection::North => "N", + UtmZoneDirection::South => "S", + } + ) + } +} + +impl UtmZone { + pub fn zones() -> impl Iterator { + (1..=60).flat_map(|zone| { + vec![ + UtmZone { + zone, + direction: UtmZoneDirection::North, + }, + UtmZone { + zone, + direction: UtmZoneDirection::South, + }, + ] + }) + } +} diff --git a/services/src/util/tests.rs b/services/src/util/tests.rs index 8eb2a9a9a..9cd7c7a42 100644 --- a/services/src/util/tests.rs +++ b/services/src/util/tests.rs @@ -51,7 +51,8 @@ use geoengine_datatypes::operations::image::RasterColorizer; use geoengine_datatypes::operations::image::RgbaColor; use geoengine_datatypes::primitives::CacheTtlSeconds; use geoengine_datatypes::primitives::Coordinate2D; -use geoengine_datatypes::primitives::SpatialResolution; +use geoengine_datatypes::raster::GeoTransform; +use geoengine_datatypes::raster::GridBoundingBox2D; use geoengine_datatypes::raster::RasterDataType; use geoengine_datatypes::raster::RenameBands; use geoengine_datatypes::spatial_reference::SpatialReference; @@ -59,12 +60,13 @@ use geoengine_datatypes::spatial_reference::SpatialReferenceOption; use geoengine_datatypes::test_data; use geoengine_datatypes::util::test::TestDefault; use geoengine_datatypes::{primitives::DateTime, raster::TilingSpecification}; -use geoengine_operators::engine::QueryContext; use geoengine_operators::engine::RasterBandDescriptor; use geoengine_operators::engine::RasterBandDescriptors; use geoengine_operators::engine::RasterResultDescriptor; +use geoengine_operators::engine::SpatialGridDescriptor; use geoengine_operators::engine::WorkflowOperatorPath; use geoengine_operators::engine::{ChunkByteSize, MultipleRasterSources}; +use geoengine_operators::engine::{QueryContext, TimeDescriptor}; use geoengine_operators::engine::{RasterOperator, TypedOperator}; use geoengine_operators::meta::quota::QuotaTracking; use geoengine_operators::processing::RasterStacker; @@ -149,7 +151,7 @@ pub async fn register_ndvi_workflow_helper_with_cache_ttl( let workflow = Workflow { operator: TypedOperator::Raster( GdalSource { - params: GdalSourceParameters { data: dataset }, + params: GdalSourceParameters::new(dataset), } .boxed(), ), @@ -208,7 +210,7 @@ pub async fn add_ndvi_to_datasets_with_cache_ttl( let ctx = app_ctx.session_context(session); let dataset_id = ctx .db() - .add_dataset(ndvi.properties, ndvi.meta_data) + .add_dataset(ndvi.properties, ndvi.meta_data, None) .await .expect("dataset db access") .id; @@ -240,7 +242,7 @@ pub async fn add_ndvi_to_datasets_with_cache_ttl( pub async fn add_land_cover_to_datasets(db: &D) -> DatasetName { let ndvi = DatasetDefinition { properties: AddDataset { - name: None, + name: Some(DatasetName::new(None, "land_cover_raster_test".to_string())), display_name: "Land Cover".to_string(), description: "Land Cover derived from MODIS/Terra+Aqua Land Cover".to_string(), source_operator: "GdalSource".to_string(), @@ -305,12 +307,11 @@ pub async fn add_land_cover_to_datasets(db: &D) -> DatasetName { result_descriptor: RasterResultDescriptor { data_type: RasterDataType::U8, spatial_reference: SpatialReferenceOption::SpatialReference(SpatialReference::epsg_4326()), - time: Some(geoengine_datatypes::primitives::TimeInterval::default()), - bbox: Some(geoengine_datatypes::primitives::SpatialPartition2D::new((-180., 90.).into(), - (180., -90.).into()).unwrap()), - resolution: Some(SpatialResolution { - x: 0.1, y: 0.1, - }), + time: TimeDescriptor::new_irregular(Some(geoengine_datatypes::primitives::TimeInterval::default())), + spatial_grid: SpatialGridDescriptor::source_from_parts( + GeoTransform::new(Coordinate2D::new(-180., 90.), 0.1, -0.1), + GridBoundingBox2D::new_min_max(0,1799, 0, 1599).unwrap(), + ), bands: RasterBandDescriptors::new(vec![RasterBandDescriptor::new("band".into(), geoengine_datatypes::primitives::Measurement::classification("Land Cover".to_string(), [ (0_u8, "Water Bodies".to_string()), @@ -336,7 +337,7 @@ pub async fn add_land_cover_to_datasets(db: &D) -> DatasetName { }), }; - db.add_dataset(ndvi.properties, ndvi.meta_data) + db.add_dataset(ndvi.properties, ndvi.meta_data, None) .await .expect("dataset db access") .name @@ -380,18 +381,21 @@ pub async fn register_ne2_multiband_workflow( GdalSource { params: GdalSourceParameters { data: blue.name.into(), + overview_level: None, }, } .boxed(), GdalSource { params: GdalSourceParameters { data: green.name.into(), + overview_level: None, }, } .boxed(), GdalSource { params: GdalSourceParameters { data: red.name.into(), + overview_level: None, }, } .boxed(), @@ -471,7 +475,7 @@ pub async fn add_file_definition_to_datasets( }; let dataset = db - .add_dataset(def.properties.clone(), def.meta_data.clone()) + .add_dataset(def.properties.clone(), def.meta_data.clone(), None) .await .unwrap(); @@ -876,7 +880,7 @@ pub async fn add_ndvi_to_datasets2> let db = app_ctx.session_context(system_session).db(); let dataset_id = db - .add_dataset(ndvi.properties, ndvi.meta_data) + .add_dataset(ndvi.properties, ndvi.meta_data, None) .await .expect("dataset db access") .id; @@ -933,7 +937,7 @@ pub async fn add_ports_to_datasets> let db = app_ctx.session_context(system_session).db(); let dataset_id = db - .add_dataset(ndvi.properties, ndvi.meta_data) + .add_dataset(ndvi.properties, ndvi.meta_data, None) .await .expect("dataset db access") .id; diff --git a/services/src/workflows/websocket_stream.rs b/services/src/workflows/websocket_stream.rs index f9aefccce..60dafcc50 100644 --- a/services/src/workflows/websocket_stream.rs +++ b/services/src/workflows/websocket_stream.rs @@ -10,8 +10,8 @@ use geoengine_datatypes::{ use geoengine_operators::{ call_on_generic_raster_processor, call_on_generic_vector_processor, engine::{ - QueryAbortTrigger, QueryContext, QueryProcessorExt, RasterOperator, VectorOperator, - WorkflowOperatorPath, + InitializedRasterOperator, QueryAbortTrigger, QueryContext, QueryProcessorExt, + RasterOperator, VectorOperator, WorkflowOperatorPath, }, }; use tokio::{ @@ -30,18 +30,11 @@ pub struct WebsocketStreamTask { } impl WebsocketStreamTask { - pub async fn new_raster( - raster_operator: Box, + pub async fn new_raster_initialized( + initialized_operator: Box, query_rectangle: RasterQueryRectangle, - execution_ctx: C::ExecutionContext, - mut query_ctx: C::QueryContext, + mut query_ctx: C, ) -> Result { - let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); - - let initialized_operator = raster_operator - .initialize(workflow_operator_path_root, &execution_ctx) - .await?; - let spatial_reference = initialized_operator.result_descriptor().spatial_reference; let query_processor = initialized_operator.query_processor()?; @@ -68,6 +61,21 @@ impl WebsocketStreamTask { )) } + pub async fn new_raster( + raster_operator: Box, + query_rectangle: RasterQueryRectangle, + execution_ctx: C::ExecutionContext, + query_ctx: C::QueryContext, + ) -> Result { + let workflow_operator_path_root = WorkflowOperatorPath::initialize_root(); + + let initialized_operator = raster_operator + .initialize(workflow_operator_path_root, &execution_ctx) + .await?; + + Self::new_raster_initialized(initialized_operator, query_rectangle, query_ctx).await + } + pub async fn new_vector( vector_operator: Box, query_rectangle: VectorQueryRectangle, diff --git a/services/src/workflows/workflow.rs b/services/src/workflows/workflow.rs index 2cbab5f5b..7f1d09b24 100644 --- a/services/src/workflows/workflow.rs +++ b/services/src/workflows/workflow.rs @@ -47,9 +47,7 @@ mod tests { let workflow = Workflow { operator: TypedOperator::Vector( MockPointSource { - params: MockPointSourceParams { - points: vec![Coordinate2D::new(1., 2.); 3], - }, + params: MockPointSourceParams::new(vec![Coordinate2D::new(1., 2.); 3]), } .boxed(), ), @@ -73,7 +71,10 @@ mod tests { }, { "x": 1.0, "y": 2.0 - }] + }], + "spatialBounds": { + "type": "none" + } } } }) diff --git a/test_data/api_calls/force/force.http b/test_data/api_calls/force/force.http new file mode 100644 index 000000000..ac0874a74 --- /dev/null +++ b/test_data/api_calls/force/force.http @@ -0,0 +1,45 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "LND05_BOA" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} +@crs = EPSG:3035 +@width = 1000 +@height = 2000 +@time = 2011-11-14T00%3A00%3A00.000Z +@minx = 4226026.3630416505 +@miny = 3044919.6079648044 + +# pixel size: 30 +@maxx = 4256026.3630416505 +@maxy = 3104919.6079648044 + +@colorizer_min = -1000 +@colorizer_max = 10000 +@styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS={{crs}}&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} + diff --git a/test_data/api_calls/force/zoomed_out.http b/test_data/api_calls/force/zoomed_out.http new file mode 100644 index 000000000..8a8582b4e --- /dev/null +++ b/test_data/api_calls/force/zoomed_out.http @@ -0,0 +1,44 @@ + +### + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "LND05_BOA" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} +@crs = EPSG:3035 +@width = 1024 +@height = 1024 +@time = 2011-11-14T00%3A00%3A00.000Z +@minx = 1908523.29 +@miny = 1137678.21 + +@maxx = 6901611.5 +@maxy = 6872461.46 + +@colorizer_min = -1000 +@colorizer_max = 10000 +@styles = custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A{{colorizer_min}}%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A{{colorizer_max}}%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES={{styles}}&TRANSPARENT=true&layers={{workflowId}}&time={{time}}&EXCEPTIONS=application%2Fjson&WIDTH={{width}}&HEIGHT={{height}}&CRS={{crs}}&BBOX={{miny}}%2C{{minx}}%2C{{maxy}}%2C{{maxx}} +Authorization: Bearer {{anonymousSession.response.body.$.id}} + diff --git a/test_data/api_calls/force/zoomed_out_4326.http b/test_data/api_calls/force/zoomed_out_4326.http new file mode 100644 index 000000000..54123501c --- /dev/null +++ b/test_data/api_calls/force/zoomed_out_4326.http @@ -0,0 +1,32 @@ + +### +### first run the time_import example: `cargo run --example tile_import` + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "LND07_BOA" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A1%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A255%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B255%2C255%2C255%2C127%5D%2C%22underColor%22%3A%5B0%2C0%2C0%2C127%5D%7D%7D&TRANSPARENT=TRUE&layers={{workflowId}}&time=2014-01-07T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=256&HEIGHT=256&CRS=EPSG%3A4326&BBOX=50.625%2C8.4375%2C52.03125%2C9.84375 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + diff --git a/test_data/api_calls/force/zoomed_out_4326_time_at_nodata.http b/test_data/api_calls/force/zoomed_out_4326_time_at_nodata.http new file mode 100644 index 000000000..7a807eb16 --- /dev/null +++ b/test_data/api_calls/force/zoomed_out_4326_time_at_nodata.http @@ -0,0 +1,32 @@ + +### +### first run the time_import example: `cargo run --example tile_import` + +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "LND07_BOA" + } + } +} + +### + +@workflowId = {{workflow.response.body.$.id}} + +GET http://localhost:3030/api/wms/{{workflowId}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A1%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A255%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B255%2C255%2C255%2C127%5D%2C%22underColor%22%3A%5B0%2C0%2C0%2C127%5D%7D%7D&TRANSPARENT=TRUE&layers={{workflowId}}&time=2014-01-01T12%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=256&HEIGHT=256&CRS=EPSG%3A4326&BBOX=50.625%2C8.4375%2C52.03125%2C9.84375 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + diff --git a/test_data/api_calls/multi_tile.http b/test_data/api_calls/multi_tile.http new file mode 100644 index 000000000..fe1da5d59 --- /dev/null +++ b/test_data/api_calls/multi_tile.http @@ -0,0 +1,47 @@ +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### +# @name dataset +POST http://localhost:3030/api/dataset +Content-Type: application/json +Authorization: Bearer {{anonymousSession.response.body.$.id}} + +< ../raster/multi_tile/metadata/dataset.json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "{{dataset.response.body.$.datasetName}}" + } + } +} + +### + +POST http://localhost:3030/api/dataset/{{dataset.response.body.$.datasetName}}/tiles +Content-Type: application/json +Authorization: Bearer {{anonymousSession.response.body.$.id}} + +< ../raster/multi_tile/metadata/loading_info.json + + +### +@colorizer = %7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22palette%22%2C%22colors%22%3A%7B%2210000%22%3A%5B31%2C119%2C180%2C255%5D%2C%2210001%22%3A%5B174%2C199%2C232%2C255%5D%2C%2210010%22%3A%5B255%2C127%2C14%2C255%5D%2C%2210011%22%3A%5B44%2C160%2C44%2C255%5D%2C%2210100%22%3A%5B152%2C223%2C138%2C255%5D%2C%2210101%22%3A%5B214%2C39%2C40%2C255%5D%2C%2210110%22%3A%5B148%2C103%2C189%2C255%5D%2C%2210111%22%3A%5B197%2C176%2C213%2C255%5D%2C%2211000%22%3A%5B140%2C86%2C75%2C255%5D%2C%2211001%22%3A%5B227%2C119%2C194%2C255%5D%2C%2211010%22%3A%5B247%2C182%2C210%2C255%5D%2C%2211011%22%3A%5B127%2C127%2C127%2C255%5D%2C%2211100%22%3A%5B188%2C189%2C34%2C255%5D%2C%2211101%22%3A%5B219%2C219%2C141%2C255%5D%2C%2211110%22%3A%5B23%2C190%2C207%2C255%5D%2C%2211111%22%3A%5B158%2C218%2C229%2C255%5D%7D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22defaultColor%22%3A%5B0%2C0%2C0%2C0%5D%7D%7D + +### + +GET http://localhost:3030/api/wms/{{workflow.response.body.$.id}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A{{colorizer}}&TRANSPARENT=true&layers={{workflow.response.body.$.id}}&time=2025-01-01T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=1800&HEIGHT=900&CRS=EPSG%3A4326&BBOX=-90%2C-180%2C90%2C180 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + + diff --git a/test_data/api_calls/multi_tile_overview.http b/test_data/api_calls/multi_tile_overview.http new file mode 100644 index 000000000..ebe2ed24e --- /dev/null +++ b/test_data/api_calls/multi_tile_overview.http @@ -0,0 +1,48 @@ +# @name anonymousSession +POST http://localhost:3030/api/anonymous +Content-Type: application/json + +### +# @name dataset +POST http://localhost:3030/api/dataset +Content-Type: application/json +Authorization: Bearer {{anonymousSession.response.body.$.id}} + +< ../raster/multi_tile/metadata/dataset.json + +### + +# @name workflow +POST http://localhost:3030/api/workflow +Authorization: Bearer {{anonymousSession.response.body.$.id}} +Content-Type: application/json + +{ + "type": "Raster", + "operator": { + "type": "MultiBandGdalSource", + "params": { + "data": "{{dataset.response.body.$.datasetName}}", + "overviewLevel": 2 + } + } +} + +### + +POST http://localhost:3030/api/dataset/{{dataset.response.body.$.datasetName}}/tiles +Content-Type: application/json +Authorization: Bearer {{anonymousSession.response.body.$.id}} + +< ../raster/multi_tile/metadata/loading_info.json + + +### +@colorizer = %7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22palette%22%2C%22colors%22%3A%7B%2210000%22%3A%5B31%2C119%2C180%2C255%5D%2C%2210001%22%3A%5B174%2C199%2C232%2C255%5D%2C%2210010%22%3A%5B255%2C127%2C14%2C255%5D%2C%2210011%22%3A%5B44%2C160%2C44%2C255%5D%2C%2210100%22%3A%5B152%2C223%2C138%2C255%5D%2C%2210101%22%3A%5B214%2C39%2C40%2C255%5D%2C%2210110%22%3A%5B148%2C103%2C189%2C255%5D%2C%2210111%22%3A%5B197%2C176%2C213%2C255%5D%2C%2211000%22%3A%5B140%2C86%2C75%2C255%5D%2C%2211001%22%3A%5B227%2C119%2C194%2C255%5D%2C%2211010%22%3A%5B247%2C182%2C210%2C255%5D%2C%2211011%22%3A%5B127%2C127%2C127%2C255%5D%2C%2211100%22%3A%5B188%2C189%2C34%2C255%5D%2C%2211101%22%3A%5B219%2C219%2C141%2C255%5D%2C%2211110%22%3A%5B23%2C190%2C207%2C255%5D%2C%2211111%22%3A%5B158%2C218%2C229%2C255%5D%7D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22defaultColor%22%3A%5B0%2C0%2C0%2C0%5D%7D%7D + +### + +GET http://localhost:3030/api/wms/{{workflow.response.body.$.id}}?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A{{colorizer}}&TRANSPARENT=true&layers={{workflow.response.body.$.id}}&time=2025-01-01T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=900&HEIGHT=450&CRS=EPSG%3A4326&BBOX=-90%2C-180%2C90%2C180 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + + diff --git a/test_data/api_calls/wcs.http b/test_data/api_calls/wcs.http index eb2e0f242..1b1babc39 100644 --- a/test_data/api_calls/wcs.http +++ b/test_data/api_calls/wcs.http @@ -23,23 +23,10 @@ Content-Type: application/json ### - -# @name workflow -POST http://localhost:3030/api/workflow +GET http://localhost:3030/api/wcs/{{workflow.response.body.$.id}}?SERVICE=WCS&REQUEST=DescribeCoverage&VERSION=1.1.1&IDENTIFIERS={{workflow.response.body.$.id}}&FORMAT=text/xml&crs=urn:ogc:def:crs:EPSG::4326 Authorization: Bearer {{anonymousSession.response.body.$.id}} -Content-Type: application/json - -{ - "type": "Raster", - "operator": { - "type": "GdalSource", - "params": { - "data": "ndvi_3857" - } - } -} ### -GET http://localhost:4200/api/wcs/{{workflow.response.body.$.id}}?SERVICE=WCS&REQUEST=DescribeCoverage&VERSION=1.1.1&IDENTIFIERS={{workflow.response.body.$.id}}&FORMAT=text/xml&crs=urn:ogc:def:crs:EPSG::4326 -Authorization: Bearer {{anonymousSession.response.body.$.id}} \ No newline at end of file +GET http://localhost:3030/api/wcs/{{workflow.response.body.$.id}}?SERVICE=WCS&REQUEST=GetCoverage&VERSION=1.1.1&IDENTIFIER={{workflow.response.body.$.id}}&FORMAT=image/tiff&CRS=urn:ogc:def:crs:EPSG::4326&BOUNDINGBOX=20,-10,80,50&GRIDCS=urn:ogc:def:cs:OGC:0.0:Grid2dSquareCS&GRIDTYPE=urn:ogc:def:method:WCS:1.1:2dSimpleGrid&GRIDORIGIN=80,-10&GRIDOFFSETS=0.7,0.7&TIME=2014-01-01T00:00:00.0Z&NODATAVALUE=0.0 +Authorization: Bearer {{anonymousSession.response.body.$.id}} diff --git a/test_data/api_calls/wms.http b/test_data/api_calls/wms.http index 509c3b7a9..b852f856c 100644 --- a/test_data/api_calls/wms.http +++ b/test_data/api_calls/wms.http @@ -22,6 +22,17 @@ Content-Type: application/json } } +### + +GET http://localhost:3030/api/wms/1e415c9c-55f3-51a2-b50b-b5053d1debbb?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A0%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A255%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D&TRANSPARENT=true&layers=1e415c9c-55f3-51a2-b50b-b5053d1debbb&time=2014-01-01T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=360&HEIGHT=180&CRS=EPSG:4326&BBOX=-90.0%2C-180.0%2C90.0%2C180.0 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + +### + + +GET http://localhost:3030/api/wms/1e415c9c-55f3-51a2-b50b-b5053d1debbb?REQUEST=GetMap&SERVICE=WMS&VERSION=1.3.0&FORMAT=image%2Fpng&STYLES=custom%3A%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22linearGradient%22%2C%22breakpoints%22%3A%5B%7B%22value%22%3A0%2C%22color%22%3A%5B0%2C0%2C0%2C255%5D%7D%2C%7B%22value%22%3A255%2C%22color%22%3A%5B255%2C255%2C255%2C255%5D%7D%5D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22overColor%22%3A%5B246%2C250%2C254%2C255%5D%2C%22underColor%22%3A%5B247%2C251%2C255%2C255%5D%7D%7D&TRANSPARENT=true&layers=1e415c9c-55f3-51a2-b50b-b5053d1debbb&time=2014-01-01T00%3A00%3A00.000Z&EXCEPTIONS=application%2Fjson&WIDTH=1800&HEIGHT=900&CRS=EPSG:4326&BBOX=-90.0%2C-180.0%2C90.0%2C180.0 +Authorization: Bearer {{anonymousSession.response.body.$.id}} + ### # @name workflow diff --git a/test_data/dataset_defs/landcover.json b/test_data/dataset_defs/landcover.json index ed33a27a8..d942e8697 100644 --- a/test_data/dataset_defs/landcover.json +++ b/test_data/dataset_defs/landcover.json @@ -49,18 +49,29 @@ "resultDescriptor": { "dataType": "U8", "spatialReference": "EPSG:4326", - "noDataValue": 255.0, "time": { - "start": "-262143-01-01T00:00:00+00:00", - "end": "+262142-12-31T23:59:59.999+00:00" - }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "-262143-01-01T00:00:00+00:00", + "end": "+262142-12-31T23:59:59.999+00:00" + }, + "dimension": "irregular" }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/mock.json b/test_data/dataset_defs/mock.json index de0245228..68376de81 100644 --- a/test_data/dataset_defs/mock.json +++ b/test_data/dataset_defs/mock.json @@ -20,10 +20,10 @@ "dataType": "MultiPoint", "spatialReference": "EPSG:4326", "columns": {}, - "time": { - "start": "-262143-01-01T00:00:00+00:00", - "end": "+262142-12-31T23:59:59.999+00:00" - }, + "time": { + "start": "-262143-01-01T00:00:00+00:00", + "end": "+262142-12-31T23:59:59.999+00:00" + }, "bbox": { "lowerLeftCoordinate": [1.0, 2.0], "upperRightCoordinate": [1.0, 2.0] diff --git a/test_data/dataset_defs/natural_earth_2_blue.json b/test_data/dataset_defs/natural_earth_2_blue.json index 13193840b..25abb8bef 100644 --- a/test_data/dataset_defs/natural_earth_2_blue.json +++ b/test_data/dataset_defs/natural_earth_2_blue.json @@ -46,16 +46,28 @@ "dataType": "U8", "spatialReference": "EPSG:4326", "time": { - "start": "-262143-01-01T00:00:00+00:00", - "end": "+262142-12-31T23:59:59.999+00:00" - }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "-262143-01-01T00:00:00+00:00", + "end": "+262142-12-31T23:59:59.999+00:00" + }, + "dimension": "irregular" }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/natural_earth_2_green.json b/test_data/dataset_defs/natural_earth_2_green.json index ffbe3f03b..ab6e86f73 100644 --- a/test_data/dataset_defs/natural_earth_2_green.json +++ b/test_data/dataset_defs/natural_earth_2_green.json @@ -46,16 +46,28 @@ "dataType": "U8", "spatialReference": "EPSG:4326", "time": { - "start": "-262143-01-01T00:00:00+00:00", - "end": "+262142-12-31T23:59:59.999+00:00" - }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "-262143-01-01T00:00:00+00:00", + "end": "+262142-12-31T23:59:59.999+00:00" + }, + "dimension": "irregular" }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/natural_earth_2_red.json b/test_data/dataset_defs/natural_earth_2_red.json index c4510eaf3..aab99785d 100644 --- a/test_data/dataset_defs/natural_earth_2_red.json +++ b/test_data/dataset_defs/natural_earth_2_red.json @@ -46,16 +46,29 @@ "dataType": "U8", "spatialReference": "EPSG:4326", "time": { - "start": "-262143-01-01T00:00:00+00:00", - "end": "+262142-12-31T23:59:59.999+00:00" - }, - "bbox": { - "upperLeftCoordinate": [-180, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "-262143-01-01T00:00:00+00:00", + "end": "+262142-12-31T23:59:59.999+00:00" + }, + "dimension": "irregular" + }, - "resolution": { - "x": 0.09999999999999, - "y": 0.09999999999999 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi (3587).json b/test_data/dataset_defs/ndvi (3587).json index fbc2ec419..e8c95df14 100644 --- a/test_data/dataset_defs/ndvi (3587).json +++ b/test_data/dataset_defs/ndvi (3587).json @@ -18,20 +18,36 @@ "dataType": "U8", "spatialReference": "EPSG:3857", "time": { - "start": "2014-01-01T00:00:00.000Z", - "end": "2014-07-01T00:00:00.000Z" - }, - "bbox": { - "upperLeftCoordinate": [ - -20037508.3427892439067364, 19971868.8804085627198219 - ], - "lowerRightCoordinate": [ - 20027452.8429077081382275, -19966571.3752283006906509 - ] + "bounds": { + "start": "2014-01-01T00:00:00.000Z", + "end": "2014-07-01T00:00:00.000Z" + }, + "dimension": { + "regular": { + "origin": "2014-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + } }, - "resolution": { - "x": 14052.95025804873876, - "y": 14057.88111778840539 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -20037508.3427892439067364, + "y": 19971868.8804085627198219 + }, + "xPixelSize":14052.95025804873876, + "yPixelSize":-14057.88111778840539 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi.json b/test_data/dataset_defs/ndvi.json index cdbc2d893..b552b2f5f 100644 --- a/test_data/dataset_defs/ndvi.json +++ b/test_data/dataset_defs/ndvi.json @@ -293,16 +293,36 @@ "measurement": "vegetation" }, "time": { - "start": "2014-01-01T00:00:00.000Z", - "end": "2014-07-01T00:00:00.000Z" - }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "2014-01-01T00:00:00.000Z", + "end": "2014-07-01T00:00:00.000Z" + }, + "dimension": { + "regular": { + "origin": "2014-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + } }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi_downscaled_3x.json b/test_data/dataset_defs/ndvi_downscaled_3x.json new file mode 100644 index 000000000..c47226ba9 --- /dev/null +++ b/test_data/dataset_defs/ndvi_downscaled_3x.json @@ -0,0 +1,369 @@ +{ + "properties": { + "name": "ndvi_downscaled_3x", + "displayName": "NDVI Downscaled 3x", + "description": "NDVI data from MODIS, downscaled 3x", + "sourceOperator": "GdalSource", + "symbology": { + "type": "raster", + "rasterColorizer": { + "type": "singleBand", + "band": 0, + "bandColorizer": { + "type": "palette", + "colors": { + "0": [236, 224, 215, 0], + "1": [235, 223, 214, 255], + "2": [234, 222, 212, 255], + "3": [234, 221, 211, 255], + "4": [233, 221, 209, 255], + "5": [232, 220, 208, 255], + "6": [231, 219, 206, 255], + "7": [231, 218, 205, 255], + "8": [230, 217, 204, 255], + "9": [229, 216, 202, 255], + "10": [228, 215, 201, 255], + "11": [227, 214, 199, 255], + "12": [227, 214, 198, 255], + "13": [226, 213, 197, 255], + "14": [225, 212, 195, 255], + "15": [224, 211, 194, 255], + "16": [224, 210, 192, 255], + "17": [223, 209, 191, 255], + "18": [222, 208, 189, 255], + "19": [221, 207, 188, 255], + "20": [221, 207, 187, 255], + "21": [220, 206, 185, 255], + "22": [219, 205, 184, 255], + "23": [218, 204, 182, 255], + "24": [217, 203, 181, 255], + "25": [217, 202, 180, 255], + "26": [216, 201, 178, 255], + "27": [215, 200, 177, 255], + "28": [214, 200, 175, 255], + "29": [214, 199, 174, 255], + "30": [213, 198, 172, 255], + "31": [212, 197, 171, 255], + "32": [211, 196, 170, 255], + "33": [210, 195, 168, 255], + "34": [209, 195, 167, 255], + "35": [209, 194, 165, 255], + "36": [208, 193, 164, 255], + "37": [207, 192, 162, 255], + "38": [206, 192, 161, 255], + "39": [205, 191, 159, 255], + "40": [204, 190, 158, 255], + "41": [204, 189, 156, 255], + "42": [203, 188, 155, 255], + "43": [202, 188, 153, 255], + "44": [201, 187, 152, 255], + "45": [200, 186, 150, 255], + "46": [199, 185, 149, 255], + "47": [199, 185, 148, 255], + "48": [198, 184, 146, 255], + "49": [197, 183, 145, 255], + "50": [196, 182, 143, 255], + "51": [195, 181, 142, 255], + "52": [194, 181, 140, 255], + "53": [193, 180, 139, 255], + "54": [193, 179, 137, 255], + "55": [192, 178, 136, 255], + "56": [191, 177, 134, 255], + "57": [190, 177, 133, 255], + "58": [189, 176, 131, 255], + "59": [188, 175, 130, 255], + "60": [188, 174, 128, 255], + "61": [187, 174, 127, 255], + "62": [186, 173, 125, 255], + "63": [185, 172, 124, 255], + "64": [184, 171, 122, 255], + "65": [183, 171, 121, 255], + "66": [182, 170, 119, 255], + "67": [181, 169, 117, 255], + "68": [180, 169, 116, 255], + "69": [179, 168, 114, 255], + "70": [178, 167, 112, 255], + "71": [177, 167, 111, 255], + "72": [176, 166, 109, 255], + "73": [175, 165, 107, 255], + "74": [174, 164, 105, 255], + "75": [173, 164, 104, 255], + "76": [172, 163, 102, 255], + "77": [171, 162, 100, 255], + "78": [170, 162, 99, 255], + "79": [170, 161, 97, 255], + "80": [169, 160, 95, 255], + "81": [168, 160, 94, 255], + "82": [167, 159, 92, 255], + "83": [166, 158, 90, 255], + "84": [165, 158, 89, 255], + "85": [164, 157, 87, 255], + "86": [163, 156, 85, 255], + "87": [162, 156, 84, 255], + "88": [161, 155, 82, 255], + "89": [160, 154, 80, 255], + "90": [159, 153, 78, 255], + "91": [158, 153, 77, 255], + "92": [157, 152, 75, 255], + "93": [156, 151, 73, 255], + "94": [155, 151, 72, 255], + "95": [154, 150, 70, 255], + "96": [153, 149, 69, 255], + "97": [152, 149, 68, 255], + "98": [151, 148, 66, 255], + "99": [150, 147, 65, 255], + "100": [149, 147, 64, 255], + "101": [147, 146, 63, 255], + "102": [146, 145, 61, 255], + "103": [145, 145, 60, 255], + "104": [144, 144, 59, 255], + "105": [143, 143, 58, 255], + "106": [142, 142, 56, 255], + "107": [141, 142, 55, 255], + "108": [140, 141, 54, 255], + "109": [139, 140, 53, 255], + "110": [138, 140, 51, 255], + "111": [137, 139, 50, 255], + "112": [135, 138, 49, 255], + "113": [134, 138, 48, 255], + "114": [133, 137, 46, 255], + "115": [132, 136, 45, 255], + "116": [131, 136, 44, 255], + "117": [130, 135, 43, 255], + "118": [129, 134, 41, 255], + "119": [128, 134, 40, 255], + "120": [127, 133, 39, 255], + "121": [126, 132, 38, 255], + "122": [124, 131, 36, 255], + "123": [123, 131, 35, 255], + "124": [122, 130, 34, 255], + "125": [121, 129, 33, 255], + "126": [120, 129, 31, 255], + "127": [119, 128, 30, 255], + "128": [118, 127, 30, 255], + "129": [117, 127, 30, 255], + "130": [116, 126, 30, 255], + "131": [115, 125, 30, 255], + "132": [114, 124, 30, 255], + "133": [113, 124, 31, 255], + "134": [112, 123, 31, 255], + "135": [111, 122, 31, 255], + "136": [110, 121, 31, 255], + "137": [109, 121, 31, 255], + "138": [108, 120, 31, 255], + "139": [107, 119, 31, 255], + "140": [106, 118, 31, 255], + "141": [105, 118, 31, 255], + "142": [104, 117, 31, 255], + "143": [103, 116, 32, 255], + "144": [102, 115, 32, 255], + "145": [101, 115, 32, 255], + "146": [100, 114, 32, 255], + "147": [99, 113, 32, 255], + "148": [98, 112, 32, 255], + "149": [97, 112, 32, 255], + "150": [96, 111, 32, 255], + "151": [95, 110, 32, 255], + "152": [94, 109, 32, 255], + "153": [93, 109, 32, 255], + "154": [92, 108, 33, 255], + "155": [91, 107, 33, 255], + "156": [90, 106, 33, 255], + "157": [89, 106, 33, 255], + "158": [88, 105, 33, 255], + "159": [87, 104, 33, 255], + "160": [86, 103, 33, 255], + "161": [85, 102, 33, 255], + "162": [85, 102, 33, 255], + "163": [84, 101, 33, 255], + "164": [83, 100, 33, 255], + "165": [82, 99, 33, 255], + "166": [81, 99, 33, 255], + "167": [81, 98, 33, 255], + "168": [80, 97, 33, 255], + "169": [79, 96, 33, 255], + "170": [78, 95, 33, 255], + "171": [77, 95, 33, 255], + "172": [76, 94, 33, 255], + "173": [76, 93, 33, 255], + "174": [75, 92, 33, 255], + "175": [74, 92, 33, 255], + "176": [73, 91, 33, 255], + "177": [72, 90, 33, 255], + "178": [72, 89, 33, 255], + "179": [71, 88, 33, 255], + "180": [70, 88, 33, 255], + "181": [69, 87, 33, 255], + "182": [68, 86, 33, 255], + "183": [68, 85, 33, 255], + "184": [67, 84, 33, 255], + "185": [66, 84, 33, 255], + "186": [65, 83, 33, 255], + "187": [64, 82, 33, 255], + "188": [63, 81, 33, 255], + "189": [63, 81, 33, 255], + "190": [62, 80, 33, 255], + "191": [61, 79, 33, 255], + "192": [60, 78, 32, 255], + "193": [59, 78, 31, 255], + "194": [58, 77, 30, 255], + "195": [57, 76, 29, 255], + "196": [56, 76, 28, 255], + "197": [55, 75, 27, 255], + "198": [54, 74, 26, 255], + "199": [53, 74, 25, 255], + "200": [52, 73, 24, 255], + "201": [51, 72, 23, 255], + "202": [50, 72, 22, 255], + "203": [49, 71, 21, 255], + "204": [48, 70, 20, 255], + "205": [47, 70, 19, 255], + "206": [46, 69, 18, 255], + "207": [46, 69, 17, 255], + "208": [45, 68, 15, 255], + "209": [44, 67, 14, 255], + "210": [43, 67, 13, 255], + "211": [42, 66, 12, 255], + "212": [41, 65, 11, 255], + "213": [40, 65, 10, 255], + "214": [39, 64, 9, 255], + "215": [38, 63, 8, 255], + "216": [37, 63, 7, 255], + "217": [36, 62, 6, 255], + "218": [35, 61, 5, 255], + "219": [34, 61, 4, 255], + "220": [33, 60, 3, 255], + "221": [32, 59, 2, 255], + "222": [31, 59, 1, 255], + "223": [30, 58, 0, 255], + "224": [29, 57, 0, 255], + "225": [29, 57, 0, 255], + "226": [28, 56, 0, 255], + "227": [28, 55, 0, 255], + "228": [27, 54, 0, 255], + "229": [26, 54, 1, 255], + "230": [26, 53, 1, 255], + "231": [25, 52, 1, 255], + "232": [24, 52, 1, 255], + "233": [24, 51, 1, 255], + "234": [23, 50, 1, 255], + "235": [23, 49, 1, 255], + "236": [22, 49, 1, 255], + "237": [21, 48, 1, 255], + "238": [21, 47, 1, 255], + "239": [20, 47, 2, 255], + "240": [19, 46, 2, 255], + "241": [19, 45, 2, 255], + "242": [18, 44, 2, 255], + "243": [18, 44, 2, 255], + "244": [17, 43, 2, 255], + "245": [16, 42, 2, 255], + "246": [16, 41, 2, 255], + "247": [15, 41, 2, 255], + "248": [14, 40, 2, 255], + "249": [14, 39, 2, 255], + "250": [13, 39, 3, 255], + "251": [13, 38, 3, 255], + "252": [12, 37, 3, 255], + "253": [11, 36, 3, 255], + "254": [11, 36, 3, 255], + "255": [0, 0, 0, 255] + }, + "noDataColor": [0, 0, 0, 0], + "defaultColor": [0, 0, 0, 0] + } + }, + "opacity": 1.0 + }, + "provenance": [ + { + "citation": "Nasa Earth Observations, MODIS Vegetation Index Products", + "license": "https://earthdata.nasa.gov/collaborate/open-data-services-and-software/data-information-policy", + "uri": "https://modis.gsfc.nasa.gov/data/dataprod/mod13.php" + } + ] + }, + "metaData": { + "type": "GdalMetaDataRegular", + "resultDescriptor": { + "dataType": "U8", + "spatialReference": "EPSG:4326", + "measurement": { + "type": "continuous", + "measurement": "vegetation" + }, + "time": { + "bounds": { + "start": "2025-01-01T00:00:00.000Z", + "end": "2025-05-01T00:00:00.000Z" + }, + "dimension": { + "regular": { + "origin": "2025-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + } + }, + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.1, + "yPixelSize": -0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [599, 1199] + } + } + }, + "bands": [ + { + "name": "ndvi", + "measurement": { + "type": "continuous", + "measurement": "vegetation" + } + } + ] + }, + "params": { + "filePath": "test_data/raster/modis_ndvi/downscaled_3x/MOD13A2_M_NDVI_%_START_TIME_%.TIFF", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.3, + "yPixelSize": -0.3 + }, + "width": 1200, + "height": 600, + "fileNotFoundHandling": "NoData", + "noDataValue": 0.0 + }, + "timePlaceholders": { + "%_START_TIME_%": { + "format": "%Y-%m-%d", + "reference": "start" + } + }, + "dataTime": { + "start": "2014-01-01T00:00:00.000Z", + "end": "2014-02-01T00:00:00.000Z" + }, + "step": { + "granularity": "months", + "step": 1 + }, + "cacheTtl": 0 + } +} diff --git a/test_data/dataset_defs/ndvi_flipped_y_axis.json b/test_data/dataset_defs/ndvi_flipped_y_axis.json index 75b10a32b..3c30d7ab1 100644 --- a/test_data/dataset_defs/ndvi_flipped_y_axis.json +++ b/test_data/dataset_defs/ndvi_flipped_y_axis.json @@ -289,16 +289,36 @@ "dataType": "U8", "spatialReference": "EPSG:4326", "time": { - "start": "2014-01-01T00:00:00.000Z", - "end": "2014-07-01T00:00:00.000Z" - }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "2014-01-01T00:00:00.000Z", + "end": "2014-07-01T00:00:00.000Z" + }, + "dimension": { + "regular": { + "origin": "2014-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + } }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": -90.0 + }, + "xPixelSize":0.1, + "yPixelSize":0.1 + }, + "gridBounds": { + "min": [-1800, 0], + "max": [-1, 3599] + } + } }, "bands": [ { diff --git a/test_data/dataset_defs/ndvi_list.json b/test_data/dataset_defs/ndvi_list.json index deb161b5a..639aa4f6a 100644 --- a/test_data/dataset_defs/ndvi_list.json +++ b/test_data/dataset_defs/ndvi_list.json @@ -290,16 +290,36 @@ "spatialReference": "EPSG:4326", "time": { - "start": "2014-01-01T00:00:00.000Z", - "end": "2014-07-01T00:00:00.000Z" - }, - "bbox": { - "upperLeftCoordinate": [-180.0, 90.0], - "lowerRightCoordinate": [180.0, -90.0] + "bounds": { + "start": "2014-01-01T00:00:00.000Z", + "end": "2014-07-01T00:00:00.000Z" + }, + "dimension": { + "regular": { + "origin": "2014-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + } }, - "resolution": { - "x": 0.1, - "y": 0.1 + "spatialGrid": { + "state": "source", + "spatialGrid":{ + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize":0.1, + "yPixelSize":-0.1 + }, + "gridBounds": { + "min": [0, 0], + "max": [1799, 3599] + } + } }, "bands": [ { diff --git a/test_data/layer_collection_defs/test_collection.json b/test_data/layer_collection_defs/test_collection.json index 955625776..90bd653d8 100644 --- a/test_data/layer_collection_defs/test_collection.json +++ b/test_data/layer_collection_defs/test_collection.json @@ -6,6 +6,7 @@ "layers": [ "b75db46e-2b9a-4a86-b33f-bc06a73cd711", "c078db52-2dc6-4838-ad75-340cefeab476", - "83866f7b-dcee-47b8-9242-e5636ceaf402" + "83866f7b-dcee-47b8-9242-e5636ceaf402", + "52ef9e16-acd1-4c61-9a80-7d5b335d0d5a" ] } diff --git a/test_data/layer_defs/natural_earth_r.json b/test_data/layer_defs/natural_earth_r.json new file mode 100644 index 000000000..42bafea32 --- /dev/null +++ b/test_data/layer_defs/natural_earth_r.json @@ -0,0 +1,42 @@ +{ + "id": "52ef9e16-acd1-4c61-9a80-7d5b335d0d5a", + "name": "Natural Earth II – R", + "description": "A raster with one band (R from RGB)", + "workflow": { + "type": "Raster", + "operator": { + "type": "GdalSource", + "params": { + "data": "ne2_raster_red" + } + } + }, + "symbology": { + "type": "raster", + "opacity": 1, + "rasterColorizer": { + "type": "singleBand", + "band": 0, + "bandColorizer": { + "type": "linearGradient", + "breakpoints": [ + { + "value": 0, + "color": [255, 245, 240, 255] + }, + { + "value": 127, + "color": [250, 105, 73, 255] + }, + { + "value": 255, + "color": [254, 244, 239, 255] + } + ], + "noDataColor": [0, 0, 0, 0], + "underColor": [255, 245, 240, 255], + "overColor": [254, 244, 239, 255] + } + } + } +} diff --git a/test_data/layer_defs/rgb.json b/test_data/layer_defs/natural_earth_rgb.json similarity index 100% rename from test_data/layer_defs/rgb.json rename to test_data/layer_defs/natural_earth_rgb.json diff --git a/test_data/provider_defs/sentinel_s2_l2a_cogs.json b/test_data/provider_defs/sentinel_s2_l2a_cogs.json index 37f2d16c2..f998e8233 100644 --- a/test_data/provider_defs/sentinel_s2_l2a_cogs.json +++ b/test_data/provider_defs/sentinel_s2_l2a_cogs.json @@ -6,60 +6,6 @@ "priority": 50, "apiUrl": "https://earth-search.aws.element84.com/v0/collections/sentinel-s2-l2a-cogs/items", "cacheTtl": 86400, - "bands": [ - { - "name": "B01", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B02", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B03", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B04", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "B08", - "noDataValue": 0, - "dataType": "U16" - }, - { - "name": "SCL", - "noDataValue": 0, - "dataType": "U8" - } - ], - "zones": [ - { - "name": "UTM32N", - "epsg": 32632 - }, - { - "name": "UTM36N", - "epsg": 32636 - }, - { - "name": "UTM36S", - "epsg": 32736 - }, - { - "name": "UTM37N", - "epsg": 32637 - }, - { - "name": "UTM37S", - "epsg": 32737 - } - ], "queryBuffer": { "startSeconds": 60, "endSeconds": 60 diff --git a/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif new file mode 100644 index 000000000..5c0b80823 Binary files /dev/null and b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30.tif differ diff --git a/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt new file mode 100644 index 000000000..d5efa4a9c --- /dev/null +++ b/test_data/raster/modis_ndvi/cropped/MOD13A2_M_NDVI_2014-04-01_30x30_bytes.txt @@ -0,0 +1,30 @@ +255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 255 180 193 202 200 201 195 191 192 174 75 255 255 255 124 +255 255 255 255 255 255 255 255 255 255 255 255 255 255 131 172 185 188 192 196 200 191 189 197 187 106 255 255 169 155 +255 255 255 255 255 255 127 107 152 179 147 177 174 175 180 191 187 185 192 198 196 193 189 181 112 42 255 255 255 255 +255 255 255 255 255 164 185 182 173 159 125 138 180 176 160 121 128 147 166 167 176 181 183 135 45 255 255 255 255 255 +255 255 255 175 186 190 167 140 123 73 46 70 151 196 190 163 185 193 191 197 207 197 142 76 255 255 255 255 255 255 +255 255 161 175 184 173 170 188 123 91 76 102 176 195 195 192 194 193 194 199 200 185 129 255 255 255 255 255 255 255 +255 255 128 177 165 145 191 174 154 179 179 120 167 191 194 195 192 187 190 185 186 191 153 84 255 255 255 255 255 255 +255 117 100 174 159 147 99 135 179 187 161 112 111 164 190 193 195 191 194 188 188 193 177 111 255 255 255 255 255 255 +255 170 166 162 163 112 79 144 189 183 172 182 164 173 185 188 191 195 197 203 202 200 193 146 170 172 191 185 132 255 +255 157 187 192 191 174 154 174 182 184 188 184 182 181 186 191 192 195 197 190 185 196 200 194 189 187 189 194 190 129 +118 101 140 180 179 175 176 181 183 176 167 171 182 186 188 189 192 199 200 204 205 207 204 201 198 196 199 199 169 77 +255 187 178 178 178 181 182 185 182 167 163 173 183 189 189 190 191 196 198 196 194 195 186 145 167 189 183 162 123 255 +255 148 168 181 183 184 184 185 184 173 169 166 172 177 182 180 188 191 198 199 188 159 104 160 164 151 148 102 255 255 +255 51 74 164 185 188 182 183 188 183 171 166 175 185 186 186 189 177 189 192 196 195 158 255 255 255 255 255 255 255 +255 37 27 102 173 183 179 178 181 181 174 168 171 180 186 192 190 191 199 200 203 203 143 128 149 128 85 110 255 255 +255 136 145 148 174 180 175 173 176 172 176 174 179 185 194 200 198 190 180 170 152 123 51 255 255 173 113 34 255 255 +255 131 179 187 184 182 187 181 177 175 172 177 180 192 197 188 192 200 212 197 115 255 138 100 255 177 87 255 197 196 +255 163 177 185 181 186 181 185 181 182 183 176 186 195 196 191 183 176 145 134 119 121 101 134 255 124 117 255 255 198 +134 158 135 160 180 184 181 181 181 183 182 184 184 189 190 193 185 147 136 181 188 190 198 195 158 140 163 89 255 255 +255 255 107 103 122 160 183 187 185 180 179 178 182 191 191 192 159 161 190 196 193 197 199 183 159 172 182 125 255 255 +255 255 255 255 92 57 164 179 185 185 189 188 190 198 203 193 105 122 174 191 197 195 191 183 186 191 200 171 101 103 +255 255 255 255 52 75 144 186 184 182 183 180 186 192 193 203 182 121 166 201 195 196 200 200 194 197 200 175 82 255 +255 255 255 255 104 115 118 185 188 191 189 178 176 183 179 148 152 86 255 255 149 177 197 200 199 192 203 164 188 160 +255 255 255 66 67 54 144 182 174 183 184 179 179 183 175 177 185 190 185 138 166 149 149 159 189 186 161 103 175 115 +255 255 255 118 255 255 174 185 185 180 179 180 182 181 190 198 198 189 197 193 136 255 164 105 80 116 134 168 154 255 +255 255 255 105 97 109 179 198 202 199 196 187 184 179 173 169 179 141 149 143 123 255 255 194 146 91 154 172 255 66 +255 255 51 77 206 135 124 189 207 207 200 198 198 192 199 215 215 209 193 161 255 255 255 255 255 255 255 137 255 174 +255 255 45 101 115 123 79 108 185 201 208 212 205 198 202 210 213 213 209 193 115 255 255 255 255 255 255 255 255 255 +255 255 255 255 44 92 143 58 106 176 204 210 211 202 197 193 189 196 210 196 118 255 255 255 255 255 255 255 255 255 +255 255 255 255 43 47 91 109 147 167 199 206 207 210 194 196 199 198 178 180 204 175 187 189 151 255 255 255 255 166 diff --git a/test_data/raster/modis_ndvi/downscaled_3x/MOD13A2_M_NDVI_2014-01-01.TIFF b/test_data/raster/modis_ndvi/downscaled_3x/MOD13A2_M_NDVI_2014-01-01.TIFF new file mode 100644 index 000000000..a072a3f3a Binary files /dev/null and b/test_data/raster/modis_ndvi/downscaled_3x/MOD13A2_M_NDVI_2014-01-01.TIFF differ diff --git a/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst b/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst new file mode 100644 index 000000000..b7eb4105f --- /dev/null +++ b/test_data/raster/modis_ndvi/projected_3857/MOD13A2_M_NDVI_2014-04-01_tile-20_v6.rst @@ -0,0 +1,68 @@ +ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿaˆoF—ªµ²ª¯µ¼º°€ÿÿÿÿÿÿÿÿ…s„}}!ÿÿÿÿÿÿÿÿL“³ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿy„™’›£¢©¬w=a—ÿÿÿÿÿÿÿÿKormou}vw~}‚~ˆ€wƒvppqox|ƒ‰–’˜‰„†…‚‚~w„’«£™‰Ÿ­›¦Ÿ‡‘}‡–·²¦·»¯œqct}…ƒ€€yz~{vv–¥’wy|vomm€Œˆvutrmo‰›˜™¤¤–”‰ƒx‡’}‡‚ƒ|zwy{vvg\^ZSWYNQLHHCADDU`RWark€xsvYBAQU]cRIC?>AEIjjm^JINLFIFEKJ@7>9307,#4$)25:Th#"/* $$(=aB#&<#' # )('89=8%0? ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4Lxs–±¹¯ª¨®¹·„ÿÿÿÿÿÿÿ¤Œÿÿÿÿ:ÿÿÿÿÿÿÿÿr—{˜|ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–— £Ÿ’œ›¡ ¡Œ\ÿÿÿÿÿÿÿÿÿM{oƒ{}~|–…|ˆ‚twysxwur{}€‚†‡‰©£…ˆ‘¦£¢”“Œ‰ˆ—¥ˆ‰–ˆ“œ•£¦£¢´ª›|‡xƒ†ƒƒ|}usu{ƒ~ooqzzxqnkkvwx}hfjfksv† “„€rkk€}orv{kdn|snisj`YYYSSMMRKMIJGFT[\UEQPVRUNEB=>jifUFAAK[hMl€va_PAA??QLPZ=983#!+1-!$//:LN3!$+# )B'+!" "(-/;IXvcUZV[V`g=@L>!!( &%  ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ39Vƒ’¬¹¶¥¨°Ÿ´°¶‹ÿÿÿ‘Æœ8ÿÿÿÿÿÿÿÿ˜‰‚vf‹»£‰^ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‘¥˜‘› ˜–¤Ÿ˜–“Yÿÿÿÿÿÿq~†…‹ˆ‰~|{zƒ†~ƒ|zƒ‚‘‘y€}Œ“„„‚ƒ…‡‹˜™±¨¥©˜‚ƒ~‘†„¦«¡zŒ˜¡¦‘›«¯›}†”…–ˆƒzzxx}y„‚|}xw||wpqmqqjl|…‡twsu…ƒ„~z~€~…pkdjon}€zqmfc^[]WOIISECCNMILGDEABGCDECD@@?A?JUOPG>8:GWV^>5/0)-.4/%## AB4*)721,!%0+$!!)) &" #$%%2LZgclU_[JKD48/&!-! ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ39Vƒ’¬¹¶¥¨°Ÿ´°¶‹ÿÿÿ‘Æœ8ÿÿÿÿÿÿÿÿ˜‰‚vf‹»£‰^ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‘¥˜‘› ˜–¤Ÿ˜–“Yÿÿÿÿÿÿq~†…‹ˆ‰~|{zƒ†~ƒ|zƒ‚‘‘y€}Œ“„„‚ƒ…‡‹˜™±¨¥©˜‚ƒ~‘†„¦«¡zŒ˜¡¦‘›«¯›}†”…–ˆƒzzxx}y„‚|}xw||wpqmqqjl|…‡twsu…ƒ„~z~€~…pkdjon}€zqmfc^[]WOIISECCNMILGDEABGCDECD@@?A?JUOPG>8:GWV^>5/0)-.4/%## AB4*)721,!%0+$!!)) &" #$%%2LZgclU_[JKD48/&!-! ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™½­µ¾¾³¬¬¾·ª´¹µj²¼C;Nÿÿÿÿÿÿÿ“¤£Í•¨Á©‘ˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‚œ¤¢©¡°­¢¡¥••”•”Œ‚u€ˆ£¬…L“”’“Œ—’”—‹Œ‹‡‡ƒ…ˆŸ²­“‰‚„‹€~Œ›‹‚…“œ¢²½¤–‡Œ˜…’‚…‹‰¢Œ›¥¡¤º¥Œœ¨¥“†‡’……‚‹†‚}‹ˆxuvnp|rnvmsy}mqƒ’™›‹„{{|qqs}ƒ~jjgfhllœœˆkrhahhcNMFFONA>DEPSYXG??@BCCAB@@=?DP`XI@<;?JJPXVei`WKC23>IGNZ<"",,!' ,?SL<6-'77*"'0:5-) ##"$++*(&,,3Hcpƒn@@UR?767-!!*%!)  ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿZ‚™±º¹»º±­²²±··¼ĂÁÄÿÿÿÿÿÿÿÿ·±½¿È²ªªVÿsÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‘¡’|Ÿ¬¢¯®±¡¬²««ŸŒ‘’–’•¯¡§¦…“–œ›”“™—•—˜–œœ˜˜‡‡•‚†„…†‹‡~€xz•›˜±­Œ‘›Œ„~|}x}¢˜”‡Ÿ­¬±‹›‡ ·µ°¥Œz‚zv{‚xztolt}x}uxqvqrz •—oqtr‡€ymffgfigk~solokcfb`_X[ZJLB?GLVPOPI?ADECBCLJJFERaWC?==?D]libVspaWK3#'5TMVU+$!)OgXWC:6@?/'?;8649+)0-0/$#$ $#'$+')5Ee„sm=\qLKP//2.! ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ:‘Ÿ±¶¸¼¸µ¯²³½¦¤´½Ă®ÿÿÿT~ˆ™»¶²¶ĂÅÇÇÆv¿¶q;ÿÿÿÿÿÿÿÿÿ™”’Ÿ¢¥¢’¢°£§§±Å½ª–“£¶™’ˆ“›¢­¯¨£“‡–›——˜§¤Ÿ“œŸŸ“‘^c£’†„ƒ–‚ƒ’‹ «¾ÄÀ¨‹…‚Œ––‘•‹„Œ‰‘ˆ†‡”ˆ„ˆ‘˜˜˜£±ª§«Ÿ|†w„„‚—~ˆwswuutwx~z|txl…‡…‡†…‘v€†œ“mgefol‰“qrcg_aefd`\XVTOOE>BECCDIC?BHCDDEdspYJ^`FFHF>;7,(]G80!*'19GOR[dG;:JI616:9"-95!(  #,+((2KYafYct\leZL2P,C<& ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ:‘Ÿ±¶¸¼¸µ¯²³½¦¤´½Ă®ÿÿÿT~ˆ™»¶²¶ĂÅÇÇÆv¿¶q;ÿÿÿÿÿÿÿÿÿ™”’Ÿ¢¥¢’¢°£§§±Å½ª–“£¶™’ˆ“›¢­¯¨£“‡–›——˜§¤Ÿ“œŸŸ“‘^c£’†„ƒ–‚ƒ’‹ «¾ÄÀ¨‹…‚Œ––‘•‹„Œ‰‘ˆ†‡”ˆ„ˆ‘˜˜˜£±ª§«Ÿ|†w„„‚—~ˆwswuutwx~z|txl…‡…‡†…‘v€†œ“mgefol‰“qrcg_aefd`\XVTOOE>BECCDIC?BHCDDEdspYJ^`FFHF>;7,(]G80!*'19GOR[dG;:JI616:9"-95!(  #,+((2KYafYct\leZL2P,C<& ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ%)‰‹¦¶µ¶µ°«¡«°²»½€/¤ÿÿˆ¬²¶¨¡µ·¾´°ÄÅÈů½¼…vxÿÿÿu›œ£¤¥§±¡˜³¿¾©¥¡§¥™‘™¾´¯®™‹™¢¦¯±§§² §œ–™•¬¡–”˜”wt~‹Œƒ…ˆƒ—‘˜¥¬——¢ª¶¾¼·±œ…–Œ…“’›¥¦€x{†~„|˜‰€‚€­«›­®¢¯¦¦‡yz{x‰}‚……‰ƒ~~ƒ…xw}‚†‰ˆyszƒƒ—’‘„~}szfdin…•Œqrthbb]gngPT[WKBACA?;BABACA>AGGDZ]YRK^[HJKC;:FbbIItt[>1"K2 +.@HQNHHX_IFEEB>&&49=4/?4)4&/,('5SKFGPs‚piN,%/# ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ13&ÿm¤Â´²º¶²ºª¹¾¶¬Ÿ·œ’²³²µ¶¯±¸½Æ½®¬±²»¾»…œ} À·‹¦­°§ª¬ª®±¬´±¯“›¤œ¦²»¬Ă¯­·Åº¨’›®²µ¯³±©¥››’”›”“‘’•”‘†ˆ†‰“«¶¾»ÈÀ¼½¯ºº±¹«—‹‰‰›‹—…Ÿª«³¸¤™”ˆsr˜™“Ÿ¢¢£©¨™Œ€~srzƒ…Œ‰€}}Œ”ˆƒ}zuni¢¯¤‚u‰’‹|ƒ’œ”qz‚€qnw{„si}}|gfmoie_OMNPPBAC@AB@@AA?>?CABJWRI@AO]cNB;=AYarwoteP=<8.%3(($%MR\G@T^^TCM_fN;78;>?@?BAA@ANPFBI>=FWG5,:KgaiqpbPQ6." !7/2;BZLMPY\K[_YikX@70==;=,A7) "+34*&*7;AWUKL?2FH=. ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(ÿÿ¡‰†“¢ÀÀ¾³®½·¸»œ¤µ»³´²­«§«¬µ±·¼À·­»¹Â»°~pjn•ª±±®¤¥¶¤©¯³¨¥ ‘›¥˜¹¼´§Ă¶³ªª¹¿¡¨¯¬­­¬°¨ ™ ˜˜›•–‘‘‘‡‹‘ˆ†‡††‘œ¯ÁŶœ¢©´²®’’‡ŒŒ„ƒ‘£˜™¡®Ă›‡‡ˆ…€ubl’•—¥™Œ“›©©© ‰€nz{‚„ˆ–’‰……’–‘•˜}pzxsw{‹”Œ’“ˆw‚…ˆ‘‹ˆ‘zjd~zqnjg`h`[]YWHGGJHKD@>>?@?BAA@ANPFBI>=FWG5,:KgaiqpbPQ6." !7/2;BZLMPY\K[_YikX@70==;=,A7) "+34*&*7;AWUKL?2FH=. ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿrZ_RL`Z#ÿÿ¾¹Á¿½¼¡’¿»½·¶·½¿É¸¹º¹¾µ±°®¬±ª¦·¸²®¸ÄÁ¼¿¾´¼¯€¥9ÿ—¶²­ª¥©¤Ÿ¡Ÿ¤“™”²¶Ÿ›«¬²»¶º»Ç·¸­œ¡°«¦«£¨³¬«¦Œ›œ‘Ÿ›¢§[?s­Ÿ—‰‰¤¢›‚~„¡¢¨§‘„••“”—¬˜‰™¸©©œ‘‹ˆ~y{|€}‡‘–‹˜œ“•—¥¢¡¯’†††……‡–™”‹~‰‡‡’•““ §rnttu†—®£¥§ˆ…‰‘‹up‡tnlprqjiggg_QMMSSGACRFDC==>>>@GEEHD?>=>=>>=77;AO`msslK:3$+5+'#,5;6<>2/9N<;]j‚paBCC7J?(& (*%'.>GGTC:KIBT@2"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>I=7mªµ¹À¿ÆŒÿPŒ»¾ÂÁ»ĂÅ”¨¥—–¯¹·´±´¼¸³¥´³´¶®³¬£¤ª¹º»À¾ÄÀ²±¸¯¯¾§{Á¸¨£¨¨¡Ÿ ˜˜œª˜“˜ª¥¨ª¡”¢ÀÁ¸¼Á´ ¥¨§¡©§¥®œ”¶²³ºœ“²¼®³½¢œ’†–¡˜—•Œ†‰„ƒ†„¢’ŒŒ¤°·‡…„ƒŸ³ ›™¡‘‰–ƒ‡‹ˆ‘ˆ¬§«©Ÿ†„…ˆ‰“Œœ—ˆ‚‚‚ˆˆ‘›—‰zrwz• °¦‰‰ƒ‰ˆ…}qmq}wig_UZ[^QSPOHB>AA?B==<<>??JE>???<49>=@:7=LLUeuud:+#)+#"!2&#'!1B8@G49''3C3[qrd`WH<5A=!#0+$$"2 '(*++8HA/:BV^Y[B*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ2'+ÿQ~®½½ÁÅÈθg½–x°¿¾½¼µ¯¦Œbs³Áµ«´¯©¹¹¦±­°°£¨³¯–œ¬ÇÅŸ¾»·´°¶¸©²”£­®§Ÿœ’›“—¬œ“£¶µ¨¦­Ÿ¤§›˜­²²¦´«¢ª®¬¡¡¤–•–¦µ´¦£¤©·¸º¡‹Œ¢”…‹ˆ”“…€ˆ’—‰‰‘¡¨­ª‰ˆˆ|z~™–¦¨˜¢››–‡„}z‡Ÿ“‹ƒ•¢›¦Ÿƒy„“¤¦–¤Œ†}„ƒ€‡ƒª˜‹{{„®«¯}…”Œ‚~”•‡wnqqsuhe^ZMV[XLKDB@CEA=>=>=>@AD><>E?>>AA?=><>@?EUq`B3@%.1#)#$"%#I>BDKO>&+0&4S_fnvuPC@;=>=>@AD><>E?>>AA?=><>@?EUq`B3@%.1#)#$"%#I>BDKO>&+0&4S_fnvuPC@;>=?@ADGB@>>===?=AXb8(+32+%"&'2R>&/7;//!/=@PRZmsCF=AYI1*&!+>;-%$"$&#%;4LSWVWzX4$))µaÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ]™pÿs…¤¹“‘›Å¸§ˆ§ĐĐĐÇĂÂËÙÀ¹¼Á»±¯²²±¨­¬¬¬¬°°©¯¯­±²´·°­¥ª£ª¬¥œ­¦¶²§¥³²ÁÄĺ¥¥¢˜™”§¯¥£¹­¦ÄÓ̀´¬®—¡¡°©¤—¢¢¥ ˜¦£•—“œ¤›”§«¯°¬ª¡¦°©Ÿ’“†“›—¨‘Œƒ„‹‹‰–”–£¬Œ‚—ƒ‰ˆ‚€~‰”®“‘ œŸŸ–•›Ÿ›”¡›•¥œ–¡°–‚|{~‰¦Œ{…†“™£¬™wy‡€£œ˜¨˜xnntnjuujdikQJFISYQTPPIB@>@DC??AC@?>CB@B?A>=<=>>ERaG+!,3 :/)"@FY_fbe`^]|;@O;06&&)- ()*(!!#$$*FGURfpbL8 ¶³Zÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ0J7Pÿ‹¯ÇÑƠÔĐÔ½Ă¾½£‹Á¼ÊÎ˾¿ÂƠ¿¸»·¾¸²´°®²¹µ°°¡­¨¯¹»¼¶¹·²«¡±µ²³¯²°·»¸º³¹ªÁ½¾°—¢«Ÿ– ¥´¼˜¥³©±ÄÄ™™œŸ¸³¢ ¢¦¢™¡˜™’•¬¥ ŸŸ©¥ª¦¥§§ª«Ÿ‘…•œŸ¯¬¤»º¡™ƒˆ’¥¡‘™ƒ|…‡†‚ƒ~‰‰’§¤ª¨“˜—˜Ÿ’†~”—œ—ˆ€ƒ„“’ª‡ƒ|}z†œ•™Ÿ¦¦”††vory‡vgs–¥¤“yqlhhhswokhgeRUNNNRSGBIJ@??CDDBC@AGBAD@@?@??=EGMLPjyA1 ('>&$5YOUiOOVFEI^.1B)'(-04# #+%'# "'3&$5YOUiOOVFEI^.1B)'(-04# #+%'# "'3=>>AHJHHE?>>=LdTJ‰c=<8)  '4% ))4K/$/:-#.HHGNFDJFGFD!(('1+# (46$'"$##'6F*;>YQK:(&,6'!|^-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ«‰"ÿÿ¥ÜÛĐÖƯƠƠļ¸­™¡ «¯­¦£«´´¸¶®«Ÿ°¸¶µ·¸¯¯µ´¡¤¡›Ÿ§¯±·¼¬®¬³°¦­·´º¿Ä°¯¯´»¼­©²«œ®´²©«¬¥¨™  °Â¨¾¸™©§£•œ˜°¯¦¨½¾·® ¢›˜–“Ÿ®§£¥™¤ œœŸ¤¬¦‘”•””‘Œ’ª¬©œ†—‰‹ˆˆ¯Ê·´”‡ƒƒ~†…u‚‡‰}„‰˜§›’˜¢œ”—™€{xƒ‡‡ˆƒ}†£®¦†{††{|x€xuwytvs†rm›£™“{~trpmuojoqqnsyfgomi[NFHQVTRIODBIIDFH>>@CUu…hJB@?>;SQE^‰xL:98$"& '*ABWJHCD<39<@B?(;<>A@7)-'7J_T9"!!#',:GPKBD!# * ?ÿÿ‚„x–TÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿfU©ÿÿH̉×ÖÓĐÙȶ¶¹²¥™¨«­°±±µ¸°´¹¿Êʵ¸µ¶³®³­° ¡˜¬¯­´¶³¬¬µ»µ±³ºµ­²­°±¾ÊÁ£Ÿ®ª­ª­­¡Ÿ¥«³§¬¢³Áµ³©™¬À´¤™ª¥¤–œœ”¨­Ê·­±”“¦©¬¬²®©£¤¤¥«¥¢œŸ –––¢”’’”‹‘œ ¤›œ§©›˜“‹”¦©§Æ§Ÿœ”™ˆ‚ƒ‚„…‡Œ–¢©¦ŸŸŸ¢¨ •…}„€‡„‘¤¦¥—z|…‰‹„{„Œ|ws{|’ŸŒ“ykronqqppsrcmmhcdkldRFEEFGVHAA?FIU]\HDA?EJgyLBD@?=NPUA" ?ÿÿ‚„x–TÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿfU©ÿÿH̉×ÖÓĐÙȶ¶¹²¥™¨«­°±±µ¸°´¹¿Êʵ¸µ¶³®³­° ¡˜¬¯­´¶³¬¬µ»µ±³ºµ­²­°±¾ÊÁ£Ÿ®ª­ª­­¡Ÿ¥«³§¬¢³Áµ³©™¬À´¤™ª¥¤–œœ”¨­Ê·­±”“¦©¬¬²®©£¤¤¥«¥¢œŸ –––¢”’’”‹‘œ ¤›œ§©›˜“‹”¦©§Æ§Ÿœ”™ˆ‚ƒ‚„…‡Œ–¢©¦ŸŸŸ¢¨ •…}„€‡„‘¤¦¥—z|…‰‹„{„Œ|ws{|’ŸŒ“ykronqqppsrcmmhcdkldRFEEFGVHAA?FIU]\HDA?EJgyLBD@?=NPUA" ju†˜§¨§§˜Qÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—ƒpÿÙ™¬»ÎÑĐľº°¦˜›¡±²«±¯«ª¬··ºĂÄÁº¸µ±¨««®ÂÀÀ¦¨³²«¶¶µ»¹´ª·¶³¼³¯·Æ¼°³³®ª©ª«»´µ±¯«¯·Á³£¤®±¹³“™œ™£ £°¦…†¦˜™—¾²’¥Ÿ£—¢§ œ£¢œ¨£ œ œŸ›“Œ‘——˜”’“¨§°¸»¬²§ ˜ˆ­¼¥£˜‚‰¨¨¬ƒœ‚’‰‡pv‡¥¢ Ÿ›™¤¦˜“˜›—Ÿˆs€‚Œ‡˜†no|„™œŸ§™›˜‚€†…˜—†€tolqkkimpmmnkggkjfYOMKLEHLA?CGDCJ\OBCCAWaSOLCBC><>BDJgkLH;3&&'//'%/.*)$ $4&%)22.9BC:/&&@>ChURT;7##+ 9!*$64;963>>9%;FG>/2# &!#$%.7B@3+!" ±±­¨º«¦œ© Lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®´Đ¦dÿÿxr„ŸÑÎÊĂ¼µª™–¥ ­²««¦¬±²µ¹ºÁ»¿ÉÇ¿½¼­ œ¤°µª±·¹³¶º»²¸µ«¶²±¹°¹©¤ ¦¸µº¸´µ¬°¥Ÿ­®¯³²¿À»´Ÿœ¢˜ ”’Ÿª¬˜›¡£›Ÿ ›¡Ÿ“”˜¡£¡œ  “’›¤¡¢¤¥œ››—“™™™“’¥³¼¿¯•‘‰—•¨ œœ¢x›¨¨¤¢¨›’’”œŸ”–“’¢¢Ÿ˜¤™‹‘‡‚ƒ…ƒ|v¦¢›‰ƒ“’„}z{z~|w‰†€v•¦›–…otyyqppvyyyocgkkria]aaJKHFCMXPZacmmUJYXNPHGFEECEA@?AAEOiYA7%$9;>"(>B<>=?D=?BLMMED?),!A9!$(*;@?>?B;=>?A?:*  "02.0+"  Ÿ³°©°°¸¹µ³Á²bÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ̀ÎÅÿÿÿ–}¹ƠÏÏ̀º³¯µ«¢¥¬ª®°¥¬ª¤±¯²¼Ă¼ÂÀº¹°º½²©¨­«³©¥´»¯·ºº³µ»µ·¹·¹¥–‚vŒ§ª¯¯²®µ«™ª³º»¿±°¨ª¦¡Ÿ¥ £›•¥¨““•“™¤‘œ–—§Ä¸«‘£¨®¢™’’•˜˜–™›Ÿ•““‘‘˜™œœª®¿»¬”ˆ‰‡Ÿ˜ Ÿ¨¾±«”œ©¡™’’™”›’”¥¬ª˜—¢–—’›—„„€}€…„”™„„‚‰‚ƒ~||‡†qs¤¯¨–ˆvv‚ƒ€{wqsxyqjlljmjgooorRMLPJPYan`[jmXMXUNRLEFIKFC?>?BLMMED?),!A9!$(*;@?>?B;=>?A?:*  "02.0+" ³²¯¸Å¯¥¶¿¼¼½¯fÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªªÇJƒtœŒĂÊƠÖÎĹ¾Å°©¹¨£°²®°·¹»¹¶¼Á¿¨•®·¡°­¨’¢®¤—¥®¯­´µ·³´³¶¡”™¤¤Ÿ¢”~Œ¤±¼·¶½¶§¸­¯·ª²°­­²¹›œ¤œ‘ƒ”––Ÿ˜–§Ÿ“Œ•›’‘’£¦‘˜¢”‘‰ƒ‰”—›—“– ‰‹Œ«¡¢—¢™’p„”¯£Ÿ¯·««Ÿ›¢‘›¢ƒv‹§° •”—™ ›˜“”—“’Ÿ˜nm•Ÿ¢¦«…~tŒ–‹‰€}y‰™”˜—ˆ€pyuvutskmnoujksrstrnoa\UN_ggf^\[j^NS`fgeSKFDHHB?>=>@HDB@>:<-<<+%(.49138>>A>?#5@>><73.  (0/85 & !2K7%Á²ª®©§¯±ºÈ½8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿp–‰ˆŒ“—™¹ÀÉ̉ÎÓ¼·ÆÉ¿¾³©««±¨«¸Áļ¶Ă¸¾½¸´ª‹œ¨©–¦¦­¬¯®µ°²¨°­±»À´©© ª ¥ ™Ÿ¤©»½¬°´§’º¬¦§«§®­¯¦³£¨£›Ÿ‰””’‘’˜ˆ‹“‘’’•¥‹Œ•Ÿ£›€…‡•Ÿ–•‘ŒŒ‘› «ŸŒ‹”—•’•› ¡›‹’©§¦©œ’¡¬›œ¤–‹ƒ‹‰ ¦  ›–•£™Ÿ–›’“•’s‚•†„‚†˜…Œ‘‰}w‡†~}’˜‘…tqqvvyvjXYrtutmjklrpsqtxnosvhhdiX^fyqb`jdbRQOCIDAC==@?@>?@>;7;5,?=9 !,:ADCC?@@;0=A;9=9?7**12./"  #,&·¸µ½­¯µ²º¼É«Mÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿd›™¾Ê«ºÁÀ¿¸¸¾¼ÊÔÅʶ§¼¹»¨°«¬ªŸ¸½º¶µ¯³»¸±¯¦©«³´­°£®±ª¥°±°²ª³¯¾º²Á½º¾º§«¬§¨ª£¥ª²¶µ±¬²¯¿µ¤¤²© ¦©µ®«©¨¨¤“–›™˜„œ›•—•“‹“”“’“ˆ‰„|tiƒ–“”–——‘•—œ™““Œ‹ƒ…ƒ…‡ˆ’Ÿ¤« ¦œ– ‘™Ÿ——••™›‰“¤˜¡›Ÿ˜™“•“¡Ÿ””™‡„€…ˆ‰…‚…ˆ‚‚’ˆzx…Œ‡}yxqvzuvyvvsquwspkjkjlouptu~uqrmgjemiekvvcxq`JLQFBEAB@>>@==<=<:9>@==<=<:9?<><;?>BDCC?::5'/2"$+-.*   ··¯¸¶¶ºµ¦£œhÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿµÉÛѳÅÇʲµ¯´ÈÉ̀ÑÊÁ·´³°­´­¸³µº´¨ ª³»¾´¯³·¯¬®¨±¯²¾ÀÂı®³¯±±¼»²²»¼½½±¥¦ªµ¸·­²± ¸²°¥©©Ÿ¥§«¬²«¬«¬£ª©£ª§œ  ª¹¦¢ ¢‘•™‹}…‹œ˜–”˜‘Œˆ†ƒƒ˜˜‘”œ’“•˜–—”†€¤¢Ÿq¢Ÿ˜¤œ‘‚“‹ˆ‹ˆ‰„}{y}••’“…“Ÿ¡ª“—–‘‡ˆŸ’¯}y‹Œ‰Œ©–‹‚˜“…}wu~ˆ‡zzrtzxqkpr}qhhsslkmvwstzmtjqijffdie__fiylc[d^NBACCBA@>=?>@?;=ED>3$%07B=56>@EEIK;:8420+EBBB9:7=A@@B65/(# ·°µ²°¶·¬vwÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ`¤«˜ŸåßÚÑÉ˾¿Ă¬¦Ä¿ĂÉÁº®²³´¸ª´°´¶±«­²±¾¿²»º¿µ¸·´³¯¦·¾¿Á¾»¸¹Ă¿·¶²²»Àº¹®­¼µ±¯«¬±¯±³¸¶¤º¸±­°£¢­±­«§«¤¨­¬«¨¤”’©¨¤­›˜¤œ¡˜–”–’”Œˆ‰‹„œ““•œ¥¡”•˜“›™•“•–¯­°£Ÿ™¤››––›››—‹‰›’’“‹‰Œ–Ÿ¤¢¥­¨— ¤¢£¤‰—”Œ”™’—†¨ ‚‹¢˜˜‹ˆ“~r…˜•“‰‚wrrtxx‡……qkkwpmktvpmoklqsvjiosleejiikfgjk‹‹w]]ZX>BPCA@B>??A=@==@FF/*4( !"!!3>B@97AED@16CA;AACAFHD>AA@@?>61:+#   ·°¶¸¸³«ª«jÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‡†˜”‹–ĐƠÈǼ¼¾¹­ µ½ºÄµ²°©±®­±¶¹»º¸²µ³°°±¶¹¾¼´Ă¸¾º¼¾ÆÇ·±·½Ç¼«¸¸´¶¬¨Ÿ±°«°«¦®°¹»·¥¡³ª¡“±ª©±©¢§ ·¼²«¦ «¯­¤¥¦ ¡¢˜‰‚€”Ÿ¢’ £¡™†ƒ’‰ˆ‹ŒŒ’ŒŸ•“–˜–•™™’’–˜›—” ®µ­©œ–—”‰”†‹‰‹‰‹’›©¡’•˜’–˜¢¬©§²§’œ£Ÿœ˜™’ˆƒ‰Œ˜œ›Œ”˜‹¡¯…€‚‹™ ¢¡¢‘{{Œwrqqvxy|{zxomotpmpqsoiilpjpsmnrsikqmkhqlirt’ƒsa``cOXZMD==@ACC>A>>@AE<AGEA?HB3@HF@GFCBDGFAC><:9:9?5     ·°¶¸¸³«ª«jÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‡†˜”‹–ĐƠÈǼ¼¾¹­ µ½ºÄµ²°©±®­±¶¹»º¸²µ³°°±¶¹¾¼´Ă¸¾º¼¾ÆÇ·±·½Ç¼«¸¸´¶¬¨Ÿ±°«°«¦®°¹»·¥¡³ª¡“±ª©±©¢§ ·¼²«¦ «¯­¤¥¦ ¡¢˜‰‚€”Ÿ¢’ £¡™†ƒ’‰ˆ‹ŒŒ’ŒŸ•“–˜–•™™’’–˜›—” ®µ­©œ–—”‰”†‹‰‹‰‹’›©¡’•˜’–˜¢¬©§²§’œ£Ÿœ˜™’ˆƒ‰Œ˜œ›Œ”˜‹¡¯…€‚‹™ ¢¡¢‘{{Œwrqqvxy|{zxomotpmpqsoiilpjpsmnrsikqmkhqlirt’ƒsa``cOXZMD==@ACC>A>>@AE<AGEA?HB3@HF@GFCBDGFAC><:9:9?5     ³¿Ă¾·™Œ†fÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~fzv¢˜µĂº·¸¹¹±¾·¾·½ÀÁ¹­§¡§«°²·±ºµ²²«¸¸µ¼¹¾Ç¶¸¶¾ºÈÄƶ¢¦µÀÁ¾¸«³µ¿µ¬¹¶º°©®·Ă¹§¬©³–¢¶²¡¡¬§£¨ªµ»±®¨ ¢£©Ÿ™¥ Ÿ™†‚’•”“£¡Ÿœmu’’“‘‰‹‡‹”–‹“™˜˜–“•›—–˜’©¢¢¡µ£§¥””¢’‰£›œ¨”›“˜—¢–©¨”Ÿ˜˜–“zˆˆ’…€z~’‘‹•ˆ““‰‡‡‘‘„y|yyqlpvvvtrqvvtupkpjpmgnqmousyvqonmnklopr\~mjaZ]`_dcD??A?AV]@@>@@FF=?I6+*30)586'%BJH4"%57=CBEFGIKDHE?@=;;37>AC?7    »º¶¼²«¢ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†y€sx“««°·³´°®¯²®µ³·ª°½·Ÿœ©¬®°­ÆÊñ½º°¿ººÂ¼·¸´´±¸¸¯®±´±²««°²¯²¹±¾¦±µ®¶¿½·²Á»¸½µ¤©ª¶²²¦¤Ÿ¨®«¿¯³´«£œ¦¬±£¤¥¥¡¡¡———¢‰ –›‰‰“•’Œˆ‹‰›——‘˜‘‘“””›Ÿ–”™¥œ˜¢©›™’”¡§™¤£™”œ¡¬¦Œ˜†‘¢§•™§œ¦›˜ª¬®¢©¡›™’ƒyŸ“‘}~‹y‚Œ…ˆ†|‚„‚“¤Ÿ{–‘ƒwuw}xyotyzqtxvmswvswuppmrptwrnjidmkohkmlgpZPggfi^][]ffAABFBS[_LDACBDD@CB99:<7-  ’¨¬¡‡‘yÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤£–„„˜œ¥°§µ±°´¸¶µ·¶¨¬®³«¥ œ‹™±¸ÈÆÆÆËÍÆĂĂ»´Ă¿»³·ª ²¸³°¯§©¨®ª©­½ÆÅÅIJ»À¿½´¿»³·¸´˜˜‰’Ÿ¶±«­ª¥¦§¤®“ ¡£©±¢›£‘¢ª£¢£©Ÿ–—–‘“””““«¦¦¡ˆ“˜ £‡Œ–‘“–”£Ÿ››• ‘¨¨ £Ÿ¨›¯Â¨Ÿ¥¥´¢œœ£˜Ÿ¢ˆ”¡ •‘œŸ§¥¤¡¤¥¥Ÿ¡«¦§¡ „‰”ƒy«vt|~{w{}{†€{xt|€Œ„x{zwkqwxxxrpttvtrsltwsvusslookmmikjnhkkgjrigfdgfb`\_gifX[LMMQ]RJ@ILOFRCEF?5*#*&,BA=6>>>@<502A=GBBCADB<:@HFCA>0#4A@929=>.(3(!  Ÿ–xnÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿWmg–lv†°¿¿¶µ­µ¶§“§ª§©§£®†‹•Ÿ¨°¹Æ¿º¾¿¾ÀÀ¼ºº¸´³¹±¸²°º·¨¨­ª­®¶®°±¯³¾­º¥ÊÆÀÆÀ¿¸¯¬±»®´«§Ÿ ‹®¦±µ¸³ ¸¿²¡¥«¥Ÿ™¶­¤ª­£­ œ›Ÿ¢—‘•–™˜œ £¡™‘„’›•”‰‹Œ‘‘‹‘¦ª¨’›£›¡²½¢¬¦¥£––– —’§¤Ÿ£«§£³³©¡ª²ª©¨§¬ª§˜Œ™ˆ…„¯r}u{‚‚ƒƒƒƒ~~}~|}„€ux|zxprrpxƒzwqntuyyxuppfpntponkdWjfqulknngjqodhgeecfhgij]cdd^f}MEDHC@?J<5-""(3=C?41=>AAA0-DB@EFBCBDC@?6::9;9,/8;;A>>6'!'  Ÿ–xnÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿWmg–lv†°¿¿¶µ­µ¶§“§ª§©§£®†‹•Ÿ¨°¹Æ¿º¾¿¾ÀÀ¼ºº¸´³¹±¸²°º·¨¨­ª­®¶®°±¯³¾­º¥ÊÆÀÆÀ¿¸¯¬±»®´«§Ÿ ‹®¦±µ¸³ ¸¿²¡¥«¥Ÿ™¶­¤ª­£­ œ›Ÿ¢—‘•–™˜œ £¡™‘„’›•”‰‹Œ‘‘‹‘¦ª¨’›£›¡²½¢¬¦¥£––– —’§¤Ÿ£«§£³³©¡ª²ª©¨§¬ª§˜Œ™ˆ…„¯r}u{‚‚ƒƒƒƒ~~}~|}„€ux|zxprrpxƒzwqntuyyxuppfpntponkdWjfqulknngjqodhgeecfhgij]cdd^f}MEDHC@?J<5-""(3=C?41=>AAA0-DB@EFBCBDC@?6::9;9,/8;;A>>6'!'  ¼¿¼¨´²¨¡¯§“ÿÿÿÿÿÿÿÿÿÿÿÿ“}Œ€pi~ms¶¸µ¯±ª°´²·´±¥™¯±œ ©°¹¹¥­®¯µ²µ²°¼Ă²´µ²¯¢¬¸¶¹®ªª®£¢¡¥±¶¿¯­¹­³¾§™±±»Å̀Ç»»¿Ă®¸¹·¨®§´½½Ă°´»²ª¬©§¥ª¤Ÿ§¡¤¦©§«— ˜—œœŸœ˜Ÿ•’œ˜““–œ§”’‚‡”Œƒ‚~†ˆŒŒ‹™• ™¨ “•ª¤¨¤««¬¨Œ”˜™˜˜‘¡—£¥ª©›“¤ª¢“‘Œ¤§§œ¢œ¬¡œª¡”‹vsz~vvƒ‚~ˆƒ{{|…zx€€{z}vt{{~~~vvsxyx|rlmvsqnomlc\hxut{nnjdjlkibikqyqnoliccgjdiz[ZRH?AIF;2&#!#6@FF><@@>?, :HGFCADHC??<9#(#&+1+,7676;;<2(7?<%!¿·¹¸µ»­²µÿÿÿÿÿÿÿÿÿÿÿ™°»»©§§˜Ÿ}g‘º¼¸·¥¢¤²¢¢¥¦¡”•‹© §½´³µ¿¼·³°³º±°¯µ°§ªµ´°°¥¦­²¬¤¦Áµ¯·¬¬¸½Á¶¨¼¿ºÀÄÉÈÀÅȺÀÅĂ¿¿½¸»»°¥¥¬°¹Ä¾¡¡®¡£“–¥©¡¦œ¥Ÿœœ¦˜˜ƒ‘§¬Ÿœ™¡·’•–“”“…€ƒ„†sy‰ˆ„˜’Œ‹‰„„†¦•‹•°·Ÿ›‹˜¦–œ ©›œ–•’©”…t|ƒ‹‰†”£‡ƒw©‚~t}€~|„}pzz€€wv{w€…|zzrq€}~smq~}{|‚uwzxsrzvlnntpsZCMatgggfmhkp~momqrkr}}mjflqqosvvihHJKKC<5!".:C;BA<@CCA3IDEHCFCBC>?95#1.-3?:5'5-))5=6+&%-)źººÄ¹­µµ¨€ÿÿÿÿÿÿÿÿÿ–³¾ºÆļº­³´«—®Á¹°¯±¨«²±£–¯¬“’•­¬²¿¸´º²²¶±¯²±®¦ª¯¬±²±°°²¦££­°±»¹º´´·±½¼¢±˜·ÀÀǺ·º»­«°·º¹À¼ÈÇĂ¯©£¤¬¨µ¾Ä¯Ă¾®¨¢‘•¤¯¬±¦¤«©•—–’”œ—— ¥§œ¤¥¹¯­¦Ÿ –‘‘‡‰Œ†‰“Œ‡†„~€„Œƒ‰„„‰Œ„™›Ÿ—™£¨¦¥¤“˜”‘ŒŒ¡¡›œ§©•ˆ– «“eJ§‚‰qvx‰‰†wxxƒ‡†}w|ww‚{smiuyxuo}†xpyx~{z‚‚vuoqloonmmjikq‚ƒulllkosnsl`gxol€kfdkoljpps„xuGDAACC@/).-)(9:9AC?=0?D>%8RGECAAAD@>FC5,&%5$#/:<3 .5-'+' ĂÄÀ¾¾µ¸‚s7ÿÿÿÿPj€’§°²±¶¾¿Â«°¹µ®¤¦¾¶¹¼­®¤¥¦£¤«³¨–œ¯ÀÆÀ¶³¶¸¹Âµ·¬¦³°««¬®®µ°®®¥£›ªµ»²Å°´½¶´º½·µ»½ĂÀÄÇÀÀŸœ­¹¿¸²¸¿É¯¨¡—œ§©¦©ÇÍÅ·»¸¥–†§¤ª®¨¯»˜–™›¢¡–¢¬¨£¤§½°µ£©¤•‹‚„Œ–“–‘’”™’ƒ†‹ˆ‹‡Œ’–¥•¥¢ œ™’’ˆ…}„€|ˆ‰‹€|Œ~$3Œ}vy„wuyzsy~y‰…}{}~uxxokmumv|„ˆ~|z}ƒ}}~€mskfetwkosv{kqjgglpspvnkoukrƒpqkmggmnmur}rrraFBCEA;1);51.0',:?:>>=836>75+0FCC?>FC<9?A@=<)%=43.06$ 6/.>=91&*.,!)ĂÄÀ¾¾µ¸‚s7ÿÿÿÿPj€’§°²±¶¾¿Â«°¹µ®¤¦¾¶¹¼­®¤¥¦£¤«³¨–œ¯ÀÆÀ¶³¶¸¹Âµ·¬¦³°««¬®®µ°®®¥£›ªµ»²Å°´½¶´º½·µ»½ĂÀÄÇÀÀŸœ­¹¿¸²¸¿É¯¨¡—œ§©¦©ÇÍÅ·»¸¥–†§¤ª®¨¯»˜–™›¢¡–¢¬¨£¤§½°µ£©¤•‹‚„Œ–“–‘’”™’ƒ†‹ˆ‹‡Œ’–¥•¥¢ œ™’’ˆ…}„€|ˆ‰‹€|Œ~$3Œ}vy„wuyzsy~y‰…}{}~uxxokmumv|„ˆ~|z}ƒ}}~€mskfetwkosv{kqjgglpspvnkoukrƒpqkmggmnmur}rrraFBCEA;1);51.0',:?:>>=836>75+0FCC?>FC<9?A@=<)%=43.06$ 6/.>=91&*.,!)ÄÅÇÀ»¨Ÿÿÿÿÿÿÿ‹“Ÿ¡™™¡µœ˜³·¶»¬­®¨£¢¨¨°®µ±¦¢ Ÿ¥™–†›™˜°Â½¹ºº©¥°°«§®µ°®°±°°®«­±¤¦³­©µ¯¯ª³«©··¶Â¿¸¶¼¿ÁºĂ¿¿¾¸¬­µ´·­¦¿©“¦¦–”——“˜²²±¯§°¤·¾©–§ Ÿ©°¬©¡’˜—™””§¡¤¤¡¤£ ¢–’‰’’‹ƒ‹…„‘‘“Œ”—‘…‰†…‡ƒ€‹…Œ—–”’——†¤¤˜‘…„ˆxyy‚€„‡ƒ€‡‡££¥Q8Ÿ‚‰€t‚vyw|}}Œuu}ƒzvlsuwsil{}x{ƒ‡x‡u€„y}rkniltupqmrrtomkheiinqsihgu~uwwnpgklhjiprruvxuRBBFA@DA9(88;@,2<4@A4(>CC>:%GEB@?>@>>?B?>=--=C-(@FA-""50+37CB>=B;>>?52%'$ À¿¶–}4ÿÿÿÿÿÿ¥¥ŸŸ¤¤””›©¤Ÿ¡¬¯´µµ±u¦ §­¬¯²¬›¤°©Ÿ•–‘–‘œ¸ĂÁ½½§£­«£¥¬§§¨²µ²²¯¬¬¯­¬¯·«¯³º·²®º¾½¬®»»¶¾´¸¶²¸³´®®²»¼µ­±£ §¥¬‡”˜œ¥¤«£¢®·¼¶¨Ÿ¬®§¯¡Ÿ›‘}–Ÿœ«¢¨”‘ˆ–œ¡œ•‡’Œ‡‡‹‰ˆ‰•”˜œ–‘‹~‡‹’†‰…‘”’œŸƒŒ™ŸŸ¦¢‰„†‚‚‹€ˆƒƒ˜ ¦¡Ÿ–);‰‰soƒ‡rzz}Œ‰uxrovzyƒ€}}ynyw|zy{„„y|…wuxorkllnrojlrpp{u{tmkllptooegfjoimmkjbkkmlhjjivw{v`C>BE@?@?C74>@>A?A@DC<)#! 5>/!EHD?AE@<;1=<2/8?EDDCCAACEBDBBCC@&!µ>ÿÿÿÿÿÿÿÿÿ²·µ¬¡ •‘˜™—”œª¶¼·¶´§¤¥§™˜¤ª ¦¼É˳¼¼¦§¨¢” °¼º·¹©«§¢££­§¨ª§°´´¯©¬±·¸´ÈÀ°´·¸¸ÀÁ½»»½¶ª±¯±¯±»°«©«’’œ£©¨¥¦£«—¥ª›x™Œ‘“£©©ª£ª¶¾¯–˜¤Ÿ’¨¢¢¢¢}“¢œ ¥œ“‹§µ°—™…ƒ€ª ˜…ˆŒ•‹’•’œˆ€{ƒ„|u„‹…‰”‰‘‘–’‰ƒƒ…‡…‰˜—Ÿ¬œ–’„hb†§{rzq{rz}}v…vvssnkkuyvpqkn{vtzxt‚z}€yptv|srozsrwrpnntqprqrlv{mrmhfim}wujjkikopvongjnmv{`DF??CDA?@DA:<:=>>=AEB:,!066%6IFD@BAD@=BC=B?B?>B1"(;BEBFFFFDGEGGFGDC?AAB@ÿÿÿÿÿÿÿÿÿÿÿz¶¸²¯©£ ™’”‰†¡§®²µ¯¬¬¥ªª—¡¦ ¦ÎÏÈÁÀ¾½µ¬¤¢¡­·¹³´¨¥£¦¥¥ª«¨¨¬®²²®²·°²¼¸¬º½¶·´¸´¸±µ½¸µ§«©«°­°·¡–“‘¦£¡­ª©©Ÿ›™ ©¢Ÿ¤§¤¡¡©¬¯›—£™¤§©®°­‹”“—¡•‰’•“’›™¦”ˆŒ~†˜²²·°™–•‘‹‘—ƒƒ…€‡‡†ˆ‰Œˆ~{z{uw~z|}†‡ˆ€„ˆ’ƒƒ„†…‚†ŒŒ’™¨˜~‰˜˜O¥y~yp…„tw„x{yx}upvvvrx~wy‹wy}q|„zwwyzvokhqtxjnm}€vpmtsnjlponrqkkh{{volmikiqsqkmvny‚vSTX@EG>?AD;:>;=?=8EFED:!)/%(@FJJHGEIFEB>CF=CB>BCAAGFDCA?EFCCDIIJJJLC:A@=@<(.,-ÿÿÿÿÿÿÿÿÿÿÿz¶¸²¯©£ ™’”‰†¡§®²µ¯¬¬¥ªª—¡¦ ¦ÎÏÈÁÀ¾½µ¬¤¢¡­·¹³´¨¥£¦¥¥ª«¨¨¬®²²®²·°²¼¸¬º½¶·´¸´¸±µ½¸µ§«©«°­°·¡–“‘¦£¡­ª©©Ÿ›™ ©¢Ÿ¤§¤¡¡©¬¯›—£™¤§©®°­‹”“—¡•‰’•“’›™¦”ˆŒ~†˜²²·°™–•‘‹‘—ƒƒ…€‡‡†ˆ‰Œˆ~{z{uw~z|}†‡ˆ€„ˆ’ƒƒ„†…‚†ŒŒ’™¨˜~‰˜˜O¥y~yp…„tw„x{yx}upvvvrx~wy‹wy}q|„zwwyzvokhqtxjnm}€vpmtsnjlponrqkkh{{volmikiqsqkmvny‚vSTX@EG>?AD;:>;=?=8EFED:!)/%(@FJJHGEIFEB>CF=CB>BCAAGFDCA?EFCCDIIJJJLC:A@=@<(.,-ÿÿÿÿÿÿÿÿÿÿÿÿ§®²¬©ª““‹‘˜Ÿ¡¬°±¯°«¥¡¨™˜›­¯®±ÄĐ¢¹º´´¹º©¬©³­±°«±°¨£§¥¯µ³¶®°°¨´´¨¦¡¡¸¶´´À¼¼¹µ»·¬±´´²°Ÿ£±Ÿuy–›˜£¬¡·³§§ªŸœŸ¢Ÿ›¦¡Ÿ™Ÿœ§³ºº©¬¼¥¤£¥©«¯¯ª¥–”–£Ÿ–‡ˆ‚‡‘œ•œ•{†‹’²¥¨¬§’Œ¥¢’‰“‹‡ˆ‰‡‰ ‡ƒ‡€vvzw|~~~„ƒ„ƒ€‰‰‰ˆ€}„˜¥¥‹‡›‹¤‰p {rp|„wrxwrotuwwz{t|}~uwzpzr„Œu}w~|z~ƒ}toytqkrvyyrlwwxujnomspimruvllsq{~murlnromprwudYn_MT\SWKC@C@CA=?<=>CIGA<.10'/6INHEHFFFICCBF@EDD?>EFAIHHFCBECEIJLEEKMFFFC>>37@?:7(&%&ÿÿÿÿÿÿÿÿÿÿÿm©©´¯¬­§“ˆ‘™¥©­¼´¯ª©¢–¥²«²½»¶»Ă½µ°Ă¸³»¹¸µ®›£®¥¦¯±§§¥¡­®­«¦¯±²¯¥Ÿ£¬³±¸¾µ¯°«µ®©¤´É̀ÄÀ¯£˜““’Ÿ›œ™¨¥¬¥™¡§§ªª°¨¤£¢¥™˜™••¡­µ¸©¨²¼±±²–†ˆ—›’‘Œ‡†‹‘™”‰œ›Ÿ«¦—˜¡¦°¡˜ˆ‘¦“…‘”‰ˆ†€{…€ƒƒ€‹‡z€†|†€‡‡„…ˆˆ†}‹œ……’•w\]—|tlw|ƒ…z}lnjvur~qxywqw}€z|uuƒ€{}}u}xwtw|xquqwsvqlruuuqroqpjgn{zkpnmprhqqmuzxuffccpplkzXK@@>@A=;9=BHNLI?@93=72):OFFJKJJJEDECGFIHELJKMONFGHHJIHKJKKMMKFGFDBA>/@=@=&&/"#ÿÿÿÿÿÿÿÿÿÿÿŒ¢ «­©¨œ———‹Œ•Ÿœ ©µ­³µ²²¹¹ºº¿¼¾½ÁÉÈÇÆÇÆ»µ¶·±­®¥˜¦¬©¨¯­­¬¨°²±®°°±ª£¥«§¥ª­²­´²°£¢ ©±°·¹µ±ªœ‘’˜—–™ £¡œ˜©£¦¬§¥¤¡¦£Ÿ¥¤¨¢²©­¶»Á¾Á»ª¤›‘’~zvy‘—‚|„‚‰‘“”‘§˜¢™’™•‘ ´¥™˜’†„‹z~€‚ˆˆ†Œ“”ˆ‰„ˆ’}‰…‚‚‹„‰‹‰‰€oxtv„‰{xrvlr{}{tsslkswijnq{‚wrmp{|x„…ws}wovzvsurrrs{€xonnpnru~y{t{tqupoqqtrnxmonlowx~l^WITnoknfeaQNECAA?>?DKGIHF?>@@.BD==/K@@DGKOMH?CEDCEMRONLJHNIHGFFHGJLJJLLGHGDDBC?3:AA@3+'1,ÿÿÿÿÿÿÿÿÿÿÿu¡§¯¯®¥œ˜‹‹—¥¬¯²½·¾¾·¿¾¾»¾ĂÁ¹ÂÀÂÄÆÅÁ¿¶¬±³«°º­«¦®«¯¯°³ªŸª¨ªµ¶³ªª¬©©¦¨ªª®±ªµ¸±¬ ¯¬¯¸¬ˆ‹Œ}”¢›”œ¢œœŸ‘”‘£©¡ª«§¡’”¤¥®±¢®·¯±³¨§¤¸À¹¦ŸŸ˜‡~vv’”‡‘…ˆ‡‡•“”–““Ÿœ™˜›–˜—‘™™”–•’ƒ†•—™€§¨¤Ÿ—™¤«¥™œ–Œƒ•“ƒˆ€¢¡˜rw{u}‡†„‚„wuwxz‚{qywvlgpnmtpturirw‘„r}{‚‚xyr|}}pssoruy‡xvpkjmstwwpzos|nouyovprvzqqlwyzvvkQ@Haoszts_xuMEFJCGGIIE?C@?DDDC?;?(:A>53JHLNLBFDDEGHLMOMMMHGEEGHIGFFHMNMMLIGFFCB:6<>>ABD0 -*"$'$$'ÿÿÿÿÿÿÿÿÿÿÿ«³¥«¥¨°¢—ƒ‚“ Ÿ¤­¿ÇÈÉĐȼ¾¹ÂÂÁ½ÂÀÅÈÆ¿¸¸¸»½¶°¦ª¯®®­°²¬«¨¦œ‘™ªµº¼½¦§£ ¥¢¤¨¯¶²³´±¥©¢¨¨«ªŸ•”©¡”’“œ—•‘›•™‚¢¢¥¥§´¯§¬£®¹µÎÇ£ ®±¯¤–ŒŒ“‹…„…Œ‰ˆ†‰pzˆŒ—‘“ ‘––—“ œ˜’—•Œ–”›™‘‚„‰Œ•”Œ‘–“~‰‰ƒ‡‰’‡|„ƒ„wxxr‹ƒŒz|di„‘€~„rhnnmljkrpownnzx‘‡…{t}ƒz|‘Œ€|tvwt}{qs‚ƒpjorrxqqlprnsnx|vttqtuwqyytxkvpkieSQXn}yuqz|ˆyBCBILHMQOGBGCAEDGF?: 1IUOIDHIJMKGJGGHS\Z^XMNOLJIIKKIIKKLNOQOLGFFAEGHCGBAD?A>:1)" *+!##111##&-+-ÿÿÿÿÿÿÿÿÿÿÿ«³¥«¥¨°¢—ƒ‚“ Ÿ¤­¿ÇÈÉĐȼ¾¹ÂÂÁ½ÂÀÅÈÆ¿¸¸¸»½¶°¦ª¯®®­°²¬«¨¦œ‘™ªµº¼½¦§£ ¥¢¤¨¯¶²³´±¥©¢¨¨«ªŸ•”©¡”’“œ—•‘›•™‚¢¢¥¥§´¯§¬£®¹µÎÇ£ ®±¯¤–ŒŒ“‹…„…Œ‰ˆ†‰pzˆŒ—‘“ ‘––—“ œ˜’—•Œ–”›™‘‚„‰Œ•”Œ‘–“~‰‰ƒ‡‰’‡|„ƒ„wxxr‹ƒŒz|di„‘€~„rhnnmljkrpownnzx‘‡…{t}ƒz|‘Œ€|tvwt}{qs‚ƒpjorrxqqlprnsnx|vttqtuwqyytxkvpkieSQXn}yuqz|ˆyBCBILHMQOGBGCAEDGF?: 1IUOIDHIJMKGJGGHS\Z^XMNOLJIIKKIIKKLNOQOLGFFAEGHCGBAD?A>:1)" *+!##111##&-+-ÿÿÿÿÿÿÿÿÿ·ª·µ¹³±´¯¤”‰’‘˜£©¿ƠÙÉÊÆ»·­¹Â·¹²¸ÁĂ¿¸¶´¶³¸¸°¯¨­©®®§ª§˜‘ˆ¡ Ÿ°»²­Ÿ—œ¢«¤§¬¯¬¡¥­®­µ¹›¡­­¾©•’Ÿ£¤› ¡™q‡¤¡ ¬   —’¡¢­«¬¥©ª­­­µ¶Ÿ› ª²¨Œ„„›˜”‚ƒ|lr‚˜—‘’‡”˜›—ˆ Ÿ”˜—–“˜”•™Œ““•—’‹‘ˆ‹ƒ}‚ˆ‡‰~{{w|„ˆ‹ƒr}zwz{suy‡€|zzwz}‚ykˆzmwzw|sroinsklwtnmsur‚xqq€x‡†~~}~lmrw€w~uwuuzuovtvupnjsxz~xt{sqtnsnkzvtknmw€i{wzxesxt\GOEGACLINGEKD@>@DB<?@DEBHDHKJHFKHFEJN`ebd]RQPLFIKHIMMDFLQOLFEBFDEJHBDEEIE@A<85-,*%$"" #035,)0.*32ÿÿÿÿÿÿÿ¯«µ¼À¹¹Ă¾¹±¨ •‹‰‹¡©ªÎÔÎÑÆž³¶¿½Á»Â¼¶µ¶·¯¦ª³©«°ªª­«©›—‘‘™¦¨Ÿ­£¢¬—’¤£¤¬Ÿ¥¤¤Ÿ£¤³´±«¬½´¤¢¤œ¬®¡›œ££””——›©¯¬©¡§­±®¡Ÿ©§­§§¢¥®¿ª Ÿ“œ¥¤„…~ƒ‰‡……ƒ†‹™“”—¡™™——”™™••—•¤§œ™”“•™‘“‰†˜ˆ†ƒ‡ˆ…Œƒ†„‰‚‚“…€ˆ|€ywzzuˆŒ„}}{z{{qw{ƒ„{}„~ƒ~v€lIlB„zyrhjovmoonfkmqlj…‡…|ru„‚…‚ƒ„swxvu`dor|uasvuwozymossumusx{~xwuvpmooni|~}„sswtruxvrqwmjLO[SKEKH>AACAA=??=9,8BHFIHHDDCHHDBDNHTX`dgj_NPIDLKGQPPPGJPKLJHFEGGFKKHKJFB=@;8:4//1%  ''*',..(,/&!!"),(14..ÿÿÿÿœŸ¨¼ÄÂÀÂÀ¿À¾®©˜‹~„ˆ“› ¦ÁÅÏ̀ƹ®°¸¾¾Âº·³·»¾¹¦­¬§·³µ²¯­¨’’’¡ ”¡««¯±£¥ —“ˆ™¡±®¦¦£§®¬¬©²¬§¨¸£œ¢£¡¡˜§—¨¦¤¦¨¬±²°°¬­¯±®ª§«¥¡¤¢£§©¥˜”—•|†ƒŒ‰Œ™–œ¡¦§£™–œœ›˜˜–”ŸŸŸ–’“’‹‚ƒvt†””‹€Œ…‚|„‡†ˆ‹‡ƒƒƒƒvutu~z}xxz|zxrv{tu‡……{~ƒ‹ˆuvyw}Tswprmswqsrmltlfm…v€xuyŒmpsux€|x|vtpz~vuuvuz‰xnmqqzww€wou}umspuruopqt~tmwqkppyqnmthU`_aTC@?ABB@@=@C@?>)=21/0100* &/ ).+,&&(3.0/0.-/-/-,,//.2,'&ÿ±¨§­¬±±¼½Ä»ÂƸ½À¸±¢Œ„‹‘˜¤›£µ¾ÂƠÅÀ¿·¼µ¸¸¶¹¯µ¹·¶º»«¡¬¶±µ¬¬«¬ª¡•¥§´¸¹§®¦Ÿ—”‹˜Ÿ¨§¨©©¤¥¥­«¤¥»«¦ÁµŸ›Ÿ›’œ©¼³°¦ª¤£©©«¢²°ª­®§¨¹›¥Ÿ£©Œ˜œ’•¡ –‘›––›¤œ–ŸŸ  ¤£§¡™”“™¡–™ «¦Ÿ–—’††€„“ˆƒ†{~……„ƒ~~x‚}}}vyˆ}„€€ƒ~‚‚u~xz{~ƒ}Œ‡†y~…€‡€ivxuz|qpqmjpsqmjzyunttt{uuux{rw}|€{y‘z{tzwuz‚~wnkkjlqpwsr|vtsutpptuqrux~vu|qrnuqkgv}xZJIJHFECC<=FMKACECAGEMLGIGILMJLHGQXfutgk_VPTQL8?KTRRRNOOMIIGHHHIKLMC@BB76;8<>?<:910177:7'((10-"!452221389665613/,+)-ÿ±¨§­¬±±¼½Ä»ÂƸ½À¸±¢Œ„‹‘˜¤›£µ¾ÂƠÅÀ¿·¼µ¸¸¶¹¯µ¹·¶º»«¡¬¶±µ¬¬«¬ª¡•¥§´¸¹§®¦Ÿ—”‹˜Ÿ¨§¨©©¤¥¥­«¤¥»«¦ÁµŸ›Ÿ›’œ©¼³°¦ª¤£©©«¢²°ª­®§¨¹›¥Ÿ£©Œ˜œ’•¡ –‘›––›¤œ–ŸŸ  ¤£§¡™”“™¡–™ «¦Ÿ–—’††€„“ˆƒ†{~……„ƒ~~x‚}}}vyˆ}„€€ƒ~‚‚u~xz{~ƒ}Œ‡†y~…€‡€ivxuz|qpqmjpsqmjzyunttt{uuux{rw}|€{y‘z{tzwuz‚~wnkkjlqpwsr|vtsutpptuqrux~vu|qrnuqkgv}xZJIJHFECC<=FMKACECAGEMLGIGILMJLHGQXfutgk_VPTQL8?KTRRRNOOMIIGHHHIKLMC@BB76;8<>?<:910177:7'((10-"!452221389665613/,+)-¯®§¨±²¯²¼»ẰÉĽ½¶´§£—›¥£§·¬™œ¬®°ËÁ¾Ä½º¼ÄÁ·µ¯¬¯° ®±­´¶²°«¨¨¸¸´ Œ•¡«ÁÂÁµ²¦œ—ˆœŸ¡°¨¦©£ µ¬¡µ¦ª°±±¦©¤§¡©®½¨©¨ª¥¨­®­©­§¤¡¬µ°£ £©¥–“¤¢§ ¢¥© ¤¤›‹¤›“œ°§£§¥¥¡ œ•‘“•˜©§Ÿ¡–—˜œ–—‹’‘‘’‡{zx€„|u{x€ƒz|vsvw}~‚~z{|{z…{ƒŒ‹y‰ƒ…ƒŒ‡|{zt|notpnnssqlujlsmiuƒvuqkows|wu|y€…q|twvyy{€tqqpknqqxyplr|usynkqoowxxzr}~wyupvluuoe_KBBBA?CD=ASLFCID70?CHNNKJIHKJLLHAEHfbYujQKJJOUJEHKDNTNLLJHJHCEIKONMKCCC64:><;:CA@52666"!%.&+2032-,,/1465899:24500/./-*.,ª¯²¶¹µ½¹´¼ÅÍĐ̀Ă¾·®©£œ£«²²¯°«­¢£¤¼¶²¶¿¿¼ĂĂ¿¼·§¢¨¨´¬¯º¶­¨¬­²²´²« v‡™¡¿Á½· œ›–”“˜¢¨²®§˜²¬§¨ª©¯·²³³²´¨¨ §¨°«¡£©££¤©£¥Ÿ««¦ ¢¤ ©£©¨¨°¤•Ÿ¦¡¢¨¥©¯ÁÊ»§Ÿ™ ª¡™››­¦¡¢©°µ¬¬ª­œ’“¦©« ™—‡‹“‰”’…”‘‚ƒx{}ƒz{qv|€~~u{……|ƒ€†„‚y{…‹|†‘•ˆ‡‚ƒ‡†‚ƒŸ…uWvyryq`oosprxormomsrq…wqsuq€trw}x{ƒ€uuynxusvvophlomqyz|ywrurknpouw{syqnrsonmgjmijgfa]RIJCFBCFGKEGCC6(IFKIHHEJEFIgdhYRXbkjkkk\_HDFGONLJKMMSPQFGFHEDEJKJOMDCB<:>B?=78773())$*.,+.(.42&*49474243//1,020/..ÇÅ¿½»ÂÀ–¯Á¾¹ÇÈǽ·¯§¥œ˜˜¡¤«±©¡¢¨œ¥Â¾»µ±º­«¥¢›ª®¨£˜«¤¡¤®±®­µ¹¦˜¥¥¥œŸ£§¢Ÿ¤¥¨¨¬­­­­´¬Ÿ¬½¬©¶¶º¹¹¯¯ª¤£Ÿ¡¡¨©ª¡¨¤£«¬§£«¥¤§ Ÿœ§§¢¤¥Ÿ¢¦°²«­·»º¯ª£²º›¨‹–†œœ¦£§¥§—œœ ›œ©®©§©ª­´±¡ƒŒ‘“—„‚‰†„ƒ~||€qrvv„ˆ‹‘‡ˆŒ†ŒˆŒ„€}†‰‚‹‰‹…€‡‰|…~|†ŒŒ£Ÿ†tY<-A,(mnpnot|wst‚†xjkoo{u}‰z„ttry{{zw}vx|rxtrvrprmsyqvstpzzsnitvtnxqrkeirwqnjlnfifjmhkkQVSEWYLBEDG<(CQ\f[LG4HFWqwvuplqw{zt|{pjaOQKFLLMPQRSQJ>FNJJJMJFDKPLOKK<;BDAE@A?:7=<86&!-111168*../-%+0654(110-0.01/10.̀ÏÈÄĂÁ½µ¯±¹·ÁÀ¼·¿±¤©¥¨§©ª¤²´¦‘­¥Ÿ·º¾À³·¶¯¬§¤˜¥¬©¤™¦ª¨§¨ª¯®†—™˜—  ¦¤©¥©©¬±«¬²°¬ •”›©°µº²°µ¾»¶±§¦¢¥¥¨ª®§¦›¢¡œ±¦¨¬›ŸŸ°«‘–¨˜–™£˜¥¬µ´°­ªŸ™›¤¤¨££¥‚iv}—”“™Ÿ˜”™•™¡¢§­¤©¨°°«§œƒŒ…†‘™™…Œ‰z}€‡…†‰Œƒs„…|~‚ˆ~†ˆvwv~“‰€„‹•‘ƒ‚€}‚…‚…ƒ‚Œ…„†?-_lnpolmwx†{vosq{u|vswsru}{qt}“€{Zptuotsoyyzsvy{tusunnn|†vx}uuz~tn{…|numlonpnnosfgfdh`VICBC2;LRP\VOPKJKRbwqvrsuvvwt_]p@:PQNJNPQRQQOLE@DFHOOMMMMOQSLONLOMKIC@D?9:>842$&(!!!/1+-347//*--../450/.,-23552-230̀ÏÈÄĂÁ½µ¯±¹·ÁÀ¼·¿±¤©¥¨§©ª¤²´¦‘­¥Ÿ·º¾À³·¶¯¬§¤˜¥¬©¤™¦ª¨§¨ª¯®†—™˜—  ¦¤©¥©©¬±«¬²°¬ •”›©°µº²°µ¾»¶±§¦¢¥¥¨ª®§¦›¢¡œ±¦¨¬›ŸŸ°«‘–¨˜–™£˜¥¬µ´°­ªŸ™›¤¤¨££¥‚iv}—”“™Ÿ˜”™•™¡¢§­¤©¨°°«§œƒŒ…†‘™™…Œ‰z}€‡…†‰Œƒs„…|~‚ˆ~†ˆvwv~“‰€„‹•‘ƒ‚€}‚…‚…ƒ‚Œ…„†?-_lnpolmwx†{vosq{u|vswsru}{qt}“€{Zptuotsoyyzsvy{tusunnn|†vx}uuz~tn{…|numlonpnnosfgfdh`VICBC2;LRP\VOPKJKRbwqvrsuvvwt_]p@:PQNJNPQRQQOLE@DFHOOMMMMOQSLONLOMKIC@D?9:>842$&(!!!/1+-347//*--../450/.,-23552-230ÄÄÉƺ¸º¼·º»µ¿¹µ²±¢¤¨­«µ±¨©¬±·²¤¢ª¡™›¬°µ¹ª¦®·±ª­–¢«« ¦®®°¯°´±¢†ˆ™ –”‘˜£¨°¢§ª¨­°°²«­§±ª¬¨¡£²³µ´¸¸¼Ă¾¸³¯°°¨¤§Ÿ£¯®©°Ÿ›¡¦¢¤¡™–…•Ÿ£¤¡¦«®¯­¡Ÿ—¡™¡•Ÿ«£wv}Pt~ƒ–’“›“—œ™™œ ™£§¡ ¬¬’ƒ‘–››˜“‰‹~†Œ‹ƒz„y…z}~‚…‚†y|Œˆˆ‹‚‡†ŒŒˆ…„|‰…€ƒˆ‚~}||{‰‚n/ $Spyvwx€wxvxtxyyvtvturyrtkvtsƒ‡‚‘wpouvv|zxu~~vzuyxr{~~‚€xw{~~snu|‹ukfo€omsp]UDCJHXhOLIE<=XT]w_FGIMPXT"k{tuy{€xsjc^eaZTOFHKNRQOMQSMJKKAGIRRSRNKMIMH;LKKHFFB@>>>:8;<@<4 ,$-,%*/2350/4.0/4554531/.,.0/0/1141,ÉÁÀÀÀ²ÂĂ·ÁÆø¼¹œœ–”¤¨²°¬­­©§§¥¡§®³®¡¢¦²¬±¬«§¬§§«¦©´­¦§­­›‰§£¤¤œ•¦´¾´¶¸µ´¬¨¦¦®­¤§¨«®¥¤•—¨°³·¿Á½·±›ª°«§¢¥™£©œª¬ª·œ£¤£¤™›‘“˜›– ¢¡Ÿ£¨£Ÿ™£¥ ¡¬³¯¢•™™ {‡ŒŸ›“››–™›—˜›˜—¥¢¦®©©¢œ‹Ÿ¤„’”ˆ†‹ˆ††„€†††…y…~}€ƒ„{}‰‚†ˆ„|‹€€€ˆˆ‡‚}€€~zwu~~ux€|q|}y_kPdjp[pzux{‚tlpjk}yuwt{ywpyzuvmnt„xqt~ƒ‹|xz{zqzx~}zzy‚€zxzx~sqt{x}wx~q{y[UPGF_i_MLH#+RirvnWHMauy{}‚…ƒs~~„~€wL2eIW]_ILLMIIIFDFTUPJJGC?BKVLJKJMD>HEFD@;769;;8575171//) $()!.' )191*-..256540)$000.1-).-.011.ÙÊ¿ÇĽ¶Ä̉ØÑ帨Ÿw{‘§¤¦ª°²µºº¥§¦ ¡§±ª¦¤¥«¢§©®±¯«®±¯±´®­ª¥¡{wˆœ¢§Ÿ˜”—œ§½ÂÁÀ¹«¦ ¨¦¤¢¦©«±ª°¯œ‰‚¨·À¼¸¾†™´¹®®©¢“›¦«·´¬¡¡¢£ ¥™™Ÿ”“¢¥©¡¥§™–˜™Ÿ•©°›«™s• œ“™˜˜—““™˜™ £  ›£§©¬®“ˆ‹£Ÿ‚‰‹ƒˆ‚r|~|‚…†y…„€~yy~„…{{yŒƒvv~‹‘„†…~•‘‹Œv„xzx}~y{vrwwuyw`_urv~ƒ|iroksu‰„|v{yzno„€y„€uuw„‰ˆzvˆ›ssw‚zy}€†~„uty€zyt†wwopt†‹…~ƒxr|kNJE[g_VJIA)L[jphf_drrsw{}wwmqyutmmmupourikf_bdSENE@@DFHGGIHB@AJMPKECCBEILH:8:<>@???:7124-%"%'!" $.*,5=4/-+*.0,,, .141101'*,--./12ÍÉÀÅǵ¸ØØ̉ºĐÇÇ·©ot‰¤–ª¨³º±§ ««§¦©¯œ­¬ª§¨ª©£°´·¶´®ª·¶¯©”•›Œ°¶¸²¢™¯µ¶³¹®«¦¢ª ¥¢©©¦¦¡“‰„‘¦»¸µ¶²´º²ª¦¤¨¬­°¶ ª£¢£ ‘‹~•›— ›˜˜–’”› ¡§µ›¤™¨©­­«¤ ©Ÿ¥¢’’”•‘—“›˜› ¥›¡›£¹Çº­«˜­…‹‘‰ƒˆƒwy}‚vƒ„}}{}}……ˆƒv‚‡zx€w€~|†„‰ˆˆ‡Œ†„„Œ‡xwyunrvyzƒvimxzqvvzyS3X~…yquzƒ|rszxuyuvmq|mmt{nqpsuloq}™£‰{swˆyzƒ~…€wz‡u‚|yvv‡‰‰„ƒyƒ}z„b[rYOVTGE2^mUuw\TSqxrpiv~ƒ‡pqvkX[e}|yv{wnc\_bbfheYRGNEEEFGHDGHHMME?>BADFE<9?<<74,!#$%'01+&#&(!"(0-1:40(,#!'&-.0(+--,*0/*,,..11ÈĂÄ¿¿ĂËÜÙĐÂ×Æ·®¦Œ‚£ª¢  ¬·ÂÀ©¡Ÿ›¨¡”›«ª©¬©¥¯¤ ›¢«°¯¯²­±¯±”€‚‚Œ’¤¾ÅÊÄ¡”™˜¥©®¾¶­¯¡¡¦¢œ •™¨›œŸ›†ˆœ³µ»º¸§¨‹›¦©£­²©¨§¡™‰}‚‰“‘—’ “—”—––’˜¦±¢–­ÆĶ««¡¨•–®¦œ‚”œ—”´©˜¥—–¢©½Æ½·£Ÿ–‰””{‹•y€ƒ‚|€sz†„ƒ…„|€‰Œ†‚}~x…†„}z‚}†}~‚€„~‚}stw|rptq‚€w{~~€~zuoziQuƒzwpx€‰€zusvmsvnnrxtwpmjqmjtyrvs|†{zvv‰€Œ‹‰}€}zƒ|„~‡{rr†…‡‡‹|{{{…†u}u[i_SB>@`ed{}rms€ƒˆ}t…xm[RORUWY^d_e`ZVQPQTYdf[Y\ccZIFC?>=DIJIC??=>CC=;4/..264;8:;<750781)*10)./**-((0./0'*124.(%"$&%*'%!"*,*)12,-,,/0/¾ĂÁÀ®¶ÈƯÛỖ×ȵ®¬¤—™ §—–˜©¸¹µ«§™“˜¢›œŸ¨§£  ¡œŸ“¥¬¥©°²µÆË“pk”»ËÑ̀½¡—œ—ª³°§¢¡ ›”¢¡Ÿ”•››—“¥¯·¸º°¹°¦¥¥ª½¯¦›’• –”Œ€…‚|ˆ˜™›’–‹—›¢Ÿ™›Ÿ  ¥½Á¯™•œ—•”‘Œ‘±–¢¥™’¡œ”¬²¿Èµ¶»¥˜•Œ—†‡„†|‚€‰t}…‰xz}|s{††ƒy„‡|ˆ†€‚zys„‰„ƒ…w{sqyortuz€tuxy{ˆ‹|xzx{vvGX{w~~‚‚‡~†{oinpppvrrsostwvqmporwpt~‚yƒuwŒ{|ƒ‚…„„€…ƒ‚~ˆ}„…‚„€z‹ˆTTd„mfW@AMZcc]bj„Œ‘‹‰{uz}ud58[UWWWW\[XUUSMJMOOS[][Z^fc\ZUIA?CMNKC@>?9===;00/0175788<;92)12-"+..0.'',+("*+*+.-022.,5-#&" !%0/,0,*---00¾ĂÁÀ®¶ÈƯÛỖ×ȵ®¬¤—™ §—–˜©¸¹µ«§™“˜¢›œŸ¨§£  ¡œŸ“¥¬¥©°²µÆË“pk”»ËÑ̀½¡—œ—ª³°§¢¡ ›”¢¡Ÿ”•››—“¥¯·¸º°¹°¦¥¥ª½¯¦›’• –”Œ€…‚|ˆ˜™›’–‹—›¢Ÿ™›Ÿ  ¥½Á¯™•œ—•”‘Œ‘±–¢¥™’¡œ”¬²¿Èµ¶»¥˜•Œ—†‡„†|‚€‰t}…‰xz}|s{††ƒy„‡|ˆ†€‚zys„‰„ƒ…w{sqyortuz€tuxy{ˆ‹|xzx{vvGX{w~~‚‚‡~†{oinpppvrrsostwvqmporwpt~‚yƒuwŒ{|ƒ‚…„„€…ƒ‚~ˆ}„…‚„€z‹ˆTTd„mfW@AMZcc]bj„Œ‘‹‰{uz}ud58[UWWWW\[XUUSMJMOOS[][Z^fc\ZUIA?CMNKC@>?9===;00/0175788<;92)12-"+..0.'',+("*+*+.-022.,5-#&" !%0/,0,*---00º¼À­©³ÄÍỠƠÎ͸·«¦³«¢¨›“—£ª²®§™——›Ÿ¥­ª¦¡¨¦£¢¬²¬Ÿ§¬®Ç̀ǘ}x›¶½Æξ«£¡£­®­¥¦¤ ¨–“¥£Ÿ«—•›Ÿ›•¤­¨Ÿ£¯£¡¢«»¼¹¹¸µ«­±¸­ª©¥«©œ›†ƒ‡Œ„• ¥˜‘’’ ™™Ÿ ™“’›Ÿ˜–…‹‘Œ‚Œ­¦”Ÿ§£œ››—”¦¨¹¯¼¸®¦£–˜„—…ƒ’Œˆ…|qny~yw}}ˆˆ†}|u}…„ƒ‹’‚„‰‡}~~yyƒ|†„nsv~}umkmtts|…’‚v}}vpyƒfWq}vr\xs{swtkslkkjmollotpqtplpwqjx}tpkuxyzxƒ{ƒ€|u|†Œƒy{z’ˆ†€…ˆˆ|…xtiy‚tj]LHC@bejg`[\b‹ˆˆ{pwvodb73SUTRVWTTRQONLKLNLOXWX\YUQ^[R<=FM@ABJB@@AAA@010-/.2455:80)/0#" !(,-.//-***#'-,.-.0/1020//*+!$(,+/2(.,-/../2³µÆÀ¿·¾ÂÍÑÔÅÉ娨°°¦¥¥¯·¼·¶¬£¡–—››¤¥©­««¤¥­© ¯«®¯¯²³½Ê¿¾™}ƒ£ÂÇĐ»±§¢¤¥Ÿ”˜œ˜ ªµ±• ¤¦£¡–¥§¥­¬¨ª®®¥–˜²¾¶²´¶³²±´·¸¸³ª¨„|ƒ‰‹”Ÿ›”’’’¥ •™ Ÿ™•–”•‘“‘‡†ˆ‰Œ‹†‘’˜˜“ –”™“œ——”–‘†¢¸ÍÅů¨ ¡”„€‰‹‹†w}y|{}zx{{|€ˆ…‹ˆ‚…ˆ‘—“‹„ƒzyy‚|suy}vxwx{~~{v~|‰Œƒ~}}…vy†‡zzseSd|v}qwuwxsws{wymlorrrwsdkytpittvwwzƒz€ytwvztw}„|†‹‰’€’Œˆ†††xeA/=‚z^fbX^cc[cdYYe`mnw€xytojhgke>?N\ZWV[SPOMJIJIIJJMSWfb\[bb\OJJGG7`b^]LKJHKHGHIKKPSS_^[[`\\XS^PGKNJGC@<=D>41/.244<;<:4.%*'&,...///01$*530--413(  (+/!"043222/µ­´½´³¶ÊËÍĵ¤¢©§¥˜¡—œ®­´¼ºµ¹½³­˜©®ª¨¯«±³³¸¹¶µµ´ª®¸ÉÏĂĂ½’„‰ÉÍÉÁ¯£¯«¤Ÿ¢£¦¤¢ª«´²·¾±•œ‹†œ­¯²¢ ¥°§¤££©¬¥¢Œ–•µ½¿¼§˜¡}~|ˆ“w‡‹–‘ˆ„••’ –”—¥¬¬™™•’’•“‰†ˆ‹›™Œ‡•Œ‚†‡“—““Œ˜˜™—’“f©»ÅÈ¡›™“¢Ÿ‘‹qw}x|z‚€swxx~}|z„‡†…Œ‰}ˆ‹††‚~€‰‡…‡…~yy„~ƒ‡‚yr{|twxuw€{{ˆŒ}„ƒ†|x~{uu|twxpv|x{yw|trttzrxpspxstrvwtjtx|€y{twx{tqxv€ƒ†{‡˜¢“ ˜NG’Œ„{{ƒ{wy‚ƒy‰‡‘†ƒpVsx…|qQOvqtq{vmmnjh`SKHLGFKLNNOQPVWYS]]aUUK?@@CEGGB777;@:19529;?=::<8385)'+""+--,01.+# ' %-(+-)*++)+"#$ '/02451/Á¹¶®¼½¬ĐÙĐĂ££ £¥µ©™•Ÿªºµº²·Â½µ·«²µ°«¢¢©¯²·ºĂ½µ°·¶¯´Á¿Æº¿³ƒ€ÅǾ»¯¤«§Ÿ¨©©²©°¹¸ÁÁ»¼ ™££—’n|£¾½½´¬§¯¹²¶À»½´ •™–•‘ª°¦Ÿ™Œ™ª­¬ ‡z„„–ˆ|ƒ‡‡‚›££Ÿ¢¨¤«˜‘™™‘‰†‰Œ”˜–‡ƒ„†€}{‹  ’‘’˜˜™–”—£™’«£¢¶£’Œ‡‚‚ˆ…‚ƒ€„y{{zyƒ‚„~ƒ‚…”–•‰‹„~…†‡|ƒ„{€}€{{}ww|…|Zp|w‡‡}{xws{wyyvtu{rw|w}ƒ|wsvrrpsvuoorqkqxtr{x~zvvrrrr„|€„‘‘•{ak‡‡z~w{{…|p’“‰ˆ‡ˆwhsuwvYQhknudl}|umiopjZKLKIJMONOQQMVZW^d\LBJOMV\DD=0.16:B?:648:=:7526544601/1,  %&&$+.-.//1,'!'))-*26.*)))**&*$!%$!$334410Á¹¶®¼½¬ĐÙĐĂ££ £¥µ©™•Ÿªºµº²·Â½µ·«²µ°«¢¢©¯²·ºĂ½µ°·¶¯´Á¿Æº¿³ƒ€ÅǾ»¯¤«§Ÿ¨©©²©°¹¸ÁÁ»¼ ™££—’n|£¾½½´¬§¯¹²¶À»½´ •™–•‘ª°¦Ÿ™Œ™ª­¬ ‡z„„–ˆ|ƒ‡‡‚›££Ÿ¢¨¤«˜‘™™‘‰†‰Œ”˜–‡ƒ„†€}{‹  ’‘’˜˜™–”—£™’«£¢¶£’Œ‡‚‚ˆ…‚ƒ€„y{{zyƒ‚„~ƒ‚…”–•‰‹„~…†‡|ƒ„{€}€{{}ww|…|Zp|w‡‡}{xws{wyyvtu{rw|w}ƒ|wsvrrpsvuoorqkqxtr{x~zvvrrrr„|€„‘‘•{ak‡‡z~w{{…|p’“‰ˆ‡ˆwhsuwvYQhknudl}|umiopjZKLKIJMONOQQMVZW^d\LBJOMV\DD=0.16:B?:648:=:7526544601/1,  %&&$+.-.//1,'!'))-*26.*)))**&*$!%$!$334410·¹µ²°»ÅÖ̉À¬© ¤±·³°‘—Ÿ¨«»µ¯·º´¯²¹£©¨ª®³¸¿º¹º¶³¸·´¼Ă»³½¥tz•—¿Á´Ä¤ ª¦¬®¨¦¨«®´¼´»½º­«œ©™§­£’¬¸Á½À³¬·¶¿¿»ÂÆ¿³—˜¦®¨µ»¹»¶º¶¸³¬£–•›“Œ–’‚z›~†˜ª®•›Ÿ˜™›’““‘’‡Œ”Œ —‡ˆŒ‡ƒ‡„•¥‘’—–›››™““´Ä¢”§¢¦À¹­¡’‘ˆ……€††Œ~w~‚€yz€‚ƒ„}ƒ„ƒˆ‰†‘‘Œ„‰‰‡‡€‡„‹ˆƒxvxˆ}‡ƒ€€n|x|x{€xruz€sxxzsq|}|tx}vxsrwnhcdmuxxnpryssppkqroqr|xvzƒ‹{z‘›•“‘wƒZ)e‚‚†‹|}ƒy‡„{‰Œˆqlp}‡xvxnmpyz{~zkddie]SQRMNLHLPQSUYYQV_gZW]cXV\LFC<5446=C@A>==<9112487434235-& ,$#*,..-./,,)$#&"**),.//.440+(*((+0//+,0*+),,'*('*,30©²¶³¸ÀÇÑÄ¿­ª´« ­­«  ²²»¶´³³µ¹¼­©¬§§«£¥°¸¹¼¸¸¹µ¯´¼µ´®±“njv•¿·¯Ä¦¨«­°¯°°±¶¹»ÈʽÍĐ´¸®¸«°½º¼ĂÈĂɪ®ÀÄÀ̀ÁÁº«²®²­³»½Ä¿Ă¿º´´±¯µ¶³¨–— „€¥«˜”œ•—™£«•¦••’–“““‘‰ƒ‰ ¡™™©……‰‚}€…’—•‘‹’™“Œ‘™££–•®¶®¿Í¼¨”~zx€‹ˆ‚v|~~|€†‰~z€ˆ…‡†‰‹‰‹—”’……‹‰~€y{˜Œ„wƒ{je|wty~wƒ‡‡~ycruywy„~z…{urnpjge~{{}x{y~x{yz|{x~~ƒ~{€‚ƒ{„ƒ‚…Œ”‘’Ÿ‚^tyGK‰•Œˆw{€~|„†z„†~ohcmoouxypojgqnhstk_e]Y[PTWTSQNNNOWVWSMTgYYTPUTTSMBE?AJL747=?B?:95421120122323300-+0%'.-...,0-).$#(+)).*-..-0---++++,--* %-6/((,-0,*#!-,-222½µµ¶ÈÆ˺¼»³¸·¬œœ›Ÿª©¤°À´°¬§«¬±¯¯¯¦ŸŸ¢£§¨²¶º¸´¶²¯¦·²¬¬³ª‚~‘´Œ”«§ª«ª¤¥§¯°·¸ÂÈÉÉÊƠ×ÁÊ¿½¹°·ÀÀĂƶµÆ|¨ºÇº²½¿·™’Ÿ£«£°®»»¼Ÿ¡±´·¸Á½¤’Ÿ†4t¡«›’–“™™–‘•—•›‘Œ‹‘µ¤œŸ¥œ—…‡„‡‡˜˜ˆ‰•–‘ˆ”“›Œ¼À®¼Å»˜‡„‰~~€†Œ„‰yl‚~y€ˆ„€~……Œ†‰…‹Œ“‚‹ƒŒ…qr|†~„€~mi}u{|~~‰“ƒ}n_uuqrru{}q}€|uuljptvo~zwxy€ywq…|zyw{€wxˆ‚}|ˆ†‰‹‹–¡™N=eK*N‹~}|„lwˆ„………‰Œ‘Œi{‚|rzdPbflsrifellgkrohd_YYRSUSOKMRSQXRSRJQ_WPMLLORQ@C>AKMNF<38;<==62112220/001/001100/'++,,./+)**,+--,-/./0/2/--10&()+,+.*-/10*'*-.,0.(*,/222·³¸µÄÉÇ°»º¶ºµ´­«§¤¦¦¥¡°¸¬µ°¬­¶´¨¯ª¥£§¤«¸ºµ¶²­¹¯³°²³¯²¨‘ˆ±»ªª­±©®¥›‹°µ³ÎÎÏÉÇÉÑ̀̀ÂĂÁ¾µº½¿À¿·˜¬›¬°¦¬’¨l~t–•£²ª©œ’¬­–†¢¹§©£¢wQ‹—‘”†‘—ˆŒ˜–¥œŸ¡•§¯¯¦° ›£˜•–|~€‹ˆŒˆ•›Œ‹‡Œ“’•–‹‘‘•£®¥©¤Ă¸²œ‰…‡wxvˆ„†ƒ„‚€{x„€‡‚„~y~†‡…ƒŒ‡††‚‹…„ŒŒˆˆ}„wvvwvuƒ€~‚„„zzŒŒˆ…†‡sxyztw{‡…ƒ{tx|vrwtgcrh{ƒ|w}€€{vˆ„sy‚sy‚†‚†ƒ‹“–‘“•™W/,Yfjƒ{„ˆ€v€|‰†ƒ†‘w‘•Œnz{wwyui]P\jv|{|vskkopkfhjf_QPOQONQSUPTTVQPVSVQJHJLLKFCAEDCKIE<4003752234310../.-.0330+%#"+,++-12-+++,,/+,+,01.,-,0.,..-+))*,,))(,(&)*..30++13232¯·µ½¾ÄĐ¯¸¶°¯°³²­©¨¨²¬§©¶·³³®«­´·³±¬ª ©­­®°±¯³²µ´©¨®œ‘‚…›¬±¸´¬¬¦¡®ª–ƒmw¡ĐĐĐỀËÄƵµ¼£©¡¬{†±œ’z€ˆŸ¤«}i€Ÿ`=;X†|˜§œˆpo–œ±³²¸»¼º¨™vŒ¡ Ÿœ˜Œ–”‘™‹‘‘•Ÿ–­±¯§¯µ±„•›Ÿ£’—Œ…‡ˆ‡§¥Ÿ’œŒ›˜™¡“‘›™†vœ˜ªÀ¯»£‘ˆ}up}€€‡Œ‰ƒƒ„…„~‚„zz{‚‰‹‰‹Œ“†‡‰‡„††„{|s~|…†„zy‹ƒy€{€wywt†„wz‚‰t1p}}yxy€z|~zyy}‚yorrvu~~{w}„Œˆw}Œ„„z‚|‡ƒ‰‘–’•’—–‹N1(3x…ƒ‡~‹‚‚}~††ˆ“}ƒ~{|twwwtuh[\cr{‚~trtiib^ec]`SNQNNJIOQSTXVUNNMMIEGEGGGKLI94=8316:47876530.---/12211+,*#''),.-.242.+/+-**(**,-+--,**(()*-,))*+-)(())((((,22.%,0211´·±µ¶³Ä²¬©ª¬±¬¬¬¤µ¹±´µ´½³¼À³¹ÆÇÁ½·¬¡›§³ªª±¶±±³³®±¯¦¡–‹¨²·³³«¨¨¯¹ÁͧYK´Ó̀¬±—–oƒ–ŸŸ~y|ª°ˆozP<^agY~ˆˆœ£Ÿ„†sUv©¸»³›Ÿ©À»¬¬³£©°¦­¡•“™‘‰x‘ˆvˆ•––œ§¦¯³¶°œ¢¤œ œ’—‘‡›¥¥Ÿ“›˜˜›¡Ÿ˜•œŸ“X\¡¤¡¨§©ˆ‚}‰ƒ„Œ‡‡€€ƒ‚‚ˆˆ|‚‘‘’“œ‡Œˆz{…‚‹{zƒˆ†ˆˆ”‰z{dx]-H30"02>_{ƒ‚|pqx{€|}yyuspnvuzzorz‡‹…‹||€v‹ˆ‰~z}ƒˆ€‚{’„„ˆVS|’ƒŒ……€‚€‡Œˆ‚ŒŒƒ}yyxz||vtyW`blsvupkkfe_\_a`\VQPROMLKOSVVTQGHFEDBB@>BDHJJBB@:97=>77589:>>8663//.-/43121/31010/( .---0010+.2*%'(&'(+,+,.+*+)((()(*++)++*)(((&&&*264*,2311±´´®ª¦±¯­¬©¤«¬¥«®¬³¶³¾º¾»½Ê¿³ÀÈɽ½»¶¥ŸŸ®ª­¬¨¦­³«±µµ®¦¥±°±®±·¹½¹®©£œ¬¾ÉËÓ˾µµÂ˜IZ>€ƒ‰‡‰CGNu›—vYWj~@=ƒ™gNŒ¥‡xEnr‚wœ»³¸¾º»¾¿¹¹±¥©¬ª¡¤«ª–—›Œ‹’Œ‡oz–•’‘›®³°§˜«¤–§ˆ’‘–†‹’“œ™–•–•™—”•˜œ™•˜–˜¤²¥®®³²®­„{ƒ…ƒ‹†‹‰‡~€ƒ~€€…’‚€zŒ{‚ˆ–‘…†………ƒŒ‹‹†††Œ…‰†”T$U…z<,@‚ƒ€z}tw€‚{tqtyyy‚‚}€u{ƒ”‹Œƒ||†sz}…~‰‚†€‚‹ƒ|w„‰ˆˆˆ†Œ†ƒ‚ˆ…€„‰ˆ†ˆ„€|{z{|wsvz{pXT`i~‚strfea`^dcV[[VKHIJRSTYTNPDABAAAADB?@FGFEB=<30129:78777;852,...332531-/0/0,0.-(%0//0.(++01/$****'(***,1-2*(())**+,*))'()*,,()(,122.//0--±´´®ª¦±¯­¬©¤«¬¥«®¬³¶³¾º¾»½Ê¿³ÀÈɽ½»¶¥ŸŸ®ª­¬¨¦­³«±µµ®¦¥±°±®±·¹½¹®©£œ¬¾ÉËÓ˾µµÂ˜IZ>€ƒ‰‡‰CGNu›—vYWj~@=ƒ™gNŒ¥‡xEnr‚wœ»³¸¾º»¾¿¹¹±¥©¬ª¡¤«ª–—›Œ‹’Œ‡oz–•’‘›®³°§˜«¤–§ˆ’‘–†‹’“œ™–•–•™—”•˜œ™•˜–˜¤²¥®®³²®­„{ƒ…ƒ‹†‹‰‡~€ƒ~€€…’‚€zŒ{‚ˆ–‘…†………ƒŒ‹‹†††Œ…‰†”T$U…z<,@‚ƒ€z}tw€‚{tqtyyy‚‚}€u{ƒ”‹Œƒ||†sz}…~‰‚†€‚‹ƒ|w„‰ˆˆˆ†Œ†ƒ‚ˆ…€„‰ˆ†ˆ„€|{z{|wsvz{pXT`i~‚strfea`^dcV[[VKHIJRSTYTNPDABAAAADB?@FGFEB=<30129:78777;852,...332531-/0/0,0.-(%0//0.(++01/$****'(***,1-2*(())**+,*))'()*,,()(,122.//0--¬­ª­—›¥´¶®«­²±­¯ª§³»Àµ±½º½ÊÊÏËÇĂ¼¼µµ¯¥¢¦¦¤§¯¯¬®¯¶·º¹¹¸Á¾º·¸º¶±°±¸ª’¬½ºÈÈÉ¿Â{`h28$CIOtcˆnPW„ 1@2D7Vg‚”}q`8QF=`tG_˜»¶º»¬¿Áº²¦¨¦¡¢¯©¥© ŸœŸ›’•—‘‰›Œœ‘—›§²·£™£ £œ‡”‡v–¥–‘”Œ‘”–•••“”¥¼œ¢›±²Ä¹ª’}}v{~{z„‡†uw|„…„‰…„xo€Œ‘€€ƒ†ˆ†{}ƒ}‡‡…ƒ†“‰†ˆ†‚ƒŒ‡‰•j^u„ˆ‘”‹ˆˆ‰ƒ~|zyqxyzuu~s‘x€ov…‚ˆ‘–‘Œ}}r‹†‰}‰˜””’‡††‰‰…„†‹„ƒƒ‘–„ˆ}y€|yvmnv{xshfY`gwokhcc_]WRTWWOMKLIJRTXXPOK=>CB@?AEFC@DFB@:572333;==7/1663161-.212345522,+-./++* ! +011/,2-+0,)+-32210*,,+766*)))(****)+'(*)/./+*'*2363210-,¦¯±­¤ªª¹Á¼´¹½¼µ®¤¥®²¿¹¦¹¹»ÄÏË˼¸Ä¼¹°® «­¬¦²±«¦¶»º¹¿¾¹¨±¦¨¬³´ĂÆÂÅŲ¤µÂ¦\L‘µ}O8&/BMyrŸ•€c0i-;_f\YWX^VR''%&+2Qn£¨¸¶³’½½Â³´¦”§¡ œ¤°¯¤¡¡›¤™¡˜‘Œ‘ª•‹¤œ¦ª²µ°­”¥±«¥¤›™•”’˜˜‘—“’†‘’‘‘“–—›¨¸Ÿ‘ ½¯³¼Ÿœ‡{sqx‚„~}z„}|}z{wŒ’ƒz|…‚†ˆˆ„†”}…‰ƒ‹‹“‘£Œ…‹Œ……‹MsŒˆ’‡yqtrswxy~‚‚vy‡Œ‘‰~…}z…hSmndhqcgqzz†„††ˆ‡“•”ˆ”–ˆ˜p{„ƒ‰„†‡Œ”Œˆ†ƒ€{vorqvw~‚€zoW^fuqlj][YXYUUUTLKOMPQQRRNMMI?CEGBDBJFB@DB><351476:=DH>52251382.1/.4202510-+--.*'&((*)+)+/.,,00--20,-.1540+,,,.70.++)))()*))*(+*')()(''+.3//51+/-­°±®®«©°´¿½º¾ÀĂ°§¦©¤©´¿¸¯ÁÈÆÍÎö¼¾¼½«©Ÿ«¦©«¡ ¯¶µ´±¼³­«~ª«¶ÅÊÉÄΰ¹¸œ¤h[”ƒy†P7"3A{fGP"-j+3a.:P0*6T~}— ¨ ˆª²«½´¯·£¢—œœ˜œ™¢¤Ÿ¢Ÿ¤’‘œ•–›² — œ——£¤µĂ¼®£¬¯®¬²¤—”™––œ“‘”•’—”“•‹Œ‘“–™Ÿ§ª„u£Á±¬³²¤ƒƒouz€†‰tszy{|€utuzw†•ƒ„‘‚ƒ††‡‘‘‘”“ƒ–Œ™ –™”Œ‚rt’ŒŒˆŒ†‹‹|qrvtwwrx}xw…vzƒ†uhcXpÿÿX`_`r†„‰paqu‡—“››–’ˆ•¦™¦„z‰’„‡‰ˆ‘Œˆ‚~xqruz{~‚~~wZW[enpmea`aZTRPUPPMMRPQTPIIHFGLI>@FFG>@AJHEABCB>?@AADF>;74623344433455532/.,+.,*++&%(,1//*),)-/.*1,))*+.6.*5..2728,+,*+++))-/-(((02.**/8B:5.24/20¯·¯«¡¥¦¼ÄÏ˹Ä̉ʳ©¢§®¬¹·¹½¾ÇÎÏÔÍÆÈÈž¡›ª¨££Ÿ—¢»¿¸¼·¯«‘™ «¯¿ÁẰϵ ¯‘61h2m”e;M&+,j8.&#+"" %0Dd|p€˜ªjv“µ¶¥§˜›”•˜›•–©²¡—˜•¥’u“©‘£¨Ÿ›¤œ”™ —›˜¢¢£³¹¯³·­°®±– —“’”•’’•”•‹––•”“”•™›¦¦¹“¢¬·¿¬˜ª´†toruˆ‡…Œ‡|vs„†uku{}„™Œ‡…‹‹‚†}‚‰~vƒ†Œˆ‘‰…”Œ“lE‡˜…Œ…†„ˆ†yxpo|€sjq{z‡‹|?>f%3ÿÿÿÿÿÿ‚lur{ƒ‘Œ†…s˜˜›‘ œ””’”‡“’‘Œ‡‹ˆ…†€~wutzuvz€…€yhdZUddiccad[TTSTPKMNRRPRPKGKND5,!%/FAFHKGFIIJJGDEA=ACBFA/-1454777652;;5/32/../,-.'%&*22123/,1--1-+)'*+092*2-62124))**)*+.574)(,5777595q¦•Ÿ£©™œ™“¡œ¡¤œ›¤§¹º°ª£¡­¯§“¢¥›œ™’”†‘”•–“•–™›¬¯“¤°£¬¹˜€‚xx‚…„‚†‚€uyvzurt}€yv‚£‡ˆ‰ƒ‚……‡ƒyeˆ‡€b„€}‰’…’Œˆ–‹Dv~ˆ•’†††‰}okrx|y{y‚haaÿÿÿÿÿÿÿÿÿÿÿ„’…‘{|}†”¥—…‚˜™¤Ÿ“£––ˆˆ‡‹”“‹‰‡Œƒ‚…„yr{}|zx{~ƒ‚€sppo_efa_^\VWRMOLJJNUVOOPNQT?!A@D=JK")#%C?DACB=8<9/-))5:801487<78422-.00(()*/220.052/--//0++(+.01572<:9=9:1*--#++,692-,059;;82..333993478:ª¦£¬¨¨­¯½½ÁÆžŲ¯¯¡¨Á˼Êɺ¼ÅÇÎÓÑɺÀ«£© ª°¥µ¹¾Â«±›{d~©ª²À¾ÂªT<ŒX.4q7`iv &0 &4AVFj_K1$AIA1?zmFQ^·º§¯³³˜¨™Œ—£¢ Ÿ›—Ÿ“”“b=œ® ™¦˜‰™‘“©¬›—µ»À½²™°²­¦§¬¡–›•–—“‘——––™›˜•™œ””¢®€’­˜›¸³¥‚‡x}ˆ„|€ƒƒ‚‚xy|xszzvv}–‹…ƒ„|{‚†…v‹ˆvvzmƒ‰”“™…ˆ`Pd€‚†˜–•‹‰w~}olrtkoq~zhÿÿÿÿÿÿÿÿÿÿu„†”“‹‰€~~‰•œ ”¥œŒ‚œ ‘‘œ²ˆ’„†‘†ˆ‰‹†…ƒ†‚~{xxz}}w{‚}ywtttm_Qb`[\]URKJLMST[ZXMEIH( ")>,  -FF?<69?=>7610-32031595312240220.--10..,*,011/.31,,,+/.,.955.+6;4513'#).202204985;>;640214300/523ª¦£¬¨¨­¯½½ÁÆžŲ¯¯¡¨Á˼Êɺ¼ÅÇÎÓÑɺÀ«£© ª°¥µ¹¾Â«±›{d~©ª²À¾ÂªT<ŒX.4q7`iv &0 &4AVFj_K1$AIA1?zmFQ^·º§¯³³˜¨™Œ—£¢ Ÿ›—Ÿ“”“b=œ® ™¦˜‰™‘“©¬›—µ»À½²™°²­¦§¬¡–›•–—“‘——––™›˜•™œ””¢®€’­˜›¸³¥‚‡x}ˆ„|€ƒƒ‚‚xy|xszzvv}–‹…ƒ„|{‚†…v‹ˆvvzmƒ‰”“™…ˆ`Pd€‚†˜–•‹‰w~}olrtkoq~zhÿÿÿÿÿÿÿÿÿÿu„†”“‹‰€~~‰•œ ”¥œŒ‚œ ‘‘œ²ˆ’„†‘†ˆ‰‹†…ƒ†‚~{xxz}}w{‚}ywtttm_Qb`[\]URKJLMST[ZXMEIH( ")>,  -FF?<69?=>7610-32031595312240220.--10..,*,011/.31,,,+/.,.955.+6;4513'#).202204985;>;640214300/523³¤«±±²Á¿¶½À¸»ÈÍʶ½ÀÈÆÎÇÄÊÅ¿ƠØÖÏÈÁ̀®¢¨¯¯«¨½»·º§££­¼°£‚­¬oT[g?(1QOr#" JI1L–{saH=>@3-OCHSp“˜m‹°¯©³²¸§˜˜¬¥¤‡Œ£¥›™ —‘oST}’›Ÿ •™Ÿ¡Ÿˆˆ§ª¦¤´°´­§˜¬¥§ª Ÿ˜‹—  µ»‹„—›£ —“‘”˜®›y…¥›©ĐŽŸ}ˆ}ƒ„€|‚|w‹w}}{z|‚{ƒ„ˆŒzq‚Œ—†yu|„gcƒŒ‡Œ††wluz—“˜—™’†‘†‡{wtvsu\Sc<ÿÿÿÿÿÿÿÿÿÿ? ¥‘—€…‰|uz„Œ’†‰w•‘–”˜—‹€‘„ˆ}…‚‹Œ‰ˆˆ†…€ƒ„}}‚|~}|‚xusmppoj^SVWYYROOLNPPUUUSPIB   + %EF>;:?674>>:71/,;94422223/--.+*((+.,++)((*,/-.0.-,-,,/-.2&'0-+.-+++((*'*23.-699=B@<;:@?AD<4255539BD877012201+.---*)%)///1/450.2..))%%.+*%+)((&'(/'.6?B?AA=;>?>>>=:01./+-»¨ ¤­¬®ĂÀÀÇ̉ƠÛÏËÏ̉Î̀ÉĐËÉ̀µÀÅÀÏÉËÎËĐ̀º­©°®µ¹³·µ¥_fŸ‹¡¸·u?-1/I)!#*13“•}A%!'1qPCchkbOouz‹Œ¦¤Ÿ²º»ÂÀ´›¨±¬¯®¢¡Ÿ›œ –œ¢œ——–†…Œ’–©›¦ ˆŒ“¡›¡¬»³±®²²±¨²±·©˜›—Ÿ©ª›™´‰Œ•œ Ÿ™””“£¦´ƒ–”¢±½§‚s|ˆ‰„x‚…‚ˆ…~‚‡€“EYb^ÿÿÿfs[JgInjnsxYx„{…†›ˆ“’–‘~H{‹Cÿÿÿ1ÿÿÿÿÿÿÿÿÿÿÿÿœŸ‘˜¦¦•Œ‡’€Œˆ…ˆ…‚™‘———‡~„xd““‹…‚z|{„ˆˆˆ…{{|}z|y~|{|yxxw{uqpnf[[\Z`Z_]^[[[Z[VUUS+ $D>=?C<=:><8>>:614207=@<;8(+-035·´¬¢µ¯·ÎÓÙØÑØÜỬÎĐÖËÏÉÏÓ×´ĂÊĐÏËÈĐÑ̉ήª¶°²¶·²µ¬}wrFVltDOE1"BP##8(#>˜t34":BOsQ?Y‚ƒkCLBXv€t‚¯¯¼¿¾µº²°ªªª§”˜”œ£¤œ•‹“–‚Œ‰“‘¡¡‘›¡¢¤³²¸°°­¬°°´°­£œ¡¡Ÿ“”˜Ÿ•‘‡‡ˆ˜‘˜™—˜’“–•”†”–²¥¡ƒwy}ƒ€†–¨¡}„„‡‡›”‰…€sw‰ƒUÿÿÿÿÿÿbOVPpfeogmvtƒ››‡”–‹~•†lJH`ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ•’«–’“—“•Œ‚‹‡‰“˜†•˜ˆ—˜”“˜‹Zmu‹‡ˆ~‹†{ƒ…}}}|sqr€‚‚„†„ƒzyzurkgfb]Z\]Ya`a[]XWZPNRVOF " $=9:<<2/-+@A>ABAA>;;//14<;·¶­§À½ĂĐÓÔÖØØÖÜÔËÈĐ¼ËÔØƠ׬·»ÓÏÑ̉ÙÖỞ£§¨µ¶µ¾µ®‡N—v©Qo?#.@KGFE&97:f8K=F6(7>'2Ol”›„MX14BC?>:ED239AAD@5/+)+))***)(((*-23?=722/01(&&31!## + + &++84/,-34)')5>>>>C?>>@ABFGE96;B?>934-)*.-*-0-.232.3<@<9:5000$ + /25.2.4 //--%#,*.)()).88;;;=?:BDB78@A@@>:9§°·ÀÇ¿Ă××Ø̉ÖÛØƠÙÓÎË̀ÑÎ̀ÊÁ¸ºÑ̉Í̉ÎÔÑĐ·©´°¦·¼£œ˜™¥¬hO.}hG0'mKYey€X]{ˆx‡|ymPi-[1J™—Z3J„nx‘gg†‰‘ £££¦´­­¦·¸±©®­­©¥ Ÿ£ŸŸ¦Ÿ˜” ˜–—œŸ–”µ´¨¢›‘–¢´·Å»¯¬º¹·¦±œ¥ŸŸ›—™™œ“‹Œˆ’“Œˆ‹‹—–“—˜••Ÿ ¥¦¤’‘Œ…zw~”‘||‡†€‚˜›¨¨ªµ³¯¢©yg]ÿÿÿÿÿÿÿÿÿÿÿve‡|xtsgct€p}ZHBC?T)ÿ\Iÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿc;kpf‹—™•‹“™††x©’y’˜›“€|ƒyŒ||XdtŒ–’‘‹ˆ…€}{|z‚€zow{ytqiaa[X]]^_WedYUUYEM?  + !" @ED=BEB7;4/3/0/5;>:<838=?@;<=;6397(8981/67)-78/#+*+,''.47278:<;:8:>?59@BAA>;;¥­³ÇÏÇÇÉĐĂ°ÀÎ×Ï̀ÎÎÆÆÈÂÅǹ³¾̀ËÂɳ̉Óѳ–›¨¢£¢²‹{¦¯²°‡mPKe/CU&$  '-! 1ED?<=@==>=<<=;79::;=BC8=<:@@BB@CB<=@@>918;>?8655@?3)15.4;A7"-.2())('267899:;;:79938=?>===;¥­³ÇÏÇÇÉĐĂ°ÀÎ×Ï̀ÎÎÆÆÈÂÅǹ³¾̀ËÂɳ̉Óѳ–›¨¢£¢²‹{¦¯²°‡mPKe/CU&$  '-! 1ED?<=@==>=<<=;79::;=BC8=<:@@BB@CB<=@@>918;>?8655@?3)15.4;A7"-.2())('267899:;;:79938=?>===;¤§®Á̉ÊÊÑǶ±ÇËÏÍÈÆÄĂ¿½ºÀ³¨¤ºÆÆÄÄÇÛ̀Æ°“˜¢‘¡¯®–¦±¡‡f8wD).;>37@==EGB?:04<:/%*=6D2:KE)2>>7>BD2)+69(-1-226978:895;<<:46:88;<<<¤ª°½ƠÉÁ̀Í­ẲĂ¼ÇÇÁ¿¾ÁÀ¿¶¥“¦»½Ä¿ÆÈ̀̀Á¹¦’‰‰€¨¯˜§´}n;B5-$Ac~…‰ ¥›–‘‡’’ZASˆv„^e‡}ƒ|”–˜œ“†…„o{‚{Œ˜£«ªª®´²°¦§¯®±³ªŸ–™• £¡› «©Œ…‡™Ÿ¡–œ“’¨›¯­¤¯ ª²¬µ»¸»©¢ ¡›Ÿœ˜’’’‘’—™Ÿ ˜•‘’•œ›”‹’¶¹»€„uvzr}z„|j~‹‡‹‘—˜œª‚Sÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ}ny‘˜UOL5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ^A'=n‘yŒ•ŸŸ—“ƒ~‰€}~…‹¤•–¢’‘‹‡‹‰„…‚r…‚~yqyv„}‹”‹‚z|‡„zljjjkhdksmqon`aic]\\UA3B  0@GE?==CCCCGE?54:68>;<;82786B>?@>>=6CB@=:9/ )<432><67!03:9:8D?=<;:4766689;6:;9;;943827=<>>¢®§ÀÔÎÇ̉ƠĐÍÊʾ·³¶«»ÂĂÄ»Ÿ¤ĂÁÄÀº¹ºÈŽp„‘›£Ÿ¥¥™±‹RD9\oun;"8N’–œ§¢¦¤¨£¡Ÿ¦¦ŒŒ”vRn\ˆ¶¬˜’—ƒjmHOoX–›Ÿ¤ §¬©œ§³©Ÿ¥«¢––’˜›¡¤§¦¦¤—ƒ““”› Ÿ “£›£«¬¥˜±§›¥¸ÀÁ±‘§²¢¡™˜¥“Œ‘”• ••™’“‹ƒ‰Œ“•“—©¹¸±¢‹|~o…v{„xyŒ”‹“‘’œ¤¨¨l4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒ™Œ’‘–™ˆ‘…K4:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ_bKZnpw˜”¢’˜›œ‰•‚Œˆ‡˜¢’œ›~‰…‚‡v~t„ˆˆy‰”‰‚~~ˆ„‹gdbgc_altnihrhZ`bWYUTMHB)     7F?=B;;@>?CDB931:>ABB><<>;8=:7;9=B82:?AB:A2),03.02931.48<9275?CB@=?9831,113876721233360/0;;;8;=›§­µÉÀ·ÄÍ̉̉̀ÆÀ°®ºÅ½½¿®¶¯¡ÅÉÇÅŒ·¸¾̀ʽ¨Œ‘˜¢¥¢±¢}yj¡$0,A; d_4h–˜£Œ‘‹•Œ”›œ”–—o‰Œ‹rŸ¬¥´•‡’Œ†socfBÿGÿÿ˜› ¦³¼½¸°ª¯±®¨¦¥¦£““ ¨§©˜¢ ¡–“ £˜•”Œ‘™„—£²¯ª¹±Åĸ¶Ă¶¦¤Ÿ¡«˜“’–˜¥¥·¶° ›™¥†ˆ °Ÿ–œ”˜°¼·»«¢”||ngu„Œvz‰„ƒ¢¥„œ¢TG+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†“œ£–˜¢¬¥œ•„t/2ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿBbl€{€—”˜¡’˜—£Ÿ‘˜›”–¡–¤§˜Ÿ“”’‚||x}€‹ˆ~‡qyˆy‡“ˆ…|nswlcffcb^gjqksuoZRD457DG=   + )>81'363=??@@;5/7DEA@>=@=:=;6#)ABAADD><0*,,+$(3?B<4;9A4@=225274463423<6159<;2./,-/1:>£¨¤¤ÄÇÇÄÅÆÆ̀ÍÇ´ºÄɸ»°u¢®©¾ÆÄÄŸ´¿¯Ăų°˜›§Ÿ¥®­»®¤]T\D%L’¦¤z‰œ––Œ’¡—˜”“–™¦hœ£±®¯³‹£ ˆŒŒaVicÿÿÿÿÿ~Ÿ¤˜›¢—³¨¦£¥®¦¦§¨£¢£–›¥°¦¢¥¡£Ÿ›œ—œ˜—›”¤“…’Ÿ¶Ÿ¬·¥¾Æ¼ÉĂ¼Ÿ˜¤–œ–•’’••›ƒˆ£d™”’kNUpg†˜ ¬®¨›¡¦¹²—‰ˆz‡Œtj{‰gcxu”‡¦‰ƒ8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿq~£¢’”˜˜©¡³¡˜†ˆy#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿX]b|xv’…‡–—‹¢™œ™“‡¢«¨«¦’‰ƒ‡‹ˆ€{w„‚~s„„ˆz|€€x|˜•”ywnszxnddekjm|rrvtK*B1'     :;1<-%##&2<@:C@6$:A88;=;93+499;;87@>>@ABB<3$../(#*/*)*26>D7.-03597-.634777923;:993777353365124=œ¦¹ÀÄÉÈĐÏÏÑËÁ»»¾À¹­ˆ¯±³ÂÇȼº½¶¡®¼¦Ÿª¦£ª¬½¿˜‰›9fCAac“—¨†jlly ”€£´ •  {¡™£¢›¡ƒˆƒƒdSU3ÿÿÿÿÿÿ¢§Ÿ™–”°¨¨©¡¤£ £¡›…”¢£¢¡œ›¡¡œ”¡¡Ÿ€‹„}z‰— œ¨ §³·¿Ê»¼Ÿœ•˜›¤®¢¤Ÿ–”—v¨¯Ÿ•™£“a‰–y”˜›“ƒ†”‹~€|‚ŒZu{{zZxvŒ…ohX&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‡’“’”œŸŸ¢•œ…–”a&$ÿÿÿ^!ÿ…slIÿvMÿÿÿÿBJo|ly}ˆ••Œ˜–Ÿ™¤˜…•’˜¡”w}‚™y|€‚x{|mqzvvz}zi„„Œ„xplpttslnrwkjkuxq_)$%(97 +   *# (1$!#$,<;>@=>;;;;<<=;;===@><=>>?AB8&/20-*'%%)'',06C=0,0//46.26558679::7593-.*127422//0/1˜–—··±ÈÇÊÈÎÎ˽¹¹»Ă¼´²_Ÿ³´ÇÑË·¹»³»Ă½”“˜Ÿ©¢¨y<_Y'Mz¥©“sagppŒ”‡‹—¢°¤¦©¦¬ª¢¦«¢Œ‘›–ƒ{ZJÿÿÿÿÿÿÿÿÿ®­ª¥œ›y‡«¢µ®–—›¤ ˜– ¡Ÿ¥¦¢£¦§¤›—¡Ÿ››¨¦˜¯Ÿ‹†ˆ„ƒˆ ˜¥ª»»·½Á›Ÿ—œ˜ˆj^’‰UA^‹– ›—”¢v‰›…„~yxx…ˆuux}c~=}‘©“j]ecÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†‹{—‡£¤ œ—“¦œŸ˜ˆ‘‰ we(*’p•˜x;!oxBVSKJWgwzzur”†‰ˆ‹™Œ‘„Œ™™ ¥¥—‘x|ƒ„~•€‰~twwwz}ytnoxx}‹qyˆryzwwsrjrjidjnxqfR&: "=6/311/  #.147BB>;==<>===<<;;;><=>>=<<;<=?-,/210.&)('(**64//3.14/.726566578865601AA6=<8578;;4,¨ª¢£²³¬Ầ̉ÑÔ˹·¹¾Äª£ª—ª®³ºÈÀ¸¶¹ÀÂÇÀ¤—•¬­±¨•“‡2^5/E3$~§«ubb_kq‚”‹‰™›¡—˜¥§¤’¬«~‚‚‚ˆ›ykNÿÿÿÿÿÿÿÿÿÿ¨¢£¥ÿÿ{¢®¯º–¢Ÿ¡›¨§¢—™§¥Ÿ£¢£Ÿ© £Ÿ ’„Œ˜™¡“±¹­¬’›˜dAOˆ“‹Œ“Œ””‘“—“™˜“”‘–›”•‹††ƒsƒ‹k€‚’‹ŒzeDiy„rbhndÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿl›Ÿnu™•Œ†¡¨¥ ‡rˆœ¢˜›Cc„†…MYn\uoo{w‘”—…€‚“”‹‡™‰—¤§¢£©¡sw{‰‡„|‚zqxƒohktpsqqjjm~y…yq†„‡~ttuakjlltv}td?6G=& + +(+:7-/7@:>=<6/-**.33,$'+2426<<8<<:<=>>==<<;;<>>=;;<<98;6!+0/09;# ('&'()01452+(+++3886/.03475658@HH?3=D4-/<=;7¨ª¢£²³¬Ầ̉ÑÔ˹·¹¾Äª£ª—ª®³ºÈÀ¸¶¹ÀÂÇÀ¤—•¬­±¨•“‡2^5/E3$~§«ubb_kq‚”‹‰™›¡—˜¥§¤’¬«~‚‚‚ˆ›ykNÿÿÿÿÿÿÿÿÿÿ¨¢£¥ÿÿ{¢®¯º–¢Ÿ¡›¨§¢—™§¥Ÿ£¢£Ÿ© £Ÿ ’„Œ˜™¡“±¹­¬’›˜dAOˆ“‹Œ“Œ””‘“—“™˜“”‘–›”•‹††ƒsƒ‹k€‚’‹ŒzeDiy„rbhndÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿl›Ÿnu™•Œ†¡¨¥ ‡rˆœ¢˜›Cc„†…MYn\uoo{w‘”—…€‚“”‹‡™‰—¤§¢£©¡sw{‰‡„|‚zqxƒohktpsqqjjm~y…yq†„‡~ttuakjlltv}td?6G=& + +(+:7-/7@:>=<6/-**.33,$'+2426<<8<<:<=>>==<<;;<>>=;;<<98;6!+0/09;# ('&'()01452+(+++3886/.03475658@HH?3=D4-/<=;7±°­¨²°°½Á¼ÊÅÍŹÀÆɾ‹Wp±²´´¹ÂÀ³·¾ĂĂÇÁ«”¥¤¢„‡•C"+psmˆ£‘¨¥†uuvx}~‡“’Ÿ‘–™—”””’€x}€‡‰ˆ‡…vÿÿÿÿÿÿÿÿÿœ³¤¢§ iW›ˆ’®¬¦•‘™¨ª››£¡ Ÿ¡œ™¦¡›¢©£–—©¨£—§™…ƒ~Œ“¥ ™›™““•’•k‚•”’“‘‘”•˜•—‹‘”‘ˆ‹‰‚ˆˆ||…†‡~Œƒq{„‡— •— ˜‘•o_fiUÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™™ _›¥˜…ˆ‰–™”§‹Ÿ•|Pt‚m3Ctx‡‰}xzx„~™‹Œ“˜wV‚“–“Œ‰”¨§Ÿ¤—œ—|vp„„‚zrrqrkgjdkkhmjmjm|‡ˆz{zzxumihhnrtsjP$ ,PXQ9*754.-(/8=;876?>74#**:@>A@=:9?@@;;;=<;=;988;:&&--2?= "&&&(((*,,--+(#'156764426:<7565;JIG;<>>??@<;1$!!.6=>94:<=><:<>><;::89:.!--/:<: &%%(((((().,(-.13899765759:76;EHECD;;@EGECEA°°°³»»³¸À½²·ÀÊÄÁÂÂõŸ†¯±±°ºÂ·¶º¯¦´±³³•‡§£‡z—‹fL-;mˆ¡—‡«¦¨£¥©¢’²¶©µ¿³£¡Â´¢–•©ª§—–€u}ˆ…h+ÿÿÿÿÿÿÿÿÿ®³§¥Ÿ“•[’™Ÿ›»”•¢§œ™›Ÿ©œ™Ÿœ¢¡•—§¡œœ£Ÿ–’…‘qm€¤¦©—™’“““–’››™‰‹ŒŒ„ˆ‡‡„†‡‰„ƒ‚’„•£¤—•”˜‹“–’y«‹™˜¡†5KnoheLÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­ ~…„‹‰ˆ…vKCÿÿÿÿÿÿÿÿÿÿÿÿÿk‹†…†‹€Œh•ui;,ƒ‹“œ¢ —Œ” ˜†z{vtwvy{moidhpojshmoesnnvƒ…„…„xuvokmqpmjbbA (1,% & #!'3;>>>HLF><>;=?AA?>"##.57>ADB??8>AA?>:69;<;:::;<<;;:98:;&&)+0/-+  %$%''''''(/66//239<=888:416>8@GA@CIJIEBFCBEK·²±²¼¼»½»¶º°¿ÆÊÄÆÇƾ¼³®³²¯£¬±©µ­“¦¬°¬¬†Uœ‘f#53&U’†œ¨¨Ÿ£§¬¦¢¡®²¸±¶¾°©±¬¤ –“••‹•|8ÿÿÿÿÿÿÿÿÿÿ¤ª‹ÿ¥„ÿkq ¬§µ¡™›˜–˜œœŸœ¢§¤¤Ÿ™Ÿ¦›› “£ŸŸª¤£‰t€‹€x‹w–œ˜”›œ˜”‘•Œ…‰ˆ‰‹z|z~}ƒ‚€~…„x}“™£¨¡œ¥¬”’‡‡„Á›˜“–”–hNB_ZS3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ£ •œ…yydcgDÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ}ƒ†‰‹Œ’Œ‘€~}|‹ŒŒ—®©’œ ˜‘€…Œ™ˆ€rt|{€‰zonorzslpkmljoxtn‰“‘‘›’tvopntlfiYV0 2) % !(*(+%!,9=>@?=:=:8>=9:>?=0CGBBBB@@?=>?@=89<;;<:9::;;<:999998:=5(+,*+%$%%%&&&&''(012/1458DFHHHGHHH=;«­«¬ÂÇ¾Ă¾¾Á³­¹̀ÆË̉̀¼·¶—¡Ÿ±¦•©®§ª‹›¦•¦£¦}xvZi#K.3'!6€ ›«›™›™›©£›˜•—ª«¿À¶¹Çµ³¤¤˜™¡œ•”“}T&ÿÿÿÿÿÿÿÿÿÿÿwOÿ£”C‡nx™”™¢¨ ¤—™—›Ÿ¡œ¢¡¢¤¥¡¡¡ ª©”¨© ¡« ›‰Œ„|‡†—•“™˜“———“‘†‡ƒ‚†yz|w{„‡}z{…ˆ||•”¢ªª¦´®ª²Ÿ™ƒ˜—Ÿ¥§£-#@ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸœ£”‹|VGÿÿ(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‰v{ˆ‚„ƒ€ƒ‚€‡‘…›¦•‘}yu‚……|u{|‚‹€tyrwowtqvu‚rgo†‰‹–™ƒuysqsushbO5*$%&&!"$5=:99:?=;:=@>.%?CDB@=?>;;>?><4/5;;;:;<>:9:::9897669/&'(,*%&&$#&&&'''(+..1246;==<<>@41654/6==65:;>DAEEE; ›¨­¸Á¹º¿»½¯¬ÅÅÓÑÄÂŽs‚•°¦…™²¨“¤¨¥´¤¡¥ ¡†k9.p7VU1%d±¯¥²™™›œ››ˆˆ}’Œ‹££¥¦ÄÀĂÁ½ª­¢Ÿ¥Ÿ¤•]Kÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°®”’pj‘™Ÿ‘ˆŸŒª¯ ˜œœ ¤Ÿ«¦Ÿ§¤¢›—¡Ÿ¤¦´° ¢£¤¤•’—“w…•™–“Œ•—–“‹‡ƒ†ˆ}˜‚Œ‘€‡ƒ…–™ ¡Ÿ¡¦®©œ®¬™–¥¨¡²¸¹*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ›¡‘‡Œ|=ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‰‰zy‹Œ„……‹‡ƒ†‹‹‹†‹‘{x†~}…uuj{v{ƒ„xuqxzzwquŒrvq€kkt}}‡…z~}{{}wqjP9#".''&))''())5<;=;9874305<@97GFFBA?><;::;:8:;;<:89::<99::999;:9878%*+*,(#! &'&$$%&)+-/112356:@?=;:<>:676548:<:789=;=<<<;:::::;:::;9787876/ '+,*)$#""# %)*+-10975667;><;:9;978898767:;:789::88t¤ ¤³Ÿ“”“}y™Ÿ£¤‹™™”‰¥¯µ®µ·¶¬§¤±ª¤¢§¡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ½£“»}fŒ‹ˆ‡ˆ‡ŸÄ̀²§Ÿ››¥¦ §¥ ¤¤™£¢¤¢£¦¡ŸŸ£­•–¦¨­³°£Ÿœ””‘“‹˜®››–™””•ˆ™™˜•¯œ™†…„‡€`†¦³«©¨º°¼¼ª¢­°œ³¶¾̀{ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿEv”‰‰fÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿˆ…~„ˆ†‡‹…‹“’ˆ}}„uw„zqssoipzy~qy}xmr~ƒƒƒ~uznz||€‹}e\Yd`fcS/ /'$"#&"%AADBDD>?FGGCBA>311(+,++!(D@>=;9:;<=@;>=<<;;<;:;::;967899998987,!&),-'&$#$%$"'),/33;8:=<;<<;;9:9769:98557865876654335œ‹ ½¾³¤§¤¦ª¬¶ÁÆÑĐÍÄÇÄø´§œ›Œ«¸˜©Âµ”—£˜’}nw:<")@‹¥ˆœ©_9h©·´¦¢u‰‹›¯··¯£ «²«¢¦¤£¡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¹‹Á…`qu‹‰|“³Ê¸Ÿ£›™®¶ ¥¡¨¦ª ¤¨›Ÿœ¡£¦©¬¬££¢¨²°ª›˜—’”›Ÿ›˜ §£™“—–••¤ ¦¨°§¦œ­¥Ÿ“•§¦¯½¯®ÀÄ̀ÊÀº˜°““¸ÄŸhÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿdZÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ’‰„‚„†•’‘‡„‚††…qu|rjpujfjuinqrwtmu{‚{}ngjupupƒ„‹…h`\`hccid\%%c<)%%#08<<53.))/:?>=;88:;;DH;=><:<:9988:8:<74467578997;-&&'*'&$##$$"()0358;;:;:<;;9:77879:777884455765533445¯¦—§³º©¡Ÿ¬«­´À¿̉ÔÓÍĐÏÅ¿µ®›˜©¬´ºŸ¯­¡›ƒuœ”Œbj‹"!@>>6W‚ˆ®nÿÿÿÿÿ v¼·¬ªªªe]˜¢¤œ¡¥¬¬ªª¨¥Lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª«”ƒ‹€}ƒ‡€†‰Œ”Ÿ«Ÿ±®¨œ–¢°¿¥µ³™˜˜ ›¡£©¬©¨­¯­””““—™™¡«±Â¯©£›œ •›¢«Ă«¬²¬¨½«¦²­¢¤³­ÄÆŸ¶¯¨µ¬¢—–Œ©¬¼›Uÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿsˆ…‹‹Œ–™”‰Œ†€ulrtwoqznR_pwpkqq€…yzzƒoqryivpipzƒƒxme_`daU^]/Z0&,3$&+%%(6EH7-15CDAA?>=88<9AA>@;40/;?=;::::<>BEE=??;::9:78999:;:633445667686.**-($$#$$%#$%'#*+/56798::::;997777668778956554454323355¯¦—§³º©¡Ÿ¬«­´À¿̉ÔÓÍĐÏÅ¿µ®›˜©¬´ºŸ¯­¡›ƒuœ”Œbj‹"!@>>6W‚ˆ®nÿÿÿÿÿ v¼·¬ªªªe]˜¢¤œ¡¥¬¬ªª¨¥Lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª«”ƒ‹€}ƒ‡€†‰Œ”Ÿ«Ÿ±®¨œ–¢°¿¥µ³™˜˜ ›¡£©¬©¨­¯­””““—™™¡«±Â¯©£›œ •›¢«Ă«¬²¬¨½«¦²­¢¤³­ÄÆŸ¶¯¨µ¬¢—–Œ©¬¼›Uÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿsˆ…‹‹Œ–™”‰Œ†€ulrtwoqznR_pwpkqq€…yzzƒoqryivpipzƒƒxme_`daU^]/Z0&,3$&+%%(6EH7-15CDAA?>=88<9AA>@;40/;?=;::::<>BEE=??;::9:78999:;:633445667686.**-($$#$$%#$%'#*+/56798::::;997777668778956554454323355¬¬¥¬›˜¦£©±±ºÀÊÊÇÉÖÙĐĂ²­´ ®Ÿ¬¼Ä®²²²—w›•†„€“—J5g…“\ÿÿÿÿÿÿÿÿÿ²ª±«ˆ™chw™£¢Ÿ¬¬°»µ {Pÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·¶³œœ“„‡ˆ‘€¦¦µ¯ŸŸ””˜¤¶­±´¥œ ’˜›˜¥ª° —•‘†–£¢­Èƾ½­¾² ­¬¤¼¶ª£ ¨¾Á¸­«£´›¬¦{†¥”¢‘ª·Á½¶iÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ›‡„•••›Œƒ~€}}wwvqmjkggummhvqt~|~…†|xwuqzphblpy{slnb_eflef]dF#F05-'((("%!9A?G4DC=?@A<8989:BADD@;<;;:9<;:9999;;<9862333212336<72/'&&%&+*%$*4&+,4567;<=;;;9776787988785426543555432233®¡¦¦§¡•§«®ºº´¶½ÍÖÖ;±£•ª¨¦®½¿¡¬±±‰€”Œ¢’ƒ„‡’™s\Qyv‹~4syhª¢ÿÿÿÿÿÿÿÿÿÿÿ¸|‘•¦”•™¤££¥Ÿ§­»½¶¥~HÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿͶ¦¢”vZwŸ›Œ©¶­¸™›°º±¼¿¼ª————›››¥§¢œ™›Œ“Œ‚’›–“˜¯º¹̀¿«›‘¤µ¹¯Æɳ±ÁÄËĂÄ·›y†•˜¥¡¢À¯ª¥­Á¾̀ÇuÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŒ“‚x\0z±¤‘…worqlgkgmmnmbdfkepv}{{‚ƒ}tƒtvqknjeb`nqocouptrk`W/"*'&'/43)&"($"!)A6:>=87;;;<:@0(,.-+@7%###?>+;8896=DB@A>@@=<;:59:=>::;;:9999::;><9<89::<>@<96511//..124F764SF82>A664.../7565788:87764313777::::4345444454323345®¦¥§œ™—˜››¯©«¨­³µỀÊ¿»¹¶²¡±µ¤£«˜—¦ ˜™™—‘—’¡†›„˜Ÿ˜¦¯­±¹²otÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”²½²Ÿ¯¯¬¯¦ œ¦¡­¶°²®›lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„–›¡—~}u€€{„Ÿ—•—«¹“•¨¡¤§–• ª¢—œ——˜—”˜¡•ª‘–‡’…‘ˆ²¹½›½ÅĹº§­½¬}°³¨ ¡›Ÿ—™¢¨®ÂÆÊÍ«jÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ“‘Œ†r\EEE[_j‹~ytnjf?AKR_e`^bdallppzkgxpvwppqqnkgggigmuxxvv_TQKY!QVY7% %+'%&)0%!/&*>358556<==;<9<>;?><<;9;@AA?<:85100.15549SEC?HB@:HMO451-5863..-15566545347667<=;:6545444555343434¨¦¡¥™ŸŸ£ £¢¤±§£ÂÇÄÀ¹´¶·¬¬¸· ™•—‘­««¢¦‰—¢”˜§­¨™€™œ}t€Xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿȯ³©¦©¤¥³°£¨  ¢¯±¬°©tQÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~sw™”–”‡…†‰•†|uƒ‰Œ‘˜˜ªªŒ‹‘”’–”™˜¤Ÿ¢˜™Ÿ˜‘‘•™œ¡§¨ ”…‰ƒ‘”‹ ¥«›µ­²¬”¤°­­§ ª­¡œ ©¥³¼ÄÁ•ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™ŸŸtZWF.!06u^asz€}ulrWB@BA><:8::8896788::;;;<977:=>=:;;30.3444556;3?NA:@B>:EG>218651/00/3565564465679;9996554444554433444®©Ÿ£« – ¯°˜£¡›¯¥•©¾¸§«©­®¦µ…”{Œ³±¬£’¡™•©©  ¦°ª­u_ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ³Ÿ¤´±§¥¬ª°—›Ÿ˜”¤«§¯«¥”m\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿx—†•–zp~‡…dj“‰gt–Œ‰–•–œ›’•’““’˜¡Ÿ“— ›˜™ˆ”‘¯– ¯¯³“¢£´§³¸«©®q˜ƒ–§Ÿ£œ ¤£¥¨¥«ª¯¯­ª¬¸Lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‰‘ˆ’‰XD(0d?:M[jjmvmc71]PUcgiosqr…tjpwolsjgnedhkjfkqx‡„z}peUJSy~<+3!$"('&#+& + 8S/244;77559;;><<KLILEG21:@;77765543333456655598:;<;;997544444544444443¥®§¬«¤œœ£§›¤Ÿ  ²°Ç¾¶±¾Ä³~¤sUQb{m=Ÿ‘’•–¡œ¡¦®¯²­³»ÁzUÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¦º°±¶¬¤¢¤®­¥£—–˜œª­©¬¥›”^ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–˜ŒŒ†ƒ€|~_l€N‚xo‚Œ“ª¸’“™›”’“”’—©—››¡¡˜•„Œ“–›™™©³§›¨¼¡˜¬­°±½ª¤ Ÿ¯£¤¤¡Ÿ§­©¥—©–®±¦  µ¾»¿3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ©¬™‚N1&'7*@A.7Qk\i?*DBOZntƒqnj^^adlq{z|yrdgkfnpmy}‘’”†j0AFDtf228*0#,+  7,+95169678:68:;@BA>CD?:;;<=<;87689:;9:86899:;:99:76435232:KFF<.7G432<;73<;98764333368686::<=<;;986655444555555555–¨¦›«©œŸ©´¾Ä¼Á¹¿«•~{}w%ÿÿ2Cxpr4{—™“œ™ ¡§¦¬§–¦8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¬[ÿÿÓÀ³±··¡±±­§œ™¢Ÿ¤ª«¥ ›|ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‹œ¡††”…‹Œ„zs‡}’†q©§–”“˜¢”–Ÿ’œ Ÿ‘ ¤¤£¤Œ‰‘‘£¦»¯°©³˜™Ÿ°¢¡³´­¯´¯´¡¢¥˜­ª¬£¡•¦®ª¦¦¯®±º“-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„‘£’)j??#""L/$fE,&,3DYQal}~Ydi^{qmhnpqmkjowy|‰‰‹ˆ‰Œ…e]N7hhQO(?H; + :(<7),8777674247;=9==5;99>>@>988:9999877:878655762 $45'…r˜©°­¬©Ÿ¤¶ŸnÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁ¸³À¹²¥­¯¤¦¢£ ¥¢¡©¨««¨¦˜ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ±¨¥¥€€…“ˆpwypUTˆ‘™™•™—‰Œ—£•”„Ÿ ™”§œ“‡‰zq‘†‘¯ª§¦¤­»¦¨©©µ§­µºº¸¡±§Ÿ˜œŸ§¨¢¢§Ÿµ©ª¥œw@Cwÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿu•Zi`LoK*)&-BO@REQ}{tjnrkeetqlmpppmkotzˆ‡‡Œ‹…{dTO@hpQG23:J + +#,2!.F@6);=979704568:9:;C4-459;<:88:::9797888698646216616:CH;@CE>984557768986424323799;;:<;::8966545544665555566§°«¨¯³±¦£¡–”•––•‰„}„{{‚ppAÿÿÿÿÿÿÿÿÿÿÿÿÿs°ª¹Á™Ăϧ‰ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­̀ͺ°­¬¥£®§§®§¥¢¤ ›¤£ª«ª¥ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡˜ƒ†‘z‹y|bz‡‹¤˜§‹•”ŒŒ†’¥¥¤¥¢–Œ‰‚‡”‹‘–¥Ÿ­¬§¢¦¢´·¶½£²Ÿ–›™¦¡—±™˜ ¦—cÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿwh€‡w" '%;54MBN{„‡zqsi^fgo}|wvqnqvw€„€}‚y‚ˆ€zjMO:T]H8+ABF     #">8-=6=:;@?:43489:=><;8/4;84689979:;::::9998865987 31,9A?G@CE@=?9:?;815731../03.389;;;::9987776656777666555566½À¾»µ¯±¬µ²¦ª¢”‰œ«•‹€Vÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿrƒ—££³§‡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÍ͸§¥¹¾­´¬•^l©¤¤ŸŸ¨¢¨«©¢tÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡™¤«­Â©˜”‘’‡‡ƒw‹p|SM~Ÿ©©‘‘ƒ‡Œ‹’ˆ‡ƒ—™—™¢©–”ƒ}z‹‰‡‡•‘’”‡–}z–Ÿ³ª°§œ•‘™™“—Œ›¢¥±¯˜‰ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¬¤§¤—‘‘jXC2G=#()<1Mf€}lecbhlt‡‡†xvvnx‚‰‚{w…u…wm{e)\?FV6'%6>2 +//84;75<:>=?<;:<<;;<=<<DG=CC<>DCCB>=D<5210201058<:::<;98965898777776766666666¯¸¸º³·ºº¾¼¹µ«£¥£‹ÂIJ°x9ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”‡|³§ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªÿÿuÿÿÿÿÿÀ̀ÏƴǪ©ª¤Ÿ¤©¨£§™Ÿ•©¤¤ˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³ÿÿÊ¡ÿÀĂ¹¤œ––ƒ‡ƒw„ƒVKYs›•’‹‡‘€€w“–—¡›•Œ‰ˆ„xˆˆ“†††„ŒŒ…›› ¡‘”›Ÿ‘‰’›’—¡ ™Œÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„§¨¦§™”‡~cI:HNQL<6%1.4HƒŒ€vkph…ˆˆŒ‡ˆŒ‰‹‰‚sntzs‡d ;Q7M1!  + &5:99>=;?==BA@;;989;>://37:98:::<9889899;77898831#*33459FHB@EJFEBEBDBF@6213333449;9:9<999:89;;8999988776776666=l‰€©œ—±µ°­«°ª«²§Ç̀ɶIÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ›ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÔªÿÿÿÿÿÿÿÄÆÀÅ¿µ´§¨¤®«²¹±¡•˜pZ¡Ÿˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?¿¼ÅÀ¦œ¢’‡ƒzyoztt0Sƒ‡––˜–†€€‰˜•‹‰›¢œ›œ—’‰‹—‘”ŒŒˆ‹…‹†‡‰Œ‘’‘–—““––‘“–”“™““|ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒ£“+ )Œt5!9.!4Z=Thlo~{‡€pgv‰ˆ„|{€ykXbaGjqvGHX-A!''   %14@BB?=83*2ED>8877689,68:999879:99879889754787223332338HKLHGLFCBDDBBE@7863344269:;99::;<::;;;:;;;:98777887756=l‰€©œ—±µ°­«°ª«²§Ç̀ɶIÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ›ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÔªÿÿÿÿÿÿÿÄÆÀÅ¿µ´§¨¤®«²¹±¡•˜pZ¡Ÿˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?¿¼ÅÀ¦œ¢’‡ƒzyoztt0Sƒ‡––˜–†€€‰˜•‹‰›¢œ›œ—’‰‹—‘”ŒŒˆ‹…‹†‡‰Œ‘’‘–—““––‘“–”“™““|ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒ£“+ )Œt5!9.!4Z=Thlo~{‡€pgv‰ˆ„|{€ykXbaGjqvGHX-A!''   %14@BB?=83*2ED>8877689,68:999879:99879889754787223332338HKLHGLFCBDDBBE@7863344269:;99::;<::;;;:;;;:98777887756=‚W}}@_|“™jÄ¿ºÁ½¶§˜^.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿɽÿÿÿÿÿ̀¢¶ÿØ̃̀ÇĂ¼¡œ­¬¬®¥¥¢¤¡—pP¥ª©¡fÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†ÿÿÿ½°²£†…~xyz…o=`l‰“•©›wy{ˆ”œ”“Œ‹•¡¡—”–Œ¡ –‹‘Œ“–‘‹Œ‘’—ŒŒ–•™›””‰ˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ“™–¦_>:9(*/8)0&*DhbPSQ^PcQETSk|nMFEC:6+AD;:999:<7+88:78;:999:7885589<93565122333356GJHI?4FCFBD<;<;8976300289:9989<;=<;9765:;;:98779988876#<6-Z[+'CG0dtz]¯¹µ¤”ƒv9ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ˜Á Gÿÿÿÿ¯¶uÿƯȺ°¼¶¬¨¶µ¬³²§¯ª©’“„…£©yÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿiÿ”¯ÿÿÿÈÁ¼»“‰–„|x~‰†aˆ‡„e“¥¢—ŸŸ’‡Œ“˜˜Ÿ™Ÿ˜“•–¢ˆŒ………‚{‚†…jb_’•›“•™“Œˆ‘¨–“ Ÿ–‹„ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ|–‘Ÿog+4!?^93 $";671$+'PQ`H9`bfb[SLOPZZX^h~x‚€6(  +  +   ( ""9&$)6@9::76;9848.49::889987998887798;9743343234457AGFGIGIFFEC;9887868741228:;;99<<>><5455336;:;<989::9888NZ-'.2KID)'+Qjƒ£®­ydÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­»ÍÉÊ’DÿÿÿÿÿÿÿÿÿÿÿÎĂº¸¾¾¬¿»¶ª±±®‘l›ªª¢hÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒ”¾•”~|{u}‘ywƒlc……\~¡¥§¢£‹‡‰‘”™Œ~¨†{Œ“‹‡ƒ†“ŒŒ~†’˜’œ¡˜”’¥¢¶½¸½º¬’@)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡­§§«¡–_NLGIQCIWdgeZZUkvyzR #    &%*($$;2( 7(88776897762 89:;8788778888:4676655765445541;>FGFGGHHJHC=:88:8745556637::9;:<<=:555795339=@@;99:98888¬~bl7FDE””\r,-Tuœ‹¤Ç¥Ÿt`ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƠ¿¾ÉÀƠ¹|ÿÿÿÿÿÿÿÿÿÿÿȽÀÁÇɼ¿Å¶¶¦ª°‹‰”’”§«§£Xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°‚Txyz”ª†l2YnzI‰­¡ª˜™³•‰ˆ•‘›“„ˆ¡™“oˆ„‚‹ˆ†ˆ’”™˜—£»¾»º̀ÈÀÉÍŸw]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤—˜¨ª’XZm}†saeI%,:$")+!3(1#.#*/0B^VGFS\gZ[Zb~g  +!!"! + +  "5351. ;1-3886657556456;;87778667535664996334688457756=FEDCIILHJD>:9::;7646667646899;:;;94455699424:@A=:;:88878§œ ”OT•^KXŒp>Fx¤¶ºº¸lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¾•[–Đ̀¼sÿÿÿÿÿÿÿÿÿÿÿ¢«ÇÆÇÉͼ»®Åµ¢|‚‘“lCN©±¬™(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿºœ™{|Ÿ~m03RW©¥Ÿœ™´••–‘Œ—Œ†¢£›—‡…‹Œ”™›£¯›–““–”•­¿ÖÉÓ̉¿¹¾Î̀ÊΠ¥cÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ§›œ££˜–””`o_PD0(OK+ #:I>YE7&5.3KU60OXY`fYVV\|ƒ~ $!   +!(%&,"54"+987876557736798683576322123433566333795554006C?A?ABGEB;:98::96567766538:99::9;633334498248>@?=9999988–³®¡––§¡¡­¬ª¦ynr~¤­ÇÑ¢ ŒAÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯Æy\Œ¸¨Ăÿÿÿÿÿ@ÿÿÿ±_™NÿÄÇĂÄ©°¼ÄÆ©œ“‘{–°¬§›Tÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¬}†•›¢’oZSWy€•§¡¡¥”— ¬ •‰„ˆ‘Œ™–’œ”’‡’“”°¶¶½¦³©©œ§ĂËĐÍÉÀÎ̀ȸÁ¾µ±o!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ•¥¤œ¡†‹Ÿ”xl~……qV}gGCKJR`c64$"?<==?BAA=:6869:8877776;:;:99<=820344275146:????=<;<;; Ÿ¥“‰«©§©©»¸´®¬ÍÔË»¥hÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÊÀ±lÍƯÅÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ´Ä¹²ÇÀ¾¾—’]tz‡ol½¯¯sÿÿÿÿÿ…ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡‡7pŸxƒyz†‹„‘yMŒ…¡­¬–‰€a¡–“…l:=<>?>4-37:877753:;;:9;<9952243340136788:<<<>?@>¥¥¡¨¥¥¨²±±¹¹³¨¬²ª¶ÊÓζ¨–ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÙÊ¿±̀¥vÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿq²ÇÎÇĂ¹¾¼¡“ƒ~˜‰`b±¶¼¡“‰dÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ›Œ›…‚†‡†ƒ}JUx‚†¬œ”‡…‹…’›–”R‡ ­¥•’“”’™¯¸¿ÄǼ©¶¹˜—­­¨¦¡–•’–™  kÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³ ÿÿÿÿÿÿÿÿÿÿ«“‚ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ±©¬¬ª¢˜™›—˜ˆpbfkizzu|{„€~l,('@J-,15QRW]o…‡Œ}1 +  + + /;:95 655322232232223321221.13/1K5'++,*+,6878:<:==;;<=?8/.044364149:;::;9:9:711330//453378877;<;=–¤°©¨©¶±²«¦±«µ¸º®ÍÎÀ²̉ɹÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÂĐÇÊ¡®Ơpÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°ÈĐ›²±©£†y‰‚ˆnuu…›§»¼³¯Œm¬’¬Àÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­©¼²wxvy…g]-V’Œ‰‚¥‹–Ÿ —‚„’¤¥¯·²´¥•µ´¨°ª°Â²·²®œ‘˜Ÿ§Ÿ›¥›’’œ™£¡eÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·µ¯®³´³±®£¨¬¹¸³Uÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¼¨©«Nk—„•Œ‘Œxt|nichi{}~…zrwuŒG%(=LWYfƒ…wsc& +   ;;9656432343232212222220/31*,.$+-,+-,,3899;56<;;;==>>=<46=9945::<;;;8>=<:41//3562334;:836639>›¢ª¬¯¢¸À³¥¥³°³µ¨²Ằ¾¶Ïµqÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”HäàÁÇHÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿw±’˜¤¸¶¹¤•€‰mCCk‚›“¢¸ÂÀ»Éµ«¹—µ¹¥ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€”“””ˆƒw12|~†‹‘–©¥Œˆ„‡˜‘’¡—¡®Ă¼«¶›º©˜‹œ²®µ«²ª§¤£«¾¸­ ›™› ¤QÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÅ¿¿´µ¶§¡¤ ´©¥¦¡¥¤œ¡©²”ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¦¶t2&8%Sa\d¡¤™uz…iht~†€~w€„|vˆ’{Z9+HQTiuo‚n0+   =;&155333532343222232100100! " ---./0/59=:54378;>??>=<=<<96=;9:;;<==:@@@<43338988;=>A@@9=>7;?‘˜°²¤ª«®Á½¯¦¬©­®µ¿ÇŸ¸µˆÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÆáÚÁ¸ÅQÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”Ä¿«¹ĂǺ–˜‹id†™ ¢®ÁÁ½È¼ºÈ¨ªµº‚ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤“˜‹ko>bƒ†…Po‹‚„‹ŒŒ‹ŒŸGŒŸªÇ±´±ª­™Ÿ ª¥¨Ÿ›¦¯ÀÄÂʨ  §¡¡£¨sÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÅ»¹·³²¯±¬¦­¢¥”¯¯©¨¨Ÿzÿÿÿÿ5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°¾¨4?(+BW’›tkA.ANdzr}|^ywlv{‚sxt‡€@$*8/PT`~‡8!   "=+'54553421143001122///412&$)(.-00.259:8568:;:>>>?>9?A>9568;<<<<76@AB>8235878:533237<@DC@I‚£¦ —¦¦©¬¨ª¨Â¸±ªÇĐ·‚ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÍÜàË¡_ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿº»¼¸¶º°º¿²® c‘¢°´½¾É¸ªÆÈÉ»ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿh¢”œo€[Tp‚€„rz‡…‹}‹ˆ‹”mo‹§©«²¥›§³ª«¥Ÿ™•¢Ÿ——« £·̀ƳŒ”¡™“\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ©¹­¬ª¡¡©¥¦œ›‡~|…“}‚–“…Œ—Ÿ¨‚¤sÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¿«‡vf8Mfppllf)^]hdz}~neg^kdkt„ƒ†’•r;@AAA@@>>@;9778;;;;<;9B@B@;8589<=5458768:AFIENq‰‡•¨£œ£¦£©†ˆ”ƒwXÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ̀ÙƠĂÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸµ®¶Á¶¶ÁŽ¾¯£¤±­¬´»°´×̉áÖÔ²Eÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¼¬ {_ƒ†\_}€ƒ‰‡†…’„‹‰‹‡’Œ•‘‹®°¤Ÿ±£¢©ª¦¦”—¥¨¤µÅ½«ª¸·­²°¡œhÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­·»¶©¤±²Ÿ¥ª¦„€ph“¦©t‡¦˜”¡À³µ˜ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ´¡p=YdH"[~jacXK ]ar~{umg]dbc]ez‚…„ˆŒ‹‰V'$AYOPXv{{„›–™‰   "%+  5=*2776445322233111210/00144*,2651144267:=;:9:9:;;?>?@@;=A<8858:=?=>=AAA?9989<>=313899:;DIONR†‹€…¢”¬¨ˆ‘˜]YÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªÊ¡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–°ºÀÄűºÂ¶´°²¨´¹¹´ÎÑÑÆÜÓ‰ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³®œ‡‚`rƒ„e†††„‘…{|‰™•‘™ˆ“œ‘‡˜›˜œ œ¢´ª ¤£›˜’› ­´ºÈº¾¼ÉÉÍ¿£ª‹vÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°­´²¶²¤§ª¤—›”’†lrqtœœ¥‚˜|ªªÂÅ3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÍ¿©vl+BI22;;'8ZdN:[kyzvozznp]ZSS\f{‹‰ƒ““‰A"&/\UOf|ˆŒŒ’. !,&4P3  #3.9986454123211122132222656;;973-/38>?>;:::::;:==AABB>;<:98:;;;<@@@@@><:;;?@;1332(16>DJPVU‰‰‹’•˜·¨“ˆ•lz¤rKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ…Œÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥³Æʶ®¦º¹¸À¹ ¾¹µ®¹ÑÙɸ¤‘}qMÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿœ«™™otqy„ˆ‡ƒ†£~„˜’™¢ œ••ƒw‡|†‡‰‰Œ“¬©§ ›“‘“›©§¯¼ÇÏÊƠÓÎĘ˜Ÿ«‰sÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁ·¯¦¨««¯±²ˆ–›¬¢“•§¶§‰g•’ƒ‹•˜¥¦rKÿœ˜gaÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ̀̀®—¤Y'T…dE- K]g]9& Ijsz~}uqyYW[QSWW„•…‚{2!7OCJSfx„‡ŒŒs")]?- 6." -57634444430,+2523014449;9534-128=:<;:::;;;<<;<;?@BB@;97:==98>=@A>;><>879:6525;>ABGIQY]u~zŒ—“©‹€{S<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒÿÿÿÿÿÊÑÁ˜„ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿµ·ººÂ½¬µ»»¸½ÂÑ̃ÛÉ®²©–‘•ƒ]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³›£““…|w~‰–©‹œkz”¨«Ÿ–•‰ƒœ¢ˆ«§¦ ˜¦““”œ ¥˜“°¤­¶´³È̀ỔĐ×ƱĂËĐÀÀµ¨¥˜Ÿ€”¼¾±…vLÿÿÿ´Åº²©¡£¡›£“…›–¡˜¢xœœ‰|p|…–™‹‡yŒ’‘”›²·µ¹¿¥±ƒÿĐÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­ÏÊÊz7M}vy5,JReUWH%/$?Zdnqqv}z…{jZgVJSPOWcciw…{\:*"9MVy}”‹a   +5LL1-/+88655656:60373223489;5949:;74565::;<<;;<<<<7688:::;+(0889<<;@@;79655;<877679:9BHKMdev„}Œ¨®weÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿkÿÿÿÄÀÉÜÓÅ™ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÅƶ”­®·´»µ·ĂÇÎĐÖÑƶ““’{oÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³½ªª£„nˆ–Xz„—™¥•¡”ª²¼²’ Ÿ¤¦ªŸ£³¯®±¨}•­Ÿ›³©—¥´¯²ÈÈÇÂÖ̃ÎÍ´®ª´”„Xl ªª¼¾ÆÏÊ;¾¿ÈÎÊ´©£©ª–™¢£”œ–’Œ…‚~th’†zwkgq‘“˜œ£½ÄÈÉÍÍʨʹ†ÿÿÿÿÿÿÿÂ¿Ă¦½˜UÿÿÿÿÿĐ¯̀ĂĂ°..,yrX' NCGQ%4-(AH.:Ubjjty„‚tu{zq]WUDDH9XX_pxq‡‰u:2'6imabp-   B41.$ 587889768972.!176589:79898456898<====<<<<==8689:<;:'(5227=@@BB=9:66679767679:8INWX[Z‰©Ÿ§rÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿºÿÿÿ«ĂÔËÓÔħSÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯¦Œ‡Ÿ ¬­¶¸ÂÆ·¾ÅÖÉÁ·µ‰}Œ‡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿO¤¯­¨£†|ƒZlIie•ª…rg§²›”£³Ă¸”˜’•˜·¦‘•ˆw}œIZƒ¤º¶³ª¬©”±¶»½Å½¿oÿ§ÿÿÿÿ_m`TZu£³°³ËÆÉ̀ÈÀ¾ÈĂÇÀµ°ª©­¤Ÿ¨¤¤yzz€vkmtw|“gk–ˆ„‡Xd}‘“…—”£¨§©·¾¾Ä¼¿¿¹›Ă²¨ĂÁ³³¯¯¨¤”¸ÏÑˤ³¡iSpfj9#331: (FC(1G96:Epv|sryy€|tzzpOHKD/&>Letu€ˆ‹‰€QBQkns„n  + + &+" :985746772/87'.887::9:@=9:=;7888=<<<<====?>;689:;<<;5597;=@DD><=65998788867==Letu€ˆ‹‰€QBQkns„n  + + &+" :985746772/87'.887::9:@=9:=;7888=<<<<====?>;689:;<<;5597;=@DD><=65998788867==<::789;:<<======?@@=789:<==;:34:99./26>=><::?BCDSZ\bfs¹º¡‘_="ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ̉įÇÏÍÇÏÅÅÏxÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿq†£§¤—­Ÿ¬³®¯³ÎÁı¸¼ÎƠͼ¨«¤Œÿÿÿÿÿÿÿÿÿÿÿ¢»¸´¤Ÿ¡“…e‚–Œx‚™yf~ªµ¦‡½·¹­¤¦¨‘˜«†ÿÿ¢²7ÿÿÿÿÿÿÿÿÿ”±¶¸»»³›rÿÿÿÿÿÿÿÿÿÿÿÿÿÿ…ic~ƒ’·Ÿ‰…½¼µ°¯«ª›¦ |rxŒ€nphepcgoc}l‰oiq€q…{bYWdqxˆ„‡–‹†…™Ÿ©œ’§Àµ¯­¹«‹•r¢£– †ŒmMy+=Q]_PU[`XW](&LSSTXTYW@)=DJWqszjv}{|wtn}ŒƒYJ`fnoz‰”™œ–‰zgaUcdX+2@ +6288 .!%  .67;:3354:449;97)08:8:=>=?<878779:<=>>??>@AB@?989;>==<;:89:ABBA99741014339;9:?EKRUT`hp­„E>&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ½ÂĂ¼½ÑÍÓ¹ÄÏÉ„ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€.ÿÿ¶¨t—¤ªµ²´¶¨­³·Àñ¶¿Ä»Æľ°¬­’Tÿÿÿÿÿÿÿÿÿ§¸´¡—Ÿ£ƒ„}‰†Œƒ~‚‰‰‰¡²«w|u©˜­ˆŒªŸÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿµµ¬£‡›½–_³¥™ÿÿÿÿÿÿÿÿ±¸œ±¶´®µ·°´±Â¾¸¯¥¦²µ¶¸»«™†‘u{‚‹ebjeXMP\h[^UQWb[nh`i_jkv|„„™—‹†„woyg–ŸŒ]YuƒqQF>LW J<AI^]WUVbgLNYe /BYWW^ZQL[A.7)246LZ]`fpwoy}vf[f`gdyomƒwu”„‹zwLGDP9$/  %%&0+++-($:9761-*.+4;<;:;96336:999;:9/)4:67?><:7889999:==>?@??AAA@>:8:;=>><;=<=:CCCA99845410//23.-)=NRU^dff–„1ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÆÖÉÊÉÑ̀ÀÀÆѱxÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿœ1ÿÿ˜°¨¨µ¦¤¦°¯¹º¯¡ª›¥¬¯¯º—ÿÿÿÿÿÿÿÿÿTx­¨£•˜zqS}…‡Œ~‰‰ –ˆ¨¬£z^|ª­¯´§™©_ÿÿÿÿÿuÿÿÿÿÿÿ ÿÿÿwQ\˜’¶Sÿÿÿ”{²ªÿÿÿÿbÿÿ´«²»¹·²§§©¬¨«µª¥˜——™˜a…´ ƒvxqqnhnUGMRWV[PLYWQ_k\j`hx|…|ƒ‘s}ww~}U-"7aT`9)Qqwf?%#)&+0 )RI6FTGVOKWL"MVXWVHFVW`D;*8XQ?:PRrwy~y…|hXTtw•}k{†ƒ~‚‘™‘qnXUOGKSKTS%  + /'$' -@99455557547899=:>:7696<::<<<7."/78>78889:999:<>>?@@ABBBA@?;9:;>>=<<>?B;@CD=9873773--,**(+7?EJU[T[a‹‘ƒNÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¶ÔĐÊÎÈËÊÊÍϯÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ°¾º«­™“ ˜ª®¬³ª›^œ†˜§¬°§©«‹ÿÿÿÿÿÿÿ ’­§§™†ˆ}|k†˜–’‰yrœ£¥®mÿ±¹½¶¢§£³ÿŒÿÿÿÿÿÿÿÿÿÿ¹Uÿÿÿÿÿìfÿªzd¦¹wÿÿÿÿÿÿ¶«®UL_«³©¡¢ŸŸœ™¢Ÿ¨œ„’†Œ™x_zrurmemUKEFNRQZkaRQXorrdwysv†‡‹‡{€Œ‹~z€…{^6$ ;?@)!=\_]L 1.>8;LRRA%>?@BA?@C<9:70-2:0+***+.8NP #!@?OU>.QRUYWTIM]]X=@A@A;9>;:<82114D;,,+,/9<?=;63*/8:469614788<;70647689898789:??@C@??A@@<<=>@@@A@?=<;861/---/.*,.3CAFFMS\nzzÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÍ̃ÓƠÍÛ¯¼·¾ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®ÂºÀ±¤”ŸŸŸ³ÿÿÿÿÿÿÿÿÿ¦¨­±°}ÿÿÿÿÿÿ{—¨ „œkivœ£‰™ˆ”¢“ £ •Ÿ«ÿÿÿÿÿÿÿ¤´²xÿ {ÿÿÿÿÿÿÿÿÿ£ …ÿª®»¸ÅÀ½¹²¸±̉¶‚¾º©qm©ª—ƒ¨£¦±ˆ{ˆ„ƒ‚†}qcV]ZU\diV[\_h_bQLNYYKL\TWZIEIS]filrwtpij{olRNchYZnYacVSZU^a^b`NQRSQUNCI1/ "KLJKPN4INKJKLORWR_UOY%%KRPRB4Tr|ƒ€©­¦„|}yyyo‰~mhZWS8  '&<93443334788?CB@;7332158704888=><<=????@@?>?=:=;74../.-.-+8KQYPJHLO[ed\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—²±Tÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³Ó××ÓÏŸ˜­Å}ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ»º¡«©¢¤¢´`ÿÿÿÿÿÿÿÿÿ‡¢«»–ÿÿÿÿÿÿÿÿÿ›‰¡tŸx‘•—•« £°ÀUbÿÿÿÿÿ¨…ÿ´tÿÿÿÿÿÿÿÿLcYÿuDÿœ›©Â¾ºÈ´Ä½º½ÀÇÇÆÈɾ·§¨©§”‚“‹ƒ||ƒ‹‰‹ˆ†nT8AJXLUaY_]Yc\aOTYX]Z\[e_ZOEN[s]Vttjfddfib]]S[cT\f5JYqhhgg=;VdXWYPG@='('%5;87#3>I^]w‰’—‡­«³£\knhhs‡““…omWJ6# #  "#0;103000138<=8::543.54158878=;8137669:=<:=??<69;<>@@@@A@BCA=<<=@@@>>?>>=;;:73//-,1/,/>=;:8223-.46554685558:<=??@@>94:67<=AAB@@@AB@===>A@@???>==<;874.---/-+,>QY[WPES[^prÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­¡‡ÿÿÿÿÿÿ¦¨ƒÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿsp›Æ̉ÍÊÅǾFÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ²°¡™­ŒÿÿÿÿÿÿÿÿÿÿÿÿŸÿÿÿÿÿÿÿ±ÿ`z£¦«¯¤ ­u¢¬¢™˜¸º«¶³£•ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªœÿÿÿÿ•³ª²´½¹´¾·¾Ê¿»½¾¿Á´§³®¡§›¤£œ¥Œ…„€|q‚qfZgruŒeLNLPTUU[Y`bYU[efSNMT\U[bSQQ[cc`hdbgdh_VWWUWW`ce`_^Y468ZF7"07(>FFBAB>=,":.!9>7" *".?JD? 6KQOS_f?U]MI<1C*Qeg`„ˆ•¡›‡¦””‘œ™ª¥™”]F8&"&4:.%10-*-02?B@::;;;83045305777767568:?>::;;67??ACACBAA?>>=??@A@?@>==>;851/./,-+*3JRXVB''UJP]eÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿµ³¯ª§•–ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿœº̀ÅÉÂÆÊÏĂÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿo¶¨³¸¼yÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„Ñ©¯°®µ·´œ—’Œ¢¢¢¨”¼±Ÿ²°«¨ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†|ÿÿÿÿÿ§±¾ÄÁ¶ÄºÁÄĂĂ½ĂÄĂ¾©®¹­—Ÿ ›˜jjldc^YYLSQNGGV[RQ\ZOUTQWfab_PRMKR`]\bhe`dodkm{wofTNUPR[^ZZZPPXRQNI3WN13K@8%=NOQM% " )! JQTLJ:.D3NU2MCEIMQTQSL4Sdna~†¤¢­ª—’…zt„‹…jT(+$09.'#! !*04311/4?GKJ<7543024677:;=:<<=:>?@A@@@??>==6640.-.-.-.>QRVXB6>EGNN_ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ½Ÿ›¯²³¨‹ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁÖ̀ÉÊÊÄË̉Ñÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯ª½º¸§˜WÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿĐ´v­±»µ­¯”£S££«²·¼±±¯¸ºµ€ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯°ÀÆĂǾ±µ¿ÁÄËÀªÈĸ¯ª°¯©¤¦¤t±¬€‹ƒ]jrlh\]`JUcHHIU]SUZUQUPTWbdc[\acST\]`]O]cln‰ˆ’’tk]OSSSTUYWRW@MGGMONANFIKIS\F7=KTNQ!)$&' 2@>C=PNOL<7HPIC8KSGOMJK6LL(($@dqxf†Œ¦¨”‹Œ§’s^`lr}kjQ%/0*#&#$((%!(&#%-305FCFEE@/./159989:=>:<>@=BBCBAA<<;;;:::;=?@:9==<=BBA??>ABA@@@@?==<5362-----20EC46:24:ADGS[ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ’f•¦§¢xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¾ÓÍÑ̀Ï×̉ÓÊÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®´À½±º¼¼ªÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿºÂ“9¼Àƹ°»¸qu„«§Ÿ¤¬ÇÓÏÀ©­¬ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ}ÿÿÿÿÿÿÿ²ºÄ¹¢”±°·¼ÇÀ·´¿µ²¨£ ©¥«°™˜‹gpp†—†xVW\QXYRHEGPJNWSYPTRRTo{h^SOOQNSURMV^ZZg¡˜y]WWPSTPRTO<8:?@AC@@@A@@A@@>==><6316.-,-JYQ1)*)126;>@@R\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿvuÿœ£ª ]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿĂÆÑÊÏÊÖĐÏÄÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ¸·ª¯¨­»§wÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿźy¾ÅĂ¼»Á±YmyŒ±¨œ˜¢Æß̉Ʋ®ÿÿÿÿÿÿÿÿÿÿÿÿÿ¬tÿÿÿÿÿÿÿ¶·¥ÿÿÿ§³¬»º¿»¸¹¯¿©»¸¸¡ª¼§¥›Œ…ph¨—wSPKMQKMUJFMUSYSUPLONTasg^]QOKLLJOY[bZX^ƒ}yLd\ZTTOHSRESYSPOJQTKGG#!QA%@JK# H:.NPGMSRL &0">;&@NHHGP\KBECG!CLet‚‰‚tz˜‰}lhmw€‘’—wd†i^#- (-*)  ,%&,.))** !4772/25664312678=<6334456789=<;<;;6312.+,.DXY.*/2>PPDDGKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿœª’ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¶ËÎ̀ĐÉÀÊÏ͸ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®«¥¡©°µ¸ƒÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁ¹ËĂÁ¼¿Ÿ»´“Œ ¯³¹¯¸ÎÈÀ…¤«¡ÿÿÿÿÿ©tÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·¹Ă¬™¶°±®½»³¬½¸±¬°¾¼¡º·œ¯”„‚€u~†}aiXFGMFEQ^RVOTUPQYTP_ZdnncXOMFDCIPVS_YW]wq^[][^TPPSQOIOPQXRNJPG7,2NZD5@+87EB%$$QPNMLMMKXV/@QJHIGGCJQFG)")U~†„uotw{gqv¡§’ˆ™dH# '+$(* "%**)""'(+.964(.RVC<:;8?EJIGFB?@@BAB9:;1/011122457;;<<;:;974077:403BE1,..%16;BDKLKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ’=ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ˜¸ÈËϹƒ¦̉̃¬ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­®£«°¹¼…ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƠƺ¼Â¸Å­³›ƒ›¬­§§¸ÉÓ¿ˆÿµ¿ÿÿÿ¡{ÿÿÿÿÿÿÿÿÿÿÿÿÿ®´ÅÁÁ±£¬ÅĂ¿´°¨ ¦½¼¯©¤«¯˜¸ –wpljopz‚rv^NEDEDINPPLRROU]aMZXdLZc_\V?AAFZYjVLSXQUXXhhZSRRVOLHMONDSSLTTO:'EULR?:L89^[_O-3CK/F=3MJLOK_c0!38GKIHNC?YWR5!"P~}ƒ}‰t{}zz}w•¡¼À¢|}ll; + !"*!$'(''%(+).2325BX\7AKKeYFNQPMKGDEDA?:::1...0112323689=:=><=@@AB@?:;<:;77420MMJ@61/-.//+.6>HIOLIÿÿÿÿÿÿÿÿÿÿÿ„ÿÿÿÿÿÿÿÿÿÿwÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿcÂËăÏQ€¯À¤ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ´²©Ÿ¥º¯{ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿˆÿÿʽĂĂÅ´§¯›–²§¢ŸÁ¸ÉÄ¥¼ª«¥£Äµ‰Iÿÿÿÿÿÿÿÿÿÿÿÿÿ±¬˜§Æ»ÿ°²º»³›››¦±´­¶¯¬²§§£–y{shditu‚obRGEHKKPRLNPQMN\PTWRAGIVVRWIMMPOMMRZX\TOSTSWUYYPRFKQSUQ6DZXPVPH\YM]W[_bhiVUK?U)IWa>*;RNNPh`eJ$7F&%%?>GFA?D5TWSE-FWƒ‹q‹†‹ˆkuy…¹ÆÁ·Ÿx)H.&   "#()**-+'%&+::9==H;<9RaTJRPLABFGFC?<;920001112322467778:=?@@AA>@<98;=:3220/@OPB2,'+/00/05=QRPUÿÿÿÿÿÿÿÿÿÿª–OÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿƒˆÊÚĐwÿÿÿvÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·´¨¿¯¤¹¶‘ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿœHƒÀ³«°³«³™ªºÄ¨©®±”¶ÿÿÆ”ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¸¼Á¹xÿÂ¥µ¸µ©±¥©¥›¡ ™•¤———‡¢†igimp‘xYSOOJEKRTU^YZRTPKJU@X^_QT[Y[aYNLCJVZ]RBQ\QMRVWPNMONQSUP=L^YY^MX_ZP^^benn`^]0K8'A_`P-UTKNVD@ (QX1366(#EFBEA?ANKKED?%DLy€‚s†yswj{£¶¼ÅÂΫ!&'1# %$ *$"*++*'&&+9=8>A?FHOL?YlGKONDEDHDA?:8733//0//012355545447]bnaYcfhphcPIKKJ^\ZRN7.LSROQMMMJKNVXSaUUWQhghY]j^`psihb[k_mic[ZWALTI:4C:4-0NO')3*6HCCCGHD;CD>ilqsƒ{wmid`^cfp”°ÂºÀÆj 5!(6,('$%%5 ())'()(8@>:EMM<>S^YeoJLMGBCDDC@>:8401/..,-/234555431116:;>?==<==><80/.--//0BLA/.141*-48IU`crÿÿÿÿÿÿÿÿˆtÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿuyFÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@ÿÿÿÿÿÿ¬­¼·fÿ'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·¿Á¿Á¨¾½ ¯¡§ƒ¤Ÿ¦ ©Â¥hÿÿÿÿÿ¦ª˜zÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥¬®´°¦©§««§¡˜“„†”¡œ‰€pcjrpaifmda[]YTUVQTNLIJ_A4RZdoll]WUWbUTMe]h__VZNMNUNOMFIMKHJQOZd\Y^MSOMMRUYcrfW1UreZ[`qo`]G;A+$"5NGBN8440F@./C9DJTKJJCGF??QYhrrbdW_h`hfib‚£¸¯ÀºY!)+;?4#   #  )++,.-/049HQOQPW[\afb3*@@<<=??=<552-,-,--./244323355201458::=;>><<8//.-,-../7J1.,14035BFRl\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”kÿÿÿÿÿ¢ÀĂ¸Å¥YÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿȘ²ÿ´»£¹È¾¹³¨µ²©­¶œ®Æ²¶¯´ºµÿÿÿÿÿÿÿÿÿÿÿÿ§Ÿ„ÿµ°œÿxi«¡©§ª®Ÿ …’›‘~pwƒ|uqldcaR_rdVSXUWUUNHIOUKJD1Ead`[ONWQQMTQaHHOMNT8IUHKOOSMMHLMNTZrpEKMY^Z^Yb\kgbfjgmswtpVWL>CI4-+G+9!"%D!+(4/849OB<5:3#!$((:KFCCH?/0HFD@;)19JVKI=RR\o[`^bt€‡€³´¶‡yx„k`TKMD2HT)"#.0+)!9&ACLG>58HD;BB?;=;:9;BC8(.$!nkUUbqŒ"$(00&&*) + +  -.-./.8DDASYY\cr‡‹|bXVagiN@AOP:LJ720/1510000003231//047545378211205///01365631//12==CGPY]cjÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡§¥B…ÿÿÿÿÿÿÿÿ¸¼®™¸Æö§®±{ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‚µ²Vÿÿÿÿª¹Áª¥­§¡‹ÿÿ|–‘®®Ÿ©¨ ’“°™œ7ÿÿÿÿÿÿÿÿÿ—ÿÿÿ¨»ÿ¼Â±ªœ´¹Ëƺ·´——£—~zlfh`tŒ‹zmbdl^mlotl„ljton]XSUVRVPQTUEFSTRVUNANMTVWZcdTq`LTlofYVWPOJ?XG@;:OckG[gm{‡ª®›•áäáÖ剆trpV>Ih)98.GP\O7+%EKJ>8A?:98@V\s}wƒ‚{VRNNRSPPBEMNQG=<33133100011245300123447649::5000.//1158799300113=CGEGYWYÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ©®—ª²²®v~´¸²´ĂĐÁ¾ĂȾ¨¼·´Ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®ÿÿÿÿ®µÈǼ¨§£¨¥²ª²­”pxdcÿ® ”ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¸¶ÄÀ¶¸¶¤œ’•™‘¦©¥‡kthZl‰‚}qlNZPP]myr}ja^XNQYYUKTTIFFEHMTYJ0NQVTKK\[[^okkIqojJSZSL=HNX;9BAY\Zey„˜¢¬§‰–¢ÙăÛÎÏÖ̉«¾¹™xsh^YY,8& 8<"1AFDFI<%.:Ga=:<<8925;/0124645<750/0014@NMHRhgÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ7œbŸ®®¨©®´¾Ă»½̉ÄĐÊÀ¸¤¦¶¶ÈªW§µ±´Yÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–ÿÿº¹´Àͽ‰«œ§¢¥¨—œ‘xWfŒ~ÿ˜Œ{ÿ}_ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ«§­¨¤®¬¢­µ®®¬˜‚nh‡ˆ†‰€|p=SVVRJtrgh`UHKYaeKUZNNNIOTOY[RQVU[:;W\x~‚€yw]mETJSWWYE@IFR[qnWƒˆƒ„…’«±¥˜ÔÛßƠÏ̉º¬¢£­‰eghdZJ4)(@15;:>CKDAGOKOPQTR#8ƒ–!+*,'/%," + +&,.//.//;DCb_[]XZUL^O:CKJOIEHEEMQOPCIGFZ7643112:?BH@952357A@@<HIJO<0EMXO2kˆ”)03$**'! (./21136CJNX[Y[d_SWCBCB8:AAEE<==AMY`cRZYXSE<223:HNFEHN@<875436743:887;856@F=@D?GKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿˆ‹”¦±µ¾»ĂÅ̀ÆÍÇÏÊÊǼ•b^iÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¼º ÿˆ¿¿¾·¿·³»±·«¯¨¨˜n›’y¢’„ÿÿÿÿ°›ÿÿÿÿÿÿÿÿÿÿÿÿº¾¹«ÿ¼°®³¸Æ¼¯¬¸£‡‡“”™’†|~MqllQz‚€V~GF}pne\PaastUPQUREZILT`RKRUSE s…‰œ™‹z’‚…x\ZMWjgdgy¹¢¦µ¤®½»´³Œ]®²¨¨°³²„klt€–xi_^ZD.O($21 0OGDPU&.(;LE!BJF?H@'!,GFOSI67~‘-&0&#&."&21004@>GRS^k}}ƒ{gL;?B?=@>GE>?<963FVlf`bXTMH635<;@BB?<5985;FQTWRJE@;7445545::789987<>FFBCG9Cÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿmo”¥¡ª¿¹¶ºÀÂÉÊËƶ½½«™’WÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÇ ÿÿÿ­±ÂÀ®À¹µ³³¾¤ª‚ÿÿ‰z¥ÿÿÿÿŒ–zÿÿÿÿÿÿ¼ª“ÿ~¯¬²º©…°¿ººº²²©¥£œ„’“}Œ‰„xxxYkgu‰…}Cp^Jnqk_aVVX`jZ[[_JEMOXOOKMNRSMZ™†‹ Ÿ¡—”œ–}dK[^wcabœ©£u|«º¸³¼¹¥\v‡ £Œ‹€phehitpp]CXbcZM)%8'%8NUdW?I@(CIHMNMMSURFSTOBRy‡{1%,'%"##85389;FLYi‚œˆzZ@>CEEHQQNRD?@>;41CMNKHLE=:<98=885533226768899?ABFIDDC@ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿgÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿyd”¤¯µ´ÁÂÇ¿¾½Æ̀ÅÀÁ©ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿĿ¹¹¸©Ÿ®¢¦±²…{ÿÿÿÿÿ•ÿÿÿÿÿÿ¡ÿÿÿÿÿÿ‡|ÿ†ÿÿÿÿÿme®²»¼½³²®œ‹Œˆ†rsy{khkmqt‡‘Ÿ¡|tjyztmp^j_RYYXRMRJMJKieXQTTWcvv’‰¤¢¬¬¢˜”™{ƒu|lmnx±s}’µÈ´·Â­˜x}‰‘‡zysnlnohnrvshsm`D#4D4)31M%&QJCSBHI97& MNNLNNQPKFMPSPGAg’¨‡O!!$.)(! #( 9:=ABDFHUt”¨´¦~_G>ONHALTPLCGDBAA>;?GHFCAA?I:?A>>>DEFCHDBC9:>VQPOLG@:87675456669?B?DFFI?2=<9ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿV…¶·±²®¨°µ²ÁÆÍÍÇ—ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿº¾¯³³¥§¥©†ˆ¨­²kÿÿ—ÿÿÿÿÿÿp†…nÿÿÿÿÿÿÿÿÿÿÿ„ÿ­´»¼³²³³¾´±£™‰u€†}p]af„w‘¤˜’~‚e}qetyjh_RVTUNZPJKHEPZZXWI@nƒv‡Ÿ›³°œŸ±´ª…–œ–‹u–¢›‚pˆgj{¬«¤¾·­’‘£•”†troosrhnv~ƒ…~~wrqlW-J)B2*-%%#+%GJL`!-';SKIGJEMIKNKOOOEGKMNP-3C|¡”‰Za[P[)&% +!)+)&.(FADIHNENo¡·¼ª|^ODMICY]NN??DJIEAA=>@AJHJJBJJ?FKG;BFHEGDFB<=>HMLHFG;;88888889:978;AHHNO>,!<:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªŒÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ®­¨¨¯¨¡¯ÇÇÍÈ·ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯ÇÅ­«¯¯œ«©ÿ®¨˜‹xSÿÿÿÿ³–ÿÿÿ•ÿ˜ªÿÿÿÿÿÿÿÿÿZ•ÿÿÿ˜ÂǺºª³º¸º¯‘wo€}|x`_h•˜Ÿ¦œoqnyjinh[c]`]_UZ[JBCFFD6*)7‰”•Ÿ°·µÄŸ|ƒ ¬©˜¦’x~ugbcsnbeYy—™¢££¸¸¬¶ÀÍ……xnknpvw|}—––—›unpcZiR-I;)1W2:7&;SQf,'.GPQJJJDAHKNNNMEFHHL8;?S›‡t„……xso)  &/#/.4SOWB;E?DIGER\RMKJNI;BCLRFKI>BE;;;=DD;<:99<;<<=@A:7;CKKNJG3.5:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~¤œĂĂˆ’˜{ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿZ€§§¨“²´³¾ÅÉÆkÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ²ÅÈ¿µÁ»¼¹¦±ÿ‡‹ÿÿÿÿÿÿÿÿvÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™ÿÿÿÿ¤ºÇ¾»¸·¢¼±‰~[}|‚uaYTyƒ¡ ©¯“ dtƒŒXt_W\fdmgXPQIEGLGC2'GK€£¤˜«¯À‹mx¹Ê¶›«¬€osvhfdkzqXVU^¦‡™³ÅÎÓÚÜɶÀ€{…‡…|˜¢²¼¸º³–yo|`Ug^LU]QDYH;BbQS;$/4GHKKIECBABGIFA;AIJIHG@<“£mly}~}z9- +ÿ +)/6++%SPIEPaa|Œ˜Ç¶ŸƒF>@@;:6?998BA:?===>AEMNOQmldLBGMBIPNKIO(<@><:6:C?=<>AA@@BBEICCDGPLOPG@@>Aÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ̀¸½º«ŸÁÅNÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ’¢¢¯¶«¥¶ÄÄÅ…oÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁĂĂÀ¹½À²­¬ÿÿÿÿÿÿÿÿÿÿÿ«’ÿÿÿÿ§ ÿqÿÿÿÿÿÿ¥–ÿÿ«›¸Èȼ­´ÀĬ‘ˆ’”nr{~MT`bx‚œ¶°®›Ÿ~‡…Na_Z[\]SfVMPRONO5$I~” ®¦§¥¶³yY_›¨²Ÿ½¤lb\U^bY]s}bV]Zt„~v… ®¶¨©ËèçÏÚɲ¨’•¢¤¶»´¨·¹¼‹~wljbndup_SM/!5JZO: (GS[HJKHA=AAB?<6:7@CDJKVI=†›¢z“˜›´ E:$ ÿÿÿÿÿÿÿ&"10%"ife{†¨ĂÁ©’o?><;9::8548=>@6;<@@D?<>BCDNUX]nnfSHEKMJPFFI?ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÊÿÿÿÿ¾Đ—ÿÿÿÿÿÿÿÿÿv•µÁÄÊƠàƠ¼Ë*ÿÿÿÿÿ—ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿFy—ª¢¿ÈÇÅŸÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ½Â讟¨®¥­¬ŒÿÿÿÿÿÿÿÿÿÿÿÿŒÿÿ—µ“x€ÿÿÿÿÿÿƒÿ³¯ÀÅĂ»¿µµÏÀ¬˜’‘|tZT\m€›¥¢§ª¢¥•“’w*DG;34KYSU[ZUC>IKJHIB>BBA@>=99;=CIMJLQRORPQY]`ZVMFNJEEIÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~[Qºÿ̀ÿÿÿÿÿÿÿÿÿÿÿÿ–ÇÖ¢£—¯•×ĐÀuÿÿ̀Ÿi°Î·¼³ÂÊÊĐÖÔỒ=Kÿÿÿ¶½wÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‰¥¿ÉƼqÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™ÀĂÿ¼°•²ªª¨­ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‡‘ÿ¤ƒÿÿÿÿÿÿwÿÿÿ}–¤¦¨¥ÿĂÀÈĂ»ª¥Ÿ†v{ijHPOp†œŸy’˜£­°¯•r*2L`jgftd`XQOLFCWœ‘}k•›’‘ª¥¤‡ptmfSOVXVr€xosaUYNd†§Ç²¶¶º¯¡¥¢¢£¨¤¨¢’‰Œ“œ‹‚{wojt€…qr£t,%W]]caWZ]hX]XVA44/@HD88>>A@@#CVYJJi£§­‡m”™Sÿ$ÿÿ  +b³»²©¨³¬²cUJD759:661356558;>?=;;<3;89@FA3'FHIEDRU@GGDFC><:FD=@?CF@16GY`OJ”ˆsY\t6;tT!#&%7U2AO@15([¿ º½¦“zNKF;78:921676563458BENNONPNPW]\[W\`driGNMWFGGÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ•em—˜´¸¿¼»¿ÅÎËÍÑĐBÿÿÿÏÏÀĂÓÁÉÓÓÏɾ¼¤»¿¶§½¼Ă½ÆÁ»ÑÔÖÓĂÁÀÇ«t]ƒ»ĂÆ¡ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‰±¤«¢ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™®ÿÿÿ·«r‡¯Ÿnÿÿÿÿÿÿÿÿˆ‡kÿÿÿ‰ÿÿ”†ÿÿÿÿÿÿÿÿ¡ÿÿĐÁ¤¨²­Œ‰´¸¨¯®¬¡jWaZS?i‘ˆÿÿÿÿÿ‹Œ´´±¤~FOTtZ]x}n_XQbr}¢œa*†lht“Œ›y{ÿÿ›–™xd^e‘‡wm|¤„‰ˆ¡€^TiŸ˜©¤—•‘¡  ¨²¬³¶—³­¯©¦¥¥oh|›–“¬¬³”—Œˆt&"-RZSSVZRNRQSR<$ !-?==9BHD>65Pg[NffgmOPQU3TqŸ†[ '*9'6Ya“™©‘“˜–«±°©§›{^V?8::97776137>DEIOK96667:98>B@:KB@DEFKD?>GHJTRRNPRTU_dartvu|mN[XLLTFÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿX·g@‹Ÿ²³·³ÂÊ¿ËÊÇË̀È®ÈÇΰª¾ÊÇÄÊź½ÅÂÄÈĂÂÍÊÅÈÆĂϹÍ˶¶ÆÎ̉Ơ«Å̀Ïɻħ¥ª»³¬lÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿY‹ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¬¦ÿÿ´•ÿÿÿÿÿÿÿÿÿuÿ˜…w¥Œÿÿÿÿÿÿÿÿpÿÿÿÿÿÿ¯±ÿ…²©Ÿÿÿm’«’¤bHugQF”œ†ÿÿÿÿÿÿÿ©Ÿ¦«ŸX[_fthrmkshqev” ‹VÿÿÿÿWq|—vÿÿÿÿœ¨jd^}ˆ˜†}rt„ƒ|…‹{“‚u_]qrqpxjz°œ˜–œ“¡¯¹¯œ‚uU—œ¥±¹²²­¢Ÿ‰u|UT9VS?W\MOSRROR@7.E23BA:8CDCDGCIW<76887661=CEMQDCDFIECESRKMIOS>:>BBDFUOOUPPRLTZ]iki‚€‹”}emqrk^YÿÿÿÿÿÿÿÿŸs´©¢®±·³·¹¿̀ÏÉÅĂÇÂÄÄÆĂÂÍļ»²´·°µ¹®¨¬º½Â¿ÀÉÅ¿ÆĂÍËÁÄÍËĐÏÈ»¸ÊÑĐâÙ̀Ă·»É¹²¦®¦ ›pÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¼ÿÿÿ™¨–ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ{rÿÿÿÿ“sÿÿÿÿŒ’ÿÿÿÿÿÿÿ”˜´(RUm\œŸ¡ÿÿÿÿÿÿÿÿÿs•±«‘|~„osxybcuhˆ’’ÿÿÿÿÿÿÿyO>ÿÿÿ¤—Ÿ³fbuw˜‰{mrŒgs|†~‡xeeXNSPMMVf|‚ˆyqs…}pv‚”¡„¢Œ¦¼£”²Àµ›—•}qXf.*^BPX\\WQRRQRK@@=$6?B>BBB@ACGB:9Pdh\YQ[^;(UFsŒ”‚t{tm\hr~ˆŸµ³¯¥”y\JF0.0/7;51342>4671//32-2=;8;845978896EOKN`XE=;DV=<JXbVahy~‘‰fÿÿÿÿÿÿÿÿÿÿÿÿÿ¬©•¨nn}…Ÿ€‡|x|ˆYoomcil`ZMPSOJLPQVUXuZWYOY_\m{x`n•ˆ…†¡²’•¥«†€{qRWM7cSNWba\UQVTMHDA- 5@>==?D@?ACF=5M]_YXGHKWNC2/*€…xolŒŸ¦‘Œy†¡“Ÿ¨­«†p98:.29:983247A6253211433./.;?748?959><(EF;7GE=9ETOC@@:9>N:<@NRZZNQQHUY^_]aex‡–œ œ«˜€i‘•…sÿÿh’­°£©¬¯­¶¹¯º¶¹Á¹²©¢«¨±´¬¯ •”ˆ“§–‰{‡‘‹’—¥«¥©·°¬ÀĂè ŸŸ¬³±«¬§ ¿ÁƼ¦²¨©¢©¢‡„bÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·¡€ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ{…¢±²“ÿÿÿÿÿÿÿÿ›’œ«ª–uÿ£ƒÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ¬••“‡4"ÿÿÿÿÿÿÿÿÿÿÿÿ¬´®©«vsqWZsskiktSNQ_g[aXGBAEH?AAAJGDWLSPJR^m ¦¡¥²³³‘y~¤‰Œ•{t†sƒ•†’™“VjwWY\]TSRUFCEE@CEDCABBB@EFF>HQLNRTSef_ODA1C.!)7.,5^ «¬¯·¬¨¦ygND9844722/0444402/-./1144<:44370.-/7:AGNM.%6BADPB<;>@AF<;<>FPT]b]]SRNPY]Umyr‡¢¤©–¢ ’…vls}†hwÿ‹—“‡’¡—µ¸Ÿ°¶¯©¯¶À¸©¨¤¢Ÿ©¨¡™ ˜‡t†œŒzxex‚„ˆ— ¥©®«±½¡°Ÿ§«¦©¨¡”€«¼Ă¼¹¨ª°Ÿ“†‹{]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿªÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ·«›xÿÿÿÿÿÿÿÿÿÿ‹–xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ£³´°˜ ‘p…{aÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ¶®‘|wwxngkƒzw=GLJT[YuaAMG><7h`Y^dhb\PRPPZY}„}|¢¡Œ“•ƒdwf^hrbi€¢—Ÿ¬ª¢—•ª ™—¤ª©¨¥£ƒ€{{„Œ‡‘‡‚w€†m~€cy€rt‰Œ¡¡¤©¢´„†™”–qlku}Ÿ¯´¥Èµ®¿»ˆsnŒ‰ˆXÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”„ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿBÿÿÿÿÿÿÿÿÿ{ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‘°©ƒŒKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª¥—‚t’’˜whn/BSeKEJNS[qlh^VFAGNH@:ADGMDIZcaf‚” §“’”xoh•’œ¯¨™…ˆ‰’x¤ˆ|_fkdM=SaVR=BEFD@>EB?>>=@CDC?DDQNOPVdbdv}]S:*!>5/,?h}‚^QA<97632/.-000-.01442/0/.1110;D59;11774>C@>C<71?E:=A;;>==D=;F]Z_gejoWURUMR^„‘¡¦›ˆ””o^eewwbUT¿Æ¶›§•ª§¤£™¢’•ˆxsj``eafimpjnxyvusne]i†wƒˆ—¤ ™”Œ”’…q^S_e…•ŸÍ¢¡¤¡Ÿ€‡ ~}vMÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ’zRÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤’˜•v‡›§jWXB!=HkSABM_ixo€jBLMfbZE@BFHKEDVFKNj|„lwsrea_†|€•¦•Œoq~€‰|ypdi_ZJURRIDGEDB?>CA@@>8;??BBB@HPQTYZVQ7Qj`J@8(8/$-;XW0-<;87443/-*)./.++-.0320///4445;;47942832:>>C=>63777:;7;D>;Xfa[hxxrqgY\W`ckk¡£–†€…™‹ybXXZZPPKC˜ ›·¢€«¹œ‘‘¢’Œ‚nlhlvrn\]UTYONR^YTz†ƒpmjusu‚}h†ois‚‡xiSZXg¡›•¡ŸŸ“z”}[qvQÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1ÿÿÿÿÿÿÿÿÿÿœqÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿdÿÿ|ÿÿÿuÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ©®©“œ—lzœ˜tAB@6=<<==;=DEDMOOXW]T/4MQKA<5.-73#"'#'26D781;:85310-,-+.0.+,--//0/110356855628657:85;760338<:6;BCEFFC:;KV…“•‚‡aOSj|||‚‰{u{tdS[SYOSOA/¯³±º»¸¯{‘¨¨„yu}sila`W\yzsm_YYPIFH]OORX^„€hlp‚|fŒvWXNu|l\dqi“z—–œ’wrnp‚‘‚‹ƒrOGÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—}ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª­­¦µ›uu”NF??=<;JWRJHFH@BDGZ^WA9<:9KPRR@?9::9DJBLKNM@FKfri`aUrŒs¥›”„†‹’‘~‚„ux}^C9&EOLEBB??A><::><=<:>CHIGEGPXS@566CL<4(=<:5/19:88640/,-/2331.1/-.-/-/243675::5677755889<812//6=C::EGEFID535_{~ldYgXflpŸƒ|}|pdadd]dQOHMKJLG/¶°®§™Ÿ—ˆ‡|}qplfa^`]Xkjqleb]VUKAK\cJGGR[]]^_w‹ˆ—„vMIz‰uu^ex}…{¤z‹ke_bswzwf`lxg]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿRÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–\ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”Ÿy?ÿÿÿÿÿÿÿ¾¨¢§¬Éqs„ƒ]J?AD@=GKJJDCB5578<:6X>>;EKH\C==:;96PC;JHCC;ABZZc\UX]{v|—”›‰xu‰}tpn}ioc]RNG=DLGCB>?B=;;;?BD>0?@E?>?IGNFE?=@PSP???B>>AC8?>:41774201310/,+/2/03,,++*,//.395785663333587<<<>>388646=:GE:G8>>?:4TVc\\uwoka’¤¬®œ‡mpuld^VWRE7/=ACE6$†…z„”’‘Œ‰…~€{tpp`c\^dfoumk^ZRLA4CH;=JGKKFXjsƒfq|jn„‘x|obcr‚„ntz{nfa]^k}Œ”[/qvxWÿÿÿÿÿÿÿÿÿÿÿ|fÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€¥—‚–ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—Œÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ —„ÿÿÿÿÿÿÿÿÿ¬œª£²½¥v‰‘^OHEKGCJMDCBB:79<9698Qg@<=<>H@<;:>95E.6H@998;;HFYOTipj‹¤’‚…Œ€ux‰„bv{xjkfYYFOTMHHGCC@>=>A@:>9>@@FKI:AFG;=AJVff88;<6:<3-.43/64-..10.....*,1.+,.-011/438778631/015515;9;6C:=@HBOL:=7PLF9?Skzƒqsi~®©«¨‚|hqos[GE8,6-,"0540†…‘‰ƒ‘•Ÿ‘‡ˆ‰€zslcb_^dmsqj^PROHD601127SHA?S\]U€‰„v{ƒ‹ˆvtbggde„peoxpnqua`kio›jbFltwÿÿÿÿÿÿÿÿÿÿÿÿMÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥½É¸®¯ÿÿÿ»±˜jÿÿÿÿÿÿÿÿÿÿÿÿupt}ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‡€ÿÿÿÿÿÿÿÿÿÿÿz™«–¿À}t€‡`SK@DC?FGDPD799;97<==8XqL<:<;997<886?QNKX‚¥‰Ÿ¤ZrŒ‰~}t”œ®gqbYgQRSMNIIGB==<==BDEEG@925;=?A@JC;=>53C8UmF:344766344221.-,-10-0123,-.-+)*0320133267430-,/32367@@;>;?>8887DMGNA30>DB=;EH_k[Zno¤¡¡€„ƒng^ML1+ $"§Ÿ¢¢–•¢˜¡•~†ˆxoegebd[\lqliVLQLLT[G?D<1PNSAFKHIqiu‡~‘kbmUZeysmslclexmnnytcv†rmd_npvyoÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ­°~X› ­¶Â°§§¥L»¯ÿÿ}ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ˜¢–©—‘}pHÿÿÿÿÿÿÿÿÿÿÿÿÿ¢«º²~t}paTG@BCBEEFFC9<8:;>>:977@^@8G457/=:=8::86434533<@CQV–««¢§¦™€€…{zk’£ª¹¿¿‰^bK\XTOSQVJA@?;;:;HKFEMKD;0-/?DAUSC;9845:9_nd8812353220.-...-+--/1/-,.----)-11000044212-,.-..1488668>8@ABE<>>FCS¤œ–ydmfjQ\_`\F*!?SY*¯¤ —˜¥§ “ƒ}{o]V[]ffcbhd[N*7QNFS^PKFEC>JJHTO]e]iek}snDI`XWRj\Y^fafkilijpt^]kfYffc^iq>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”“‚f®¿¸­}—ÈÆ·©§Ÿÿÿ„‡Xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‹—Œ„““oNÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ–”¯À’„•‹^OLBCCBJEGBB?:<85:=>=;9;>eJG755+;><8:<6123151179ALn•¥¥ §°w’{ttz’œ¤´¼©DTb]=RZHOXOIAF==:;AIIFVJEDC96@@?B=98?>611NrV4152AY91../..---+*++)++-/21/./.--++*+.21000.247;7?NL>0458<@><<=:<48F53DSYq_ZsqaRDLQIGIQZYQZdgaem_\fgca[h_jeOTT^mÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿmf¤£®˜Ÿ¿Æ¶™©¥®¨xÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ³¦“}tzs‡™ŸsQÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—’£© z†f]WMFEDDCDMB@C>869>>=>988kv>43342<97796033273198@JUky‡•˜­—q{twzxvu‚‹–™Œlada6O]JTWRNEFC@>>HGBENG??=@6;?95BA>??;9?KPOY_a`NHJLKMMLKJJJLPQT_d_qƒuhhbX[TRPRTWeÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ|À¿ĂÀ²›¢Ÿ “ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¨°¸¬œ‡‰’ˆeÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸŸ›¡œ’}l`^KEB@BABECEAE?=<>>=;;;;@o_964253125560244543F;AOQD?C=:1*--./463/-,10..00////00//0//./000/0//.()))(*..-.-,,+,+.13116;;:2147D>>;;CE:<8;5138?A9=AA8Cb|migHEL^†licVIB9&)/œ–—’‹„…xohf_caUOEETUJ:NUPhbWRONLTHFA=?@?><:;;=@URLLF?DHHJROGMQKJPQOKVacmbda_[SGbRKWZY-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿXÿÿÿÿÿ”•ˆ€£ÿÿaÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¯´±´Ÿœ›‹†czmj$ÿÿÿÿÿÿÿÿÿÿÿÿÿ¢¡©´¦xxkcYGA>=B@BDABCA@?==?97679BXv966432224102244477==?G]POmyˆ §§|~monsqˆ~ynR`[=[N^]YVTQPIVAILVDBCB@A:?DHZN?@3)*+-,-.*+-..,./0//..000/1000../111000/+*)*+//+-/--/--+,//-,49:512117>>==BH9;9342323998:=@ELadigLNHRYhi„j]N7 #(-/…ˆzkfhgeh__UKGFITTKH`b`eURMJGDJF@@?=@EFEA?<=???<=>;;>>ACEIMRHGMLMNUZ_bc`[[`QKYXUX[Mÿ(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿpkÿÿÿÿÿÿÿÿÿÿÿrÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ– Ÿ“‘‘„hÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿb¡¢¦¥¯°—vm[ZWB>=?ECGJ?@EJOLD@:<::989;[J:5411013333422976A<;Hb_iL]{‡›|‡{zluy“}ghSgw]]h_TUVVRPNRFLJE=99EB@;88=>@Chwiq@2//+,-.-/0/..,/0/.../0//011.-.00000/00,+*,-/0.263011')*)--.0////12124;=8;J=4D6851025D?;?>FDHHHKOEHGLXe„cOE)#"/0"…‰‚}mmhc_^^VSQKJHHSPUZVSTRSTQJEBNDA>=;9;DZL==CE@A>9867999AELZZRNMNWUOSXZZ\Z\\QRRNVfG0ÿF7ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª£ŸvtkkQÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¢§µº­¤xmmXUQHAFEECBECDLKPOKCJJBDGB89BIZfzxifh˜u|€YS_cxechTNQWOSQv^E2BB=:L@>>>;9;;=HOaE25532/25:A>=@AGDIIDHINMNTnQ=6,-rysgZYWTUUZVTVUTKKSW\_YbkYMKHDGEF><====GFQJ:>KG<=ABC?E567?>BCNNRNTYUWMNSSYSWYONKKWG+ÿL-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ‰ryRÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ™µ·±©™z^UJCDD@FFFJGFHNVJMMJFNDCJGG=A?XzZ=5335F=769779<68;@IS\zjjk_m{˜llx†…€‰aZacl‚sglk[_^DAQod]T#8<=XFC;;99761AC=98PE++)))'&+,20010/101/./00110/.10/./01/,,+,)+.0,-.1-,+,.4125230/02>>1248@75D:>5523/0506769?FFOD>=-40;@I*-%- #"ZVSNPNC=CKGTa]TWMKU\_redVOHECDEE@><=;:;9<@999;=637:8;K984677LJE9=>=98112<972BCA5223311/113=@>>==?8;:/%'2653)(OMMJME;=ROQY\ZTRQMRWZ[dUKDA?@EG@>;:<999;777568:401832=9ID469;@BI@EECSLLSIIL@@LQG>3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿt’£mhroXVP88:9CAGAF?:==@ITX^ah\HEBABA>:>dQRSPPRQFFD>95787JORZqyqGRˆd‘†£—€}€©›v`aaj}{€eADh`\Wp\fTHPC?8323265/125??@B?;,@A?=#(*)-144/( JRVOC>8CTVO[Z[QVRQLOZYEDJ>@@AB??=9::78:666555681-.00365>:467:99=;:99BEFGGC=8??BGMPQFAASUYROVH@>>CHA@BFSSLFBBIAB+-(<8==4EVXnw{¡„OWœ¢‚w{‡o£{~k|”|cebavuqZgegYVRfgUNOTOD;BI;=IB568;:85;7982,*%''(///0///0112455.-.-.//.../-,,,+)+-.0.4=650-././269.7;.43205;9343/./2>;73185=4A\WMG@8-6MTNF=;8=72/*'GCB;AJMJUWPV[YSTUTUVYX@IMBB@@?@A=;9777666665778. !004557=55=F457676767999::;?E<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤œ}9XbI?:67899?A?CJKMNKMMRVN\FGD@?DGIGGGOt`D@>BD==;5=>@<3BWXbol–•xe?Fnrzlˆ„nheagtkr}tg[QKXplVJEGJ@?I9=P@>:83148457>86111.----.../--22454--.-...-.0-,,,+)**.//-/5453-2./13444,,-/,/5/39021/--0433442147;QkbXCCJMFD:0:85+*!@:<=?GIXZQQVYRTWWPC@DKAFBEC?>??>;988766666669=351-1342544334332676679:97768:=A+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ|RMbP=<88:798>?BGOTNJMKGHFTfPMEFKTTPPMNOLUDA?B?<=>;@CD><8GRXg}‘ˆƒ„†_uT/~kowo†|ngjfg`kcjgkUVUPHZ^DEEGG?<8:32/.,--0.,,+,,233410/0.//114+(*++(,+./21061..,0/2212342104/2123=000/,+,./20643669?><:?BBC;;:81,.,+,)**'OLEFIHJRPNGHQOTHDG;>@ACB@?>=>==;;:9955544547<9I+011123443121277B.-,-31232239@<:7544689;CBBDF>=?>>@RLJJPPXNSMEDBDAAFA@@@>>ADVOBF3tKTZu‡””ŒzpŸ–“zgrpbarnodb_lj`ja^^URTZVMB6`ZPCCDU@?7;5-488954/6=JC<712020..313.-221111188;8;;9)%$$)**,,0439<98A933/,-.127:9009<3564573,*/1/04545>@C@?AB>2#"/242541$"!!',51"MOMIIIJQOOGNSKDDED;>=BELO;:==<;;;;::8653556666;55323335422322763-+-06521133=AAH#ÿÿÿHLIÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿn‡jxsKEH?>84555889:>A@B@<>B?=@NQRRKJMW`KCAAE>CD>B@??BFCSROC5M,ZP_t‹‰w’˜|tkhV_`uessjljhf^jPPPVp‰a"7qcHFD62-001-3511/3332158ACDC84/$&$#)**.-025==<9:H310---79357428:78;98841-/3//2224?DGGC@@:;:9=:65)%$.$!$*+,,KIIJHHMLXLDG?AA@BJU@;:;<;;;;;;::876767564244423465301493--,+,..G@754457<@C7+ÿÿCHDÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ… }§‚`BIMD976569:>;<>=<<>>>CCCEVZWOTX``WTJDAEJDAB;;<>?CCFNGA>;;XXXr~q……†q–‡™‚Tk[M_cinj~seo‚olkWZXV[bULJcgOLJD=<9>?=6738885-+8632>9,+///313330234137>>>;<96&'&$++,0-/13:::=<<111--,1<0381269659C::93200-*('.4;ABECAB<<=68-)"'&$),*(!%$HHIHHLHLMCDGA==<==A@DB@?><;;<;;:;;;::98777765254443474720044420/+,./5A966558:>?MG;@&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—œ}‡e^HNN9;742469;==>>?@D@BGHJNKNQLRXaYXUQOLHHGB>:>?=<>@::D:;BHVLRjx|}›€‹z–]I[]RWjq`T^ˆ†rnhabc`sONkj\ZEUUNJCEA=FIG<6556562-2044;M7,+&)--//1.0227767;;<788*,*%*,.2///,8:8=>93110-,.01653378;:6D:;;3160,'$)/38:>B@?=A;;A@@?===<;;;<<<:;;::987776741933459;88100122451,,-.EJ:65559=CIIGB==ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„£|vffL]ˆD6347:7=GLDFFAEPKUW`[UQWVURA><>=>B>;;QQPl\^Zy‡z|qgggl…uucY[PZYFFH@;<>9TQ9-315774.68<7C97'!!',-..-.2.25347;;93%+++,-0122/,3;5;<52201./.1//11342588;>:51///-*(.1234<@=<=;??<**38443..044% #%'12FEC@A>@?>@ADA=<:;<>==>=<<<;;;;;;;::;;:96665:40723335:9310///1323.-.09G955458:>FHHHE9>;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”~WSXRdsF887832ECD?<@A=>=>?@?B@EHSX_caWRONJH=::<=<@@<;<<9;NI]h}gi{•†UeeT\bTE33gHMmcTTRr…ywkl‚‡zmaOIJ\UIG@::53_G1,3624353./866B?20-*+,00/00.02238:750$%+.,,/11532,35;<93213/../-.012567548C@63/,2-./012248?9<=>:9::8762-*321/+'+%1/(,>?@>>>=@@CL@A@gdea[QPOJFB;:;<<=>?><=9;:8:9)>_^Txk:URLWLEXD9:>@==?@???=;:::;<;<;;<=====>===;:::;853331./522233443110/0211//.//02577679:;GGKEECB%)5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†™†ˆ~KH@3348:419:::<;;=??>ACKPA?@jcd\UUUPNF>::;<==>??@?:999994@_ˆŒg[cpfZOFK::<2JLU[\Y_aRszvxo^YTHQA^j:'-CB8528/.001213334666578970/.+)%',.20-$#-/,&#'(-10010./04.(-3484001./011211135323268>52/00001010386E469:8988975421123##&29?B@>>;<<<<<=:8::;;;;;<==<====;;;:99:876422./6132332432112111111002345677778;FFD?BCCGH@0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŸ™–¦½¥•QOF31136363547;;ABKEC@@ZRXXSWRTKC=?=<=<<>CBDC:9:?;:::=G]{˜‰Œ€fœ|dB7003PbjZUWh\W[b„—~|xb^UA\^d_<+&2B>-;7%)-/0.-,33245653874622-,+''*/066,(*+(%%)+-,.0/,.//751-4473..0/012256206620/221422///0/1003642665:=@?>==624841*.)118==>=;:;;<=<;:;:;;:;<=<<>=<<;::;:99766531.021222224431122222212245656687789>?><=A@AEA9998ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¦¢Ÿ§É»®–cOK72159@O4456;;>BGEGF=;<<@<;58BFO}•…‹£k51.0KJCeLLQ^`a`acdtmhaVWK_`PGNC7B0$:967//..-,8736671642/11++**)+.442,/,,/&',11.,.--12165101133/-0/./147320251../1//00/,.110.-7843;<<>BB?@B@;92<,#'%-19CMH;RZB@?BCFFDC@@IJMHFDBCABFA??A>@CEBHH?;;=?AF94;?Kfb’}‹txB104KbERKNJXT_TQgu]lqlidW[db_[WH$#2,"(>6(*301,,434465757320-*(),,*.1++(()('$%-230),.-23232332/0/0./.--13512010....-.2372./01/..1348?=9>@;=?@=>:46!3:8,7CHGB>==<;989::96555113411111223433222343344456667888788:9979:;@@A7?DMMJ@[:ÿÿ\ULYcVÿ.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿc§¨¡¦£…:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿgb˜¥k¿¨–—mVQPI66;:9;8:99;;;=@?DFFFA>?ORKPDB?BCBA@CEA=AFCEGG@;;=@C;-/5;@Zs—œ‡“‰vZa616R\>FRMLMPUOV^sgiacjhii^[ce_BAT9?<;6<:4/-+2CD93156575/0)*+*#*1.,+*,)('',,/1),-/23353203./10--.-,213221..-/0..-/3530001//028?>::7;:8<;99997659668>AHJHE]]?=>===<<:9<<9:965532233200222234333223444445566777778898677665>>:7=6EZbkvtuv„~~{msvscÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”†‘±Ÿ¤¬ ‡}cO[ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ|®œ¹{Ÿƒ’•^SZ\P;7;;;8474988;??@@A@FIKIB<:;=>?&1KL^o‰“–¤™xsgV?2.19JA8HFOMLERWrpTZkƒ}cbZ\WG?IK8,?86475133?YL;6./21464-*)**').+.),*,*'*-**0),-/23454/14200.,,.-,*-23110///2..03102////0587:88:4776=AERTLLVX><<;;;;:;;<:::;;;::::9<=>@@?==<<;::::9967773233321222233323122344455556667888886555468=@<<<4;NUp„‹‹}€{q‚zqouKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~‹´Ä¬¥¨¬°¡„fUUWZbPL;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ•¬¶¥´¦zzz[UM]SB8:997674:9;==>><=ACA@GQ_ZPMKI@@BBB?@BCB?CIGLB@><;=?$%Dhƒ}z¢–xuYJD9?Z\D6;LGEeh_YWqtWQVZdbeja`^( # 6?<9<536CBDP637;=?><:98<;6;CHLYORUab9;<====<<<<=<<;;;;:978:=>A@AA;;;===<::;99965311344333333444433335688655677:::::7654458:<@:88@>FOaipmdqlhc]\\UQLNfVOTF>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿD_̉± te_VE>;::889=AC0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿs”’‘•‰zkO>98JD=532:275469<>=>>9:<;@DFBDD@@B?=??@=??=<>CQ@BHIGD>AFBB>=AFY{ep}‰‹z~‰‘^]^¢oYZF77ZRShYw|TUu„i_iyrvi]i^=-(-47@C725:G75713346,+../2100.0143.2-)+.+*-5-*,/3345332010-.//-,,.0105330.12.0303143017335-,5J=276?ABBEDB9;?87>FYTQniiUQN;;>=====<<===<=<;<;:879>??@B@:::<<;<<;;::9454333344343343444433367887656769:;::8865568:;;::;;=FJLN`qXccZba^NF@<>@H]nkU4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿZfœ¸‹›YIBA=99887768=>=ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿw¡•Œw|„siZ9:<><72217214549===>>::=>>?G=>>>@EG?>>=<;;<;;ARHEHFDA?CB>BA;@AMfƒ…v€z|Lz¢Zf]zSL[93KRUYeZ‘‹ºÆ˜‹ujnvxwsjskS)-4AD9:>=DH><7343421.,0..871-4,-./,*(*)*/,//,,-3636411011+*//,+++---023/,.-001/./00.320982/EH-15;@BACA?>;:779=BZ[a‡[@RI<<<>=<<======<<;;:99889>?=@B@;:;:;;;=;;;96454444444433343435433467787765568<<;;;;96679:::::<>@HHRLL\U^VUZYP?<;:;;;?FSc7!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ_x­‹zA=<<:>8:875589=AD5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ•„msz}p\9698410334334569<=>??>>@A@ACA=>BCDA?@===?@A>=>@BDFCD@=?@?BC>?ADW|zxxnvi_XlqY]Ynm\QV@8DPIJo_Qi–¾¯—„’€hmoptuo;D%"!HIQD977B;7;?7686/2#&100563-8.00)))'''(0102//-43384//030-+,.+-+)*),.--.+-.,132001,,11078-.171338;BGU@:;A;<97;CWcjeaVZPJ=<==========<<:99988899>?CCE@;::;:::<<<::5445554444444444444434466777765566:=<<;;:889:;:;BBCCGMIIOQLa`SOLCF?<9976679?@AB?><==?CDD??B?@?DFD@>=@CD@?=?@G•zkwkx{lb82[Z[YTSaYMCBKDg^Tid€¢• o}jueƒ€lv)?>0E?IE<68::798789758+"/1015621/-030))')-.10301/158;82/.//.,+-,++)'+)++,..++&*1410/((*-/42/11110078>B]WB;:98:==HT^\dNSV?>===<====>==<;;<;:98777:>@@BA>:;;;:::<=;9:88565666545554444444445666776745568;<<:;<;9:<>>AHB@FLEDLYUQlRLQ>8:<;9866676695.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8HdjhcTG<<7787986698677=CECBBDG?74$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿs†‚rY\ueVA332///0545855659>@AAAACDA>@AA>>>>=><<<=ADED?<=?ADBCBA@=?D@=><83*…¢ˆ`bn{ycZfcc_UXW^ˆsQ@ZHKc]`al’}yqfiletqszX7C6>NILA:::;9:>;8:793'#'-.0033116242/-)(.422./0-./79::62//0-,,--+*(&()+,-,0/--,081//-*),---0000-.03248Q<68988;;9;AJGNHTUBD=>>><=>>=<;;;:=<<:88789<=AA@?:;<::::<;;89<755666555555554445545556677765568;::;;:;>?@AACDHKHK\NQWQMMXLCB7489;;:88676660(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿCI[[VNG<96767?:64557778;@?@ECCDGCACFFI@?<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?…~|dMWpZEA3568125:9787999=>>@@A?BCBAAAB@??@>>>@??ACCE?=-YHJGHDD=>>>>=<;<:9888;;;:88889=>>@@?:;<;:::<<;9834556655455555555555555556777744469889;<;=>BCKLPQOMNIFE@A???<;9679;<=<8877765-/#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿKHY?DG<:847778755567778:::;>=<:==>@A@:=@A@A@?CEFECAGGCBBCB@?>>EEEDBCD@ACCEEC?;75{™ccnrrypYf^jtt`ugC?>@9HWJQ^ˆ˜~‘iYhn^nƒŒiL:!>>>>><;:95658:;99888:<;AA@><;<<:::<<<97546656554555555555555555556778544577789;=;=>>ACECB@=>;:=?=?<:<=9;;:<;8;=;666602+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿD>KC;B?:;778999877777899998:::9=><=;;<>B?ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ{§ªƒK;,baB44765456666:?<93;=>>=8;??>>>?A@B?ACKRKI@CCBA@?BAABECCDBBC@CD?><:‚•zhdf]€u~vTf\mt‡‡…x\bhuu>>=>?>=<;877688868888:98>???<;=<:;;=<9635455553444555556555555556556775445:::;<;;<<<<<=>=<<;;;:<;<<::=><=<;;:<987:766332ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=:7?;><;989::997:::88999::99999:8889:<<@FÿÿÿÿÿB:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿx¦¸®ƒjE$ruM;458:;<764.7:<94688:89:;==<=><<>A?A@HHGF@BC@?A?@??@FDFECEDAAA?><:Sl{dgjrnRHHWbnp~‹ae„mmzw”}aFE„˜œ‹lSPggZiuQAjbc]YA8MF@?:8;37632332.-,02585;73123*)-13174686259;93210042/-+*)()(),,30+)(*+*.740/-+)-.3/0/0010/.2;/024454Id|xUTKG_J79H>>=<=>=><;888888768889:;>==<==<;;<;8345455544444555556665655555566665434:;::<::::::9:<<<<<<;;;::;;<==>===;99>:6887766<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿIÿ69459@;:?89::;98999:::9:99879:899988889;A4/0=MRNMIG;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ3.7km:ÿÿ/&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿT€¥­’n^9$fU@6247<==;3251345698479999;;;::;9<><=>@CBC@ABA>>>=>@AEDCDHEDACC?@>>:8=77641452687/+,.156618;.052,)01059567428;9850000110+++)()*)+/.++(-1-.,+0//.''./,///0//097./026643TrmW:<;IGGXZI==<<<>?>=;99988877988999;>==<<=><;;<:::5433444444545555666666666665656666549999::999:;::::::;<;;:::;;:<<>>==?<:9999888796 ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4485:=>:;999::998878:999988899899988889:G?=>@?ACDBEDCABCDBAA@=:=TON[_WY4^Yfyƒks…or‚zp—°„erxx˜aTnMTSZUTenc?+KXYNDJ<3;?084065445460-.-123564=44:61)*+05=723539;;:72/0-020,++)&&***(.),6*/.--*/430-'/---.././0/././35255_oUs==>8==GFM;::;=??>==;99988888888989=;:;<<<=;;;;::5423434555555556666666666666656666556788:89899:;::999;;;;::;:;<<===<<;9899999:9<8773-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ39678;9:9999;;;8889:;;<:9;:;999898888999;;?ADCFHNNIGBPA:44ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(ÿ†¾̀«–“­̀À¹ÆÆ”F ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿz™ •meeW6/[?21237>?@=753/.03122035778899<:9;<;A@?AAACB=<=@>==>?@BCCBBBBCFGEDCA>><@=GSSWb>BUqrˆ}‰~¦Â‘~g_DZlj‹oFSfFOSgm`^deTB:HOA*4N==D?;95;;7552260,-//.444875:5-*--.074126877777310/000,**++)+))**)*,+.-)//.3132*0-23,+---,,/00343210?J[Y=;:===>;;9:;<>=<<:99:99888878888;<<;;:<<;;::9998544333335656666666666666666665556665444588889:9::9:::8:;:9::877999:;:988;<;9::<9:87755:>><=9778898888889:<;;;<=<;<=@>==;:=:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿF—¿Ñ̉ØĐƠÍ¿¸ÉÑÈÉÆÎň‰<ÿÿÿ!-A=>?=<:732/22./07:;78:<<===?BCC?@><+-;P93457:>?=>??>>><2041/.-3765689<;D@;;<9:@GDC@CCCEC@A@BBACB@;?<=:978974018((,*,0/0/.353-&*21530/1127432301,-.--+,110,,-+*$*,,*)-++,,*))8@A;971*+,-,-.//./.-..BX8:;:68:==>68:;;<<::998878777777898::;:;<<;;:;;9953343333345566666666666666666666666654357789:::::::::::;:98789::::8899989:999999999987888899779::<=87ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>=::;;87899::;<<<==;=@@A?@@<<<;9:;<;889888799888898779999:;;;899;763ÿÿÿÿ#BN¦®¾¿Ç̉̉ÊÀÅÈ»¾Â»²©ksY?6976543444246=D=8;:::2?=<71388:>@?>@@>:77410,.+,1687766:;@JIDBA@<97>>DB==<;=GF@BA??AABBECBBBDB@=<=?@O:*AQL`W^N]YWbeYO611:9G_JF88<>?nMF\U_aƒokLYK\Q*-??@?;;67733107('*,),,,--0.)'+*/796/-..08885313.0..,,,00/--,*+&,/-,,-,,.,*+,9DQ<:.**,-./0/.../--.0Q;7;<:68::CD57:::;;;:9987777777778;9:99:;<<:;:8976534433333444556666666666666666667666544447999:;;:::::::;::9:;:;;:9988:8689::;99:999999899889889988::::+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ:>=;9;=<8778::;=====<;;<<:99998777776777667777788899998899+.=Ba`}­“¦©¯®Â¾¹À¾ºÄÄÂÅżn]L755643332443345778788637C@76489:=?A@@>>?=87,-*-/.17:8558:=B=<:;<<<=B@?=>;:?ABF>?@BBBEFCCCFCC@=>??=?J?.1LD=W|k\FfF8=0/)6DIRL@Z>FTkJJJW_jpˆ™“Z@;Z]N'B:EH==@737922,*//1*(')*,,'&)7**154606;68875113/100///0/..10*.333-,,.036**)-3?790**+,--..../.--..3P66;:78:<>=>?BA@@@A?>=<;;<<<;;;:998998756665665667776677666569CF@cw€r†™˜Ÿ¨µ½¶Â¹ÎÈÁ½¤msD66554446673576458769;:67EC8849<;>@AA@@?@B@873+10147865789<=??=;<>=<:=>??>?>=:;AFEDEFDDB?ADEDGGA@B@;;<>AL3+494F+'324D@1$6@U[YNQYbD?DJUYwz~–hL8J_^N(6IM>6B:26<:50+.2//)(*+,,**)15,*;22613;57::2272210..-/./00..2:87.,/./630,++,/#&-+*,-,--..-,-.///7H5557889<;?@8987998987676655556677=;99999:;;;;:976546533333444455566666666666677777766544579:<;;;::;;;;<<:;;;;<<=<;987976899::;:99989::9999:999999888888887.ÿÿÿÿÿÿÿÿÿÿÿ52@?::=<;;:9:<=======>?A@@?>>=<;<<==<<=<;:99:97766555554566666555654334456432;ix‚~€”Ÿ¾ÂẦÍ̀Ÿ ˆ``H65433456754674468956::47C<8;89:;?@@AB?<@AA>ET614598766689<<==;:;?><:==>A@<;:9;>FFHDACBB@A>BCHFBB@A><>>>@A9476749=>976BF42BkVPIDF=CA;LX_ƒ~z†ƒh^K?FSW.;JIIOJ>>==;;8353-.0+,,-00/,//41/-.482/98686452211//-..//--04=;8/.---421-)))*++,*+---....-...///I;5678;8:<>AB8878888776655555556676<=8::99:;:7:997655654333445445566666666666777788876653358:;<;;;;;<<<=<;<<<;<=<==<;9898899:::;;99988::98::9977999988899887.#ÿÿÿÿÿÿÿÿ5457<:7:;:6599:;<=<<<<>??@?=<===;;<<>>>>>>>;:::988765545444445555212334433333233<68B56;;:<<;::;=@>;;<<@@=<=:;<=@CH?;;?@=779=EBC<@B3/7;##'HSG;,*4AD\sbYc„qvzqwkSNAB=@FGKNM;@<>;:77101-1/),/4541-//9327.4744>>457636410//-/-.-036=;8+))),142.*))*++-++,--.//...3?>BC955747;<>??CC8778887655544445556777688999:::;668776767654445555455666666666677888998765434478::=<<;==<<=;<<=<;<>>>>=<998:999:;:::;;897;:879:;99778898888999885&ÿÿÿÿÿÿ/;;:8::9986424899:;;<;<<==<;<>=<;;<==>>>?==>=<==9877775333233344223244333343353334=RcVLsµĂ»¿Æʘ…zbDCf85445677872158668:<::66=H579>=<<@?@@@?@=<>@GXKE=LT>;888:9:99889===;<=>>>@<9@A@==CGA<>BIKHJMIIHHLEDDCA>=;8::9785.4668+.+$ ,EOlfr{[sltvkd‚txiD;96IHMIEDE<=@C443/,/6&+.07754212><<2/5667>7:94169110/-..-.047=:4))''*.20-)(**+,.,,--..///.0<84446554468;>@@BD7778776555444455567775<76;99::;;989978877765556655556666666666778999:98765421558::<==:>=<;==>==<>>>>>>=;;99:::;::::;;<977;<9899<998798888788889877-ÿÿÿ%02;<:88:98652/056899:::9998::;;;;:;<=<===<<<=>=<>;988785223332233555773556555785445AKGJJgŸÁ½»¿F:M=69#>6457878983346878;;:868@F789;946;<=?@@@<<=>FnV@Q;D?8879::888545;<<<:99::;;;?>===??B>=;9:;:88==@C@>==B8/+1<21*$$(+ORMjPp_^fkwƒ‹–—~k43(8RVQJHIGHDG>774.-.((+.62458411.5108369A@712/31.10/-..--257;82++*&),..+))**++.--.//0000171222234212357:?@BD667665554443445666777:876::;;<<<:79;9887777777666655667666666778:;;;:986544524379:<<;:;;===>>>=>?>>>>>=;<::::;;::::;;<=:8:=<;;:;=;98:;9888888898774233:;767888999750..1477787776769999:::;<;;;;<<<<;?==><;::87412354339;9834799:977887766?R]GLgX\ŒµÂB5654326065677778756856777<:656AC7:;:348;<==???@?>@T‡€d<6;C688:;@8884:;;;<;<::;:9::;;<;==>@AAFELLPLKJQNNEEEFF?>=;<=;>;<;:>A?B@KA5)*,:$/,' ($;JPc;DN[Y`_lywˆ…P=CHNBeVKHLHKDD??90+/-))+1658<82/.,+-3878;98--26/,100/-,,,038851--,%)).-++++++,,01/./02433011010231122357:;A@555434434445677777:=@=<9:;:::<;;:;;::988888877766666666677667::;;;::87644666542479;::;<<=>>>>>>>=>==<<<=<<<<=<;9:;;:;<=<:;:<=:;<9:<:::87654778:9::5544346765567787/02234566777777888888788999:::;;<;;<<==<<<<:65766777778;;;;;98966777665456788998IB:74224679E75666825457764357:896?C?89;<<;==<::999=<<;LwŒ|>349;9ADD668::;@:::978997979<9:<<>>=?<8;<;;:<:8:<<;9889989887766666666777789;;;;;:97653445664455799;=;><@?????>====><;==<====;:;;;;<<<<;86:<<;<:8::::868<8789:8:::85333345655678730-033665455566778887788789:::::;;:9::==;==:666677785688999;978876677887667997955g551138=?A,857658454369<57<7;89187<;==><@@<;84538::;;<99:;8885578889<===>DIJGOSWQHKOURNIEECE8659<@?A@@@4237CF"!"./$ÿÿÿÿÿ ÿÿÿÿ7WVXW_lpqttQR\[LWan{rkSNNDEGD:9340+-3413449158735622252850.0////01001521/-**),+&)--,**-,)*+++)'(.0012322100.-./334579::943334545677786669B??@@9:>?>>;:;;;;<;999999988777665666676789:;::99887444455664466699:>>?>>=>==<>=<=====><;<<<<<=<<<=878;===9:8989749:86889878:986431/13689:9641///245455555567777677766789;;9:;:;;;===<:84563787899;;8777787:97778888887987644s<67447;97632568<<<;>=<=??CHLRSNNGIPNMKAA@C:76:;9;?@?;112245%%'&ÿÿÿÿÿÿÿÿÿÿÿÿÿHM`W[]eqpnq[?GVJJPgrul^Z]JIH@F;63.,//211126<:::946;626/570.212/.000..10.-+..+-+)().-*(,,+*)))&')00001331.--,,-,0136999::3334576565678777?B@@>;@>::<<<:;::::99998877666666666789:::99878956876776556568:;<=?@@>?>?>>>>=======>>====<;<<<==<=<=9998<<=;<<:98769:857889778998554./2689:9556110264444445555555456565434689;=;;=>??>:73474279;<=;:?8877797997888668988886444xI6::;6743ÿ/69;95.12::==59>:98/587868;;344344589::::<<@XZLFg;68988;8888;<<=9888;997679::<;>>>=>?;;>KNQOIJLJQOI=@AB=87<866=BC2/00/22*%)"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿHNTP\]`ibnqMFJLGK^`zuN]D"?=BD=0...+++-./4:<<>2/59//27662/33//./0//.--,-)*/-+,,&'/0,(,+(&()))+,/0002332-.-.///223577788345566655688:9;=A?BC=@<;;;;;:::998877666666666789::8887999679:8668667659;:<>>?@@??>>>>>=<<=>>=>>====:;;<<===<==:::9<<<=<<:88779;;768:98667889640015::978774423345333455677775544553122337;;=@@@?<842560.477:<:7<;<=7789888778877767865432Y8:;<575-ÿ 65;83-.6@>>=9:;:66.45565445233344677897677:>L?YF999988879;;:88889::<<===<;:9===D0//014:1"'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿIMKSZUirolhRNQTOS]hv\L<>B81::>/022*+,)/35A=>>;1445434683453...-----,-+)((.1,-+)*+./*&(*('((*+-01124453-/01///25635786434666667666:?>@C@@A;:?BAAB@=<>BC><<<<;;:9:98887766666677799887778886798;987779978:;==??AA?>>>>======<===>><<;:;::;<<<<=>=::<:<<;<<;:6775888:6899865666887556899999885653332333455566666555543233334:;<>><9764571/./3158976<=?9::979966789876542(BiSƒL7<=<:=>5ÿÿ43993/18=@=:9:985//3487;654200455677674448789898988899989;<:9;:;;;<>?@BA@>@@>?@?>=<;=<=??HMPLKKIHE?CEE>96:==A8:A1/033:9%$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>@e^Y]xjit]IYWWdU\fVPW76@B46?4593,.0(1=<8@>>9()8555-/10262--.-.,++++)*)(031.+..)*+*&$(+)())*,243334312321134<<8-./31-466666688778:@AA?@=>AADAC@?>?>AB><<=<<;:99:988776676667778776678455478::<:999:;::;<=?@AAA??>>>=>>>===>>>==<;;<<:;<<<<=;<;::98;<;;;;;6897887:9687664444788758::;;:999767732222334556666555554444444578:::888;:780./3543689;=:>>>=;:8876687764@>o¤“F6<==>>?A$ÿ13:985539@?98:::)138;:86782/046799:74589;;996798877:99878::;<;<;;CDCEDDF?@DCDCA>=<==<;;<>EONKHEHG?CDC97;ED=79?800/131ÿÿ%'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ5HW\fTTQx‚pTU^^]5$POLRC>=E.0;6588321%'4<>=ACCB@BAB@A>>><<<===;:::9888876667677776666787655667;;;<<=<:9::;<>@@?@@??>>==>>>>===>>==;;<;::;<<<<;;;;=:96<;::;<<9:97789998864452243777867::9:::9745733222234456666555555555444455668898:9895;>;<;82489989>=<<;;:979:87434E}¶²±f‰66=>@B@>>4/ÿ2699864:?><:<<<ÿ/25;><97772014589::88898::997877668:;;99:::;;;<<;><>LLJHIII;>AB9@KD>96@:310//4.+3$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ6J\rU\XYmrpkMO^jXKKPRRQJ?+")1;89778: ,1?<>@>;:/398%362.02/0,--,*)*('*)).243-)++('')())+-,+)+2312345832679:;506)*)*+-5765456;88:=AA?=9<@ACEAAACAAA@>><;<=>=;:;:9888877677677776655557:64146;>==?=====::<>@A>>>??>>==>==>>===<=>=:::;;:;=;99;<9=9759;99::;99888998976:634422//64440::::::9:787643222344556665555555555544437,58659;978:><:852247::9=<<<=<;;99986434$3¾_u[28ABBCBAA8/ÿÿ5889866A@@<<;9ÿ2148<<;6641014787:;:54446776889:988:<<:;<;::<<<<<;<>@CEE@AEDAA@>>@@?@?BD=:>GKJJJI:;>?BE;<87<>4211154E994ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>ÿÿÿ|lf\`cVurj\Y[`aYZGLODGDG706B=<7:;=)#45?9:>E=89;@>896420-*-,)+))*(''))'*/1-,+./.*&&)**+,,,-.0/.-24463568><9/1;')*+--232169:=@A@AA?<<;;ABCCBBBA>==@?=;;;<=<;<;;::998777777766653434696489367<=;:>@?>?>;;=<======>>>>>>>>==>>>>>?:;;::;896568:9D<986:877879766579888557753455555432389988999:>9953222344554455555555566666655566745651/-00/../122789;:79;<=<:986512256yÉ°215;=@A888:+ÿÿÿ55668BA><<62-11348::8643421235643433154675323:;;<<=>>======<>>>>>>>?????A?A?>@?>?B?>A>=;;>=<889755<:6956:)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ'SU[dhXjfehhbXTaSTVUZTJQNTWQHE>1323348,6:;@>9774:A><8JF<.+0+-./&$%')))...*+/0,)(&%')(++)+,,(+1.+..---/20002.,,,-/0.3335=A@ACEEA@;:?>??@@BAAA@?==>=;:@>><;;;<==>?????>>>>??>?>::;9:9785538766:879?@=;988656767;9:64441345665245468::98799:9::63322344555555555555566777666666676666431211///0478;<825;<<<<;9631223F²Äd1349<:96<:@/ÿÿÿ23;<89>;;<43//4663398965<223332544211045775326:<===>=>>>?@?>?=??>??>?@?@?>@@A==<=ABA>==?>;:975555458:;8854776-CJV_aPT\d^^e\ZOMc`TNVLNye^OJIH91436>>65>>?<;:45::CFNLF66)(;KNC$%$)**)+01.-..))('*'))**+*))&)+((+-*-///..-,-.,,-.105568DFDB@?A?=;>@AC>@AAB@@A@>=>@B@;;=<;;;:9::99888777774477766=9:;998567<==>>>==>>>>=<:;=>>==?==<=>@@@@?@?@<;:9:8976544663789:86432/115766666679:<:98:99988743334444545555555556667777777766666443333222111588<=9324:=<><;960.03}¾j6124::88;<>?8#ÿÿ35<>:78:643!022.2189:87Y7222224533232668:86546::;<<;<<=>=<=>>@@???AAAA@>><<<<;:9>AAA@>>><99765454368964465547556793342ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ6&<<649<<:B><5;=<9>;DD?BL<66.;%(7/'-+),(.360./.*('(('%-+++++*&('&'*-,.-,,,+-/.-8;D?1067:=EA@?>?@>=BA?AA@DDCAAA@A@AA??AB@?<9::98999998777765465568<<>?<985467;=>>=>>??>=<:88;;7<>>>;;<==??@AA@=>=:::88754467:9879::@@@AA?;<<97567A?757530.0678:7666779=;:9:974998532234444455544555666677788877778895443886321015789:83223:<<<;;92002ªÇN3358:869@>>;=4+ÿÿ583;752052-%0/-06.2::97MJ44333334323367::89978978889:;=<<;<=??AABBBAB@>===>>>=;:;>???>=<;:7765564479855767663334522100-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿCCOZ`cYVUZY`hTYSPSQ>I]TEIeobKMFGG9369:=>?ACCA@@;?@D=?GDB:;-%$5761+(*(.583/231.,+*+$2*(*+,,('('(*,,()*-..-/04559:/19:>B@@A>?@><<>?>;::87445766;=;;9:??>?A@BA==>764454545676559:;>@???B??A>87355>GA;5431.1655;=<8:6668;:9998799864323344444444444555667778888888876477655431100224777444379;<>==8536Ǧ:348;:988?=>556+ÿÿ3687252.11+.,*,50-;;98ZYQ7334334434367778::87776659::;:<<==?@CCDCAAA@?>>??>@?=<;:;;===987655665468:87998776333483<461)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿDFQ_`V\\UTLPVZ_QZYGAARBCHWhrSKJJD=8946=;D>@=A?><@KAYNGGD<.*&&H:1*'((/4:43557:7600)(1,**+,+)('(+,,)(+1/...-+,//,03:;>E>>?><<=;?CDA???BA@DECDCCB?==@@@A::96888886766566875775689;;;855425:;:<:=;987764223676545854::;=>??@>;:877753455654455:;==>==>?>>==:67459>BC9761.05544;89>:788999::9988875433344444433444455667778888888778889676543210223444376515;<=><:745Æ’82678:988?>=2342ÿÿ/6/./62320"ÿ/+-.251<;;=P]V844346764445545865445655599:::<=>?@ACDBBA@@A@??@?@>==<>><;=?<85697668557997;:869953344?9?JE85ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿF89@=;8:>@@mlAF1;53*&%(')&((+179246:A>854+'+1.++++*(((*++,*.0-,--*+.-46<><=>B@?>:7;<>ADACA?>ABDEFECCBB@>>>>@@:866554455456789978656579874/13544423433356553123688876334356329=>>>97568:845334421368:9:;<<<<<<;:89565679:=;730/55434569<<=;888999898775443334445444455566677777888888888:9:9676622565443455>77225;<==:98FÅ{53699::89?;94323)ÿÿ51+,5101,"ÿ01-.2546:;:6E5LP=<4653345412242234555569988====?@ABBCBA@?@????>?@><=?>><;;:53378768555945976476433445688874$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿL?JMOXZMLKRQVRUlSGOQQIE?@E=;DPPOIHB<687FF99A@:5<<>Cgt^FICC@7,')(+()*.006585=B=>76,&$#$')++*+)')--/,+,,+-*//,--/:5=<<@?>>=??=@CB?@AAAADEDDFECDB@>=AB@@78:644455557787;875786465,-03355456667666732113555788877676542439998212678823302236665689:;;<=;;7439665664677640045444336;;=?;989988987765433444555555666667887787888888888987998659:<;;96557G=;4349;;=:98VŒ635579::=<=794334. ÿ/3,+751,%ÿÿ13..656/9<:6349_jK22202546421/013354455657:<>??@?@ABBABA@@@@??=<==<<>>?<;;;74467788756:789644443543333310//#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿEOYMLOVQJHGMKVm]_\GINMKRLGFDBIKUMNGB?F@:B?>>@AC@??>>??>B?ADCBDEDEECA@A@=78::67797778754333324577776555554545666534324688776778999888865779766443678522223336687447<>><;4610/01537;<<6464555655334499;<<<<:9889:9987533444555566666677787778888888889:;;:::;;87878;98756201142+ÿÿ6/022/'ÿ-02-0./748><74483356657623455433222234459;;;<=???ACEDDCEDBAB@@@?===<8:::<=;;:97656767775775555433333333310011ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=LMN[ZWQSJJLLKWTV\OPMMJCLDDG><=6=BC@FP^9<<<>D?dq[GHLA>?@??<@A??F:.-@A54847200362126*&)''-/*&*)(+220..140-+/011/-9?@AB@????@ADCBABCBADCEEAB@<867897777877777642356777776655454355656543445457:9:8788899::9886555:7664439;:86444355456557;>?>=<211/0/0168??@8345678775323499::==<::98::9997634444557666667777777778888888889:::889:98888:;:8::9867665;;9=<:7¯^33897<;::CA42442340+ÿÿ2./13(#"(+,1+1:?8759:9799866541/0232111223459:;;=?@@A@A@B@AMVU@@?@??>???>569::::::8536657664555555453454333321111+%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ,@GTRZ^UKMNMLLMQNUNIFVRBB@>?4<>:983HFIHH>@8:9>VOOIHVKD:<:>9??@@EA>8EE@9>670/1/14300-,()&&)*%.1--11244440/04111/-:>?A@=?>ACB@>??B=AEEDCCB866798875689977743256677776555556655679987766774688;;989999:::99768876654428876556555676666:<===;200////12344:=734778865633346899<===<::8999875444445577667678877777788887788889:9999988889::88898:;77569<9==>;¿›93798;<;;AC833420110"ÿÿ0/06#&"ÿÿ%!,,*.*2;=@=9:;9::9656843204-//1123469;<<>?@@?@@?>>@QONUO=7A>;648988;::8675644555655555545874332221113)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ!.?I?LUWYVSTOJINNQVS?EEE=;99,37??@?HFBGKIJB?=9IEI_NAGVB9<::=@>;=A@>EDA<>:851013761202*(''(&(.232012245323241/.--;>>@?>>?=ACAA@C>89?><:889ADBCBA@645447867888778643235577755554544577:9::;986788778:;<<999:999:::867765655436886433445666677;<:;84//0//1222112:;633777765633334469;;<<;;:999887644444557666779<;:887777788777778999988889:999:779899:;986699<=;:­‚64799::;;@B:6100/0/.*Dÿ311'ÿÿÿÿÿÿÿ..,545?=CG988896766453544041/133457;<<==>>=>@?==>@M?@[>6@:78=@>=<;9579:;;;98764345776556544459963322221112!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ,738EYaXSXZQOOKKIJL6:A@><:287<;;?@CCBDCB:889987788<>@BA?:53313447:;87678533123466555555543459<;8<<;:98799;<;;<<:999999:::977896555434564333345544468;<:::-+00/123520.089443566565544334455;;;;:99998877644444558668:;<>==<:977777777777788888888999:9986889:;==:76669;;:±E468::99:=?>81.../4..ÿÿÿ+ÿÿÿÿÿÿÿ +0-+2347>=E=957996445467:61200233578<<<<<:9599;=>=9:=I<@:\;28;:<<><<8998:::9977645686656654465<>;>:5222111/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿLR]f_UPIFIMRGC>=>?A=7:<:=>A>DCACBFD@BFJQLREBBPK7>GG@E75449753212789:98>36983,/'&&&%&%%$')*+,../0.+*,-0.-/<>>=<:9>=ABBDD<88887657657;>=:763211224:8678677221112344565554433227::;>>>=<89:<<=<;;;98899999:::866776655322351136556344579:<:<,,11122211//0255865555655443344549<;999:87888774444456668:<<<====>=;7666776777788888888899:99888979:=>=96665:;;ÁªF5789:999=@@9320/-//D3ÿÿÿÿÿÿÿÿÿÿÿÿ//84337<@>;7784410348;535301234669;::98:5567;><;:89999:FMH:::::=<:988899:999:6764885456644757;>OQ;3323000&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ2:DfsaX<=>AEECFEB===:<<29899;>?@CCC98876555554887665422223337668989322222335665543433443368:>=<:::<=>>=:::::::89::::::96455665555557<:344415544678>==86666666777778877799999997887699<>?<65668;<±†7689989::;=?9411/--4&0"ÿÿÿÿÿÿÿÿÿÿÿ./532489>@=:84352357633200024456699::864668;=;==645;FH<{a@57:=;::::77799;:::9886577546533446:>@@DHF=9>?==>?9;=5;=BAEIZRMKOUaHJGD2200,+-'-<:/-+....2010257=7/21-,*,-*)%&&&'-..//,+)(,/.+*--.;;=>=<;66;>>=97666885522100.1211223455644343211221145566643224443344565655566>=;965655667777878878667424577:;86:::<6555789Z¾¬sA>:9;;77788768999:899865466555454446>72111471010/(ÿ#"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿCGLNTE=IFIRC=<><<;67::37;><=JLFBBMZKoGCFEEBEF2)+,))*+,-02H3-,/00137:88;3:9055-.,*)''*.*+.,+,.///-,-,1/:89:8764476788667797421121/..1122233566543423322212445466533344333334556656547<867889888999:;<===<;:766654432223443:J;3237667=>>4001//110//0132697975544444324454677777777787664445556789::99:;;<>==:9655556677777777776663205246988:9:;86557898Q™Ç“H79>?>@@A@71012/-.21'ÿÿÿÿÿÿÿÿÿÿÿÿÿ10-/113>FB@B<876;743111001224866866333489;:87654445843TeXKD989::999:9664565555567555321112310010/ ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@HIGEFJC>?@BC:120/-.0/0 ÿÿÿÿÿÿÿÿÿÿÿÿÿ6.--2138ELFB@?;7753221010112378885233489;946554133:;75@Q?;5?=7D86689:;:::::999:865356554555423221111651110/ /ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8=DDFFHC>B@:>BGC;@;64356/*"%+1),TdFUrIGFD?87854296*(0271275:GM/2147746445420-./.,.2/.-.0/..344//03/15544556977776666533322121.0/,1223333445563622120125646359888655432344555666765888867:989::;;;<==<<;;:9876554434<;B67463337755;>C=20/431222210//4:89786456752245756656667777766644455567789:::;;;;<<<765555566667778777799G034896477889:::6556789788 ¼\9<<<>>AB?33/.222.0ÿÿÿÿÿÿÿÿÿÿÿÿÿ#+0.---49CFE@B=:6142312321122478433356689974564224864439F99F=6>>5679<<<:;::;9::976456655484432222115321121024(-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ79BFDEB874799:;886848(!ÿÿ- ÿ9EPPOHLQIAC>;4488@K;8955643150/25996:462365312110211/,-/0243540/201/756556656666555234353212200/-.1122344344656623441026764647=<;:86642225545688666886565:8;<<=;::;<<;;::::987654444>QK4454444865479@B>435754322/..27:;;:96567763235667766666667876643455567789:;;;<<<==<65555556666677776768692588875777:9;;:6655689889WÊŸ;;;>??@AA83.,0/1//*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ0-0./4566JC@C>98775633332222323345577:9744556434974126C;PPD4786458:;=<;::;;::98644556645543222243;5643210/10ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<=:;>;<6668745440)$.,ÿÿÿÿ,:IGKENPEDHE>99501332036764723;98549::97675533452210--/6631/./31012466555555544433134442121110.02221233334679<547883009;:6776?=>=:9844445654555654676666::=>><:::::9988899997655446^TA45654555555456?=?;4455010//0689:;;:7565752246777777666677777643455567899;<=<===>>;66655566666666565666<[5656777666:9;<;8655699889:Ë:;?A>?;2.-/--/1)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ&//12547A?B@@:6;7986433333433444376685545698434411003=K:97735=555:<==<9;8:::::854555754444332594513512914:30.!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;AAH=<>;<952555201,,)+.23#ÿÿÿÿ:G?AF>69?>E@?879445:<667645456414775789754435643431..0.//03231010315555235444422333544332002++.5221011166:6545789:820<<<:588:==>=;;55554334343434565569<><<;:;;:998777677898668:97@875665555544434379=>5442.,/13789:;;::856573225667777767767887764345556789:<=>==>>>=;66665666666666665657<<556666656699:;;:7557899897Q»\;9<@@A@>:2.////01&0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿA/633668BA=;;;9:9895422235542592/034644356795453333013>547@7687439;<=;8858:;;::7655655545544336832251292110-5&"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿB3ÿ688/52,.-,53/$!$/--%ÿÿÿÿÿÿÿ><=@C>448@?:=B;:56688857683243356887687777797665431*)./244232.-./225555344322334421232111.01/01.15440/06786/1654779:44779:;89;:77554543452.01448768:><:;::9875574349>;9=>:55555K957777777765434554227<740-..1315678998556343366666656665688666754455669;=?@@@==866655566666666666667656=63124556675479999::77787756766±\:=BCBA878///-+,3.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ3331/36FC@=?>=458:9766863567752./4223545544453/-/000147BB@C:4345=;<>;:87889====<;:998556676544334431221222.>3$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿDÿÿ481ÿÿÿÿÿÿÿÿÿÿÿÿÿ0845=?4-.110329?@@C<<:99:;::;;;>>?=;=<:932467775443001210000/,*-0..554324443231111121100//0.-.413633-/12:53.3:;9:=92711;>97;;;;<77736655554401442257789<::;:9864322577898765444557?D767776656543233222018636./14324654577555442245556555554577566754455667889<;;:87755555566666666666666679>53214556774467989:87777665765P¶e9>=<:9866767764444432222222203/#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ%8018<8.,/1.0573668:5:566999:<>B:998:;8868<:656200/121/000/-*-0461/44433345334.11110111..02/,.542140/165=30-3:9<<<<897218::;<;9<=<<86788756435543466789<;9:96432459:99:9856F655566FA86566656564434432200211;30033433334555555222234555555345665677544556667889:99878655555566666665556677664565335557654468989766566546767x¦i:???=:58340,)(20(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ3/00/,?>A546>31-,0785,.0258=6372223453356510/-.--//.48HO|\E78976;>99796768988==>=<;:7788777456554422223210004ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+ÿÿÿ$103=;96/52,-3314641552333368::;89;==<5518875554310./0//1/+)*.1-+-444555544332/00123311..11//088443237<6>//.23.35::=8;:98<:;><::>=<8777853541566563568:=:9974214;:::88865557556665DF6755;44433444554332../0872123212234445664223323455553356666777444555567985766778765555566666665556666563355622446776566888655467656775?‡´c<>B=56;66-,+)5.1ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/,../7988;>41.,.0643/32.17;77522113111654/-.-,,,.00468FiT?97769<<76677779768;==<<;9899764656633323333211.16ÿÿ@/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=ÿÿÿÿÿ2ÿÿÿÿÿÿÿ3*ÿÿÿÿÿ+,+4-34,,+++1/202/..05233556434578577025;;447<930,*,,-./,,-/.-*+19445542344210113443231.00103784016::;89+--+-/0046:<9<;=<<>?>;;<<;88777655657543421369<::86316:;;::9875555;>:5666;;4F;8@44444445554442////4523321233344545633222234555434566667764445567896643156888755565555655544455433642445523456677767995544556566778:f~BoŸrB;66/-*+-75ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ'*,031496:D9//,-//2122300086530./0/102411.--+,--/000.0;OBD@63555;D:5367797998:<==<<:988776554444333334011/ +L.: ÿÿÿ1&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8/(ÿÿÿÿ*)(/,0%111789=82///..232987<=<;<:;<;:5=7872223/.,**,,-,-,)(,*)*,26444423433111223332343/1223659533<:78=7)*+.20//./9=8:<=:9=<<;99:88966665689:854312357;:875369:;;:9876D=555635666644=>5444544445664440//.012/222223344444554333223355444556666776344567855521014565::8555666655554343355255332176444455566779444455666678887C8qu:760.+.*+.)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ-./.0/0;=@A/.//2130012411422/-1001-/.//..-,+.0./000002>;<=66@878@;6545788889:CB<<;::98877455566554743011-"ÿB6!ÿÿ4<-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;4,ÿÿÿÿÿ++('-.1=?>3:743-40025322133498656744421//04(:?5/22122021-./,..,+0654333333211102334343432223378767977;85567798989;;;<<::998887665557895433320*#ÿ03ÿÿ9<:.&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ6[ÿÿÿÿÿÿÿÿÿÿÿ95,ÿÿÿÿÿ-.)+4549751-+*124-/1534430--12120143320-+*-$.0//.--./220///010,+0/444343331114344555455442379235868=<=80*-//.0005//0;:<=4568;=9;hl:97202/./-/./)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(),-016751),24./,9:9;<52+"#)120,./12//00,,*+,,0112223239:9;@<;7;=967676:;;;:78?<<;::::9887656675333331222+ÿÿÿÿV/4785/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;A78'ÿÿÿÿÿÿÿ" 1812@51../,.54..168531006B4-2<3110.++,/0.--///01110.-.10-*((*/443333211011324444454541666//479=<<;12),-./561222//05:9:9<;=>;5268:833686467648622225775468:97459<=<;;;:7653322344545544453575556687445301534545555554554332222222235222566543344556754/++/347988999:7567765554444444402354331018P574577753112345676678998>?>u>8:9842--.-.-.-!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*++/41/00/+.6*+.9;97214....2/-+.1210//1//-./13133111221A:<;9<7:<<59987899;89:==<;:;;::9976455453431263240 ÿÿÿ,1475%ÿÿÿÿÿÿ<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1:9:9;=88<;:9=;<>=:;:::==<;::8875665444425;6234ÿÿÿ(2463 ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ9==7..2ÿÿÿÿÿÿÿÿÿÿÿÿ"+*).8../-89*50-123010-,+13132.../0/00124/,**(*,+*++153333320-0/1114/145215443561008898862,.---.0156.442.-0059<:=<:875433344555444479878747653666841253543244467543222222222222234456422243334764465355:87::99765677554434555544112255421.2HI6:045756233D46677777899886c:::972752-....-)(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ$$+-3764./010,-,2997985420/40//,..,./.--.,,/4420.12211034BJE:9::9;=:=<;:;9<>8:;:;?==;;:988788545442>=<:975443445555455489789646442587744564443455664332222223332323345456422345544556666687:999985555577644344455554422255520.12541/03763223<5756777789978:‡?978:2761--/-.*(*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ'*-865330/100/./367753101/31..,+-.0..---,.13111/143/0/137FGB9:<:7=877<9788;;79=8@D==<::989;;6553435B91343/ÿ/004/2-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ49A;:;440–ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ÿ+#/$!ÿ;,(*-)(&,/*,+++*,)(&&&+#'#%%()&''*25-((443332342/.2450.36366054443129873861-01.-..0340*'02101322213-51016767677899;::6122448:87657869:::;;=>>=<9875443556455444699779345421565534565464456544333233333333333455445223334566576666898989:852445678644444445454553345552..345100.0/0324B6866778889:88:jW774567541,,.,++.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1/54345.*-./-,02354621/..22.,++----.,..-/051110123-./115FKL=:;=6=;65::6779<77:69:>>><;;99;:75555549423441!ÿ-.1222/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+ÿÿÿÿÿÿÿ<=?8;:1-.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ2/(()$%%*+)+3/##% #**ÿÿÿÿÿ!%( ÿÿÿÿÿÿÿ43334444443432.122356456622207632320/1/0/68.00-./21333221120.,-/58:<;875556:9<8431538:565588689:;;==;:996554445665444455777767454544343344665553664344332333433333334455443333444543/.468798888888874677786544444444445543334443133333331235336347756889::;885=]l:797441((+**-.)'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<534772433.043313//021//-/0--.0///0--,...2//-//1.1///001:8A799<<7;<49=4676>:69878>??<<;:7896365654243342,//01--+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<568<6458,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ. ÿ6$ÿÿÿ/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ33344432444462///23245555101074221/.011/058.00.-/30443/02121-/3533468::865589;:21377795546776889:;;:9877544455556954445477876455344422333/%255467643332233333433333344443333334445432034775779778<::99987765444444554455443445543323234424344352245678899999765Df98987741+*&'++,&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ'/0155962400672321./1332020.//,-...0.-./-0200/0330/.../..?@=57<9;6244397476997656:<>>><<;66872554433433433)-/0/..*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1//567:6553,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4444454645555560464148551001082210.-/00107611//12444321002322354111143588666899224678655566778879:99775544455544584454346875543354330011.(2444755433233344334443333444433333344456776559:8878779<:9888887655445554575544433454552333354343544723457789999988456N<89:8751-++)*),/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ0)01228520-33131202013:2111/00.-0./0,,/0/.10./0342--/.//.9>;888679546248795676:658;;>>=<;64773454543444432/-1110.&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<94266892245..ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ444455444455657646636856301316210/...10115842/.4344322212333223312113339757578;34358766567778878999855554455544555454443585542333221000.*)+34566543223344443344444444443333444445566669:::7799:99:7666898765445466446666444344553443555444354473356678899:889456:8887765123*./,*.,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ./0/44564.16@:5433./37100/11110/1/.++..//01001332.-//0/01:?>;7757=::73::78118:636;:===<;856832545545444421*201/)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ13;74567::78:66!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ444455523507433666436779554433231/.0434101663//44443222123232344122234457345479433467487667889999887555444444445555544258654323210010.,+*,045654443333444444454444444443333444445799789::977:;:9:945556877765555554445674344344333336547354544502466668888889467:;987666331,,-.+++ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ2,--/3423338=92/09201/.012/1141////-,-.///3124531///0////03G?@6578=<@;78950/78454:;=>=<;956733545554444432-11(% ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*.;464568;<8:9;93ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ5555565534312411674335<>845352241//0444021461072124311024223444334222233672546853312758746777999777654444445445444435547854422100000.-,+,0245543445344454454555555443443344343445679889898688;;:9;89235667766545554345465446457645336656555433431455655654689267GeA56344510+++,,*,)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*.-,/15;A4655.,./753253230.1620//...-.00/56541000//0.-././;B314568A@=:87352595659:<>==<965532555554444431,.2/$&Mÿÿÿÿ0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1-7845588@:623:201/0/1646/015620621342/0//221334444432112455343568621267764767889876544443444445555532117861110/-/0/..-,++023554335564444445355555555444433444443443/46788776778:99983544467767556554334456546445332245766555333420345434435796356MuoZ812554/--+-/..,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ--0-.215/1282/11053/110//.023/122/-..00/0351//00100...//.35413344?BA>;=;558::57579==<<964622595654444441.+//(/Fÿÿÿÿ*ÿ0ÿÿÿÿÿÿÿÿÿÿÿÿ!?56445589:<;86=:312ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ66666676555443334663496>>5234422000/975./46431431110/,.13112333333330025434015645241686456998887443333334445544555244540-,,,+*.0.--,,./02454334444555423465544765777544444414445434446555566678:7710523767876556655433433433344432356655334323324535521134695223>687877851../2130,+)!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ.'++--./24==9222653.,,--/0//0000...-./0///1001/00.,-0/./)+/1+*.35>;159987<@J>=@:@=>=<864922345554554442//00/0%#ÿÿRÿÿÿ2ÿÿÿÿ+ÿÿ4/ÿ.../58698:<<@43/24152Bÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ677777765655552334766868823444222//0672/17554211201///023533322212322014422/024454314954678998764333333344444332221.00/--+++)-0/-,,,/002345344445466432456646466776554444445545554458>95544447797760622564777656775443533344466434444455334323434445401453578523;366798810/-../1//**$!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ&12/,./29686/09553+,,,/0/..------,-011/00.120//-.-...-+-/2..-.47=:986557JE78777E?==754A314555555444420020.-+%ÿTÿÿ$)+ $#!(/+*/222126985423345225003153//34211214701265541235552420033243222/--3661325777986765533332333233132200/0224.-,',+,..---./01234444445555322112/47656435565444444455554444344544435779:8554020586798756665576684656455645445444454363322456632344477035222453352.-/-+*(+--)&!)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ%+,,.062143,)32342,-/-,,//--+,-.-02//./012/01/--+-+**.1223313797696577I823237;<=<75:939B45445554444/021.-,,*/--001112000.0/03442357878;?>90.497543209Yÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8888877778777655775679655223443310122640/35202145466763650135430212244233210/--.361222877875655433223333332333200/1333/--(*,,.--..//012443444554441/11220056621345564444444566444433345444367688620,+,05777:9766884786464646455646434434653344212467752444475135*2224334640./-,**)-,$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ&,,/0/-,,'(3367.,/--././--+,-.-//./.00/1./10./--+,,,/.24242555675456F634138;<=<75339=@45545554444002//,+--.-100233232233333445679899;@A<335>57=5434K`Hÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ788888777787665664446976632233330.0/33353663342322655662.234321112232443321101-..36644466777656422123332233112211233210.-,*,-.//00011244444455522212223201343588665544440443654333443455447898854)02.,,3634578767;6765455545444545336555664442323666864565553)+-%54234303677553///./(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(+,*/4-8:/64551(+0/0-+,+**+)-0.+-.,/0.2233/./../0.,./.132334545;456;5440259<=<9754112124453443331/221/--.0102322234556877566788888886331>>:5<;8685.,05879941ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ888777777776665566569=8652222222221323076761032444554632324734320/1110133333431..13677654565665311233221212444234554311/.//0000010002235333333330/01230013353266744555.+2443566333665556886697644223456665888775677755445545434333446544655335#747776656682248-3(45444551/288771/0/0)(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ%(,//,3746754.*,0//..0-,,-++..-),.+-.-1240//0///.,,..012/03357657743330259<=<9654B92011244233332/0110.,--./133333556679999988899758<=;51/.<<=6134::<9;;A86<743"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8888777777777655665569:76113323212134727745/.//2565754/04446642300241102333222122311377655555542232212212445422565543210//0111111111123333323330-0.-.30022261477754453,+244356633366556577:9786651335566439:866568765544455544454344546678656458555322444653336A$.43455630.8;7750../,"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*,1./00478751.-/-./0000-/,,.,,*,++.01/1420//.//-,+,./01331356545533330248<=;85435=4222132122232122110,,,-.1233345666798::999::9755:>>70//458114<:9A9:B74380111ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ777776777788775656778;8630223332221289638511-.12445764..03366673325311022232112322134576545454333322222245444335665434421//01/11111112223332222.32-/,10131366879644464.-355566533454455575:9875364346667557;73667766544444454455533454456:775591244565666644426 +221324311564674/,-,'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ#+.-,-168867820-./030///-,*+++**++-.0/131010-/1-,+,-002422248>76833320238==<854321/23324313333322321/,,,+/13344567677::9::::::8556786112/3:<87::?@>B6:34665225ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ6677666777877766649;:;875122232133138:44:64./234558755/.021666413652/00222232224435655685443354333233224556654598644345320010/10111022222221211.22-./0012257766744456654557665433444445556:97643554566766217547566565554444544444333444458852339::58;676756653,232/322311363423/-+-*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ**..0895888734,.--2100.,-(+,))*,-+-0,0/1011--.-,++./01300255456833310247<<;865203/22234544444333321/+,,,01234556678222334447:<:654433343324444 /232220//2..14011.*+**)*+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4*(/,/1;987988.0111/.0*-+*))+++.-.0.-,//0/--/02/.001/1300643643432103569<;;8863200466535544444333210,,--2334566779====<<<:887545311100..//2357:54584733485973AA?.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ66778887777776788776554332222234221/432302767757673566/022235776123454454444445553345544566644322345543366788779765453211010//002410011102/011101300.//.067678776567421366666445554554566986555775466227877666766555577666446C?S?22344449;;;7645333434"(*/%./4534/...21123221,.,)(**(''(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ**+089:9:8777.../11..*),,,*+)*---///,-,-00.//0///00/1200643644321114459;;;9853311366535544444433211,-,-233456677:?>>>>=<=<;9779532220-..01357:6241353221241/>A@8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ7777777677766666677665432322243034439864467777623675352223321332123354555654444443344444555845313445444566887778655552111121011123111111./12223100/010--07888866466362/06766545665554445798655566626724866676566665556666644RG?P42234435;;::6445333324,.''035633364126423100-,*(**(((++%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ++*/717799:96///0/.**'&+-++*(),/.///.++,,//./100///2540342464422111544::::9853312455445544455443221---.223567778;>???>>>???=;9:65220/.-../2347502165134311448?@6693ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ7777777766666666566655432334431/3432387996654552203424225211012333456555555444444313443434564553444345666777699655543110131000//22111111.03533211/-*.0./5799776726746412566654555455544579765456644753587667656555555566665AF54<62233335;9:64444333324* +(.//1366544/1121-0/.../+*%*&(**+,*'&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ.,01078;==95100//.-,('*-++*('*./0///,+**-00/0/1201452/2535573310125338;:88754322344445544555443321..-.123568878:>@@@>?@BBBA><:864210/../1/14620/722033446;63683983ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ455566666655656666666644444331//32125521001223122502342211233324665466555555334554423433233574645555777666678984445546322244312223321000102120241//0//0257777777563/4654567554565554446669755562.267999986666667775555666544454?;8310/.--,-+--*)((()/0.1/./,+**).0//.035540262577843//23436:;967555333444355544555443321/.-.1347889989?>@@@BACBCCDA@;82110/./2302115990.../02689:7:83359ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ444576666775566666656643545422212212121//00000020302433322454334776666675555333555333433333365654256667878799964444434543434534334421/12110311251111030155576776673035455675454544545666697666731667999976666667875555666544544843332248996443334324 32//14454345565235.11/20,24.-3.++,-,,.+,' ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ)+,-0037;>>;931/.--,+.--,,*),,+/0///..+,,+,*,2./06652/155667345//22447<:766445433443335444554443321/.-.1347899::;@=?@ABBBCDCBBA;83210//0000000243/.--,/279996<<207%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ323456656676666665566665543223233211001/010././12212431233454446767776675454434565443443324355553424677788:998644444444443445443222332234201123301//..12443767766643466566545434455555667;76567770/4899666777667766666666545554433333338765443344234044345557534445538510/,,++12/-20/,--,//---+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ),.,./66:<;9520/..-,+.,-.,,,**+///./..-,,,---10./555102465785670122357:9755335433333346444554433322/.-/13479::;;<@@@@ABBABCBA@?<93100////000011110..,//1699966=530ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ3124456566665666554464555232222144322220010/../232334433345555677777666554454445555444333234345611477877:::9776343455454543457554443332355331421//1-/203544667666665556566544434555555569<8667874/0555457778777876566776654465444332234977564443345.3544565775344433./:31//-,02751430/+,.2/0..)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ)/-22244787460/...-,+.,-/./0,++.-,..--0.,,-../010562/1555675443222346798642244434333255555665433322/-./23479::;<=@A@@BBBAAAB???=:52010///0/011111//-(/0/4799666360ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ53441256656666676554426642332233443232211110///1222334444656667777776655445555555555543334322447534678899::7665343453354332245435543311267631321000-./1255667766765545546543344566555457:<976886302445577777788876566776655565333222246978665432344 2555566654555531,.2455564.28626521,+,.,0.-( ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ).,02647761152-.-./3.1-.0/11-,+/.+-/..0.--/0/...2640/6457793432122356887532444444443355555564433322/../2347:;;<=>CB@ABCB@@@A>??=:52010000000011110/.+.0146797533-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ333322445556657777665465522222124433222223201102213344555666776789887665434567666655445533211236867788988986554453334423211254435443333321512121101/01664466665676543455544444554456556787767665355677877776676655556666665555323332448788976444444112332354434564423-0//8:115/56530./)+.2261**(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/-+/0-/17;40,,+,+2520124112.,,+.,*0211000///1//12/.246664332233343788752133444442334555555543333220//023479;:>?ACAAABAB>=>??==><960001000/000110000/./257987343ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ33444455555655666776555662122113443423222232111321334456666676778876655444445666665555554322124686577898:87553345544332231026533434424..0/23421.120/0467266566667544446665534553334444456677763335425677666777555455556765555433322245877786655444522133224543246552,,-:378-+,,<8683/++*+/130,*' ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ50-0-./1=;42/-.,+24302622220.-,-,*-220100//001121004555534313333447985211443344422455555555543322210/023479:9>@AC@AABB@>=<>>>==<:71112//100100001000/0176687352ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ44434444445445666676555673222223332333222322222333043356766777767766654343445666665554554223224277677999976543445544353122026432123343/0.-/221/**+112541165575567542346667754433333333567778851154686776666866655455556655555323322235886677766435130/122235535566530.5862/62-1<;55/-,-,+/013,)'*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*/42+,/5C>46/.+,-22112633332/-,-,,.34/012//123331015758233233234459975212333334323555555555543332210013346878>@AB@AAA@@=<::====;:81/3300001110000000./17666633 ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ34334334445455666665545553322223322323323322222433234444654567777876544555446666655544554333314478778876865544445533362022/0552//0234442**,0-,.),.31361//6569766755456766654433444344455666883324686777679:7776445555665555543343333467766687653550.,-110/3545666433/04:764=775<945+**4/-1431-)''ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ#,-*,/26>=97--,,,132045333312/,..-.33.01200020240034668452123345569864213333333435545555555443322210013346766>@AA@ABA@?;:;;<<<<=<85123-/0012211000001/167765'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ333444344345564556644544443222353333333233332223334455544534566766655557544556666555455443332124778787687544433345334511101238410/032011,++--.+)*,2156112878988775537776654443354344555555777775566768887:;9876455566665554443333444355467677545310*+-12114444631323/567<=<66729754**++-.2343.))''ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ*+**3.28L<7931/-+,/01764445020-+,-,-0//120//1/240044433542212345579762133332233355545445544443322110123346644=@AA@B@??>;999<=<<==<9732-.2311130021000/25566$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4455314554544444455445554423334433332223344433444445554445543656667766655444445555555554544322122467678754411223333554543531/25610./,.,0*+*--..*,,4796.0456657899877765655444444444455455677788789997778:9::9654456776545433334333334644799652212-*.,(.010343322.0//,.1.29=1/6<83///13,-2512/..-*&$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(,)+/4=JXWSC630,)/59;:4344..4.+**++-200./..00430246534332233456795421233224534654554333343333222111223346534>@AA@AA?=<:877:;;;=<==977533444221//111111466*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ3465345435643345644434455522334433332234445543545445554446444554566543553333444555555555544322213455777754412204445564432342/01701///-+.*().//0*),4673/34476567885567566454344434445555556777689::989:;;999888844666665444432333433346449875512---***)-./-3433221......-3=>2/3:74/2/02--3;/,,11-*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ++-46@ACLNL>41.+089997581..1,,***+,/5//1//13320446523344333466873211221246766544543333333333222111223346645=@AA?@@?>=:8889::<<;==<;;55323552104333322463ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ45453454345433456553343444223334333343356654445554455553354343344545445433344445555555554543211244325655543232134366443442322/232200///0&(,)+/0,*1440-05657555677445677544433333334565566666688:;899<<<:998<;7666556665444333334433455448765511..-*(,+,--/2212011-..//..06984577540.,,,03:.*.32/,!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ),-0255?P\A80.49667553..,--,*-,+-,-/010145795557433212333575331001014676655454332222233233221122223346567=?A??@><;:8888775888865425565766443322323010%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ665455545435548874;55555510232444445545766555555554444553334664444344443333344444555555545542222334554445332023344665524443210254565400.''&+-.-)0642/-45665446577556665544433333344444555566899999:;;;<<:9:;;:85555544444444344443347;7575535....,+,,0,-/0232210251/+,.---0429189;:.--.0084/0/0/,'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/4,*,356=FSTK404:776340.---+,-0//../013246@B>==<;;;89868863575220002223444431233/..,"ÿ'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ7755467656577479:9<4667653101/145555567765555555544433453324532343445554444444444544455545654334444443333321233545444455543341554445454.))////-,/4542/5557544566565554445544444334443321245556667778999836799:7555454433333333344457876675430..*)*,,-/,-..1202122330-,.,.083//.8:;/...1234:7322/,)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(/+,-135<>AcYA74363/110,/-,++/-,,-..-0ARKFKK7>957569:;9:7;78664213124566656766665665544443332225443203365555555544433334555544555445444443211222333244453455535531134222344420/.1/22).0220004575446666665444455444433333232434555445546657764556987566554433444445444444565555440-62-*,-.//.-../112002131/.../04>70232-.,-/2967872831+'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(-007::=yZ@2/0/-.-.,..++,///.11033@fjB8;5664:20135797321333456555455532111111112222222222333334579;=>@@?;:52235<;86532222200//00/333343123'ÿÿÿÿÿ&"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ5675789A?;5887768899=8:5772232235656667656766443445543331464441587765666655444433355554445555444454412112223433545545544444001242101344430/1111)),0/1223654456666655555543314333343444445555445456556632456975555554333554334333334566665442/02-,-,/01.-..01121223310////026103/.-.-//3<9:86573/-&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ0+-489;hfM;01/--,+-,,**-////11344:LhF@9556432268888555765566555655531111111111222222223333444678:<=>@@>;86366786761020011//.../0434450033%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ6665797:<:7797446::???>:88487;5674.//0000.--,-.0644650144ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ56557875:>=<;988677:955345443344545654455555544455665442025866:;7777556445533323443444455455577::21122221223324656543334454112441/3/0003662033/..*//03/154456665655555544333334446455556664656555444544445776554444332454444444444467655433130/.......-../01222223322221/0.///000143405<88688773/+#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ,-.19EG>SQMA5203-.++*+-0/-0477565bgDQ6679;;;;:9878887655655553111111111111112222233444456679::==>??=;88:;:<;95542100010--.-/1577674433*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ77558579>?<::888566966555666577687765445544555456666544310375464333445644554333233234455445569;=51101111134434565544444454212554215000026640231/-)00140114556555555555543443332455555556665455444553433444589677663333554444344444466545433170/--......-./12223333733220.0.022167;:76/57636895331..ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+423=<:=QNA0111-/,-,-0.-,1579888==6579:<<<<::876776655555643211111111111111212233444456778::;=>>??<;;:;;7796554211//1-,,--0177777542/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4533358;?=888888677<4656678999888888776677764666676655232317933542132454434443323223445456667:=:31100111235545655543445565324853756200/03553343/.-,./233344565455545544434434556565555566654554445544334455899987633345443444555444665564330GL0,-/./..///012313337:42220/-.001138=8531A7863384471/$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/1189;=??==>=>>>656566511000.-,,--1368777662.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4545567<=<;88776778;5766789::9888888887777766777776665332553733441222454334443333333454556789=8621101111334555555533345575445644753000/.1331232/0//,0-34565555555455544454244455585556664555544444445444456789966744444443455555555655673232@\4-..-//.../012122237991110/-/001114945002/552133232+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ,05857=8EA2330./132../,/57777669978:;<<;;;877656666655663211112111111121112233334556789::<<<>???=?BBA>;533765311/0/-,++,-56888886781ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ2455879>>>:899887656556678888888888888777778777777665522245565;94133343432333333333456555789:87621101111334545555433345544555554520./..-0001112//2//1/145666555545554444432455654765665556443444434455555676688777444544345445433765557643307S:0././//../011111237781001//13134653400/3214226763..+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ27756;;>H5230/0040.2/,14778866::87:;;<:::8776666665566522111222111111111322223456778::;;<==?@AC@?BDB?;43565111100.-++,-/86788886788+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ556777779<:;=8;655777777788666655555665556777778786654332248788958<=;74753333323444555776765544432111122333445644334564433333434410/..-,,,-10110.-00233465545655554444553334454566665555555544444444556666567446656544444565532358873444411,/FD-/000/////0/0111/.0..83120//32138420336;7545885745,,&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ377789?AEEFGGFDDA>:5663222110/---,-0277899:;::75.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ566888:9::;9;7;7667666778887767656766666667677888876542333479>9;9;<:;9556543343444455677644443422111122234445543334554211222222230//0//--././///0/10246664545555554555544444444323545666655544443344555644453455667544445566632358:74444421+/=@../00/0///0/00(.00.//0700/./34348432225::767754331+)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ12357D9CN843-,02-042008;877563567;;:;8998777666666653322223333322222233333446789:;<===>@BCEEEFFED@?;88881121010.-,,-0136779;:;<;865ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ7889:79<;:9767:7558777888678887766667777666667788887653332485;9:9;=:96666763333344445676543332221111322244444433225421011111222210//00001210/0///021335654456677644554453320034233313566555445444344555444564556665544455566533448:85443421,/5:./12000///0/01,.110///60/001124355412565464598483-)(&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8756?;>Q<32.-./-/220/9::96563898;:::898877776566664432223444544444556654455789:<=?@?@@BDDDDDEBCB@<:88850011110.,,,/132567;;9;;9889$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ688997:=9:9767;977978889987776676666666676777776788876543248;<9:878;9997688544433333456543333222222132234444333224310001111121110///0111220100/1003334555446666654665555431204445564233555444544333334335665455777675445566654444788444432/,045/013////000 10522..09001002210134669;<789978:72/+*'ÿÿ"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<;6379EVC34211.0//2/.67986654996:;:999977877666665333233444554567899987656699;?ACCCBABDCBBDDC@B?;::97430001322.-+-0244577<97841169ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8778;87<:987789777989899988887776666667667777776778888775238:<88865;;:96;;;444334433466543333433221132235443322233000/011111211100//0000011221001/3434654456666553455555775247535676457765555543322223336655456575665455666554445676644332..1262223101001. &(120//.-/9110/0110112367<=<==96446421/*,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ÿÿÿ631255:RQ95643110/40056686654895:;;999977777666665433344545566679:<<:988899<=>CCDDDDCBCBAACDBC@=99985331122454.++.233237764/%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ799:;:9:99:9995::7789::9877888779777778678777777886655888885::9:;7678778;<;74333253444444222233333323344332122200000011110111111////00//0123234./.256754566557544555667888756999878887765555543343332234656555544466566666679445566954442101005333333221220100121671-.4100110/32.0677:4777::841.150-4+/"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+30283;JJ9333101213/,563013458:68899867767777565444455566677889;<<;99::;<;==??===<9::9787789;=;8767775545433543/.11356676"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8::9999;<<:8997::8789:987776676797676787777767778766555689868:9899587689;;=:54233323444321222232344333332211210000111110011111110/..../01223234113467644466556644555678866689:86875624654575533443323344566765444466776656778665568:54442212115334233422GC2101331870..62/021./23036499157842330242/-/9:?ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ,16:58<=0/341221340.342067659;879898875466777653455566667888:;=<<:9::::9::;;7666655566666677:975576786333445532124687561ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8:9;<99;<=:889:9:779:;8876677796877868:88787666777665557:8739:999738977;;:<:443332223421122222223444332211110000000111211211222211////0112232333445665465666566675557776677877899::85254666544433333444445554556656776666677767557:944442222223@44333537?2121430201/048611000/26566573458531133331.*,=Ha6-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ59:869=7.153400213442238677;<:::998776446777534556677889999:<=;:9::9976777755554545565556678635598896324466557645798982ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ9::<:98;:;:888;:9979;:777777677578787667789777787666557799966=:89989989:86::5432332344111211222233333222111000000001111112212212100../022223222233554457855676666346767689988::;977657766654444333334443434455566677667766777787689744443334334D5533444G52121.24500/345:32100/2448664476773001..-1-*,7Gc9ÿ(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ789=>BJJ920452/125620327779<;;;9:9988766566644557778999;:::;;::99988756776544644455465546779575587875344378466847799:;0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ::9<;98989:88;:99769:87887656555766677766888898776766656697:9<:8;;:9898984555433333332111111123333322211000000000000011012222222211//011111333234433455665566667667889:::;:99988777876566544334432344444334455555675567765767997787744445434344A444456D8212100333200//36F2/221366765435664301.,/31-..@BHA75621310643467:878306779999986545678:;;:<==<;::88776897873454644555435455444545;775766888:99=7658999;9:;=9ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;:<>;:98;;::;;:9:98999898:99897777677777788;;<;9879998888998988899:88775345543233322111110011222322111111100000001121112334434343222210000022332326689544455666656665666897766644432344443322332334455555445555765555665567787767864555577877665566?54=65544464321121335:5122316423221121.04445771/36.0.3/,/*!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<:59;:<::977666898875423754454444555354444865468;8:;;99>7679:99::;9=ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ8;>;::89:9:<<9999:9999999:88877866557887999;<=<;87999988888::;9::988966744444333432111110001122210/0111111000001112222122334343432110000//122234234555444444476555545565554433312223124431223422244455555554456675556466557787767755786467776655555344565345488742122244:=223226514454585324587891132-15:3/3/%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;;;689:@=;=;;8213023122668986467778877655679;=<=>>>==999:5466477665422564156443446556543466555788;::99=7:8<1'$'.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;>989999;9;:9998999998:988988676545788889988;=<;:999999898::8<<99878:766644333443311110010111211..//00111100000001122233222223332100000/00012333333565455543364443334555442433433323333231333322334555556555456655566467767886778855565467777666555456445444446676322323>E4334354435458665787878426310121@97//)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ667:7>><<=<=133/1220.0345667898998766568;<===>>>=<<8999777557855442774546645446555444575558778687:;<=::9:&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?:79988:9:;;;;;;9;:99::::99878979<<979:8:;<;=?@==<<<::788777:9<;7549473402223554321010111000///////0120011111112223333433322210011111111...223333445475566666564222123222303311221422211/122234335555567655555666678866777888778:655555577799875685467653346776654343443I>:33212456556889866887421620213??D8.*+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ+386>@UGHFA:53054;320256657888786546::;89;;::;;:888<:97644446976899679:867656769:989;<98578=@@?A@>ÿÿÿÿÿÿ- Pÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ;8:;:877899;;;;;;;::::::9;;;;9:9<:88999:9::;=>?>@>:;:;99::8:;:9966677652122244543211100000//000/000011111111122222334554432221001122////001223333455786645556553221111324513221/0102110/023234344555566655455557777787777888878886665456778::8654;7477665246789765554554?A;21012367867;897789886400.34249<@6.**#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ#;>==:22;<501233557998778458:;<;359::998758><::899:;;;;:899897887554312234243322110000/0000//00000000112122222233444332222100000/.//11112234345577655555554433222211323222101/000000022323445455566655445666877776678987877778766555688:;946;;<84666653687::98556644B641DK812465;8::97568866330204456<7I81,,&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/2<9?SRPQ=<:85531241146897666569;;<<:3588898546?:AA@> ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>:9;;9998:;:::<<<;<===;:;<<;;::;:9:;;<<>??>==@@<;=689:<;:89<;9::9;;85322232554543421210000000//000000001111122223333323321001//...//23332223334555767755644343344333242355433134232001223334445545456656555678777787777898888787656655679:<:9649;<95567665999:;:9566565R0010?A=3666;;:<5665688312222946752781.,+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ12>:=[VFM=;8456343304678765555:<;;;;94578975339=DCCBBBB@<97577:=<9999<9::9767===<;::::9:=<<;==<:88;===<9;:;;;;==<>==?><>@>?=968:=<;;;<;:;:;;;8421332253344332211000000/00000000001112233333223322000////..012334111..00//4666676456444433322343454554324434453114223445555556666666668987788:777689998898766665678:;;8978><935667665989689945954AI101./-48895:;::766665933343233561/1300+,"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ(-C=>IMMD><7357431057876455559;;;:::7336843336<:DDBBBBBB=978:89;77:;;:<;;;887?>>?@@A?BBCBB?\UWTSIA91+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?@===@>;9:;79;8:9;<;:9?@@???>;:988:;=;=<=>>?=?@>===<:5::;?>=;;899;85112444453454321111553////11111111112333444323333232223332110../0...,*+****+.047764565444544444435457544687388865563333445666667766776678887988777777787777666556688965573575;936668789879677787755KA011/./2289854885993124578543753231262/.*(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ#,'ÿÿÿÿ;?>DR]O:AD8963442303232239;;::9764202112334<9?CBCCECBA;879:9997<=<<:<=;:768=>=ABCCGC=<>HB-&ÿÿÿÿ*%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=>==<>;;8768;:;<<=;:=>><9?@?>9:::<==<;<<>==>===?<;<:::59:?>>?:;:95421244545444543320123340000122111123443334433222222221122211110/../-+*++*+++-1577645667546666555445666577:77648:64452433457666677666766888877998888888777776656667788666564676681654887788968779::6<[944400/324876539973437578864277304469;1.*)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/"+,ÿÿÿ496?Pem=;<984288411431334::98642222001223398?A?@ABADBE:3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>>986<@;:89;=<>>=<=<<<>======<=>=>>>;;>?<2.*+&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ--+. ÿ597@:;;<<<<====?ET]\Bÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ??<898;:;:;<===>:=;<>=:>?>=?>>>>>===@?>>>?>@?>=<<;<=;==;<9;;:44336666566555554312232223211232222344444544422222211211122110000000/../12233557654555565666766565789::9778997876566774443677666665665459::88779998888866766665435546655665:87896:<780588999989D:8;G@?<9>87012/644625445897587787535632:7<>M:3.++'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ#*1/ÿÿÿ48B=DQ;99;;:736622320366964531111201214==@BBBCCCD=;:998678866889;<=>@>>><;<<=>EMMM:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ?=;9459==98:5216><4:>=:<=>;86=>=>???>>??@?@>?@??>><::;>=;=<<:754568856767666554332433332233331223333445554444332221111111222211111102422434566655556776677887788766688:;=;<86868665556556676666653676889:8998889987799866666655445334565422898987567759;88=>;99MM:89;<=:9851312354243559<:989;79===;9610/1133433330000000001=CCCDDDDDDADCC=;65447784478:=>=??>?=::7)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ9:99467;=::32:6564:5;=>633>>??@@@==??@AA?@?==??<;;:<<;97886667875677777665543344434331543332245544544333433322200001012222222222134444444456665578976688888776889:9:::<:669988632653456767665578889:9:98::889996799876566655554345654343799;:9787856988<;:8>J999:<<;;795543224423557=>?E:988976696569<<==;72.-(ÿÿ-ÿ%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ 'ÿÿ;7;;:@<<><>=?>><101023241120000000./13ADDDDCFDDDDEDCDCCC:445656679=?>??==;6'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ977:7679::;4566;@@11238::?525?@@BA@??>>A@A@???>>>==;::9;::988888876687777765543344555533366432335554333333343333210011001112233333344445434455686647888778999877689<:;;9:8889988763035436766776568978999::9:9889:9667887656655555985776444689:;<;686736798;99;@::9:=;<:;9;7996555434558=>@RC>;:88778457;>?<:662-,+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ #ÿÿÿ89<;@:@@=>>>@B91.002223111//00000.14>DCCCDDDFFEECBADEFE@633354879=<:;5586/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ99:9953555443576:>24339::>435<>@B@@@?>?????>??>=><><=:999<;:9988876878877765432454565542265444444444333333333333321111000112233344555556445666677556889988898768:9:;<=;899888::7876545446757866788:879:;::::977::96777777776555455587876426789;;87997259:9988JC;9::<;;<;;<9:::775555759>?BIJ<<;9;87:55:CFD@>9653/+$*ÿÿÿ)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ/ ÿÿ79;=;3,0..233001100000/149BBBCDDDCEDEDDDBDDEDE?;5445567=?:92458!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=:998679:;979;99<=51696:;A737;=;ABA>@>A@>>@???>=<<9:<<99::::8789::9888766654455556566652/868654445444554554433332322221110012333344445776678754666778999888878:<::::;<;:99;:;;989:55446564445879::9:998;;79999:::9767877865665666665678887767677:78:<5699;;9:L<;;:9:<>=<>=;9;:8889;;J;7=>ALC?;;<;<9=988:EGNH:?<32/-ÿ%.ÿG1&+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ::==QHBA=<=<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=@;8687:;<98;95:>?;77<98;A::<;<:@BB@>AB>?B@?@@@?>=;;=><;;<==;99:::9998986565555556567665276776665555456555544443332233334333333333444555778755767788899888988:;<::;<<:;98899989989:8322556668759::89::9=8569;;;::99868777567866675546688::978987::9::979:<99:V@;:;;:<=<=??=;::7;;:=EI9:?=AQO>;;:><<=7:99<>>?6/10./15100122/./0012267>@>>DCBCCBDEHIKKFDFGGGA?=;<=>=>?AFÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<=BA998967955456:?>>;8=@@A?@A=>@@@@A?@><:9:;=<<<>>>;;<;9:9;<;997777655657787666777776656654566555444333322234455554533223354557896776787899999789:::<<:;:::<;:899888<:9:;:548<>@;787547:99;;8757::;;<;:::8796666677666864557779877768:9::;<;789>:;><==>=@@>9:<9=<==DGA=@?>LL><:;?:==:88;DAAC@FIM8210ÿÿÿ2:.ÿ;ÿÿÿÿÿÿÿÿÿÿÿÿ)-=7;<>B?KFD@D9<<;=664+-./3-,,/00/.-022336689BFA-ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>??F98;:;78:65856;?>:69:;@;;>?@?;4425A>AABA@?<998;<=>=?>===<;;::::;9977888668788876666787866655554565554443344333344555655332236766678787667799;::9989:;<=>>>:::;<=;:<9:99=:;<==77;867::7677799899654779;<=;;<;:87876666766688656797768679:88:<:<;898;9;:PJ=?=;>=?=<@?=9;>:=>?@QRK;@AAMO=<;;=?>@NOJ„>73,ÿÿÿ1:9&ÿÿ ÿÿÿÿÿÿÿÿÿÿÿÿ %55NM==NNE???:;888966457><66:88@?A?@BB>>AB?200637<<>?@?<:::<=>>>?><;<=;;<<<<:::>;89999:888666676776666665555566655555454444455466666663347676577786767899998999;;?=>=;:::<<=;:99;;;::;>?=>9<;889777788999::9647:9<99<===;866:765877667:8656788878979:8;;?;;:5459;;=<>?=<>??GVb9@BBBK?<;:=<>;=@B@>>?=EJG~ZC:29ÿ5/.,8 ÿ(ÿÿÿÿÿÿÿÿÿÿÿÿÿ/;==?APPI@DME9;76570185/++,//00112?;8?=@@@AA@AACA>==?>:9/+ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>?>@BCA??>====>:98:?<:<=7@@>8ABB=@A<><:/3A=?@@A@?>?><>?=>?><<=>>;<<<=<::987767667877787877888676655555465565556777887:869766765466777889:;9:<<>;<<;=<=<<===>;;<;<:;>??>=>==;=87888<;;<;;<:8:66:=:749;<98986887788787::86577:<989;<<=;7779:53288;==?BAAGS@=BDBCJBB@@CBC@><<>>@@LJGM]lu=<0?B<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ"3GLShBDB?<:::=:64546/.-11/*))/116CBDCBA?CBCBA@B@@ADHDEDA@ABC?=<:8=?;0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<>>@?><=:995<><965<>>==85@B?=>??>>>?><=:>A;::;=>>><<;===<<;:9898788887788888877777656555566656655687878;:;:777755687799;:<:::<;=@>=?=>;<>><=?=?>=<=>>?@>>=??>><:8988<<=<:::::<8;<;743:;547766999988898;;96656=<999:=??85766965667;<;HOXOK@@?ACCBADBBB@@ABBA?GJ;@AB@ECC@?=GG?>ECFFC?OCAIJBB@9?4+(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ (4BOTN@BBQ@?89:97665/1/00/-+)*.245CBA@B=???CB@@A@?BFECCB@>=>===??>;8,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=@BDD@;===:8<::8669?>9><8AB?;;<<:;7657827>9CB@A@>>>>@>>??>>?@>>==<;:;<;<>=<;>=;<;;<<;:999::9887788898877877655677776666666678988;:;:8766457699:=>;=;<>@?@@@<>>===>>?>?>>A>>?@@@>=>@>>>>;9998:;<;<:::<=;98==6553344:435779:88::9=;96777>=99:=?@=868877879:8:<:@ACEIMGB?BAABGGOKGKPGJDGDBlI=A?21ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ'!::>EBKLGFJE<6899877102/1//,+,-56:AB>?:>B=>@BAA@@?AA?>???==<>B=:1/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<@>CA><=>>;7<:5668:@?6>>:AAA;=<:=76657<8;=AD=???=@@A@@?@@=>@>>?>?<8;;=<>>=><=<:;<=<<<;:9:::988889::987878877676888987777888889:9:9:<96566769===<;;?>?>@@B>@;A=;>@==?AA@?>?=<;?==>><;;87998;:988<>=>;>?978;69::71148879:::;><9867;><97:=<7<976789958798<78?Q_eTIHAABDCBCCCCDCBAFDC@F<;AEEEDD?EAAEDEELOMNLJJGJIILKdIB>,*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ563<=?>;><;>AA@@A?><=<>@>=?=99/4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ:956:=?;966669987579<588>@>:;:;:><@=997>888>=<<:@DBA@AAAAABCC@@A><;:>=>ADAB@@>=>??>=;<;;:;;;::::::;::;;:::88998::::::::;::::;<;;:;;:678;;9:;=>=;<<===?==?<==<<=>=>E?=@?ACDC@AAA>99==;8;:;<:8;<9;<;79:=;:>;55898:;=;<@<<<<<<<=:<<=;877645<;779:;:0-;EO_ZVXCA@ADEBCBCABCCDEEEA54@FIHIBAGIKMIJLQSPFCDEINKLGE?B3477752ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4=EIG><@CDE9:;656978<955251+,88@@?@@@@??@A?>>=;99;<:::+1.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿBCEECBA?@=9;;<=88<8=<564<:??DG@C?<>AAABA@??@@?@>===<;::;;;;<;;;;;;;:;;::;;;;;;;;;<<<;;;;<=<<<=<;87:<;;:=?@><>>>>=?=?A@?>=:<@?A@A@ACCEDC@@@?@<<<=<<::;:;==<<=;;899=<>===<>@A??=;>=83488<>>?A=;<=>==>><=<<<<:8875<<<8;;;8.=<=ES_[SBAA@EDCCCCDCEDHFEEC<6CHKHGDFFHLMKMNNUQEFGGEFDIGLUD3,15461 "&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@C=AYD=DIKG<8966999:9:57651,*9?@??@?>?@AA@<89603*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿBB>ACCIA?<698<:65779<4;;;<878::=<><79=;9=;:<=<<@BCDBABDBACCDDDDDA=@??????@BA@@>>??@?=>==<;<<;<<<;;<<;;;;;;:<<=<<;;<;;<<;;<<<<=>?>>=;8:<=<<=@??@A@@>@@??AB>>=;>?@AABBCBAEIHDBBABA?;;=?>><;;=>><>=;:88;<;;:67:==@??<89;:99:9>>=A@><<=>?=?><<<==;:9778;;>;;;=:C=>?DOcg^HABCDCBDDDEHGFGHFEC?ADKNJJHHJFJKLTTSQPMKKJHHEDE@JB4,1052-0000ÿÿÿÿÿÿÿÿÿÿÿÿÿÿ<;DEhXLHIHIA=;::<:::993442/-,2;>??>?@AACA?@BA@B989,!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿADADDCDFE@<::@=;=>@;<>?CABBBBABAABCEDDEECBDC?@AA@BDBA?>??=>====<<<===;;;<<<;;:99<>@?>=<<=<==;<<;<=?A@?><;<<==>@@?@@@BBA?AA>?@@A@>>?AB?A@?>@FKIDBC@ABB?=>??@<>???>><=>=99;=<=>879<:>><==88:7<<;<>>>@A@>>??>?A><<:>?<<=>=<=<>;;;>C0>?=88?;B>;=BCBFGM==;@>:;;:;9?::;@?9???>>?>>==>?>??>===<99<@AA@@??>>>>=>??==<<<<<>>@BCA@B@ABCBDB>A@@??=>?@A?>@BAAEEBEIFDBCBBBACB@?@?AAABCE@<=:<9:9<;:<=@<><=<=;8:879;<:A?@ACBA>>BAA@?@?;@A@@@@AA===@@AB9ADOUUVY\acMBBQDCEFFGGGHGFEFIGEGHJKLPNMFJNKIIKSORRECCOKKIWTG@5.).,1.184,%ÿÿÿÿÿÿÿÿÿÿÿA5EEFEFFCEHDC=<:999;A?::;::78667<<;9/007:==:8/3"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ>?=>CEEFDEG<8<@;;89:8898CB@DCD@ACC@A@BBABEEBAA@@@@@ADD@@>????>??>??=>?@@?>>>>==<6<>FTKG88;?BAAA???>?>>=>><<><<<=@AABCBAAAABEDB?@C@CB?>?ACA@BDBABDEFEEEDGECCCDGBBA?BCBCEEG??@;=:8:;;<>===<<=>?>:989:;9;?@@ACCC?>B@@CAB?@AAABACCDA@AACCCH6EMQSVEOYhcHCTHBFHHHFFFFGFKRG@CGHIINOKLQLMLMOTRPMFMO\V[\^[_K71-2.-2002-(#ÿÿÿÿÿÿÿÿÿÿ00;>JAFDBNPLH><9;:9:=>;::9877567=>==3001.42868.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ=>?>@@A?>@EA;?B=A<:=:;:89F@<;B=>=<<>:66>=;9=IZZYH98:<<=;<==ABBABBBCBABEGC@BACBCB>@CBGFEFAAFFGDEEDFHEFDDDEEDBADDFFGEDACCA;;8;=>?;;<>=?>>=;<<;;<=9>>@BABDC@ABBDCBECAAABBBBBCCDDECCDJ9HMTWWCBHYaM@LNEEGGGHHGFFGEKEAEIJEHJMMTOMSTRTX]V[]_PVU^[XUUK=2/30-1.+4,)$$ÿÿÿÿÿÿÿÿÿÿÿA@;A:88810///09;.$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿEFGFD@A=>CBD:9B;=>=>;=9;;@>@?A<;BA<==999@@>B@BH<@?ADCECCDDDDCABECCCCCBAB@AEBCDCBABBAABA@@@AAA???>=<=>@C?9:;CBBCBA@@??@??>==>:=<=BCBAABBCDEDBDCBACEABB?EIDEHHGDIEDDCDEDKJLPHDBCDCABEBBFCFGADFC==>==??>=>?>?<98:==<:>@=??AA@BECBDBAECCCCB?ABCCCBCDEFDCADGE5PQRVABAX^F@BNPGJKJHGGFEEEHDABHIILOSOOQSTTPRT]Y]YNHHOHCKPVa?40*)0..,1-,()ÿÿÿÿÿÿÿÿ)ÿÿ<24?=AEV]^PIB?>>>;888468FDJG8799;48620/0/-'/(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿCACEDDCEDCED<:@==><>>=AAABB?FHG??C>>;75CIDECEEDEEEAEDFDCBDEEFDDEECCDBADCCBCDECBBEDBDECBAAA@?@@@@@BA?==@PVS_GOGABCCDCCBABBA?>?A?=>=>AABCBDDCCFHHFGGCDCCBABDDEFEEDCEBBDFEFCDLNNOIECAAABFDDFFEIHNGCA?@@A@B>A@@@>=?><;<9;A?@?@AB@EDDDFFHFIGEBAC?BCAACCBDGD@DGHFNNEHQWCDC`XQOBBFIGIGFGFGFCDJFDDGHFMMPSaj`OQX`WVYYVUQOGBHPXWnA.-0/(*-00./.+,+'ÿÿÿÿÿÿÿÿ88>>GemaVIFC@?=A@?<<;888732111..-4%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿDDEFDB=A?@BBD;:@??>@=>F??BCGJHI@?DEB??<@KFCFBFEGHE@DDDDCDDCEDFEEDCBBABCCBDDDDCDDDDDEDCBAB@???@@AAB@@==?C‰‡\\TEBBCDDCCBEDAA@AB>>??ABCDEDFGEEGHKGIGFDBDGCBCCBCECCGABCFGFEEEMLLLLFC@ACIHCEGEFGCFOB@?@@BBA=BCA>AA?<9:;;@B?@?AAA@EDBEJIHHIFEBBB?CDBBBDDGHGBFHGFJW@NOWDDDNWTQCABJFFGFJIIKGEIECDEIHLOTS_ehSQO[]WZSRUWOKRPNNSX6.0/0$$)//./23+.,ÿÿÿÿÿ&ÿÿÿ54ABMUOJJGC@==:96876530/-+)++(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿCCDCDA@?=B;=;:>E@@EFHJHJADDDGHB>CEEFCGEFXSJHHHGFECDDGEGCDCA?>BDDCCEEDFEFEEDECBCCBABA@??@ABABA>@@J±¹„kOICCCCDDCDECBCACBBACDEEDEFFFFGFFGKIIGEDDHDGGDEEDBABFDDGEGEDDDKIHGJDCCDGHKEGEDEEEGBA@=@ABCC?AAB?@C?;9::=@@@ABBCDBFDFEJIGGGHGCCCAEDDBCFEHGGEGGFFHXLMPSEEJ_[[ID>AFYMGHGFFHGECCABDIIPUTWilt[TU\VSURPROL\ZQRLK\;30/=*%**---17%$$'ÿÿÿ ÿÿÿÿ6>DFFG@ICA?;<9?IB64656442/0,*(%!!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿFGCCDA??<;?;<>AB@=?A@AF@CFHJKIIFFGLIFA>BJDCAJ?'HHMLIHHMDBACFFHHDCEACDDFCBEDEEGHFGFEDDDCBCCCBAABBBBCCGJHJ|³¶€hQMPQLLKECFEDCBBBDDCDDECEEFEFGIGHHGIFEECDDGEFIFCDBCCFFEHEFFCEFGJIHGHGEFEBBGEDAEGIMGE=@@ACA@?CBC<>??F@?CCCDDFFEEGGGHGHHEDCCECDACDGGGHHIHGGFG]JGGCDCGLNLLBAVOOHFEFFDDEBA@BGIJSSYZacof[Z_Z_hrFHEJTV[KMIM@7.*3,%+()0205)" &)ÿÿÿÿÿÿÿÿ8@@@NQFBC@:8>>4J=45599931/2+!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿNJEBEBDHECFA?ACD@F>@ACFGIKIGIEEEDJQPNICCHHNB;;BGIHLKLGHKECEJHGHIHGHIHJGGGFEFFGIIGFEEDEFGFEDECEGFEEHNEEDBK’y’‹xvZZXSDCCEDCDDDDCDFFEFEEFGHIIHFIJIHHGHHGHGFEFGHEGIGFEFHHGJMKJMNHIGDDDCFNJIGKMHCIHA@>@@B@F?DA@>>DBBBB?@=@@@FEEIJHIDBDEDEDGHGFFEEEDDGFEFFHJJFFEHLJ[CCCDEIPLGDHGFQFEEDABB@BBCOSQUXZZioxncdjm{{kZE>>DNONKI@<50++/.,"0%,78=1,&#-ÿÿÿÿÿÿ?;00&"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿKGHJHEFEGMJEAABEBB>BBEGIKKIJKDEFFKKOTHEAIOG@>AHHLKUSMRGFGQOIJKIHGHHHFGGIHEFHIJJKJJFFEFFFFDFFFFGHFPQFFFEDFh¡Ÿ§§€gNxf_SDEDFEEEEEFFEFEFFFGHIIJKKHFGHFGGHEGHDHJLLFHHKKGEKLILMNKJMLLIFFEGIGHGMKNRMBDFB@B@ABBBBDB=@FCBEDCA>AA@ACCGJJJIEBACEEFGGGDFGIGEDDEFFFGGHGEFHU[ZJEEGLCCWGGIHKILHBBABCGE@DSUTTUX[^`fjhhx~vabU>4NINVMIH=>62//25%2():<;1+-*. ÿÿÿÿÿ916DGH@605+++O3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿGLKILKJKMMLIHDFECECFFFEGFLLKEBHELIHKKHFQOLFIJTSTVKKJKSHH@GHFIJKKGHLLIJJKJJIHJIIJIIJJGFGGHIIIHNOQULIFGHGDEFHe”Œ–DdygKFGHHHHHHIKLMLHOdYIHIRMFHGHHOMPPLIICEIIJNIFKLIKIGFGJHJKMNRLKIIHIGHHIHOU]QEIB@ACEDECBBA@@CEEBDDBCBAEEEFHJJJG>??DDCEEFFEECFGFEHDDDEDFHEFGGGIGkNFGEDDRFDDDDCRaHFDGJ?CFFUVWg[[_f[]bcjp``[^;9BLGP`[\aFP>9212.443)'*:64:7965-&1ÿÿÿ047;;=>B10.13+*.)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿNHHNSOOTPMLONJKMLIFKKHIJLLLHHJFEGGDMGIKIFFEMKO]U^ZFGFKJHJILEFGFHHJLKMKKJIIIHIIJP^jJVSMILMOONPKKKLMKIIJIJJHGr”‚WcpRQOMKKKKJHJLJJIFHFGIVsGOMKKJNMKLNONNKJKGJKKJKLHHKHEHIIPKHJKOMLKJLGBEEHGGGKTNJKIBBECCDCBBBDCECAFFCDDDCBBBEEHFFEFBBF>@BDBBFIHEFIIHJHJFGFFGFGGFFFIFFGHKGDGPEEDBIGLQTHHECGKQUZ[\hdeiieki[VGGOHB=?MTWbez‡A;@2A68;50+3/-78<>5777-,+ÿÿÿ/14740***)'.$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿKJKNTSSRQPRTOOLKLNHMPLJLPOQKJFGGDCKLGJKFBFITULQVHHFGFJNJKGEEFGGHLMMNMLLKKJIJJKLNkY]_WLFGHIJJKKMMNNMLJJKLJIIoŸ›cmZNKLLKJJHHIKLJLJMGDLPSe*LLLIKNOMJMQONLMJHLPPKJKIMMLDAGGMGFEGRSJKOKKNOJJLJHFFKIIEDCDCCEECBEABBECEFEEEEEAABIFHBDGIBBEBBDDDFFGFHMHIIGGFCGFECCDEFGKGFFGLIEEFGQEEDEdUSRRNLHKNSVXX\ebjoosv^XVCJMHIJIHOXafytLCD8E9889636:984<<876744-0ÿÿ,+.411)'ÿ%/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿLNRO\WSQPTXTSUMLOOQQTNLQTPNMLIGJIEHDMMGAEHDLXONGFIGIHKTKHHGFHIGHMNQONLKMLKIJKJK_qUaZPHGIKLLKIKKNPQOPJKLMKIKTƒNMKPKHJKJJIHIIKJIJLKFLTOTYQNQNOOOPMVWYULJJLMSVNMJJHIIKKOKKIECIRPJNNQPXUNNRQOMDDEEDDCBDEFDECABCCCDDFEGHHDBBCFGGC>CGHFDFIGEFHHHFGHIJIHGGEFGFGDEFJHOHCEFLKGDFJQPEBF^a[SSTPOMNQW[`^hjiemm]^]LEIMLLQPJMUbj•ySJA@N77568779<93<:7555450*&ÿÿ!ÿ)1/&&#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿQE50[O@ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿLQUUXTROOUVXSSOPUZTQONOUUQPQNIMKJKFDIKGSMHHUYYMB8NGGGK\VMGGIIJLMLUQPNOMLLLKJKLKchV[OJIJIIIIHHKLKPPONNLNMKIKLS`gKJGGJGHKIHIIIIJLLILLGKRPNRTQNNMLPQXVW[VIHJNRRWNNKMLMMOOMJLMLPSROKLLOPPMSSSSQJEEEDBEBBEDDCCBBDEDCEDHGIIHEGGEEEELHICBCCEEDAFIHIKOLIJHHFGFEEGGEDDGHLMGGFJIFCGJNIIOYTWcYZ\ZXVSSX[^cklj[\^ibWMHAHJLORWUVZ^fXM>7@:933573879;=:7643454..$ÿÿÿÿ!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@OIELxaK:2ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿQPPUPSWRYTSSQTXTUV[YXWQRWWUQPTMNOIIHGJPSSPXWLJLGHGEFGEV]N?>GHIJJJKPLOOOMNMLRLSQNQRJKLOLOJKJIKKNNROQPPPOMNKJGEDGFFDEFHEMMMKKILIKKLMNTJKOUQQJJMQPOQPPNTSHONSUSPSSPOLMNOPONNQPTUSMHFJNMLPW[_YOOHIHGEFEEFGEFFEABCEFEFEIHFFHGGFHHKLOOJFGEFEFFKJMN\VLLNKIJHIFFFEHGGEHUJEGGLMIEKGGRIJZTQ]QQS\TVZ]f[Y^^hgZ[XS^YMLGGIRZYdic^qn\^G?=;:78846996777857776101/-$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿISQI<.!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿQONPRRUY\\ZVYXRTROTSVVUTZTQSSTOKNQOLIQZMQR[NHIKJHGFHHFL[JKKIJIJKILLJKOPLLOTVNJINQIJJLKNJKKKIJLNOPQQQPQPKMKLIFDEHHGEHHGLKMNKKLNRLOOOSLLQYONHDFJIJJLRPLQUWSSTTOQPQRKONPQOKPOONOOJFEHKJKV_dcZNLIJJHFEDCEFGGEGFGGFGIGEHGGEDEGFGGILLJIEFFEHLGIMMTRNUKOLNKHFEEFEFIGEISGKIIKIFEGGFQIKUXOZSSU[SVZcvfY_`ca5T&2^_\RKIMTVVabbddgSZBDB;>888666769;;:889;31/261)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ. ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿPMLPSUQY\ZZWVWTUTVUWUUVTWTRUSUQPMOSMTROIIMMKKKLLKIGGJIISOLKIIHIIIJKEJJLRQTRKLJILJKMKKNLKMJJIKOPPQSRMQRMHQJKHGFFFHGFIJEHIMNMKLNPQPMQQMMPOMKLKFFGNMMSQNOPVUUTPPPMPROQQPNMLOMNJPLIFFIHILTbc]SKJIIIIDEDCEEHGHILHHEEHHGHHEFJIFIJFHIGHHFDEOOOKOMNVTMORPOSOIIHIFEFEEILYJJIKJLHGGHGMMLNWQSURTVYZY`ogdib]T  PcfUWSNOSXa_jrao[rCB>=;98368767:>98:654210351/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1/ÿÿÿÿÿÿSÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿMONQPMUZQQTVYVUXYTTVZWWVTQXXWURRLKPQ\TOMNOOOMOMPNJIIJKDMMGFHJHJKLFFHLQOOONPLKJLKKKJJOOKNOMLLMNPQQQPOTTMHMKJFFJFFFGMYKHJILLMMMNPQRPPQPMOPLMKNIKLKLLVVPQVSTTTPQSMPRSQQPPNQQOQNMHIKIKHHKMU[UKIKJLKHHDHDGFHJIIIHGDGHGHGFHILHGJICBDDDHFHJRUUMTILPMKMXPNQQOKIHGGCACFG_KJIMKIJGDDGRJLLTSQSTVYTUYYZZ[eaVNQYXUXVSOTT]ecezmgBA9=>66352989;:8586592159530ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿTSQQQPTUTVUUPNOPVX]a_[UTQTRVUUUWVRTWa_\ZZSUQOPVZURMIHFIKKHFGIJKKMRPMPNPSTSSNPRSNLIOPJLPNMJLMMLNOPOOOOMOROMHIIHHLGFZibXLMQQMMONRRPPPPRQOQRSQROPOQSPPOPRVVURX[YVUTVXVSRNRTWURTNLQNMOMNMNTSOOJILIHGIJILLIIJKNJIHGHIHHIKKMONNLHA@DEFJHPTSXVRJIJLOOQOOMOSXPKKIIFHIJJ`VGIGFFFHHDFJDMORUWWY\ac`b]b[Zic[W \jairkff[aTVdhk‘pucPC>@=7:6/.0/47157718=7<75%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ:?BHF/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿRTSSPUTSWX\TSPQSW]dd`[XWQTSWSSWZWUUYYVVXUQTRQNYTZWMKJIHIFGGHIC@KRcTPNMOPOSRNQTSNMKRKKLKMLIKMLKNOQPPRNLMOOMKKHIJKMJ__`\OQQPMNMNPRSOOOROQPTQQPQPOQQPRRSTYWSR[^WUUUSSQSRQQSRNOQNUPONMPRQRSOMOJILJIHKHILIJIIKJNFGHJJKKKNMNPMMLFDDEGKLOQRV[YRRNLQSUTMOOPNJKTQKIEIMLKYUGIGGHIJHEC1CMPTW[^_dbil_a[Zc^f[id0RRXbhwwv`_eattny†ohVIEIC<979PF26966756/8PE7ÿ ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ@ABCE8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿQQQUTURWZYXTQORY[dYY]\VWUUWXURRQTRNSVXXUUORRPOQPQPLLOSIIIHHKKCBMPcUPMMNMNTWNOSNMMLKKNMHLIJGHGILQQOPSONNKPNNKMKLMNK\Z[\PSRPNONOQTRQONRPRRVTTQPQVSTTUWVQXTRUX[WRTVUSURTTYSRPNPNOPTOSRSOPUWMJNLJMKHIJKMGKKKLJKHIJJMMNOONNOJLIJDBLJJOQQSSSSPONNQPQTPPMQKROLKIIHLONM[ROOGFHJMHGF2ENUTYa`_felkif^[_id[k]?YWY\glha[U`jjmhau€d\JJDA?43-5Yp>Q78335)"42ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿDPWTIJL7ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿUUVWUSRUYYRSQQU_giVSY^YYZTTUUUVSSPLQUUPUSKKQPNURTRNR\YKIJKKKIJNPKMLLLMNORSOOPRNIHHLNNNKMOHHFJMOQOPQROPMLNNOJNONRPLY\T[UTSQONONQRSSORUTPQSXWURTWXVVY]XVUUUZ\XUTVWSSRSURUUWRPTNRQXTTRPQRXUKKJKLOKKHKFKHIKKLKLHJKMOPQNLPKOMHEGCHJJLMMORQQPTNMOSVRSRTQNLJMNNNJLMMNW\IKNKLHGFKOL;#CNUUY\]_bqvq{ud_fm^XfadaZWT[Z`WNKQX``^\ite_KGA<:44&8Qnt7441140/0100"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ68ADFJNMJGÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿUVUWVWUSY^URUW`baaVWXZWUTQW\Yc^XWSMJUTSWVPRPOOOUUPUjgbSOLJLMILMPF;FNMLMLOPNPNOLJMKJMOQMOOIIHLNMPSTXTRSTUUVTKNKM[WT[^TSbTTTQTUURQSUTXW[STXVTZUWVWYX[[[[YZWPLRTTTTRTUWXVYUTSSTTSTROPTORRTSPNOLLSMIIKKMMLJMMMMMOPTQNPNOMLLKIFHJIJIMMLJMOLLNONPPVWUQSRMLKOOQNMLRTL`RP[OKEHHL``RNMLDRX_a_be^sˆtjdfihceprm]WTOYUKKFELRXdf`ehpaTSN;899`574_k887/5365215554 ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ78(ÿÿÿ9@DKOHABCBCFNNDÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿTVYUVWVSWVWUZ[ghc_YZYYTYXU[YYfcZTSQMVSSQPQPOORPTWSYpb\OOMKNMLSNQKKNZONNMMLLNOSPPNQQQMPRNJJIFKMMORPRRSTXUWSPHHJNVXVVZPUbZUVZWWWVSQSWVXZZV\[X\Z\^_\]\[[[YXTUSQRPPQST]`SUSRSXXWVRPSRRQSVRSOPNJONMMMMLLLLMLNOOOMOPRQNNSRRNPMKJMMJKKLOLMMNLMKNMRSSZXRSRNKMOMLORPOQQdWXXKIIHKPRONOOLMPSTY_Z`Zfekgiilipfsphb[SQSRMJGLTd`agfu‚xhaYK>9:M>65/DP#,97464/336633ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿAA>@=68;?CCCJLDDDA@DGH=ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿSUWX@TXSSQTWW`glaZYZZVUXa\YWZ[[XYYWSTWYUUQQRRUR[nmgl_VQQPSPNMRQQNOOaTUWZQVPOPSQRNKWSOQURPJHHJJLNOPNORQWVVSRMJKNSU\STOQ_WUXWWWUZXSRTVYXZYZ_Y]\^[`]YZY[ZZXUWXRTOSTTYb]QUSSTXWVSQRQRQOSVUTROMKNRPLMQJLQNPPOPPPMOQTUVYPUTTPONJOOKKLLMMLMKQTNONOTSWTSUTMNMSLOOSRNOU_UTZSMKLMRNPOSOQOVSU\]dija`kyjiktvuvsm[]X[WSTSNRVcheonjxne_fF>;;67977445965343377462ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ155%229747559:32434.344346PekAA?@?>?BBEE@?>DFFFJKLIFFEAA>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿWTTROOKPV_ezz„i_`cdYWX_dc]cgj^\[WW[aP[^dVUY^Z`VVguhdglmkea[^X\]SW[YXmlfgdg^WWX]SSXUSU]a_TPIJQLLNMMMPOOPRQUYRTSNNNPUYhdR^Y[`]XZYZXYZ[UZXY^\\]\^]__`VUXX[\UYYSPOOSVVTY\UUVRX_]XWWZVWXXXT^\[YWTTZXWW\XWOORVXRQRTUTYVVSQPTUTTW\YPOLOLRYRKMMNRRRRRSPLONMRUSUU[[RX]WQTYPLOSSSTPWSY[_a^jjkhlˆoozkol{mnszylz€ˆ’\[XPR_jjcoscZ`‚ˆˆ¤VQ`AC?544676998965./53337=EHÿÿÿÿÿÿÿÿÿÿÿÿ772'ÿ485Gƒ_H9769745;>??>;;??@EIGJHEDC?A<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿZXWVLOQT[`k†ƒoggqc]XYkqmdmrnjcjYYVXR]]\ZXX\Z^]XYT[jnfngc[\]^WUWVfZQcl`cbaZVW[_YXQZYYYj^S8IIONJNNPRQOPRNRWYSSTMOTTYX]\QZ]dhee[XXZ\Y`Z[][[ZXZYVX^_`WVWV\\W][TYPVTRW[XXTTVV]_ZZWUXTOVYWX^^\]UTU]^YYVX[VSW_`\URWYU[YWSRTWVUUWYTQXUPQQRRMRWRORPWUJLNOPNPVOU]]XWZVPQRSNPOQSPPRSR]]cl[cdcjq‡•„{‚wmkaaigk{o~z‡[WWNTY\\fjf][VƒyŒ¦P_TDA<9658866877762254348>DHD)ÿÿÿÿÿÿÿÿ116566666479;@=<:8<@>CGEEDB?@A<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿVTUYMQT[`kj‚ˆ{}ort`_Y[swltrnnbaZY>Ydb[Z\Y[^_]\^]h|rmmmdPbfbb\X_bWO]m`hg_ZWZ\\ZWP\[XUZSN-OPNOLLMSTSRSRQ]YTQPNMNSQTZ[WQV_ba`aXUWV[[[Y[_dbWZZ\UR_[_VVWXX]YYUSSTXVR\YYUUUWY``TWUXTRSZZ^[[a_dTTU\\XSWXWWXad]`^UUZYZXXUUYXUVVWWRS[]TSQQQLQXYPQRXQMOOMNOS[UP\\WaURQRTTMUSQPOROPUZ^ioima\mu|’~sˆƒ}‚v]^knju~to^`WST[[bd]X`e]}zSJLJ?<>7:;:98583134507678>EMMO.ÿÿÿÿÿÿÿ687:>>FHB=B?=AFB<:9576989:;77:<=;69;<@ACBBA?A@8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿVSTRQVWZqu|}|pptihckklpeqrpqjptY_>_]TTTRVZ]Z[_Z`qsnixptppf\z{ir]WSjhf][ZUNVZVPaSONLPNMPOSRNORRSRSTURPZWXUQPNQSVVZX[\T\\_aa^\W^^[_ba]``caa^\UW]aa^]YZ[`VRPPX^\`]^_YWZZab\WXYY^]egde`^W\c_UN[]`a_^\adccc`^UY`bdd_`c^YXWZXYVV^US[ab^P[YOYZYNSSQWQPOVYUZMJT^e]WX\VRVPPPQRa[\fwsihgjvnf‰—‡vmng_abgfdkkmce\VVXXXUZ\^ebcrdu@Ci=@E>98;9B>:98867:?AC=8436=8<=9>>AA><;9965;;>>ABA7-9:ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿRONOORUZl}‚~~tmqujlomheeqlooopqa`S[\T[WOV\YY\ZYb|rwwxry‚ui‹{vdXZ]iafYZZSRVWTTWPNOPKONNQOSOTXUTSQUSSP]WUWSONNPRSXWWYVT^_^^ZXXWZ\Z_ca]afdc\YXYb^db\YWZWQPRS^a__]a`XX_bb^a_[\eWOUc[[YWUX\ZW^]\e\[__\__aa_]]Z\]bb_aca_ZZZ\XWW`XUY^]]W_YZWSSQSWYZSSRSU]`QJS]aXRVVSVOOTQMPX_^dw‘~idensh]gˆ’†…okfeV^^\Y_\[YU[\\[Y]Z]cjhebiF[TEED;8;:?KC<><<9;>@BDFIEGE>CA@=:<679;;:=>;CMA<;>:59;==<;;<>;@BDC>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿTOMPWU_]krh…„„‡yv„rtlhfcgoptukuulpY3UUTWYO[c]]YZ]epijo‚”Œ‘‡|}‰ka\be__bZY\[a]XY\XPNVUOOLJNQPMVQSSOOYSTY[ZUXSOQSSTQVXZVWT\\ZXZYYXZXZ]^cc_ac\\\ZXZ`c`[UP8XSSXY`_[X]]YZ]dh`ec_Z^`USVbc`YZZ]]Yabcda[\Z_YY_\YZ]`\WX[agl``a]YXZ[YYd]VZ^`_^\[\c[VWWXYQTXUSQXdd^a_baSWXXTTRROPRSVU`jjoneuhd`fiw‰‡›lpr_[__\]W\Z\\^]XX[][_``|hppBZ†KG@=<>;9>BCA<>9:88:;><=A<768;<>===@ABA?@A>?@@BBBBBÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿSRORYY]^pnhy{‚Œ‚†’tpvistnijokrtneT/\RMXUMVj\\[^erkecApv”‡”ƒƒz`Ygeeab[XWYXZ\^[YPQZRONLKKKJKVRVUQSTVVbXWYVUTUQRRRRUWSZ[UU[[`[VV[[Za``bb`a^da_Ya_]YX^]?_`YY[`^]`[XXW_fb`ad``de_a^eed^]][]ac`]a^\[X^[__\XZZ_b^YWX[Z\``c]\^][Z]a]ZbgiZa[PYTZUZWUXYY^UW\W]jadg[WSTVY\\TRSXRV[]nph{wxlhhd^nz…˜¦©™Œˆ‹c\^][VW__b`a[Y^c`^]]Vwam„Lq‹QGGC=A><>?:<;<<@BHHNA@KIDKNLDC@=@OLFYnO>;=I8>A?AA?98<>>:679;==<=>??BBBBBCCC@@AB@DÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿTSTVZ]cppli~stŒw…}€}soomvvz~]\`Towx{†\Yah`[ajlaUdUi€•†”ƒq|ri]ahRYlg]\ZZ[\dj`YWSPMNOPWZWLSRSUSQ]b]\[]VXSSTSTXZVTVUYZ[SROX]WVWXSU]_\]`a^me]`d]Z`cbY[`^djda`^`[\\aY_`\^^g`hnlpmljllhhhhj`baca^\Z^_^^[][XZ\`__WW]ljpr][Z^]][^_ad`ibbLKPSNPOa[XXZWWU[XL[[W^\VHITX^ecguYQUX]nh{zgfx‰nppjri¦´¯§~~ˆ|rdkogkgdip[^`]_^X^xoqwbOOCFID;===CB?CDJWLJOJFCGFCC=?@?>=>@>??B>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿRTUZ^]iuigf~u}€‘~‹‚~rrs|rsxbca^`mpieZUU[d^]gl\\h^f‡ƒtx~|empkVVgd^TV\Y\ee^ZSZONPXY_QRU;QSRPPUfd][ZXWRRTUW\_WVVYZ[YPQNM[ST\URR\^\`\`bfdd^][]e]X^V]Xegfa]^^ZVee_jef[\dfiqonokmklopklscbja]]^[\bZY_^YVWXZY\ZZ_fgkb[WXWZ[[^^\`^aabaTRRSNN__Z[Z_KJUSWZY]WSOMKV]WchiYQRW[_wsz…wrr‘|ynjhh˜yw‹{{qsx}orfijehi\[Z`_i‡‹‡gfH?AAFF=@A@HJGFDDNTTYOHFGEDCBAA@>A@@ACAB@BBDFFFDBC>=>758;==<6556::;<<>>;<>=<>@@?@C@ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿRQ[Xacnxsln~~…ƒ„•˜’†jxnuukoqobVSV^fdZnU_iY[ah[tlgjpzƒqi]hdka^RWZ`aWVYUYb`]bXTQQNUXX[]PKQVTTQPZXVYWWZU[\C[_d]VUX[]]SOMMMQUVMRX^_\^]^n^ag^b``W[[Y`ddfcca\^aX^c_jl^`bdopsvptmumihnroqnkldYY[a]^c``aa]ZX\[VWba]adaa][WXZZYaged^_\djacc[SOTWVb`]TSRRZZ\SXONMJQ[bol_TJMT]f{tv}tns‚yykqstxtpctˆ–{~‚psozmsrlheaig]bof`™ˆxYwLA?IGD>@DCPXSX_fw^VUOKIKHFGIECA@BA?@??CCEECBFFECD;;<8779=<;;>>;?BBCACGDCAB@:::788:<;<=;9;<<;>?==<=?==<=>@Eÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ_cfh”Ÿ—¥¦ª–†‘–”qƒ‹”‰lirmlulR[a^pwuswic^gfelpq~va‡…vxprqs‡[OI`ab\WUTTb^\ZafcVTWYSRUZRRS\_YXVXUTUTXXXX^WTVZZZ[`W_[[\RMV`acb_^bfaabhmfhk_Zdgefd__]Z\bgc`dcfhdemuxotvyrptsnusz}qhbbahpijlhheh^]^[Z\\aa`gcb_]\[U]ff[SR_peeSgltxxnx™‰ilfidgbSSVRRSLKMMV_UM\Zg_Z`o˜•‚ˆ†’›˜ƒ’`bacm“¬‰”|€tyŒws|‡}nb^geiiw{hV_†HO`o|z…‡~{sorpom_`TMMORNMLJIBDB@>ABBC?AACEFFFBAA??>:;89;;:;@@A@=;;<>>>>>===>>>>@D!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿgmmz”§©¡¦±©ª¥›”“”‹~~~‹ƒnjiuuox0dakrju„‚ecdcbbbgkmvsy€€h‹…‚wmz‘^[Uiy…oWQSW`^^Z]b\b\SUTYYVTUPZVZTY]ZWXWV[_^YXV[a^[giU]X\^[SO\]]`^_``_`acbjkjhmqlmfdaa[bjf]Z]gkkhxz{}{uvtvww{zvux†’uunqgegopjstjcnnhgjdacaba\bbaa\``^jup]ZkƒudW]mmjjW`n‘”‡mryrmib\ccdUQQMQUYkjcsnh{c\d¥¥”xƒ†©¤¡”‹`m{¨¥†u„w‹}†~{z{ikuca^dYuou_yoI\|pqvwƒ}vs|ijkcf^dYKQPPPPKJFDA@@@A??@>EEEHEHE=>?=<::<;;>>:=BDB@@A>;>?=>@?==>?@?B;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿgls‘“’”˜—¥Ÿ™›£Œ“•™Ÿ›ˆ}€|y{uervquhoW^cihnz„~bjabeeaesllll~xl‡v†rtutfk|dV_]c``bb]VURXYXUSUYZYXY^VbcdXXV\][[WUZfkjdhg`_VV[YSQY^_abb`^ecgkheddlikfed]_^eigeojhxzyx…|yxxz|yyvv„ƒ€•›¢~y~qpqruqonp^ekuqllhie]Z[[\^^^^\]]cb`[^s†faZ_fh^afle‰–˜uynliimmhaVRROXXZ}xl[VVSV[qœ…‘ƒ˜¡¤ ¨¯£•‰i¥­œ–jr|ry{rpirhk}g`ZeengoblW^mxlqnt€zpowmggneye\QSPPNPMKA@>??ABBCA?FGHHCHDBCCC>;<;>><<=>BA?=>@B?>>=?@??=>@ABD1ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿlh|“†›ªŸ—•Ÿ—¤«§‡„‹|xtwlryz…ngc^sorntm\h`acmell\ba]b_p€~‚…“‹¦g`dg^Y^[^aakiZ]T[^TRMQQVV]cW\U[`eVSGZ[ZZXVbikhechge^^YUZZX^^dkld_iiiqllnllnqkffccbhnwu}k`o|ƒ‚krvx|ƒ~{…†ˆ”‘•˜™˜~}|uvz~wxqdhjpnqtnd_ll`ZZZY]XZX[W[b_fk|‡‹ikam`\Zojk‰˜’“’—Œymlqlrgec`\Xhrp‰}‚[P^Zah”œ‹”’‡‰“¦¤”|q–xuvhszx{okzhgpkq\WUXf__jutWm„nx†}wxzquvto^toriccTOPNFHGBA@@ABEEFDDGIJJHEICCAA>==9<=9?BACCA?=ADF@@=?B@A>@ABC>3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿq‚„¡£””“Ÿ¡¢§¦¤™‡‚~‰‰ˆ„‹‹„‘†{rpeol_bYQS\Ycwjhq_ncwjjiuŒ~ƒ{otqdaXT\WWVcdcekqgY[^cSSSVW__b^dagkh[_Z-"R\ZX[lngf`cbbaZZUVTV`___b__dffhomjhghumhcnrv{„|vy†„{|rw{……‡‡†‘…•ˆ•™¢Ÿ–€{‡ƒ………w{zsqvtrrieaejfjiga\_]bca^]]mj}ˆ†ƒgZknda•”‘™¡œ•qkfQYfokUj}sv|ƒ~‚e_n`im‚¥°È²”— ±£¡¢«…ˆyw‚jn‹ƒ~kdY]\\if^ZaUU[b„aiekw|~wtuqrmxvscx|saa_\ZWPLGHHBCBAABEGGGGILMMNMIEIEA=;76769968;;;>@CCDECBAABA@BD,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿm€’­ª¢•„ˆ «¤¢¥¥™‘•…‹‰’‡Ÿ“†vnqqo}jZ]ciain†~w}€ˆvutrw‹‘™whb‰{Z[[T\YW\a^[da`[^du}fiWYV[[Zcqedgk`V\YRZWZ[_gc`d`_]ff`_]WWWbab^b]]chjkmnljlhkoem€vx‡qvsw}€|~~ˆˆ‰‚ˆ‰•“™•“¤§¨¦›Œ††‰’›Œ‰……xsqvytmjlrmkriji^W`fgaa]_`bjs•†‡fsnhc•”Ÿ ocZahgboY`€|{‹kgnysem`^ad‡½Î„y„¡²°¸É§ —…v‡|–­¨œ’odYZX[^_[UhZbjtemipqwvrlnmberpth|pda\a\`TKHIJGBBCFDFHIJJKKMOMKIFED@=:87658768:::<>@AECEABCCCAAAÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿtŒ‘·²¤ˆª†—®­§¢‘ˆ†ƒˆ†§«’…†„€„|‹‡ˆ‹yrrddrjmv„†‡gcpnq†uje^^ngXX[U\WU]\`hVYY]oi}²œ{fWVX\rkqb]^kbV`jh_ZYZ]dd^^b]]^]a^\SW]`^^`fdgmkhqsomsrsqq{rsqs}‚‚€}…ƒ}„‡‘‹”—¦¨³«¬¥Ÿ–’”•”••–…ˆ„swrtvy}kuwomsmkfke``^]cafafdfƒ•kgvks—’–hWUhff_e]^toqd][k‚ldlbX`’ÎÈqs{œ£¸±³Ÿ±Œ„…‚˜‚po^Y[Zj^]XS`a|cv…{rvhnndmpjl|rooƒwoeaYZY_XOMMGFBBEEEGJKKJMMKNLMMIHG@?>:98898779<:=@@@BBBBBCEFDE7ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿrr›º¬|‚ˆ˜——•–„›••˜„……§£•œ†`‚†…›¡¢œ–‹„szˆ„—–np‡‹†iie\V^_^\]_Y\X\lYXTX`ikrƒª †TXd_XV]onndZ^hnjsqajhfgaWodd\^flb\Zglrk\[bntqn{npv€ˆ†‰ˆy‚usnz†…†ˆ„‘‰€¥¢’‡”˜Ÿ®°»¶¼­»·¯¢’–™”“†{tvwy}{ryyxxxsptuvuyycmcZbeammikc`d…ˆ~ƒjphv””““‘˜”hYb[fZfbPR[pcc`md\cssyl`r}¯x|‚‹Ÿ©¹ÅÀ½¼°©—†®·‰»ta_bmŒ‰xq}r`lXHydt‚™dhk~zukhx{xra{vtmi^VNPU[RROGGIIGEHJNKLMMNNQPMOMLKGA?>?;;;;::<=>>AABACDBAAABDD$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ_\†•Â½¤ˆ‡…§—‡…˜•“–Œ€~” ¤Œ’“„‘¤’›­¡¤››–˜vzxr{‡‡ƒej^Z]__]\`a^Z`he[[U_luzˆ«¯§…^_xa]]]ju}}teypubY^Z_elarnj]\^tdoqrvwmbYcjvrnx{u€x|‰›Œ…uxxxzz~{€ƒ…“”¥Ÿ˜£°­£«®¹·¶»¯‘’••“›£”€€yz‡xzzwz‚|}{{ykhccbgffc^e^[ek}z…„|dj”‘“‹ˆpcW[\abYTPNdae_hk`kjfbh`faov•{…Œ³Â»ÖÓÎ˺³§‘’­µ˜Đa^i{|zn^^/Qq^i„‹™•ocdurphfsqszqejptseQTUQPPTQMGMPSKHJMKLLMLMPPOOLNNMIDC@>??=<;;<=>@@ABADEC@?ACD<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ[w‰º¶¬‚}’“¥¡•“Œ’‹‘‹~‚“‘‰‚—–¤Ÿœ¤“ªŸ››¦Ÿvw|•—Œ€}‡x{ykivofifd\aadmfYU[W\dqxv–Ÿ©«°£”^cpjgd]hjhc|rcukhoeeckjefppnfihuhkoyurpegrutu€‡ƒ|x†–‰r‰†ˆ„‹‰‰ˆ‰—°°¤™zƒ–§£ ¦ª©´µ³®˜™—•œ™Œ’‡ˆ„…|z€‘‰zvtswyy|‚zlhggbfhfafa_^^dovŒ‰q’ˆ€vkU\V[]VVX[RU_ec]cce[dcbcjlmukvˆÍȼÆÏ̃ÚÙÁ´®—›¥«¿yjm…—“‰i[`W_iPVbo˜‰nggikefefuv†}kife\[QSaSMKMUOJJLRQQJMKIHKILONNOLLNMJJGDA@AA>>=<>@@ADDEGGEEEDDE<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿXz‘ÁĹª…{”‘„—–¡ˆu¡™›†‘˜“ƒ|ˆ€ƒjƒ•¢‰–¤¡©¬¤—ª Ÿ–œ›‡zu‹¡–ƒw€||}…{vrmghd^ejh^_a__a`gpuk™™¨¢•]hnrsrpokpr|rvzoppjlouunfboqkgizkghloqri{‡‡uuw~ƒˆ‹Œ‡‹…ƒ€€‡‰‚ˆŒ˜Ÿ¥³¶¢Œ–Ÿ¨§¨«­²§¥¤œ¡˜™“˜”‹‹…‰…ˆ‡“ˆ†prv}|y€}}nlljdmrhkia`_]]_V_„{‚†‘‘„†ojd[VYWT]`PONQdbaimmfvtpjjfph|Ÿ«ĂÎÁĐáçÛÀĂÀ¬”¢¢«¸‰sy€‘“›‹f]eeZ]Ped`yx’…ntidkjrvno{„{lcY[ULOZWOHHKMIONHKNLMKMHIILMMNNNPMLEIIFEDBB?@=>>@BDDDFGGDDEDD6*ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿm‚¡Èȸ |z|Œ•Œ™|rs›Ÿ¢™”Œ‘†“–‰„„¨¯·¶²ÉÔ´¿º¾½½Ä­§–›¡œŒytw˜”‘ƒyp„vjhrgddgiifkkvk|w|upidqœ”’ˆwr{‡‰…xrssƒxw‘‹‘‹Œˆ…~~wccnpchkem}qmfz}†|xy‘–“‰ˆ’–‚ŒŒ’‘¡§­¸¹¤“£ª¯µ¼²¦¦ Ÿ¡›—¢§¡ ˜¡–˜™•›”•™–™”zy~w‚…‰‚€yrrtrkoljppdbc^VZubbs|–œ©¤•‚xldMKUfXZ[RUWRfmwƒg}~spqk À̃èåååÓâææÉÎÓ³¸®ĂÀœ‡‰h¤‡‚…{b[/Rp‡h^v}zu‚onyxtƒŒ~~z[TNUMIGIMTLGFGQMLMLLMLJKJHKOOQRQQNMLKIIJHHGEDD@@BCEGIKIHHHGGIÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ1c›Đ̀¦‡xx~–‘™”’£¦¬˜‚ƒ’’†u–®ª«῭ÂÆÍÎËË»̀Á«¢‘y{tŒ¥œ¡‘„€xzstqqf`fecajnok†x“vsegy…¤‘‘”‡„‡€ƒ–txy”{mŒ“‹‰‰€|kadgdolinlu…ˆ{ƒ„€}}‹…‡‘‰”˜–“”‘ƒ†‡™’’––™Ÿ¨ªÄ´­˜§¦°µ¸Á¡Ÿª­¡—¨£ œ›¥¦¤˜˜™Ÿ–•Ÿ”—”†††—ˆ‚}xrtztlkoqsegfdZYdx^\k}‹‘¢ªˆuutdW[SZSUTQSQ[ajjrv}pcdsŒ±ÏàåäƯẫÖÔÓ½¹Ä®¸¼ÊË”qs€l–˜or_XVXfknk^ao—ltwvvy{u‰“{gdVNJMMJJLPZRGDGINOOILONMMKKLMOOORROOPOMIIJHGDDEABDDEFIJKIHFHFDÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ58Ÿ×ƠĹ©‡‹ Ÿ¨›ŒŸ¡ª“•†Œ”•›¡¥“ÊÅɼÅĐƠÙÉÇÀ¢—œ¤›vˆ–ª©§›”“…|kfcjebjecffhikj‚‡‡€ “——¤¥›ŒŒ……˜’‡†}}‰–„—†‹’ˆrkvwjie^cckwzƒ~y€ƒƒ•”‰ˆ‰’—™•‘‡‘‹’—’”™™œ  ¥­Ë»±§¡¬º´³³À¼¨«˜œ—Ÿª¡›˜Ÿ¤™•–œ¡›—›•’’„‘‘’“ˆ…t~xxvenqsjafea]a{f^X[j‚’ ª›|~yxakZVTPSQSRMYfe`_jv}}swv¯¹ÆáææåâØáÔÍÓÜÓ˹¾¼¶ƒemvy‘”xtkeYYbSU]g€ec^]c{zyr†„w|•‚q`_SKHLGLIOQUPFECDGMRMJOMNOQOONOQRRSSPNMNKJJIGFEDCDEEGGIJKIGGFEFÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ9E¨ÜÚÇÆŃ…†——¤œ”“œ§“‡‚Œ”œ¨²¬ ·¤¶Ç½¨ÊÔ×Ç»»¨–›§“ˆ¨®«¦ª£¡†ƒ}z}~}qjihjqjnpmg}‚‚—£´¢Ÿ§©ª¸Ÿ–†“„„‘–™““‡€ƒƒpik{}vprebcrwq‡ƒx~—‡”‘–››™”„‘”—™—— ›œ¡¢©»¸°¤«É½¸µ½Ä°¤››¥³­ª¢¥¤ ›¡£˜—’¦¥–ŒŒ“•‘‡‘••‚w††wnvssmgfksvmhbaXX[|•¦®‘†‚…_s`UQXWRRPS_aTa`mlfvu‰¥ÀÎßáèçæÜÖ̀Ñ̉ÜåçǸ¹«—|z€€‡‹ois]4:e^hmpfgrqczˆvr“„y‚††sa[MHJNIJKZYSOHECLHLQPOOOMTWRSQQUTTSSPONNMIGGGIGCCEHJKIJJIGFCDGBÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ4,¢Â×Ë¿¶¦p›¥£—ŒŒŒ›®®°œƒ¢µ³‘…’¢©ª”©©·Â´¾¯Î´·¼­¢—“•«½©®§œœœ€„}‰wqhinghlmrstq‚{®®¹´·’‰¹´À¡±¡‘–™—•’|py—‘‹……~z~d\tƒ„…ˆ…€„œ˜’ƒ‰““™”‘›¡°¦¡¥²°§¬ª¯±¬««§œ¦°²²¨¸¼·½µ¹»¹´²¾¾¶½¾¸º³±´§¥£¤°·¬ ¤¢œ‘ˆ—‹‹‚€‡ˆ‰†‘†wx}ytmavusap`WUR\][q’§’£¦¦sQWROPSRSTTTm``i_^`xvjun~§ËÜáèäÚÜ̉ĐØÁ ¦Æ¿³—¬ª‘oƒ‡lfccL_œ~}}ggqvsw€‹u›‡yu]duk]dlZLKJMQPXSY]SXGCEEFRRX^ZSSRRVUVTSSSRNMMLOMKKGCBGGIIJKJIIGEDH8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿG6—ÉÏʼ°˜‚¥§“ˆ¥ª³·§¢…“ª³©ug—¦³©§¶³°£®½¸º̀ĐÅ®¶ª”‰”¤¶±³¦¤¦«”Œ‡{xvxmkolkksvƒs’z¦Ç¼± ˜À°£ ÁÁ …ˆƒ™œ‘rt”…‹…†ˆ‚|}}no„€‰˜ƒ…‡„“Ÿ••¡™œ›­¶¯µ±µ½¹µ«®²«±¬££¡ ¬ªµ¶´»¼»Í˼¿Ă¾¹Å½¿Á¾»±µµ®©§¬«²ª¦ ŸŸ…Œ‹‰‰…‡Œ‹Œ‹‹z{ztsqkoodcyjXeNZ\mg”«¢£¨„‘jOXRORQQSSUXe^nfke_kzpgr|–´ƠÛăëá̀ĂÉÓĂÍÊ´ÂÅ»©™‡“•‹tgabidr˜‚g[]nh\Wdfq[‚~„Z]e]QTZXNMNWYQWQTaXPUIIKLSVW`SVUSPRUSQPRPOLLLNMJJJDDDIHHILLLKDFFHA$ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿT>±Ø̉Æ»¨Ÿ‘–¢§£•’‰”œ´¸º›¤™™¡§­œ˜¦§¡Ÿ¢´©¦–›·¹­¸»È³¸© ˜’©¨£§°§¤¨œ• vyzonppmpnx…†x“ÍÍ­œ©£¸¾ˆ ¢yƒ•—œŸ—‡}‹‡|Œ‡“w|†‡‹Œ™„}¤œ˜•˜’‹¢˜—°¬¹­³·¯¯ª³±¯±«¬©¤¦¤«¶¸¸¸ÄÈÇÈʸ¾ÈÉ¿¿ÀÅÊľ¿º²­¶±¥¢¡±¡¥”¡¤¤›™’”‘…ˆxŒ†ƒ||}uukslm_ee[Xa[[z‚{‚·©³®‹“eQSSPTRUXYVZil|‰zrj}wn~w²Ëż°²ÆÜÑÖØ¿¹˜–›’•ˆˆ‰‘ocehss•~gVUZ€_UUXhq]|…Œzh]_VQRPOOQsxTJSVX\gYWYJOSXWV\XYTQPRRPMNPONMLLLLJIHDFGIIIJKJJGEIE@.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ$SÀ¸ÊÁ¡–—­¨ ˆ™˜‹’¦§£{«¡œ¥­¥¦¢Ÿ¬©°¶¼±¡¨¾¼«¯±œ”ƒ™™•¡®¬«¨—‹‚‘{tpmcjru…‡Œ’ ³œ¿Æ¬Ÿ¯¯ÅÎÄ™–„Œ’—Ÿ˜˜¢¥£¦œ‘yu—‘ˆ—†‡˜˜™‘§œ”…¦¬¤˜™œ–†‰–¢­¦£¸µµ¶¦«±¬§ÁÁ¿º¬·³¨¤©°®µº¼ÁÅĂĂÇÇË¿Ăž½ÊĐÀ¾À½¾¹¹°¢–¤§ª¢§«ª¢¨¤£—¢˜–”„‰†ƒƒ}’‰Œ‡‚zsrwrokkga_f[W‚Œ‚v«°¥³†[RSQURSXWYZXnmyœ”„uuv|xx £¯µÁÏͯ»ÑÏƯáßʼ£¤‡…ˆ‹e`yƒ€™¦u`TKjƒn^Y]v‚zvgvaWPPRROP^xmNGR]]]qieXUMSZZ\WUYWSPPRMKPQPNLLJKJFFFBEIIIIIJJKEDCC3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ%ƒ¶ÄĂ·½±©¦³…Œœ ¤—¤™ £˜¬Ÿ›•”¨¥««µ·³¬¶­¨¶¹Éµº³¹ x“¬¬³±®¦ ˆ”‡ˆ„yq}xjvˆ¢œ§ª·Æ»¼¶Æ»npƒ¤£œ¨£¥«¦¦¤Ÿ¨°®´±§ ¦¢˜‚}‡ˆx‡™ ¢¡¢¤£’™¨§¨£“§¥—“Ÿ¥¶¸µº¬¢¥§¬¿Â½¾¼¹¯·´²¬ª±³¾½ĂĂÉËÉÊÏÉÊÍÏÍÆ¿¾·»¼¯¬§¤«¢Ÿ£œ¤ª³¥¤œ˜£‹‘w†’•™•€~†‚€|popmfzw]‡‹€tx¢¸À¦rNNPOQVSVXUTQn`mkhj^pdhnjcqÈẩĐÛƠÁÀ³»Æ×Û̃ÔÈÍÀ“{|™k{”ÆÉĘx~‰”y•‡ƒsl_a[WRNNMMQNReUIDMXX`qhd`WQNTSVVUVSRQOPNNONLHIHFFEFDFEIIHJIILJHEDA>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥³̉ÈÆ·¾²¯¯”‘™•¦¤¢©«¶±²›–‰„”¥®¨©¦´Â±¨˜ ­·¼ÈÀ½¶½§xy‹¢§¦Ÿ¯‘‰€‚|||…ˆ›¡¦¶ĂƱÓÊ˦†lju‘Ÿ¦¥¥ª¥¥ª¡¡¨¹²´¤¡¥Œ“•„ƒˆ¢¯¦¥¦¡¥©¬¡Ÿ˜‹”“•˜£´¯¼µ®¯° ®©¾ÁÄĺºº±¸»¸®±²±¸¹»ÀÇĂÄÎÑÎÄÅÅÄÁÂÁ·´®¨ª¤¯­¤£§ŸŸ£¤±© ©¢¡”‡”’‹—œ‰‰€—‘zlqqncoj\ƒ•ykc¡µ‚KNMMNZSRVVVRkkewp_jscqS[is²̃́ÉÄÀ·©¨À̀ËÎÍ˺׹‘——”xeŒ¬ÈÇDZµ¶¼¤‹j°¿¶™Œ„xjccimzRMLSPOUYLINZ`VZfcaWNKOSTRIMRRRPQMLLMKGKCFDCFDCIHHHHGHLJIFF@9ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿO}®¾Â¾¹·Ÿœª£œ£±®œ¦¥©©°‘‡ov˜±Ÿ–¦¢¦ª®´¢¡ª̉̉À»²¶¸±z{¢¥©¶›†„†z‚‡˜y¢•“¤©µ¸ÀÁ«‘ÈĐ¯Œƒw„“Œ ³¥Ÿ¯¸´°§¬¬º´« Ÿ¬£¢¥¢ ‰‡©±¬¯§¬³—›¤©§•”Œ™•¤¬©»½»¸°®­§²®¿º½Ă¿»»¸½Â¼±¶½Ä¾ÄƼÅÆÅ̀Ï̀ÊÏÂÆ¿¹¼·»®«³±°·¨¬«ª¤­¡¬·²¦©®£•ƒ€’‰ˆ‡~‚Œ™ƒ~{xrpi|gbh}cbd^ix¶œ^MNNL\URYWSRkh^{~riqgoPZbi Đ麬¾²±Ÿ§±µÁÉ·§»Ÿ“‰‡ˆ}‚‹‰v‹¢ÁÄͽ¹±Â¶´€œ´œ™˜‰…lojq}ŸƒMIGooMO\UGMTW[LXZRNMJKPPROPSQONRMLLKIIICB>?FB?IJIEFEAIHGGC<3ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿˆÇ³¨¬¼Æ¼¸´À›œ¬°¹¤«¢—‘EPyn’‡±—¡¡³¦·´µ¶´ĂÇĐÓȱ}”®Á¦˜„xvmxvv}¡§©¼££¶º½»ªÀ¬¦—’˜œ™Ÿ›¥« «­­³³ª­ªªº­¬›­µ¶¸¶°Ç¸¯°®´º¥´°µ±Ÿ—¬®´®¤¨®’£ª¨—”™ª¶¿»²³³½ÅÇÅ¿ĂÇ»¼½¼½ÄÉË̀ÄÄÈɹ̀ÙÏĂÂÈÍĐÊż²ª³¿º«ÂĂ¯§¬¯·±½º°¹µ±²µ²©–”˜–œ—““ŒzqpqogefakikWd[ih´“_QXZWQRSNTYTTYao{h\SSPY\\s‚‡¦Ă²¨«©™¯ª²¼»›’‡ƒyws9Wvwµ´°¹¾ËËÏ̀ÏỮ³™™osyqaaaa[RFCBHDCJOPRJMNFFGHGGGFRJMTSROLMONLMMOLJKGCAEHIGEEFCDDCFGDC@0ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ ¨¯±·´µ°¸¾Æ› ¬{¸» /w—ª§œ…R@Y~h•‰Œ›®»›·ÀÄ®»½ÎÊȽȖ‹˜£‹w‘wxx|u|‡©©´¿°³ĂÄÁ´²®ªœ“– ¬©±§£Ÿ¨ «´º¬¢¨¢´¦°¶«³³¼·¬¹­µÅÉĹÀÀ¦²·±œ£¨±µ»²¨§¡™¡ ˜¥£§´»ÁÁ¹©ÊÅÇÇÈÇĺºĂĂº¸·¿ÇÆÂĂÎ̀ÂÎËÓÇÅÈỞÊÁ¿¶ºÂż¥Å¾°§ºÀ·²¤¨µ³»¼º¼®¡£™–›™—“—“‰‹–€€}zrkinrbkfee_`^`k¥u[\^WRSZQSYPSVWhaaQUSZjh[hp…’— ©¦ª­®©± ª§|pj~‡`N|°¥£•¹ÂÅ̀ÍÖϪ€q{zmikibYcacjKCBGD@DHHKHGIGFEGJKMKQKLRPMMKJJIKONJJJLJHCIJIFFDCDECBECCG; ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¬¢»°¸²©¥«–¡Ÿ¨ƒ;o1ÿCÿÿÿÿÿÿœ™ ˜©»¹ÔÏÈ{©ÊÍÆÆ“”œ¥—–~‘ƒxtxƒ}|ƒ·²ºÊ¬¯ÈÏÎʲ¹°Ÿ–—¡¬«ª§±·¬¥§°¼¶º˜¦¬´··®¬®µ½¾«°®²Ă½ÂŽĥ²²ª¢«—³·»´°º¼±¦©›©­©¯µ»¾Â¸¤®ÇÈÎÊÔÉÅÀÄÅÀ»¸³»µ²¶ËÏÏĂÄÇÏÄÊÎÛ̉źº¶±¹Å±­Äļ¬¾Æ¹º·¬®·´³³»¶°¥¡  £ ˜™•˜••…‰…|wnpstule_dOJOV^~¸hq^WZ]VPPT^OVRWgb[Za\`l{†hk†”’œ«¢©–¤ ¥—˜„{kp†o†€£¼—–’²¯Áălj†sq†ti_aiZ`ji~RFECJJGFHGDFHHBIGJLONJIMMOLLLKJJNMMKJLNMKEJJJFEDEEFBBCCGK27ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ­‘Ÿ¹¶µ¶©z˜‚tŒ:#ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¡¯·ËÑǘÁÏɹ³£˜‘†‘•§‚‰kr‚••¸¸³µ¾§ÇÇÈÀȶºµ‘˜£©¨—¸¿º¿²¬º¶¹º»¶²¼Äº¯±¸º¿´¸Âų¿ÇÁÀÇ°°¹¯©±¾¾Á¿¯²¹¥«ª³Á¼½ÀÄÊÀDzµÅÆÓ̀ÍĂÁ»¾Ă¼¸¹¹³ºº·ËÍÏÈËÓƠÆ̀Ñ×Ờº±«­ºÅ¿¿ÁĂ¼¿¿¸´´®³¼´³µ³¹µ¢¥››§¨›Œ•¡’‹–™“™’‰‹~vy|x}g`cWQONW^ngihTVTPNOVRKOWV\[ba_ZŒ…}­³£¥˜¬‰µ³¡„zw|“Ÿ™ƒ1£È¶›— ¸ÈÆ™‹‹ˆwryp`\\d]hgizybfINSOPNDEEGHDCBNPOLJKLLKLLLMMLPNNMKLKMKHJHHHFGFGECDDFBG8ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¢¥›¢«¶¨nhÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ©ÀºÇ×Ï®¼̉Ѻ¬´¶Ă Ÿ’’›ˆx|{…†œ°¸´º¼ÁÉÎĐʸ®¨®„¨´·£ºÍȹ¾´¯Â¿ÂÅÏĂÄÆƾµ¼¸¶¸­¶Á¿¾¼Âº¶¼··°´²ººË½¿¹¯¯»»º½µÇÄÂÇÇĂ¾Â¿¿¾¾Æ²ÈÁ´¸°¶´°º»¹¼½µÁÅÎÎÆÄËÑÉÍÈÁ¬±¥¤©¡¶º¸³¸Âȶ»ÀŹ¹´Æ¾­©«±©£Ÿª¬¤§ ¨˜­ª¥¤£› Œ€††vqxwpl`\`b`]__lfPS[UXPMKGQ_QRY[VPP]{„{u‡‚•’—ˆw‹¡mo{´¬‘‡ˆ¢ŸŒ+W”¾¸³²¬´¢˜¨•†„umrforZW]^Xcji“™“u_]MIKKMCGFHKKJLOGGJLMKIGMMNLKIIFCKJKGHDDECBBDECDC@Bÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ—¬u5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ««°°¿ÉÏÓ­«¯··¤°¥¡¡Ÿ”€{z‡’˜§Â·³ÀĐÔÇÊÚ¶Ă³¦”‘¦º´½°«ÏÔȲ­­µ̉À¿ºăÊÇÂÆ´´´·«§©¦ÀÅĂ¿±ª«¤­¾½¹ÈÄÇƼ¸±¨µ¸¾³¯ºº»¾ÉÇÀº´ÂÅÅÇÉÆÀ¾¹¸µ®¤­³»§ °µ·ÀÁ̀̀ƠÑÉÊÉĐÈÁ¾Ç«­ª´¦±¼»»À±«¾¼¾³«°Æ¾«°²°±©¨³´¯¯Ÿ˜§¯®¬ ¥“’–w}rsssrm`_S_^]epXQVYVPOKLIIRPUW[RPX]u‚rjhk|‰}††{ ª`…~œª¦¡™œ‹uq¤¦Ÿ§¹ÂĂ·½³‘§´–‹†˜‡z{hj{dUSW^ccc~|—vsydSKJIFEFEDFJLIJLGHJMNLOPOONJKIIMJLLNIFEBGBCBCC@ECA?ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„…ƒZ’]ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ€‚–´³ĂÀª¶µ´œ¬­—Ÿ¦³’……œŸ­¯»®·ºĐ¼¿̀ÂĂ¹›Œ‡dœ²­´¶¿Ä»Ë¸£·̀ËĂĐÓäá̀ÀÅ¿»³ª¹¼¦¢™½·ºº£³µÁÇÆÂÈÀÄɾ±¦¬«¯´¶¼±³·¶ÉË°­º½ÅÀ½¸¸·¶¯¸»«¥£©¦¨§¨¤³ÁẦÎÓÎỜÎËÍÑÅÂÁ¹¥̉ŵ¹½²¿¶½Ç¶°²¥´¶··º·­¸µ±¶µ³µ²°§¤®ÁÀ©£²­¤œ‡’“Œ}}uvyvcb^bY]]ˆ^SWUKKMMMQLO\XZ\URX^s{w|ig{ˆ|‹“¬­}—¡ª›¨’“m ´§” ®Æ¾¼« °­–ƒ„‹‹rlk\VRR^nwimuyqljrTLKJHGHDDDLGFFJHHKKNOPPNLLJIIKJKIKKHFDBBCDDDE>DBA5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿo]fNw‡pÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ‚mŒ•Å¿´­¯·°´¤µ±‘ŸœŸ”¬¥£—µ¸¹ÉÏÂDZÄƨ€ƒ–¡¶ºÄÍĐ̀º¿àÑÁÈƯâÎăƠÍŸĿ¶³±Ÿ¬¸³ª´¯œ™¹´ÆÈ·½Á¶Ä¾·±¡«ª´Á¾¾¢°»ÁĐ´¸¼È¾³½³³«±« Á¼­§¤ªª¤¤³¼ÇÏÎ̉×ÓÊÊÖ̀ÅÉÉÓÜÎÅ´ĺº¹Æ¹²º¯³º´´Ë̀˳Ľ¼¼»¶³«ª­ª¸Å°µ¯¬°¬©—¢ “…tˆƒadcdXWbn—mUSNPPLNPOOKZccinZSWr|‡yla†•t…‰¥¤—‘¡•ªw˜H‹¥¬¡ŸÑÍ·¥’‰‘”Œ}{xn„—‚™™f]WWdzwmcninbekLKJIJFECEIEEGGIEELNMMNPPLLHIFHJHLKMHCED??BDHKCFB;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ«­uw‡µ̀ÁÊϱÀ®±«´­¢•ª»¸²œˆ££·»¹·´¼Ö̀¬‘‘œ¸¶²«ÊÛÛÁ²«ÚͽÏÏÙÜßßçáĐÇȹ««§¢µº·¯¬®¶Â½·ĂÇ̀¸È̉¸Á°»¢®°®·¿¿½À»´»¼¾ÄÅ̀Á¹³¯©±©±ÆÁ¯°°½´§ ¥¨¹ÂÍÉ¿ÀËĐÑÎ̀ƠÓ×ÜâáäÜÉÎÍĂÆÄ®²´¶¶¾µ»¸ÉËžÆŹ¸ÎƠÑ¿»½µª°ÈÂÀÀ¾¼µª££¯±“†|x…‡rmfaib^ar~WNGNQMKLRR^z{tit§o€{ˆ~“˜¤†Ÿ™§ªŸ€| –„zŸ±„‡±Âœ¥–™°“‹{•~x„’‹sqe[WhsjZ^`qvvbTTQMLLILMMEDKSSEEFDHKMNNMKHGLGFLQQPIHGGCACKM@=:ZTJDCGJMLNMNJHLJGMSSPLJJHCCDJLHF;,ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ”|»»º¿µ­¾É´·±´¥ ¥µ³§Ÿuz‹­¸ÅĂÎÔỬÆÁ¨›™§«½¿Â»½Íļ±ÜÖÆÑéèëéæăÜÍÏ̀¨°¬™™¤¬¿¶±¶¶Çʺ®ĂºÄĂÅË´¼²¹Á½ĂÆ¿ƿ¾Ä¼Â¿¼ÂÆ°³·«°²»ÈĐľÂº³°¥­¾ÈÊĂÆÄËßâäƯăæÔáäØÙỜĂÎÆĐľ¼¸À¿°·¹¼¶µ´©­Á¶¾ÍËÆÈÄÍ̀ÇÀ»¸»ÁÊĬ±¿²®ª¤•‘•vqjtrccc„oQIV[JNLKfktwiiYal†sj……zr|‘zqw{~w…ytwg‰{–”ªŸ—‚…‡–§±¥f\bx|{yyuwq`hadZYRSSQYTPPKKMJFGD=;:9AiNHAEFMKMLLJHMKKMOSLNPNJCJIIPMJ4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ´Ÿ¹µ´¯½³º¿´´²µ²ª‚œ•œ–™ƒ« ÖÙ×ÏÎỜ稤œ¨´ÂÄÏÂÀ;ż·´̉Ü̃àåÔàÚÚâÑĂ¹² ”¨¶³·¸¾¹ÁÄÁ¾¾»̀Ͻ¶²ÅÈÅÁ¾¾¿½ÀÈÉĂÆ̉ƠÈÉν³¦¥¢®«ªÎÂÀ¸ÄÁ¾½¾¸¶ÂÄÅ̀̀àêèää×ÓÙƠàÛÚØÅÅÁÅÉÀij¼Á°»Æ¹»¯¹¿º¼ĐÆÊ¿ÇÑĐÉÊÈÈÉ»µÅ»´³©±¬²·Çª§§­¯£˜•‘„baeo|]QMQX^MKSbr[bb^ext…u|{e`[XX[_MbZ[q™nw‹§–~Ÿ’invx‹˜¤¨µ©›}w]qlelspr_[_g\S_]UXXJINOMKJIDCB@@?<:;:<@EPHDCGIJJLNMHFIJKMKKMQNOQNVIK<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¥ª¯«µº¯­²¶»µ¦ —€u’“’™œ©¢àØÔÁÇÀÁÔѾ§¶±½É¹ÀÄÈĂÆË­®½ÖƠÓÖÚỔÚÜßăÆϺ¬¦§«¬¸¼ÊÈÀĂÉÍÁẦÈÀ´±½ËÄÄ¿·Áȹ̀ËÊỀËÅÏɽ©··½§°¹Đ̀À½Æȳ¹ºÊιÄÄÉËƠßçååÜààÏàÜÍÅÎÑÏ¿ÅÉ̉ɶµ¶­»³Äº¸Ă¾»ÈÉĂÉÅ¿ÄĐÆ̀ÇÆĺ²³·º°¬¶¯ª±Ä¼½Á¿›¨¤™‡‰ƒ{tqtkv^UJOcbMORe‡iddlpzrr‰xug\ZaVTSU[U`\w„„tw~¦‘tŒy‰‰grtƒ©§´¯£œw€{vkkw‚yl\cotYNQXRR]KIJLGEEDED@>?><;;=@@ATLDBFIIIMMNLIJHNMLOSSPOQTUS@5ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿº•¨ÁÀ®¢µ¶¯—zz•‘…‡‹‹‘w‰œ¥¡–’½À¿¸ÓÇÅĐĐÆĂÊ·¼ÊÓÊÂƺÇ̉¬¤½ÈĂ½ĂÊÊÖÏÜÙܹÀµ¾¾ºœ©­»ÖĐÆξ¯²½ĂËÇÁ»¶ÏĂ¼¾½ÇѶÊØÑÄËÂÍʼ°·½¿¿¾®©ÏĐľ»º·¼Á̀ØĐȽÈ̉̀×êêáÔáÜăà̃ØÎÆËÄĂÈÉÔÊÈÄÔĐ¼»¿¿ÂÉÊÅĐ̀×ỀÎÏÓÖÏÅʽ¹¯­¬°µ©¬±­¶½Ă¶±´©£›‰‰‹qfcpkdcVUZmdYSVuxqlojz|z‹‹zba\rl\fyLJLabˆ‘y¡£†“†ˆ“wxxsy…’®…—†—pukh†lemgotjZTSURNLIFEEEDFEB?=>>@?@CBBBDHCBEIMONPQMLHGMMLOSUSESW[A1ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†Œ”­ÎĂǾ²§¥“Yz‡ Ÿœ‘¤¡¤­¥º¤µ±Çº¸½ÈØÆÛÔÎÊÄÅĶµ³º­»̀½Đ̉Ç×ƯØ×Ưå½»™¶¬”’¡§ÂÑÍÉ̀ÉÄÁÊÄſͽºÈºĂ»È̀̀ÍÈ̀̀ÁÍÅȽ¶´¨º½ÄËÊÅÆÎÈ̀ËÊÀ¼ÇÎÛ×ĐÁ¼ÆÉÙçàƠÎÍØâÜØƠÈ;ÂÇĐÎØƠØƯĐÓËÂÁ̀¼¿ÇÈ»ÉÜØÖÑ×ÚĐƠÙ̉Ă¾¹²¯»¸°±ºËÁÍÑ®¸³«°««¨“‰‰nb_seofa_^z‡kQTpj€ntqvwpƒsj\Xspbu—B@?Qj€ˆ{oZ²¨–‰b…•ˆzoorw˜x€›vfl{w`f^[^YZVSRQMJCEADEDGC@=>=?IJHGDEDCDFFFFKLNRSOOLJONLUVVSPYFP:4ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿˆ›¡€v_i\|ƒ„…{t`N|{Œ¤¼¸´¨®®®¡µØÖ̉ƠÙÍË«¼½´µ«»Ä´¨ÈƠľ϶»¾ÊƠȬ‘›¤²ª—–¥¾ÇÇÆÄÆ̀ËÖ̉̉ÄŹÀÍ̀¿ÀĐÓ̉ĐÇĶ·º¸»¯»¦ª®¿ÍØÑÓ̀À±ÀÅ̀ËÅỀÊÑÉÑÆÛĐÜÑØÛÙÙÔÏ̉ÑƠÛÎÊÊÈÆÆĐåăàăØÙ×ÓÍÁµµ·±«¯ÍÔÔÑØ̉ÏÑËɵ¢±³¼ÀÄÁ¹É̉ÓÑÊĂ³´´©·´¸„sjjk}„sob_iƒ‡umw`towusmnYXYJLWaQPWEAN«_]]t?4@ˆ¥”—œœovˆ¤¥•”yz{ˆilee_]TRJJIDBA?=CDEB=:HQX]XVTOKJGEEDEINPORRPNLFPWTPRRSZQA9 ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ˜–ˆ†‡‘~bj^`dS}Œ˜”‘ƒIÿ}x ÅÇ̀·½Ă«½È¼ÇÑ̉ÈΫ¨¥½½ ª³—ºÓØÇĐƠ¾²ĂÍĂ®œ˜™¯Âº®²£®¿½ÁÆÓÓƯăßƠĐÊÅÆÈÓżÅÎÍÔÉÇ»¼¸´·¹½²§¯±ÂỜÄĐÀĂµÂ¿ÁÂÇÎ̀Ñ̀ÍÚÖ̉×â×̃̃ÔÓ×Ü̉ĂÑä̃Ù×ÊĂÈÄâçèçăÔÚà̉ͺ°­²·¿ÉÈ̀ÏƠÔÖ̉Đ̀½¸¯¸ĂÏÇ¿¾ỜʹË˽´´¯µº¹—tuhp‘–€}ogxŒaqik|€„’‚hTRJDGMTK@GJLt§\c\f:;h–”£–›ˆ••“udƒ‰Œˆ†|‘Ÿ¢jiuiirc^]^ZRKKHIB@ADBJHC?;=R[^]YYURPPKHDCDKLQTPSQNPNQSOPPTVYCB"ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÍ»q€vnx€y}vvÿÿÿÿÿÿ_ÿ–¤Êļ¼¹«¤¤¹¶¹·¶ÇÍ¡›¬»¶°®‹¸ÀËÑáƠÈÅƠẵ˶¬¬ÁºÍǶ±¹ºª´Å̉̉ÓÚ̃Û×Ï̉ÑÉÇ̀ŽȺĐÑÏÆκ¬¿ÓÆ»²²³¿×ÚÓ̉¿¹¶¬­²¹ÑÍÑÔÓ̉ØăßâáÛÚĐÓ̉á́ÖƠÓà̃̃æÍÏƯßƯåéäăææÚ̉̉¹§¥½±µÄÈÍÏÔÖÖÔÎÈÆÀº¾ĂÑÇỀÉÁ¼´Ëϲ»Ä³·½®plfl›Œ„pno}zf]h„”“r~j_VOIOOS@?KaZP-JX]nS~‹¥Ÿ˜—¡¥€rrnƒ†ˆˆy‚‰Œ†o_paaqqfa\\YQMHIJC@ADBIE?<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ»zLÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿµ…–§œ”¦¥˜¥±¸ÆÈ»ªÁÅ·°³¼ÄĂÊË××ÙÏÚàăĐ×ß̉Ñ×æâáØÊÆ¿Á¿¹¶Å¼ĐåÜß×æäăåÔÏÏĐÏƠàÜáâÑŸ¾­³´»ÁÆÙÓÖÏϹ½º¿ÊÎÏÎ̀ÈËÓÍƯ×ÏØà̃ßçëíïëååăäáƠỖçÛ́èåä̉ÑÏ̀ÄÀ¿¿ÁĂººÉÊÇƾÀÇ̀ÍÇÅ̀ÊÁÆĂ¿À¾¯´¹°¾·À¼º¸¨®¤¡~tuz’¥¦œ˜›‚Ÿ˜ˆ™™Ÿ˜Œ‹€i[JPSWKJDDY[HBJ`p‚}€z|ƒ£‘qu~––Œ|dknpwi[iyncd[WTYMMLKGDDC@F@>>?EGJLNRROORSSOLHIFGIHTTSYYVYQNS[M<;(ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤’”’[†‰•…¨£°µ²ÄĂÍÍÛĐ¼ÉÈÅÖȽ·¿Ä̀Óăßâäåăåååäååß̃ÖỞâáäăÔáå̃åäẫâÚ̉Ơ×ÛäÛØà̃ß̀­¶³±®½ÇÊÚÔ¹ÓÇ·´ÀÉçêêééåææåÜÔÍËàæîđííếḉăâåØÖÑØƯåääÏÅĂÊÄÎÎÇÈɽỂϼ´½ÀÄÁÏỞÊÈÎÍ̀ÂÄ›­¾¼¼½Á¶¿´¯´®¨–—”‹¨ ÉÁ©µ~wmmqmŒ’ƒpxZHTHBB?EXM85@Gbo{VRR[VWW­™|s‚ˆ…‘r{vekmjcmiZXSPGMHHJFGGGHDBGD@A=DNMMPQNSMNPTOPKNJLRVST]ZYVUWWWGBC.ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ„€{ƒ|‡”—¨½°²²¹Á̀¢Å¾¸ÁÁ°Å·«½ÎƠÏÎàäăåâäääæåèéêæßàÜƯßåèàáäâƯâăàéèäƯàßăåàßâß¼»¹±´¶¼ÈÜ×̀ÂÀø²ÊÖâêèèèâáåèæä×ÅØà́ííëçăăăççåäåääßÚç̃ÎÍÑ̀ÎÑÙÓĹÍĐÎÀ®³¹¹»º»¹É̀̉ÊÉÎÀºµ¤£¹½ÀĂÁ¾°·¶¿±±›’¡‘’‘ˆ‰­ÇÙÇ›piepgas„z†cHOFEE@K^P-:A@]`joNJGUM]bw‹†}w{‰ˆ…‚udhcg]df[SRNPMKOPQQIFFIEEIGDGKNOOKNSSUVS[WTLQLHQWTTZ^\T^WcMDB>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÊ¡‘ÿÿÿÿ|uˆ——” “¼Ç¾ ÀÈĐÈÎ××Ⱥ´»À½ÁÔÛßÙØƠââççéæèèçéêéææéáäéçæäååăâçéèæẫåäåáÙáăÜضº¸¹¸¹½Ă¾À»ÀĽÆ×ÚØåèêæêéåçăà̃ƠËÎâßæè́ëæƯâäăëíí́èíçĐÎ×ßƠÓØÜÖÊϽ»Á̉Óằ±½ÆÀ°µµÏËÉÅ˸²­®ºĂÁÀ¶²·ĐÖ©©¸º››¤‰‹‡x•™®ª§meg ¤‰}™{~y_WIDkTABHQ. +#IGFEFGMCFM^b_nbo„}vp~vrsefhdytZQQQSZUTRPMMHHEGNBORPTZ[[dfcaa[b^[\YPOKT[^_SPSUhgU@:>ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¹£“ÿÿÿÿÿ”¦ª£ª·¶»½ÖØƯƠÍÖÏÂÇÄÄÅÈÚÛÜÛÜÛÜåèèèéçæêë́èéëíëêæçæåçâåçå̃áâƯÜæèåÜƯỮÚÎÓ¿¾»º»¼ÇÀ¿¼»Ă¿ÂÆÄÜäæææëëêåÛăäăăáççæèæéêéèëçâîíéêëëÜăàßéßá̃ëƯỜ̀ÈÈÉ̉æ¹ÇÔ¿ÆĐÇÈÁẰỜÇÂÆÀÀ¼À¼¿·Ñɬ¨«®›¦”‰˜†|v}˜¤fmg“¦™w€p{c\NOHBBDFN'7HE@@@=GAHMOV_\bk†Ÿ„wsrzyrrlhhic{‚kXUXWYYYYWRSLKJHHKDS[ZW`e`ikjeccf[ZZ[MLKO`^`RTUYk_K72ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ£¯ÿÿÿÿÿÿ’‘™²«¥§¡µ³¥ÀÏÀÏÑÆÆÆÇÆÇÆÆ̀̃ààÚÜâÚääççäăåêë́ßéë́êëéåăƯçåååäåăçáäæèçæåæçà̉Í¿·¸¿Ê×ÓÍÇȺ¹¼¾ËÍăææåăàèáëëî́ëëè́æêçæééæèçéåèíñêåéÔØÙßä̃éçççÖÄÄƠ×ÓÚáÓ×ÑËÎËƠĐ̉ÓÖËÉĂÀÅƼÀº¿Ïžµ»­¡œ–ˆŒ”•Œˆ€w”–—ˆ››ˆ{pg|yhMPLDBDBC@> AA<869:=@HGP^f[`lu˜“ƒƒt‚…yrkddgkqegc\_]]\ZXWVSRVLHKJWdclshthiihfgbj^YVWRLIQeb\`^_\YP<;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÅǽªÿÿÿÿÿÿ°¥³½´¤§±«·ÂÎÖàÙÊÎØÎÆÅÈÏ×Üăâàääáåååăääêéêḉêè́ëééØ̉äåäăææäáçÙé́ëéçèçầËÅ»¹̀×ß×Ü×ͺĂÊÉ̀̀àâçâçëêé́èïïíîîïíççèéçàæè́îîååæăåèëë́åëêëêçàÇĂƠăÎÍÚáÙƠĐËÑ×ÏƠáăÔÎÍÀÂÅ¿Áº¾Í¼¸·¸¸£“ˆz{ˆ|‹‡†‘‡Œ˜{™–…r_ypjcV^E>B<=:; 7>94358:@HJ\q|dS^p‡…”—ƒ‚}od]d`]]kef^a_ad[XXUSRLIKUX^kxwtzmjgeehgga]WVUSNX`_YdgdbRC;ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¹¢ÿÿÿÿÿÿÿÿÿ³º±̀Àª̀×»äåâÊàäÛâáăËÉÍ̀Üáåæèçèçéçäâá̃çéé́́ëëíîïêØ̉éäæäæếëëëçëêééíèăÇĐĐÓÏÊĐ̉ÄÆØÖĐÈÑËÏÔƠèèçæèî́́çåëëïî́ÜÔä́êêëáéé́́ééđđé́́çạ̊đî́ëæé̃̉̀̉̃ÔăëểÔËÔØæâââƠỜÎĐÈÏÈÀÀĂ½Å®œ”•€™‚‡““†“„„—Œ‚vpevjg]bli\Œ^B@;:56;!4:64215?ERfd[ZIJOxltƒ—unliYUXXeaaabk`Z^a]YYXSNLITc`\_dVN\s‡udbgheb`\[`^\T_cfmlfY@/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¶¹ÄÓ´×ÏĐ̃ÙƯå̉̃̃ĐÛƯßÅÈÊÎÖÜ̃åçæääçæäáßàæéëîîî́ë́́́ăèèåéèççèë́êè́èéëëåÓËÜÙÑĐĐÙ×̉ÎƯØÛÇÑÉÓƯéåäéæäêçëëëíîîëëâƯåëîïîèééëêêçđïéëëêẹ̈ọ́îîêé́àƠÛáçêééåỪ̉ƠéçÜ×̉ÎÎÖÚ»ĂÆÀÆÇ¿¶¯y‚‡‚¯°º®“›˜’¤—|rw_`Vcmlaz’—I>>>>BCL9:7639:==DCEZUUYyprrzefc]ZVVXW^X]`\[^\_^_]_]XUNNVggc`jkOYtdbghc_][W`k_X_cjdTG7ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¸¹ĂÑ̃Ôº¾âÍËÍØ̃̃ÏÜăÚÆÄÇÉ̉ÙØƯççêèèåæăäăçëë́íîí́íííáăèèèéèéèçèíåîèææèÛÈäØ̉ÖääăØÛÖÙÙä×Ö̀ÙâëçâÜææë́îííçèááëé́íë́ë́éçèëêçåçèèë́í́îđñîèâçêåàâăßäèèææçßĐÙÚ×ÓÚÑ×äâĐÓÓÊÆÂÁ°®¢”“ƒ…©Ăº¼¶{ƒ†¥¢“—§¡™‡–†vYUTQVkibemE=NZsªRUPTQR`YVUVZZUab_\]_`_|jd]\]\opbjkfmbbk`Z\`ZWWQRLJNz{l]U<ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿĂƠØØĐƠÔăèéåÙ̃ä̃ÓáÖ̃ăÊÎÍÎæÜ̃éíí́́êëæåéêéêéèêëêéæåæÛçêäé́íïîíḯ́íäêÉăƯáâäâàÙàƯâàǻèäçÔÏèéëèéêâèåẩØ̃ÚÖÜ×ÚăÎ̀ä̃Ư×ÍØàÚÜæî́×ÊçƯçßƯØÊƯéíëƯÔƯâăÑÛÄÆÑâêéíéîññîîîî́êéộçÑÑË¿°°¦¡ukg|†Œ¤‘“‚ˆ‹‰Œ‹…‰ˆsmnTS_ht‚¡h]??977AaRPYURKMQULRf˜¢¦ƒtrXNWNSNNTUST^YWUXW[dba\\a`dYRQXbZka^`a_a_a`^d_\XRQNXjebX6/ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿºÄ̀ÜƯƠØƯăâļÙÖÊÚáƠÏÊÊĐƠßƯØè́íîîíí́íåëéèèæçéëëäæâßÖßäÚæåêïïîđíîîà̉Û́ĐÜåääß̃ååèääåáăàÜƯêë́êææÛÜâăÍÄ̀ÑÓÙÖ̉ÛÈĐ̉̉ØèÆÑâÙ×ë́úîƯäâéÚÛäØÔéåæÙÜèăáÙÊÎƯÜ×êíđë́óñëêëåếéị̈ñàƠÖʦ¤«w_ˆ „x“ ‘„‘“’…Œ–•xka]]nsyŒ’paPE?86D{ej]”£xVWRO[]eU]\UFGCBACCIQSRQRNQTUZY`b_goniec`gvyprqo{|vtwwmdhMÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ«Å»°Ö½ÄÍÎƠâÏÊ°Æ̃ƯƯçëÙßåëäçỖÜÖÏçïàĐ̀̉ß××đëắäàïñđïïààăééîíèâáÙƯáææĐííÛéèØ©ØăêêăßßǻëëêèéëíéëđåđđëíâẫØÑÚđđđíỨùöơñïêîăäèååäñđîăấƯĐÚáæăåïíë́çáßåèçßÚáä×ǻéíéæăººÀ±¦†o•¹»¨–‘˜ƒ™©M 'AKJKLHKWORU^jjddhc^]\bpikpzyvstuuffZH2ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¢Ă§¼é»É̃νÛÜ̉¹ĂàáƯåÛÙäçë́ä×âă×êíẩÏÇÅÜÍ̃ïéåǽïđïëññíëâéêçéêÍØàØäăÍêíéêîé«Üéæèæâåäååàáè́îïóóị̂ẹ́öơơơèÜƠĐ̃ññïô¼ïôơñđíăăỮâñóăÚåäÛçèÎÖÏßßäíñîîîăƯäæƯÚÜăêÜ×éåèíëååǛĐº¤0 \̉³»¢˜£›”‹™‘Œkœ€€„‹ƒµ¬£€r‡¢·±`†•YGZƒ¡ra|Ç›miMRiufYHD?@FD>@@DHBHHLU`TUZhfrmhzveadiogilt{xppuocICÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ†“ĂắëÎ̀êËĂÄÈÔÁÂÙååêßÜâëêèääßá̃ƯæßÊÇĂÏƠÑÚæâßáđîïîñđóí́äççăî̃è́íåèåÊæéçêë¼ííèéåçèỮæëí́íïîí́đóöèôôñçëóăƯÜǗïñöâ´Úçæ̣óíƯÑÏçđ̣ƯÖåçèññÑÚß×ÖäîîññîëáĐƯàÖÚéåæçééïçââæäàß±ea¬Đɺºµ›“›‘–£¦©¢‘‹vˆ•…Œ”¯ÖĂÆot¡©ĂÂl†Tae†º¬pƒ̀¿qj_`l[TSGACkaD>EDSGIFXX[Yb`\aihhu|ufepqmhsu‰wtsnjP:%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¹¹ÂÛëäƠÚíË·»¹¼¸¸ƯàæäâƯÖèíêç̃̉ÑỪâêË̀ÉÖÏÉÇâăàäđïîîïïôîéæ̃ƯàåÖặç̃ßåÛđ́ăë»Ïîđêếêèêéçæííë́đïîđ̣̣íöøôïóñị̂́êëïđñæàÊÓóùøàæƠỠâñ×Îë́èá̃ÖßáÙÙâçíî́î́̃ÎäỮÖçăçîḯ́âàßèåƠ͹¤} ÆÑÑŵ½®¤¥¨ ™¯´µ³£ £™””‡‚‰¼ÂÅsrz­µ°Ç}‚‚Wad‚¢° iiÅdd`fm^RKMG‡ia\XEJGLPPW^TPffYVaiq~pkjgpyr‹„y‚||{yqP8&ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ~ÿ‰¾´ÆÑỪ¾Å»»ĂÓĂÖ×ăÛẫ×̀ÓäÚÎÎÍÎ×çá×ÊàÓ̀ÚÙÖâéééèèïííḯéáßÖçâƯÍẮæäçäëçỮºđđẹ́ộđçéèîđî́îíëô÷ơëêääđïïíị̈ôèÛßẹ̈đí×ÙÖåñéÈôóæé××ÙßƯïíêçíçẵÙææêîéë́ÛÛÜÙåæáíîïđëǻàêáäĐẳ‡¤¬ÈÊ»¼¼¾²¥£¥³®³¶¼®«£¤œ¤•¬¢–“¬ƯăÖ„‡ˆ¡¼„^Tn|ƒx±PU™j]_^`V`V^TLOJA@ALMS[gj]TUQVUYTV_orn_ghomŒ€{Œ†~vfUL!ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÓÙ̃É˾½ÉƠļẲ̉×ÓÛ̃ÛÙØ×ÑË×ÓàæÜáåØáÜÛº×êæèçæéè́ïêêéáßàÚèăß×Ứ̃íéếí×́âíîóé́ïí́ïííñîíôôóôäçợëéïñïộ̣ëÚĐ̃îóơïÙ́đọ́óîđêûư×̃Ưáîçêïñ́åäăÙăèçêííîăé́åéèàéḉîñïëí̃×̣áßá©»ÏÇÍ˼Ƚ¼±ªªµ··ºÂ»­²¯¢°«¥ª¦¤“‹«—ª‹‰‡v…¼•‰m[|‰„ƒpˆ”SNTbkb`Y\_RKIJHHI=PT^gqm[RPKNRV^`noqa]osyu‹ƒƒ‹„€mVL)ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÆÙåÆÖµ²±ÆϽ¿ÔĐËÅÏáÖƠÛÈÅẲßÚÛêà̃êƯÁ̉عÙêëàÙèííïëîâáäÛÜèáăäƯÍÛêçé́àÑåéäó÷ḯ́ëíííî́ñđđ÷÷øôïîâÖÖÏØđñöñđæÛÏÖị́́ẹ̀èçíçÜå̀̀ÛüÍÖÖëñîíñôđăßêÔåèêëñïïêàßßêééâèǻđïèäßåơäçÙ¹̉ƯĐËËÍÄ®µÀ¢ª­¥Ă½¹µ§ª¬ ¨¦£œ•¥©—”˜’‘nu ’’zz™• y_u¥TNP\Q_a_eY\TJGIJJKHEUolk`RLIIJNUXaaY[Y`gquv˜‰~ƒ„vhM'ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ§”¸¢°¿ –­ËáÖ¯ÆÎ×̃ÚÉêåÑÍÏÀÀ̀ØƯÙîíèäÚƠÆáëæàĂÏƯáèçíị̈í̃ØÏ×̉âßàÜ¿¶åàâÉƠèåéîôîđñññđ́íđññơöö÷øơôêñêî̀ßïññëé̉ÊÂçäô÷äØàåáêÜåăÛâ¿ËÆƠîíëïïïçâà̉ÜàÚ́íîóçêçââçíëëàỨîêàÖåèÑÄ·̀ÚæËÅÑĐж°¤³¬°²¬´¶´®¯¸ĂÀ²¦›’¯µ¦ “™  —–{‹Ÿ®™ƒK›}tozjhmYi^}ly‘‚h`VQGLNQDMJ[`X]ZVSGIPUWVSRYW\kyŒƒ„˜‡wfaKÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿª[ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¤È³¢™›™§Ä×Û³¨ÊÓØÔÓÄäÛƠÑÏÈĐÜàäßæíå̀̉ÜÏ̃ààå×ØáßÜáæ̃îđáåà̃à̃ß̃äÔ´ç̃ßÂƯçÚâèị̣̈ọ́ñïíđ̣̣̣÷ơôñöçÛî́đñÓÜïđëçƯÙĐÖ́åáàäÜÑÔßêÜ̃ƯễÄÉÏÑáđọ̣̣́ßëÜ̃Û̃Øæïîæëîèèèị́æíêăé́éçÊÎâ`oªÁàăĐ·½ÍÀ¶¿´°¶È»²±œ¦µÁă·o¦©œª¯Ÿ¢™–›¦ €–Á¬sƒ—zr„l`‰¦^}nve‰~†~j\TKMNNONEHU\`VZUORVTe_X[`dm~‡‹m_R ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÁƠ‰ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿqs¬´¦«›·¡¡­¶À¥½¹ÆÙÙÚâăƯÏÉÅÍâççăëÜâßÖØÛââá×àØÚíëÜßçæèăèèèèâèâÑä̃ƠäấÇæåÂáçëñđ́ñëị́î́ïị̣̈öø̀ÛÛîđđëÜƠƠäâçƯĐÍ̉ĐÄË̀ÓÄ×ÑîöçæåääÅÊÆƯÛÜæ̣ơđéèáäƠÜÜàéïéơêçèçëëë́ëæçíÙßÅĂáKi¸¸â¿¾±µÂµ¿¿Í¹´xu¯®­v«½µ¢O„£‘ ›”›Â¸¡…¬§Ÿ}’–…}yqhk¬—jpu©}|je[YRMOX``PKL[VYRPMS\ec_jiw~…„‹…™eW%ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ˜Ù¦ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿŒeo¬£¯²­¯ ©¦¯¤³©ÂáäàƯÛÅÀÇÎàäæêî́ÓÚÑÎÑÜæçâäẫÑÙÚÙßäâíèåæåéẫ̃Ô×âàßẫäéê³ăâï̃é́îđđ̣́îîị̂́éÙÇ×ßîêçîáÓØàƠÑ̃ÑÏÍáÔÙÙÅÁÈÈù́ëăßẫÎʺÍĂƠåṇ̣̃́éççâåîđị̂îîíêèééëëéăçêë̀ÙÉáá^€¾ÎÍ·³Í²«¼³µ½¶¨¡¶ª«ªˆ‹¿Çªh”| 8‚r˜Ÿ’””ÆÖų•®±‘„ƒ¢•’‚|rn|…ˆ”‰±¦{{plifZVTW^hYUYSGZXTW[_jd_bpf„q†y…ƒ”ya1ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÄ°ÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ¾®¹¬¦«¾³¯¥ªº¸¤Ÿ¸·Ç¤œ¥¬àäăäăÜÙ̀ƯçåÏÓÙăèëêăáá̃̃̃à̃äàäáààååæçàÜßâçƯÁßéÚ¬ắçíḯđḯóđïñṇ̃íÖÛéØôôôñîêÙĐ̃Û̀ɱ´ÇđïäÛăÙƠç₫̣ïïçæ̃̀ÎÚ̉åèØđḯíêṇ̃́êïîôöđó́ị̈́çè́́ëëđç̃ÊÄζ¼´¨s½»º±±³¶»·½À±³°¢E(03)Y?#**;¢‹…’”•῭ÉÇÇÑ°‹•Ÿ’’œ”¤ÍÄŸ“‰„}pdjaTSZ[WSOKOVWRZ\`kolldr‚}‰}qpqvkUÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿÿ \ No newline at end of file diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_1.tif new file mode 100644 index 000000000..cddd6dce1 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_2.tif new file mode 100644 index 000000000..78efca3f9 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-01-01_1_2.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_1.tif new file mode 100644 index 000000000..3bad0f7e5 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_2.tif new file mode 100644 index 000000000..2dfb3a71d Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-02-01_1_2.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_1.tif new file mode 100644 index 000000000..ba98f7f30 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_2.tif new file mode 100644 index 000000000..d8a5d5318 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-03-01_1_2.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_1.tif new file mode 100644 index 000000000..8d2453140 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_2.tif new file mode 100644 index 000000000..39302eb1d Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-04-01_1_2.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_1.tif new file mode 100644 index 000000000..186d4cd8b Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_2.tif new file mode 100644 index 000000000..d6d0328f2 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-05-01_1_2.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_1.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_1.tif new file mode 100644 index 000000000..084f666d1 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_1.tif differ diff --git a/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_2.tif b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_2.tif new file mode 100644 index 000000000..d9f940f32 Binary files /dev/null and b/test_data/raster/modis_ndvi/tiled/MOD13A2_M_NDVI_2014-06-01_1_2.tif differ diff --git a/test_data/raster/multi_tile/.gitignore b/test_data/raster/multi_tile/.gitignore new file mode 100644 index 000000000..8fa5b33d8 --- /dev/null +++ b/test_data/raster/multi_tile/.gitignore @@ -0,0 +1 @@ +env \ No newline at end of file diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b0.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b0.tif new file mode 100644 index 000000000..6d4f257c1 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b1.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b1.tif new file mode 100644 index 000000000..a9d851746 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b0.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b0.tif new file mode 100644 index 000000000..bd515f97b Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b1.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b1.tif new file mode 100644 index 000000000..200e170fe Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x0_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b0.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b0.tif new file mode 100644 index 000000000..78cfd259a Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b1.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b1.tif new file mode 100644 index 000000000..29cf95b22 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b0.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b0.tif new file mode 100644 index 000000000..90c83512c Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b1.tif b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b1.tif new file mode 100644 index 000000000..e443ae24d Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-01-01_tile_x1_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b0.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b0.tif new file mode 100644 index 000000000..1fa837928 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b1.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b1.tif new file mode 100644 index 000000000..6c2a86362 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b0.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b0.tif new file mode 100644 index 000000000..8ce8dea94 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b1.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b1.tif new file mode 100644 index 000000000..c32e72af4 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x0_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b0.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b0.tif new file mode 100644 index 000000000..0123512e6 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b1.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b1.tif new file mode 100644 index 000000000..c9a90553d Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b0.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b0.tif new file mode 100644 index 000000000..e3c442292 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b1.tif b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b1.tif new file mode 100644 index 000000000..113ab1323 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-02-01_tile_x1_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b0.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b0.tif new file mode 100644 index 000000000..6f31e0169 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b1.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b1.tif new file mode 100644 index 000000000..b8553f7c0 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b0.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b0.tif new file mode 100644 index 000000000..1d35822b1 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b1.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b1.tif new file mode 100644 index 000000000..215e99809 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x0_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b0.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b0.tif new file mode 100644 index 000000000..bad7720b4 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b1.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b1.tif new file mode 100644 index 000000000..5277bf740 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y0_b1.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b0.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b0.tif new file mode 100644 index 000000000..9a52acc29 Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b0.tif differ diff --git a/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b1.tif b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b1.tif new file mode 100644 index 000000000..ec6034b8a Binary files /dev/null and b/test_data/raster/multi_tile/data/2025-04-01_tile_x1_y1_b1.tif differ diff --git a/test_data/raster/multi_tile/generate_geoengine_tiles.py b/test_data/raster/multi_tile/generate_geoengine_tiles.py new file mode 100644 index 000000000..fd3067147 --- /dev/null +++ b/test_data/raster/multi_tile/generate_geoengine_tiles.py @@ -0,0 +1,110 @@ +from osgeo import gdal +import numpy as np +import os +import glob + +# TODO: turn input bands into own tiffs, one tiff for each geoengine tile in the order of the engine production +def create_geoengine_tiles_from_raster(filename, out_dir=".", tile_size_px=512, origin=(0, 0)): + """ + Create tiles from a raster file like they would be produced by Geo Engine. + """ + ds = gdal.Open(filename) + + if ds is None: + raise RuntimeError(f"Could not open {filename}") + + width = ds.RasterXSize + height = ds.RasterYSize + bands = ds.RasterCount + + # Get geotransform to map between pixel and geo coordinates + gt = ds.GetGeoTransform() + + # Compute the pixel coordinate of (0,0) in the dataset + # x_geo = gt[0] + px * gt[1] + py * gt[2] + # y_geo = gt[3] + px * gt[4] + py * gt[5] + # For north-up images, gt[2] and gt[4] are 0 + px0 = int(round((origin[0] - gt[0]) / gt[1])) + py0 = int(round((origin[1] - gt[3]) / gt[5])) + + min_tile_x = (-(px0)) // tile_size_px + max_tile_x = (width - px0 - 1) // tile_size_px + + min_tile_y = (-(py0)) // tile_size_px + max_tile_y = (height - py0 - 1) // tile_size_px + + tile_index = 0 + for tile_y in range(min_tile_y, max_tile_y + 1): + for tile_x in range(min_tile_x, max_tile_x + 1): + # Compute pixel window in the dataset for this tile + px_start = px0 + tile_x * tile_size_px + py_start = py0 + tile_y * tile_size_px + + px_end = px_start + tile_size_px + py_end = py_start + tile_size_px + + # Clip to dataset bounds + read_xoff = max(px_start, 0) + read_yoff = max(py_start, 0) + read_xsize = min(px_end, width) - read_xoff + read_ysize = min(py_end, height) - read_yoff + + # Prepare output array, fill with zeros (no data) + dtype = gdal.GetDataTypeName(ds.GetRasterBand(1).DataType) + np_dtype = { + 'Byte': np.uint8, + 'UInt16': np.uint16, + 'Int16': np.int16, + 'UInt32': np.uint32, + 'Int32': np.int32, + 'Float32': np.float32, + 'Float64': np.float64, + }.get(dtype, np.uint16) + tile_array = np.zeros((tile_size_px, tile_size_px), dtype=np_dtype) + + if read_xsize > 0 and read_ysize > 0: + # Read data from source + data = ds.GetRasterBand(1).ReadAsArray(read_xoff, read_yoff, read_xsize, read_ysize) + # Place into output array + x_insert = read_xoff - px_start + y_insert = read_yoff - py_start + tile_array[y_insert:y_insert+read_ysize, x_insert:x_insert+read_xsize] = data + + # Write tile to file + base_name = os.path.splitext(os.path.basename(filename))[0] + out_filename = f"{out_dir}/{base_name}_tile_{tile_index}.tif" + driver = gdal.GetDriverByName('GTiff') + out_ds = driver.Create( + out_filename, tile_size_px, tile_size_px, 1, ds.GetRasterBand(1).DataType, + options=["COMPRESS=DEFLATE"] + ) + # Compute geotransform for this tile + tile_origin_x = gt[0] + (px0 + tile_x * tile_size_px) * gt[1] + tile_origin_y = gt[3] + (py0 + tile_y * tile_size_px) * gt[5] + out_gt = (tile_origin_x, gt[1], 0, tile_origin_y, 0, gt[5]) + out_ds.SetGeoTransform(out_gt) + out_ds.SetProjection(ds.GetProjection()) + out_ds.GetRasterBand(1).WriteArray(tile_array) + out_ds.GetRasterBand(1).SetNoDataValue(0) + out_ds.FlushCache() + out_ds = None + + tile_index += 1 + + ds = None + +if __name__ == "__main__": + input_dir = "results/z_index/global" + for filepath in glob.glob(os.path.join(input_dir, "*.tif")): + if os.path.isfile(filepath): + create_geoengine_tiles_from_raster(filepath, "results/z_index/tiles", tile_size_px=512, origin=(0, 0)) + + input_dir = "results/z_index_reversed/global" + for filepath in glob.glob(os.path.join(input_dir, "*.tif")): + if os.path.isfile(filepath): + create_geoengine_tiles_from_raster(filepath, "results/z_index_reversed/tiles", tile_size_px=512, origin=(0, 0)) + + input_dir = "results/overview_level_2/global" + for filepath in glob.glob(os.path.join(input_dir, "*.tif")): + if os.path.isfile(filepath): + create_geoengine_tiles_from_raster(filepath, "results/overview_level_2/tiles", tile_size_px=512, origin=(0, 0)) \ No newline at end of file diff --git a/test_data/raster/multi_tile/generate_test_data.py b/test_data/raster/multi_tile/generate_test_data.py new file mode 100644 index 000000000..9f8ba0a42 --- /dev/null +++ b/test_data/raster/multi_tile/generate_test_data.py @@ -0,0 +1,242 @@ +import os +import numpy as np +from osgeo import gdal, osr +import datetime +import json +import random +import matplotlib.pyplot as plt + +# World extent in WGS84 +minx, maxx = -180, 180 +miny, maxy = -90, 90 + +# 2x2 tiles +tiles_x, tiles_y = 2, 2 + +# Tile size in pixels (non-overlapping) +base_tile_width = int(1800 / tiles_x) +base_tile_height = int(900 / tiles_y) + +# Overlap fraction +overlap_frac = 0.25 + +# Adjusted tile size for overlap +tile_width = int(base_tile_width * (1 + overlap_frac)) +tile_height = int(base_tile_height * (1 + overlap_frac)) + +geo_engine_tile_size_px = 512 + +# Bands +bands = 2 + +dates = ["2025-01-01", "2025-02-01", "2025-04-01"] + +def create_tile_tiff(filename, width, height, data_array, geotransform, projection): + driver = gdal.GetDriverByName('GTiff') + ds = driver.Create( + filename, width, height, 1, gdal.GDT_UInt16, + options=["COMPRESS=DEFLATE"] + ) + ds.SetGeoTransform(geotransform) + ds.SetProjection(projection) + ds.GetRasterBand(1).WriteArray(data_array) + ds.FlushCache() + ds = None + + +# WGS84 projection +srs = osr.SpatialReference() +srs.ImportFromEPSG(4326) +proj = srs.ExportToWkt() + +# Pixel size (deg per pixel) +px_size_x = (maxx - minx) / (base_tile_width * tiles_x) +px_size_y = (miny - maxy) / (base_tile_height * tiles_y) # negative + +# Overlap in pixels +overlap_px_x = int(base_tile_width * overlap_frac) +overlap_px_y = int(base_tile_height * overlap_frac) + +loading_info = [] +loading_info_rev = [] +pixel_values = set() + +for date_idx, date in enumerate(dates): + for band in range(bands): + # create global raster for each date and band + global_filename = f"results/z_index/global/{date}_global_b{band}.tif" + global_width = base_tile_width * tiles_x + global_height = base_tile_height * tiles_y + global_gt = (minx, px_size_x, 0, maxy, 0, px_size_y) + driver = gdal.GetDriverByName('GTiff') + + print(f"Creating global raster: {global_filename} with dimensions {global_width}x{global_height}") + global_ds = driver.Create( + global_filename, global_width, global_height, 1, gdal.GDT_UInt16, + options=["COMPRESS=DEFLATE"] + ) + global_ds.SetGeoTransform(global_gt) + global_ds.SetProjection(proj) + + global_rev_filename = f"results/z_index_reversed/global/{date}_global_b{band}.tif" + global_ds_rev = driver.Create( + global_rev_filename, global_width, global_height, 1, gdal.GDT_UInt16, + options=["COMPRESS=DEFLATE"] + ) + global_ds_rev.SetGeoTransform(global_gt) + global_ds_rev.SetProjection(proj) + + global_ds_overview_filename = f"results/overview_level_2/global/{date}_global_b{band}.tif" + global_ds_overview = driver.Create( + global_ds_overview_filename, global_width // 2, global_height // 2, 1, gdal.GDT_UInt16, + options=["COMPRESS=DEFLATE"] + ) + global_ds_overview.SetGeoTransform((minx, px_size_x * 2, 0, maxy, 0, px_size_y * 2)) + global_ds_overview.SetProjection(proj) + + global_tiles = [] + + for i in range(tiles_x): + for j in range(tiles_y): + # TODO: compute this like our engine (what is even the difference?) + # Calculate top-left pixel position in the global raster + start_px_x = i * (base_tile_width - overlap_px_x) + start_px_y = j * (base_tile_height - overlap_px_y) + + # Calculate geotransform for this tile + xmin = minx + start_px_x * px_size_x + ymax = maxy + start_px_y * px_size_y + + gt = (xmin, px_size_x, 0, ymax, 0, px_size_y) + + value = 10000 + date_idx * 1000 + band * 100 + i * 10 + j + pixel_values.add(value) + data = np.full((tile_height, tile_width), value, dtype=np.float32) + + filename = f"data/{date}_tile_x{i}_y{j}_b{band}.tif" + create_tile_tiff(filename, tile_width, tile_height, data, gt, proj) + + # Calculate spatial partition + upper_left_x = xmin + upper_left_y = ymax + lower_right_x = xmin + tile_width * px_size_x + lower_right_y = ymax + tile_height * px_size_y + + # Calculate time in ms since epoch + def date_to_ms(date_str): + dt = datetime.datetime.strptime(date_str, "%Y-%m-%d") + dt = dt.replace(tzinfo=datetime.timezone.utc) # Treat as UTC + return int(dt.timestamp() * 1000) + + time_start = date_to_ms(date) + + next_month = (datetime.datetime.strptime(date, "%Y-%m-%d") + datetime.timedelta(days=31)).replace(day=1) + next_month = next_month.replace(tzinfo=datetime.timezone.utc) # Treat as UTC + time_end = int(next_month.timestamp() * 1000) + + meta = { + "time": {"start": time_start, "end": time_end}, + "spatial_partition": { + "upperLeftCoordinate": {"x": upper_left_x, "y": upper_left_y}, + "lowerRightCoordinate": {"x": lower_right_x, "y": lower_right_y} + }, + "band": band, + "z_index": i + j, # todo + "params": { + "filePath": f"test_data/raster/multi_tile/{filename}", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": {"x": xmin, "y": ymax}, + "xPixelSize": px_size_x, + "yPixelSize": px_size_y + }, + "width": tile_width, + "height": tile_height, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": None, + "gdalOpenOptions": None, + "gdalConfigOptions": None, + "allowAlphabandAsMask": True + } + } + + loading_info.append(meta) + + # copy meta and reverse r_index + meta_rev = meta.copy() + meta_rev["z_index"] = 3- meta_rev["z_index"] + loading_info_rev.append(meta_rev) + + global_tiles.append({"data": data.astype(np.uint16), "xoff": start_px_x, "yoff": start_px_y}) + + for tile in global_tiles: + global_ds.GetRasterBand(1).WriteArray( + tile["data"], + xoff=tile["xoff"], + yoff=tile["yoff"] + ) + + for tile in reversed(global_tiles): + global_ds_rev.GetRasterBand(1).WriteArray( + tile["data"], + xoff=tile["xoff"], + yoff=tile["yoff"] + ) + + + # overview + band = global_ds.GetRasterBand(1) + overview_data_downsampled = band.ReadRaster( + 0, 0, global_width, global_height, + int(global_width / 2), int(global_height / 2), + buf_type=gdal.GDT_UInt16 + ) + overview_array = np.frombuffer(overview_data_downsampled, dtype=np.uint16).reshape(int(global_height / 2), int(global_width / 2)) + global_ds_overview.GetRasterBand(1).WriteArray(overview_array) + global_ds_overview.FlushCache() + + global_ds_overview.FlushCache() + global_ds_overview = None + global_ds.FlushCache() + global_ds = None + global_ds_rev.FlushCache() + global_ds_rev = None + + + + +with open("metadata/loading_info.json", "w") as f: + json.dump(loading_info, f, indent=2) + +with open("metadata/loading_info_rev.json", "w") as f: + json.dump(loading_info_rev, f, indent=2) + +cmap = plt.get_cmap('tab20') +values = sorted(pixel_values) +n = len(values) + +def colormap_color(idx): + rgba = cmap(idx / max(n - 1, 1)) + return [int(rgba[0] * 255), int(rgba[1] * 255), int(rgba[2] * 255), int(rgba[3] * 255)] + +import urllib.parse + +colorizer = { + "type": "singleBand", + "band": 0, + "bandColorizer": { + "type": "palette", + "colors": { + str(v): colormap_color(i) for i, v in enumerate(values) + }, + "noDataColor": [0, 0, 0, 0], + "defaultColor": [0, 0, 0, 0] + } +} + +colorizer_json = json.dumps(colorizer, separators=(',', ':')) +colorizer_urlencoded = urllib.parse.quote(colorizer_json) + +with open("metadata/colorizer_urlencoded.txt", "w") as f: + f.write(colorizer_urlencoded) \ No newline at end of file diff --git a/test_data/raster/multi_tile/metadata/colorizer_urlencoded.txt b/test_data/raster/multi_tile/metadata/colorizer_urlencoded.txt new file mode 100644 index 000000000..8dfd1ee3e --- /dev/null +++ b/test_data/raster/multi_tile/metadata/colorizer_urlencoded.txt @@ -0,0 +1 @@ +%7B%22type%22%3A%22singleBand%22%2C%22band%22%3A0%2C%22bandColorizer%22%3A%7B%22type%22%3A%22palette%22%2C%22colors%22%3A%7B%2210000%22%3A%5B31%2C119%2C180%2C255%5D%2C%2210001%22%3A%5B31%2C119%2C180%2C255%5D%2C%2210010%22%3A%5B174%2C199%2C232%2C255%5D%2C%2210011%22%3A%5B255%2C127%2C14%2C255%5D%2C%2210100%22%3A%5B255%2C187%2C120%2C255%5D%2C%2210101%22%3A%5B44%2C160%2C44%2C255%5D%2C%2210110%22%3A%5B152%2C223%2C138%2C255%5D%2C%2210111%22%3A%5B214%2C39%2C40%2C255%5D%2C%2211000%22%3A%5B214%2C39%2C40%2C255%5D%2C%2211001%22%3A%5B255%2C152%2C150%2C255%5D%2C%2211010%22%3A%5B148%2C103%2C189%2C255%5D%2C%2211011%22%3A%5B197%2C176%2C213%2C255%5D%2C%2211100%22%3A%5B140%2C86%2C75%2C255%5D%2C%2211101%22%3A%5B196%2C156%2C148%2C255%5D%2C%2211110%22%3A%5B227%2C119%2C194%2C255%5D%2C%2211111%22%3A%5B247%2C182%2C210%2C255%5D%2C%2212000%22%3A%5B247%2C182%2C210%2C255%5D%2C%2212001%22%3A%5B127%2C127%2C127%2C255%5D%2C%2212010%22%3A%5B199%2C199%2C199%2C255%5D%2C%2212011%22%3A%5B188%2C189%2C34%2C255%5D%2C%2212100%22%3A%5B219%2C219%2C141%2C255%5D%2C%2212101%22%3A%5B23%2C190%2C207%2C255%5D%2C%2212110%22%3A%5B158%2C218%2C229%2C255%5D%2C%2212111%22%3A%5B158%2C218%2C229%2C255%5D%7D%2C%22noDataColor%22%3A%5B0%2C0%2C0%2C0%5D%2C%22defaultColor%22%3A%5B0%2C0%2C0%2C0%5D%7D%7D \ No newline at end of file diff --git a/test_data/raster/multi_tile/metadata/dataset_irregular.json b/test_data/raster/multi_tile/metadata/dataset_irregular.json new file mode 100644 index 000000000..4b1a3832e --- /dev/null +++ b/test_data/raster/multi_tile/metadata/dataset_irregular.json @@ -0,0 +1,69 @@ +{ + "dataPath": { + "volume": "test_data" + }, + "definition": { + "properties": { + "name": null, + "displayName": "multi tile multi band", + "description": "", + "sourceOperator": "MultiBandGdalSource", + "symbology": null, + "provenance": null, + "tags": ["upload", "test"] + }, + "metaData": { + "type": "GdalMultiBand", + "resultDescriptor": { + "dataType": "U16", + "spatialReference": "EPSG:4326", + "time": { + "bounds": { + "start": "2025-01-01T00:00:00.000Z", + "end": "2025-05-01T00:00:00.000Z" + }, + "dimension": { + "type": "irregular" + } + }, + "spatialGrid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "gridBounds": { + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 899, + "xIdx": 1799 + } + } + }, + "descriptor": "source" + }, + "bands": [ + { + "name": "band 0", + "measurement": { + "type": "unitless" + } + }, + { + "name": "band 1", + "measurement": { + "type": "unitless" + } + } + ] + } + } + } +} diff --git a/test_data/raster/multi_tile/metadata/dataset_regular.json b/test_data/raster/multi_tile/metadata/dataset_regular.json new file mode 100644 index 000000000..ff7329038 --- /dev/null +++ b/test_data/raster/multi_tile/metadata/dataset_regular.json @@ -0,0 +1,74 @@ +{ + "dataPath": { + "volume": "test_data" + }, + "definition": { + "properties": { + "name": null, + "displayName": "multi tile multi band", + "description": "", + "sourceOperator": "MultiBandGdalSource", + "symbology": null, + "provenance": null, + "tags": ["upload", "test"] + }, + "metaData": { + "type": "GdalMultiBand", + "resultDescriptor": { + "dataType": "U16", + "spatialReference": "EPSG:4326", + "time": { + "bounds": { + "start": "2025-01-01T00:00:00.000Z", + "end": "2025-05-01T00:00:00.000Z" + }, + "dimension": { + "type": "regular", + "origin": "2014-01-01T00:00:00.000Z", + "step": { + "granularity": "months", + "step": 1 + } + } + }, + "spatialGrid": { + "spatialGrid": { + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "gridBounds": { + "topLeftIdx": { + "yIdx": 0, + "xIdx": 0 + }, + "bottomRightIdx": { + "yIdx": 899, + "xIdx": 1799 + } + } + }, + "descriptor": "source" + }, + "bands": [ + { + "name": "band 0", + "measurement": { + "type": "unitless" + } + }, + { + "name": "band 1", + "measurement": { + "type": "unitless" + } + } + ] + } + } + } +} diff --git a/test_data/raster/multi_tile/metadata/loading_info.json b/test_data/raster/multi_tile/metadata/loading_info.json new file mode 100644 index 000000000..5e2e9c44b --- /dev/null +++ b/test_data/raster/multi_tile/metadata/loading_info.json @@ -0,0 +1,914 @@ +[ + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 0, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + } +] diff --git a/test_data/raster/multi_tile/metadata/loading_info_rev.json b/test_data/raster/multi_tile/metadata/loading_info_rev.json new file mode 100644 index 000000000..ca982b735 --- /dev/null +++ b/test_data/raster/multi_tile/metadata/loading_info_rev.json @@ -0,0 +1,914 @@ +[ + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1735689600000, + "end": 1738368000000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-01-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1738368000000, + "end": 1740787200000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-02-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 0, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y0_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 0, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y1_b0.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 3, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 45.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x0_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -180.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -22.400000000000006 + } + }, + "band": 1, + "z_index": 2, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y0_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 90.0 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + }, + { + "time": { + "start": 1743465600000, + "end": 1746057600000 + }, + "spatial_partition": { + "upperLeftCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "lowerRightCoordinate": { + "x": 180.0, + "y": -90.00000000000001 + } + }, + "band": 1, + "z_index": 1, + "params": { + "filePath": "raster/multi_tile/data/2025-04-01_tile_x1_y1_b1.tif", + "rasterbandChannel": 1, + "geoTransform": { + "originCoordinate": { + "x": -45.0, + "y": 22.39999999999999 + }, + "xPixelSize": 0.2, + "yPixelSize": -0.2 + }, + "width": 1125, + "height": 562, + "fileNotFoundHandling": "Error", + "noDataValue": 0.0, + "propertiesMapping": null, + "gdalOpenOptions": null, + "gdalConfigOptions": null, + "allowAlphabandAsMask": true + } + } +] diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b0.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b0.tif new file mode 100644 index 000000000..832abc8b9 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b1.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b1.tif new file mode 100644 index 000000000..e4898062a Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-01-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b0.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b0.tif new file mode 100644 index 000000000..6287a0939 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b1.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b1.tif new file mode 100644 index 000000000..22874703c Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-02-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b0.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b0.tif new file mode 100644 index 000000000..56728f17e Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b1.tif b/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b1.tif new file mode 100644 index 000000000..9a0978dba Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/global/2025-04-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_0.tif new file mode 100644 index 000000000..5973beb2c Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_1.tif new file mode 100644 index 000000000..92744d328 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_2.tif new file mode 100644 index 000000000..8cf5341f9 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_3.tif new file mode 100644 index 000000000..52645d390 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_0.tif new file mode 100644 index 000000000..3a3fc63b8 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_1.tif new file mode 100644 index 000000000..b186ee867 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_2.tif new file mode 100644 index 000000000..c12da5778 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_3.tif new file mode 100644 index 000000000..dc0560614 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-01-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_0.tif new file mode 100644 index 000000000..a25160081 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_1.tif new file mode 100644 index 000000000..60b735e9d Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_2.tif new file mode 100644 index 000000000..280d36a2c Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_3.tif new file mode 100644 index 000000000..4db23f137 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_0.tif new file mode 100644 index 000000000..894f22bb3 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_1.tif new file mode 100644 index 000000000..7e4d9d35f Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_2.tif new file mode 100644 index 000000000..f89e6e846 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_3.tif new file mode 100644 index 000000000..d579d2e34 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-02-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_0.tif new file mode 100644 index 000000000..0c654f2ff Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_1.tif new file mode 100644 index 000000000..c10a64811 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_2.tif new file mode 100644 index 000000000..3b2f54cf8 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_3.tif new file mode 100644 index 000000000..f0d0d1a0c Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_0.tif new file mode 100644 index 000000000..00d633860 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_1.tif new file mode 100644 index 000000000..569e2341b Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_2.tif new file mode 100644 index 000000000..9480eb67d Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_3.tif new file mode 100644 index 000000000..01b85ed23 Binary files /dev/null and b/test_data/raster/multi_tile/results/overview_level_2/tiles/2025-04-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b0.tif new file mode 100644 index 000000000..9256fb393 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b1.tif new file mode 100644 index 000000000..3983d5fa4 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-01-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b0.tif new file mode 100644 index 000000000..2fe54a1ae Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b1.tif new file mode 100644 index 000000000..8ce361e89 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-02-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b0.tif new file mode 100644 index 000000000..f65d494af Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b1.tif new file mode 100644 index 000000000..3913d305d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/global/2025-04-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_0.tif new file mode 100644 index 000000000..1e97d24e1 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_1.tif new file mode 100644 index 000000000..4a20e2228 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_2.tif new file mode 100644 index 000000000..2f5a7d951 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_3.tif new file mode 100644 index 000000000..b5ebd0194 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_4.tif new file mode 100644 index 000000000..bb6f5155d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_5.tif new file mode 100644 index 000000000..202a2ea32 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_6.tif new file mode 100644 index 000000000..8769a0f60 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_7.tif new file mode 100644 index 000000000..f35a3ac28 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_0.tif new file mode 100644 index 000000000..205c7baae Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_1.tif new file mode 100644 index 000000000..fbf30045d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_2.tif new file mode 100644 index 000000000..f2ae7b4ae Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_3.tif new file mode 100644 index 000000000..fe030fef6 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_4.tif new file mode 100644 index 000000000..d3d19e9a4 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_5.tif new file mode 100644 index 000000000..d2e768717 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_6.tif new file mode 100644 index 000000000..04f38d8ac Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_7.tif new file mode 100644 index 000000000..1256064d7 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-01-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_0.tif new file mode 100644 index 000000000..53899c9de Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_1.tif new file mode 100644 index 000000000..dbada29c7 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_2.tif new file mode 100644 index 000000000..e451d77b7 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_3.tif new file mode 100644 index 000000000..c7ece1a30 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_4.tif new file mode 100644 index 000000000..cdd9402fc Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_5.tif new file mode 100644 index 000000000..fec0703ca Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_6.tif new file mode 100644 index 000000000..990b859b3 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_7.tif new file mode 100644 index 000000000..be93b985e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_0.tif new file mode 100644 index 000000000..166cca7cb Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_1.tif new file mode 100644 index 000000000..483321e5b Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_2.tif new file mode 100644 index 000000000..3cc019b35 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_3.tif new file mode 100644 index 000000000..5d0d48914 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_4.tif new file mode 100644 index 000000000..f57114aaf Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_5.tif new file mode 100644 index 000000000..3a98b6aa5 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_6.tif new file mode 100644 index 000000000..da00443b6 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_7.tif new file mode 100644 index 000000000..5489ede3f Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-02-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_0.tif new file mode 100644 index 000000000..db8b72b91 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_1.tif new file mode 100644 index 000000000..032f5f8ac Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_2.tif new file mode 100644 index 000000000..4bdd15509 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_3.tif new file mode 100644 index 000000000..7d8dbf03a Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_4.tif new file mode 100644 index 000000000..410e6ddcc Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_5.tif new file mode 100644 index 000000000..1e834437a Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_6.tif new file mode 100644 index 000000000..c56444e1a Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_7.tif new file mode 100644 index 000000000..ff7afb06d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_0.tif new file mode 100644 index 000000000..5c6bb927d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_1.tif new file mode 100644 index 000000000..900083758 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_2.tif new file mode 100644 index 000000000..f3358217f Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_3.tif new file mode 100644 index 000000000..fbfb9352e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_4.tif new file mode 100644 index 000000000..5dbad73eb Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_5.tif new file mode 100644 index 000000000..e066351c5 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_6.tif new file mode 100644 index 000000000..2712b9057 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_7.tif new file mode 100644 index 000000000..31782a673 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index/tiles/2025-04-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b0.tif new file mode 100644 index 000000000..3004519e6 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b1.tif new file mode 100644 index 000000000..8e394dbb3 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-01-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b0.tif new file mode 100644 index 000000000..3c4969bc5 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b1.tif new file mode 100644 index 000000000..93f377a04 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-02-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b0.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b0.tif new file mode 100644 index 000000000..b96564774 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b1.tif b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b1.tif new file mode 100644 index 000000000..ce99b5bb9 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/global/2025-04-01_global_b1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_0.tif new file mode 100644 index 000000000..1bf79706c Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_1.tif new file mode 100644 index 000000000..089effbba Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_2.tif new file mode 100644 index 000000000..32eb1ec1d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_3.tif new file mode 100644 index 000000000..3f6a4945d Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_4.tif new file mode 100644 index 000000000..8e49a1048 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_5.tif new file mode 100644 index 000000000..57884aa82 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_6.tif new file mode 100644 index 000000000..7b3324b7e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_7.tif new file mode 100644 index 000000000..14ee80540 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_0.tif new file mode 100644 index 000000000..fa2a0f5b2 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_1.tif new file mode 100644 index 000000000..73688df22 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_2.tif new file mode 100644 index 000000000..a52120b54 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_3.tif new file mode 100644 index 000000000..0a0a105de Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_4.tif new file mode 100644 index 000000000..0cbe5e4c4 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_5.tif new file mode 100644 index 000000000..5751842ce Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_6.tif new file mode 100644 index 000000000..84ab4e7d5 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_7.tif new file mode 100644 index 000000000..4f6965ebe Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-01-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_0.tif new file mode 100644 index 000000000..8847fc51e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_1.tif new file mode 100644 index 000000000..6a60cd303 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_2.tif new file mode 100644 index 000000000..702f9632f Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_3.tif new file mode 100644 index 000000000..e6e91cf67 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_4.tif new file mode 100644 index 000000000..24dd2a6c8 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_5.tif new file mode 100644 index 000000000..f1377ec4a Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_6.tif new file mode 100644 index 000000000..6c80d13d1 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_7.tif new file mode 100644 index 000000000..5a161775e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_0.tif new file mode 100644 index 000000000..e367a2436 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_1.tif new file mode 100644 index 000000000..656f4c5fb Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_2.tif new file mode 100644 index 000000000..6dbb5c7fc Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_3.tif new file mode 100644 index 000000000..5694feb30 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_4.tif new file mode 100644 index 000000000..ee36c8a35 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_5.tif new file mode 100644 index 000000000..7c0332863 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_6.tif new file mode 100644 index 000000000..5f03b2eb3 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_7.tif new file mode 100644 index 000000000..aa508b6a3 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-02-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_0.tif new file mode 100644 index 000000000..716bbb744 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_1.tif new file mode 100644 index 000000000..89c42b60f Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_2.tif new file mode 100644 index 000000000..0cdac64bb Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_3.tif new file mode 100644 index 000000000..c18c6529e Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_4.tif new file mode 100644 index 000000000..53062b102 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_5.tif new file mode 100644 index 000000000..d8dacf5b3 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_6.tif new file mode 100644 index 000000000..7e0791b9f Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_7.tif new file mode 100644 index 000000000..f9cd0446c Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b0_tile_7.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_0.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_0.tif new file mode 100644 index 000000000..071846bc0 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_0.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_1.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_1.tif new file mode 100644 index 000000000..8d234eaff Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_1.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_2.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_2.tif new file mode 100644 index 000000000..acdd4f624 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_2.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_3.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_3.tif new file mode 100644 index 000000000..35717e595 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_3.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_4.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_4.tif new file mode 100644 index 000000000..577e404eb Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_4.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_5.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_5.tif new file mode 100644 index 000000000..71119cff4 Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_5.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_6.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_6.tif new file mode 100644 index 000000000..92e8b0e4a Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_6.tif differ diff --git a/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_7.tif b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_7.tif new file mode 100644 index 000000000..83dbd096b Binary files /dev/null and b/test_data/raster/multi_tile/results/z_index_reversed/tiles/2025-04-01_global_b1_tile_7.tif differ diff --git a/test_data/raster/multi_tile/wms.png b/test_data/raster/multi_tile/wms.png new file mode 100644 index 000000000..f41d3602e Binary files /dev/null and b/test_data/raster/multi_tile/wms.png differ diff --git a/test_data/raster/png/png_from_stream.png b/test_data/raster/png/png_from_stream.png index 0d8ff3c1d..8c15e1b28 100644 Binary files a/test_data/raster/png/png_from_stream.png and b/test_data/raster/png/png_from_stream.png differ diff --git a/test_data/wms/get_map_ndvi.png b/test_data/wms/get_map_ndvi.png index 8bd0cb2ed..7b80fd7f0 100644 Binary files a/test_data/wms/get_map_ndvi.png and b/test_data/wms/get_map_ndvi.png differ diff --git a/test_data/wms/partial_derivative.png b/test_data/wms/partial_derivative.png index a01846230..91ea226fb 100644 Binary files a/test_data/wms/partial_derivative.png and b/test_data/wms/partial_derivative.png differ