diff --git a/Cargo.lock b/Cargo.lock index 21003ea2e..43964ac16 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,6 +53,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "actix-cors" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e772b3bcafe335042b5db010ab7c09013dad6eac4915c91d8d50902769f331" +dependencies = [ + "actix-utils", + "actix-web", + "derive_more", + "futures-util", + "log", + "once_cell", + "smallvec", +] + [[package]] name = "actix-files" version = "0.6.5" @@ -2688,6 +2703,7 @@ dependencies = [ "postgres-types", "proj", "rayon", + "schemars", "serde", "serde_json", "serde_with", @@ -2752,6 +2768,7 @@ dependencies = [ "geoengine-datatypes", "geoengine-expression", "httptest", + "inventory", "itertools 0.12.1", "libloading", "log", @@ -2769,6 +2786,7 @@ dependencies = [ "rand", "rayon", "rustc-hash", + "schemars", "serde", "serde_json", "snafu", @@ -2787,6 +2805,7 @@ name = "geoengine-services" version = "0.8.0" dependencies = [ "actix", + "actix-cors", "actix-files", "actix-http", "actix-multipart", @@ -6312,6 +6331,30 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "schemars" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45a28f4c49489add4ce10783f7911893516f15afe45d015608d41faca6bc4d29" +dependencies = [ + "dyn-clone", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c767fd6fa65d9ccf9cf026122c1b555f2ef9a4f0cea69da4d7dbc3e258d30967" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -6414,6 +6457,17 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_ignored" version = "0.1.10" diff --git a/datatypes/Cargo.toml b/datatypes/Cargo.toml index 3d4001150..72c6c174e 100644 --- a/datatypes/Cargo.toml +++ b/datatypes/Cargo.toml @@ -37,6 +37,7 @@ postgres-types = { version = "0.2", features = [ ] } proj = "0.22" # needs to stay fixed to use fixed proj version rayon = "1.8" +schemars = "0.8.16" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_with = "3.6" diff --git a/datatypes/src/dataset.rs b/datatypes/src/dataset.rs index dfefc9903..d411f0f8e 100644 --- a/datatypes/src/dataset.rs +++ b/datatypes/src/dataset.rs @@ -1,4 +1,7 @@ -use crate::identifier; +use std::borrow::Cow; + +use crate::{identifier, util::helpers::json_schema_help_link}; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::{de::Visitor, Deserialize, Serialize}; identifier!(DataProviderId); @@ -86,6 +89,45 @@ pub struct NamedData { pub name: String, } +impl JsonSchema for NamedData { + fn schema_name() -> String { + "NamedData".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::NamedData")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + description: Some("The user-facing identifier for loadable data. +It can be resolved into a [`DataId`]. + +It is a triple of namespace, provider and name. +The namespace and provider are optional and default to the system namespace and provider. + +# Examples + +* `dataset` -> `NamedData { namespace: None, provider: None, name: \"dataset\" }` +* `namespace:dataset` -> `NamedData { namespace: Some(\"namespace\"), provider: None, name: \"dataset\" }` +* `namespace:provider:dataset` -> `NamedData { namespace: Some(\"namespace\"), provider: Some(\"provider\"), name: \"dataset\" }`".to_owned()), +..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + extensions: schemars::Map::from([ + json_schema_help_link("https://docs.geoengine.io/geoengine/datasets.html") + ]), + ..Default::default() + }) + } +} + impl NamedData { /// Canonicalize a name that reflects the system namespace and provider. fn canonicalize + PartialEq<&'static str>>( diff --git a/datatypes/src/primitives/coordinate.rs b/datatypes/src/primitives/coordinate.rs index cfe886045..ea115ea4d 100644 --- a/datatypes/src/primitives/coordinate.rs +++ b/datatypes/src/primitives/coordinate.rs @@ -6,6 +6,7 @@ use float_cmp::ApproxEq; use postgres_types::{FromSql, ToSql}; use proj::Coord; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::sync::Arc; use std::{ @@ -15,7 +16,17 @@ use std::{ }; #[derive( - Clone, Copy, Debug, Deserialize, PartialEq, PartialOrd, Serialize, Default, ToSql, FromSql, + Clone, + Copy, + Debug, + Deserialize, + PartialEq, + PartialOrd, + Serialize, + Default, + ToSql, + FromSql, + JsonSchema, )] #[repr(C)] pub struct Coordinate2D { diff --git a/datatypes/src/primitives/measurement.rs b/datatypes/src/primitives/measurement.rs index 0067b066a..7c463a95a 100644 --- a/datatypes/src/primitives/measurement.rs +++ b/datatypes/src/primitives/measurement.rs @@ -1,14 +1,16 @@ use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::str::FromStr; -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type")] pub enum Measurement { Unitless, Continuous(ContinuousMeasurement), + #[schemars(with = "SerializableClassificationMeasurement")] Classification(ClassificationMeasurement), } @@ -31,7 +33,7 @@ impl Default for Measurement { } } -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, FromSql, ToSql)] +#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize, FromSql, ToSql, JsonSchema)] pub struct ContinuousMeasurement { pub measurement: String, pub unit: Option, @@ -47,9 +49,12 @@ pub struct ClassificationMeasurement { pub classes: HashMap, } -/// A type that is solely for serde's serializability. -/// You cannot serialize floats as JSON map keys. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +// A type that is solely for serde's serializability. +// You cannot serialize floats as JSON map keys. +// +// Note: Do not use a doc comment here, because otherwise +// internal details would be shown in workflow editor. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] pub struct SerializableClassificationMeasurement { pub measurement: String, // use a BTreeMap to preserve the order of the keys diff --git a/datatypes/src/primitives/spatial_resolution.rs b/datatypes/src/primitives/spatial_resolution.rs index b892ec046..dac09908a 100644 --- a/datatypes/src/primitives/spatial_resolution.rs +++ b/datatypes/src/primitives/spatial_resolution.rs @@ -3,11 +3,12 @@ use std::{convert::TryFrom, ops::Add, ops::Div, ops::Mul, ops::Sub}; use crate::primitives::error; use crate::util::Result; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; /// The spatial resolution in SRS units -#[derive(Copy, Clone, Debug, PartialEq, Deserialize, Serialize, ToSql, FromSql)] +#[derive(Copy, Clone, Debug, PartialEq, Deserialize, Serialize, ToSql, FromSql, JsonSchema)] pub struct SpatialResolution { pub x: f64, pub y: f64, diff --git a/datatypes/src/primitives/time_instance.rs b/datatypes/src/primitives/time_instance.rs index 397416902..d3e34eadb 100644 --- a/datatypes/src/primitives/time_instance.rs +++ b/datatypes/src/primitives/time_instance.rs @@ -1,10 +1,15 @@ use super::datetime::DateTimeError; use super::{DateTime, Duration}; use crate::primitives::error; +use crate::util::helpers::json_schema_help_link; use crate::util::Result; use postgres_types::{FromSql, ToSql}; +use schemars::gen::SchemaGenerator; +use schemars::schema::Schema; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; +use std::borrow::Cow; use std::ops::AddAssign; use std::{ convert::TryFrom, @@ -18,6 +23,56 @@ use std::{ #[postgres(transparent)] pub struct TimeInstance(i64); +impl JsonSchema for TimeInstance { + fn schema_name() -> String { + "TimeInstance".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::TimeInstance")) + } + + fn is_referenceable() -> bool { + true + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + subschemas: Some(Box::new(SubschemaValidation { + one_of: Some(vec![ + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Unix Timestamp".to_owned()), + description: Some("Unix timestamp in milliseconds".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Integer))), + ..Default::default() + }), + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Datetime String".to_owned()), + description: Some( + "Date and time as defined in RFC 3339, section 5.6".to_owned(), + ), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + format: Some("date-time".to_owned()), + extensions: schemars::Map::from([json_schema_help_link( + "http://tools.ietf.org/html/rfc3339", + )]), + ..Default::default() + }), + ]), + ..Default::default() + })), + ..Default::default() + }) + } +} + impl TimeInstance { pub fn from_millis(millis: i64) -> Result { ensure!( diff --git a/datatypes/src/primitives/time_interval.rs b/datatypes/src/primitives/time_interval.rs index b0c2d35ba..02bf75a67 100755 --- a/datatypes/src/primitives/time_interval.rs +++ b/datatypes/src/primitives/time_interval.rs @@ -7,6 +7,7 @@ use arrow::datatypes::{DataType, Field}; use arrow::error::ArrowError; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; use std::fmt::{Debug, Display}; @@ -14,7 +15,7 @@ use std::sync::Arc; use std::{cmp::Ordering, convert::TryInto}; /// Stores time intervals in ms in close-open semantic [start, end) -#[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq, ToSql, FromSql)] +#[derive(Clone, Copy, Deserialize, Serialize, PartialEq, Eq, ToSql, FromSql, JsonSchema)] #[repr(C)] pub struct TimeInterval { start: TimeInstance, diff --git a/datatypes/src/primitives/time_step.rs b/datatypes/src/primitives/time_step.rs index 9d5cc0fc6..82683ef8d 100644 --- a/datatypes/src/primitives/time_step.rs +++ b/datatypes/src/primitives/time_step.rs @@ -1,6 +1,7 @@ use std::ops::{Mul, Sub}; use std::{cmp::max, convert::TryInto, ops::Add}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use postgres_types::{FromSql, ToSql}; @@ -13,7 +14,7 @@ use crate::util::Result; use super::{DateTime, Duration, TimeInterval}; /// A time granularity. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ToSql, FromSql)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ToSql, FromSql, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum TimeGranularity { Millis, @@ -26,7 +27,7 @@ pub enum TimeGranularity { } /// A step in time. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ToSql, FromSql)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, ToSql, FromSql, JsonSchema)] pub struct TimeStep { pub granularity: TimeGranularity, pub step: u32, // TODO: ensure on deserialization it is > 0 diff --git a/datatypes/src/raster/band_names.rs b/datatypes/src/raster/band_names.rs index eccf730ec..1793dc355 100644 --- a/datatypes/src/raster/band_names.rs +++ b/datatypes/src/raster/band_names.rs @@ -1,3 +1,4 @@ +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -7,12 +8,15 @@ use crate::error::{ }; use crate::util::Result; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type", content = "values")] pub enum RenameBands { - Default, // append " (n)" to the band name for the `n`-th conflict, - Suffix(Vec), // A suffix for every input, to be appended to the original band names - Rename(Vec), // A new name for each band, to be used instead of the original band names + /// Append " (n)" to the band name for the `n`-th conflict + Default, + /// A suffix for every input, to be appended to the original band names + Suffix(Vec), + /// A new name for each band, to be used instead of the original band names + Rename(Vec), } impl RenameBands { diff --git a/datatypes/src/raster/data_type.rs b/datatypes/src/raster/data_type.rs index e11a6a9d8..4dc10153c 100644 --- a/datatypes/src/raster/data_type.rs +++ b/datatypes/src/raster/data_type.rs @@ -6,6 +6,7 @@ use crate::util::Result; use gdal::raster::GdalDataType; use num_traits::{AsPrimitive, Bounded, Num}; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; @@ -142,7 +143,19 @@ impl Pixel for f32 {} impl Pixel for f64 {} #[derive( - Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Deserialize, Serialize, Copy, Clone, FromSql, ToSql, + Debug, + Ord, + PartialOrd, + Eq, + PartialEq, + Hash, + Deserialize, + Serialize, + Copy, + Clone, + FromSql, + ToSql, + JsonSchema, )] pub enum RasterDataType { U8, diff --git a/datatypes/src/raster/raster_properties.rs b/datatypes/src/raster/raster_properties.rs index 0c8c963aa..19557ca37 100644 --- a/datatypes/src/raster/raster_properties.rs +++ b/datatypes/src/raster/raster_properties.rs @@ -3,6 +3,7 @@ use crate::util::{ByteSize, Result}; use num_traits::FromPrimitive; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use serde_with::serde_as; use std::collections::HashMap; @@ -120,7 +121,18 @@ impl RasterProperties { } #[derive( - Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq, PartialOrd, Ord, FromSql, ToSql, + Debug, + Clone, + Serialize, + Deserialize, + PartialEq, + Hash, + Eq, + PartialOrd, + Ord, + FromSql, + ToSql, + JsonSchema, )] pub struct RasterPropertiesKey { pub domain: Option, diff --git a/datatypes/src/spatial_reference.rs b/datatypes/src/spatial_reference.rs index 5cd05b435..721401f7e 100644 --- a/datatypes/src/spatial_reference.rs +++ b/datatypes/src/spatial_reference.rs @@ -2,7 +2,7 @@ use crate::{ error, operations::reproject::{CoordinateProjection, CoordinateProjector, Reproject}, primitives::AxisAlignedRectangle, - util::Result, + util::{helpers::json_schema_help_link, Result}, }; use gdal::spatial_ref::SpatialRef; @@ -10,12 +10,13 @@ use postgres_types::private::BytesMut; use postgres_types::{FromSql, IsNull, ToSql, Type}; use proj::Proj; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::de::Visitor; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use snafu::Error; use snafu::ResultExt; -use std::str::FromStr; +use std::{borrow::Cow, str::FromStr}; use std::{convert::TryFrom, fmt::Formatter}; /// A spatial reference authority that is part of a spatial reference definition @@ -52,6 +53,29 @@ pub struct SpatialReference { code: u32, } +impl JsonSchema for SpatialReference { + fn schema_name() -> String { + "SpatialReference".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::SpatialReference")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + extensions: schemars::Map::from([json_schema_help_link("https://epsg.io")]), + ..Default::default() + }) + } +} + impl SpatialReference { pub fn new(authority: SpatialReferenceAuthority, code: u32) -> Self { Self { authority, code } diff --git a/datatypes/src/util/helpers.rs b/datatypes/src/util/helpers.rs index 924219684..e3a343819 100644 --- a/datatypes/src/util/helpers.rs +++ b/datatypes/src/util/helpers.rs @@ -245,6 +245,18 @@ pub fn indices_for_split_at( (left_index, left_length_right_index, right_length) } +pub fn json_schema_help_link(url: &str) -> (String, serde_json::Value) { + ( + "links".to_owned(), + serde_json::json!([ + { + "rel": "external help", + "href": url + } + ]), + ) +} + #[cfg(test)] mod tests { use std::collections::HashMap; diff --git a/operators/Cargo.toml b/operators/Cargo.toml index 7decb9e68..f395f6d49 100644 --- a/operators/Cargo.toml +++ b/operators/Cargo.toml @@ -60,6 +60,8 @@ tracing = "0.1" typetag = "0.2" uuid = { version = "1.7", features = ["serde", "v4", "v5"] } xgboost-rs = { version = "0.3", optional = true } +schemars = "0.8.16" +inventory = "0.3.15" [dev-dependencies] async-stream = "0.3" diff --git a/operators/src/engine/mod.rs b/operators/src/engine/mod.rs index b4b2947a4..ee252a714 100644 --- a/operators/src/engine/mod.rs +++ b/operators/src/engine/mod.rs @@ -6,8 +6,9 @@ pub use execution_context::{ ExecutionContext, ExecutionContextExtensions, MetaData, MetaDataProvider, MockExecutionContext, StaticMetaData, }; +pub(crate) use operator::OperatorSchema; pub use operator::{ - CanonicOperatorName, InitializedPlotOperator, InitializedRasterOperator, + build_workflow_schema, CanonicOperatorName, InitializedPlotOperator, InitializedRasterOperator, InitializedVectorOperator, OperatorData, OperatorName, PlotOperator, RasterOperator, TypedOperator, VectorOperator, }; diff --git a/operators/src/engine/operator.rs b/operators/src/engine/operator.rs index 5a2254f79..fd7ef830d 100644 --- a/operators/src/engine/operator.rs +++ b/operators/src/engine/operator.rs @@ -1,10 +1,20 @@ +use std::borrow::Cow; + +use schemars::{ + gen::{SchemaGenerator, SchemaSettings}, + schema::{InstanceType, RootSchema, Schema, SchemaObject, SingleOrVec}, + JsonSchema, +}; use serde::{Deserialize, Serialize}; use tracing::debug; use crate::error; use crate::util::Result; use async_trait::async_trait; -use geoengine_datatypes::{dataset::NamedData, util::ByteSize}; +use geoengine_datatypes::{ + dataset::NamedData, + util::{helpers::json_schema_help_link, ByteSize}, +}; use super::{ query_processor::{TypedRasterQueryProcessor, TypedVectorQueryProcessor}, @@ -64,6 +74,28 @@ pub trait RasterOperator: fn span(&self) -> CreateSpan; } +impl JsonSchema for dyn RasterOperator { + fn schema_name() -> String { + "RasterOperator".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::RasterOperator")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + reference: Some("#/definitions/raster".to_owned()), + ..Default::default() + }) + } +} + /// Common methods for `VectorOperator`s #[typetag::serde(tag = "type")] #[async_trait] @@ -98,6 +130,28 @@ pub trait VectorOperator: fn span(&self) -> CreateSpan; } +impl JsonSchema for dyn VectorOperator { + fn schema_name() -> String { + "VectorOperator".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::VectorOperator")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + reference: Some("#/definitions/vector".to_owned()), + ..Default::default() + }) + } +} + /// Common methods for `PlotOperator`s #[typetag::serde(tag = "type")] #[async_trait] @@ -132,6 +186,28 @@ pub trait PlotOperator: fn span(&self) -> CreateSpan; } +impl JsonSchema for dyn PlotOperator { + fn schema_name() -> String { + "PlotOperator".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::PlotOperator")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + reference: Some("#/definitions/plot".to_owned()), + ..Default::default() + }) + } +} + pub trait InitializedRasterOperator: Send + Sync { /// Get the result descriptor of the `Operator` fn result_descriptor(&self) -> &RasterResultDescriptor; @@ -391,6 +467,163 @@ pub trait OperatorName { const TYPE_NAME: &'static str; } +pub(crate) struct OperatorSchema { + gen: fn() -> RootSchema, + output_type: &'static str, + help_url: &'static str, +} + +impl OperatorSchema { + pub(crate) const fn new(output_type: &'static str, help_url: &'static str) -> Self + where + T: JsonSchema + OperatorName, + { + Self { + gen: || { + let settings = SchemaSettings::draft07().with(|s| { + s.inline_subschemas = true; + }); + let gen = settings.into_generator(); + let mut schema = gen.into_root_schema_for::(); + + let metadata = schema.schema.metadata(); + match metadata.title.as_ref() { + Some(title) if title.contains("_for_") => { + // do not use autogenerated title containing generic parameters + metadata.title = None; + } + _ => {} + } + + let object = schema.schema.object(); + object.properties.insert( + "type".to_owned(), + Schema::Object(SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + enum_values: Some(vec![serde_json::Value::String(T::TYPE_NAME.to_owned())]), + ..Default::default() + }), + ); + object.required.insert("type".to_owned()); + + schema + }, + output_type, + help_url, + } + } + + fn get_schema(&self) -> serde_json::Value { + assert!( + self.help_url.starts_with("http"), + "all operator help_url's must be urls" + ); + + let mut schema = (self.gen)(); + + let help_url = json_schema_help_link(self.help_url); + schema.schema.extensions.insert(help_url.0, help_url.1); + + serde_json::to_value(schema).unwrap() + } + + fn get_output_type(&self) -> &'static str { + self.output_type + } +} + +inventory::collect!(OperatorSchema); + +#[macro_export] +macro_rules! define_operator { + ($operator_name:ident, $operator_params:ident, output_type = $output_type:literal, help_url = $help_url:literal) => { + pub type $operator_name = $crate::engine::SourceOperator<$operator_params>; + + impl $crate::engine::OperatorName for $operator_name { + const TYPE_NAME: &'static str = stringify!($operator_name); + } + + inventory::submit! { + $crate::engine::OperatorSchema::new::<$operator_name>($output_type, $help_url) + } + }; + ($operator_name:ident, $operator_params:ident, $operator_sources:ident, output_type = $output_type:literal, help_url = $help_url:literal) => { + pub type $operator_name = $crate::engine::Operator<$operator_params, $operator_sources>; + + impl $crate::engine::OperatorName for $operator_name { + const TYPE_NAME: &'static str = stringify!($operator_name); + } + + inventory::submit! { + $crate::engine::OperatorSchema::new::<$operator_name>($output_type, $help_url) + } + }; +} + +pub fn build_workflow_schema() -> serde_json::Value { + let mut workflow_schema = serde_json::json!({ + "$schema": "http://json-schema.org/draft-07/schema", + "definitions": {}, + "oneOf": [] + }); + let definitions = workflow_schema + .get_mut("definitions") + .unwrap() + .as_object_mut() + .unwrap(); + let mut output_types: Vec<&str> = Vec::new(); + + for operator in inventory::iter:: { + let operator_schema = operator.get_schema(); + let operator_name = operator_schema + .get("properties") + .unwrap() + .get("type") + .unwrap() + .get("enum") + .unwrap() + .get(0) + .unwrap() + .as_str() + .unwrap() + .to_owned(); + definitions.insert(operator_name.clone(), operator_schema); + + let operator_output_type = operator.get_output_type(); + + if !output_types.contains(&operator_output_type) { + output_types.push(operator_output_type); + definitions.insert( + operator_output_type.to_owned(), + serde_json::json!({ + "oneOf": [] + }), + ); + } + definitions + .get_mut(operator_output_type) + .unwrap() + .get_mut("oneOf") + .unwrap() + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "$ref": format!("#/definitions/{}", operator_name) + })); + } + for operator_output_type in output_types { + workflow_schema + .get_mut("oneOf") + .unwrap() + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "$ref": format!("#/definitions/{}", operator_output_type) + })); + } + workflow_schema +} + #[cfg(test)] mod tests { use serde_json::json; diff --git a/operators/src/engine/operator_impl.rs b/operators/src/engine/operator_impl.rs index 2f007f7b2..e2ab042ea 100644 --- a/operators/src/engine/operator_impl.rs +++ b/operators/src/engine/operator_impl.rs @@ -1,36 +1,37 @@ use geoengine_datatypes::dataset::NamedData; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use crate::util::input::{MultiRasterOrVectorOperator, RasterOrVectorOperator}; use super::{OperatorData, RasterOperator, VectorOperator}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct Operator { pub params: Params, pub sources: Sources, } -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SourceOperator { pub params: Params, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SingleRasterSource { pub raster: Box, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SingleVectorSource { pub vector: Box, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SingleRasterOrVectorSource { pub source: RasterOrVectorOperator, @@ -74,19 +75,19 @@ impl MultipleRasterOrSingleVectorSource { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MultipleRasterSources { pub rasters: Vec>, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct MultipleVectorSources { pub vectors: Vec>, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct SingleVectorMultipleRasterSources { pub vector: Box, diff --git a/operators/src/engine/result_descriptor.rs b/operators/src/engine/result_descriptor.rs index 84bd2124b..5f623a0a1 100644 --- a/operators/src/engine/result_descriptor.rs +++ b/operators/src/engine/result_descriptor.rs @@ -8,6 +8,7 @@ use geoengine_datatypes::{ collections::VectorDataType, raster::RasterDataType, spatial_reference::SpatialReferenceOption, }; use postgres_types::{FromSql, IsNull, ToSql, Type}; +use schemars::JsonSchema; use serde::{Deserialize, Deserializer, Serialize}; use snafu::ensure; use std::collections::{HashMap, HashSet}; @@ -225,7 +226,7 @@ impl ToSql for RasterBandDescriptors { } } -#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql)] +#[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, ToSql, FromSql, JsonSchema)] pub struct RasterBandDescriptor { pub name: String, pub measurement: Measurement, diff --git a/operators/src/plot/class_histogram.rs b/operators/src/plot/class_histogram.rs index aaecb225a..7b206d8a7 100644 --- a/operators/src/plot/class_histogram.rs +++ b/operators/src/plot/class_histogram.rs @@ -1,11 +1,11 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, - InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, + InitializedVectorOperator, OperatorName, PlotOperator, PlotQueryProcessor, PlotResultDescriptor, QueryContext, SingleRasterOrVectorSource, TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, }; use crate::engine::{QueryProcessor, WorkflowOperatorPath}; -use crate::error; +use crate::{define_operator, error}; use crate::error::Error; use crate::util::input::RasterOrVectorOperator; use crate::util::Result; @@ -18,6 +18,7 @@ use geoengine_datatypes::primitives::{ Measurement, PlotQueryRectangle, RasterQueryRectangle, }; use num_traits::AsPrimitive; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::{ensure, OptionExt}; use std::collections::HashMap; @@ -28,14 +29,16 @@ pub const CLASS_HISTOGRAM_OPERATOR_NAME: &str = "ClassHistogram"; /// /// For vector inputs, it calculates the histogram on one of its attributes. /// -pub type ClassHistogram = Operator; - -impl OperatorName for ClassHistogram { - const TYPE_NAME: &'static str = "ClassHistogram"; -} +define_operator!( + ClassHistogram, + ClassHistogramParams, + SingleRasterOrVectorSource, + output_type = "plot", + help_url = "https://docs.geoengine.io/plots/classHistogram.html" +); /// The parameter spec for `Histogram` -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ClassHistogramParams { /// Name of the (numeric) attribute to compute the histogram on. Fails if set for rasters. diff --git a/operators/src/plot/histogram.rs b/operators/src/plot/histogram.rs index 04b961ee9..efbe190d8 100644 --- a/operators/src/plot/histogram.rs +++ b/operators/src/plot/histogram.rs @@ -1,11 +1,11 @@ use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedPlotOperator, InitializedRasterOperator, - InitializedVectorOperator, Operator, OperatorName, PlotOperator, PlotQueryProcessor, + InitializedVectorOperator, OperatorName, PlotOperator, PlotQueryProcessor, PlotResultDescriptor, QueryContext, SingleRasterOrVectorSource, TypedPlotQueryProcessor, TypedRasterQueryProcessor, TypedVectorQueryProcessor, }; use crate::engine::{QueryProcessor, WorkflowOperatorPath}; -use crate::error; +use crate::{define_operator, error}; use crate::error::Error; use crate::string_token; use crate::util::input::RasterOrVectorOperator; @@ -24,6 +24,7 @@ use geoengine_datatypes::{ collections::{FeatureCollection, FeatureCollectionInfos}, raster::GridSize, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; use std::convert::TryFrom; @@ -34,14 +35,16 @@ pub const HISTOGRAM_OPERATOR_NAME: &str = "Histogram"; /// /// For vector inputs, it calculates the histogram on one of its attributes. /// -pub type Histogram = Operator; - -impl OperatorName for Histogram { - const TYPE_NAME: &'static str = "Histogram"; -} +define_operator!( + Histogram, + HistogramParams, + SingleRasterOrVectorSource, + output_type = "plot", + help_url = "https://docs.geoengine.io/plots/histogram.html" +); /// The parameter spec for `Histogram` -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct HistogramParams { /// Name of the (numeric) vector attribute or raster band to compute the histogram on. @@ -56,7 +59,7 @@ pub struct HistogramParams { } /// Options for how to derive the histogram's number of buckets. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type")] pub enum HistogramBuckets { #[serde(rename_all = "camelCase")] @@ -74,8 +77,33 @@ fn default_max_number_of_buckets() -> u8 { string_token!(Data, "data"); +impl schemars::JsonSchema for Data { + fn schema_name() -> String { + "Data".to_owned() + } + + fn schema_id() -> std::borrow::Cow<'static, str> { + std::borrow::Cow::Borrowed(concat!(module_path!(), "::Data")) + } + + fn is_referenceable() -> bool { + false + } + + fn json_schema(_gen: &mut schemars::gen::SchemaGenerator) -> schemars::schema::Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::String))), + enum_values: Some(vec![ + serde_json::Value::String("data".to_owned()) + ]), + ..Default::default() + }) + } +} + /// Let the bounds either be computed or given. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] pub enum HistogramBounds { Data(Data), diff --git a/operators/src/processing/column_range_filter.rs b/operators/src/processing/column_range_filter.rs index d6858a57f..9a80fa623 100644 --- a/operators/src/processing/column_range_filter.rs +++ b/operators/src/processing/column_range_filter.rs @@ -1,12 +1,12 @@ use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, Operator, + CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, OperatorName, QueryContext, QueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; -use crate::error; use crate::util::input::StringOrNumberRange; use crate::util::Result; use crate::{adapters::FeatureCollectionChunkMerger, engine::SingleVectorSource}; +use crate::{define_operator, error}; use async_trait::async_trait; use futures::stream::BoxStream; use futures::StreamExt; @@ -18,11 +18,12 @@ use geoengine_datatypes::primitives::{ VectorQueryRectangle, }; use geoengine_datatypes::util::arrow::ArrowTyped; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use std::ops::RangeInclusive; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ColumnRangeFilterParams { pub column: String, @@ -30,11 +31,13 @@ pub struct ColumnRangeFilterParams { pub keep_nulls: bool, } -pub type ColumnRangeFilter = Operator; - -impl OperatorName for ColumnRangeFilter { - const TYPE_NAME: &'static str = "ColumnRangeFilter"; -} +define_operator!( + ColumnRangeFilter, + ColumnRangeFilterParams, + SingleVectorSource, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/columnrangefilter.html" +); #[typetag::serde] #[async_trait] diff --git a/operators/src/processing/expression/raster_operator.rs b/operators/src/processing/expression/raster_operator.rs index 7c6cd58cb..650e18a26 100644 --- a/operators/src/processing/expression/raster_operator.rs +++ b/operators/src/processing/expression/raster_operator.rs @@ -4,8 +4,9 @@ use super::{ RasterExpressionError, }; use crate::{ + define_operator, engine::{ - CanonicOperatorName, InitializedRasterOperator, InitializedSources, Operator, OperatorName, + CanonicOperatorName, InitializedRasterOperator, InitializedSources, OperatorName, RasterBandDescriptor, RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, @@ -18,6 +19,7 @@ use geoengine_datatypes::raster::RasterDataType; use geoengine_expression::{ DataType, ExpressionAst, ExpressionParser, LinkedExpression, Parameter, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -27,7 +29,7 @@ use snafu::ensure; /// * `output_type` is the data type of the produced raster tiles. /// * `output_no_data_value` is the no data value of the output raster /// * `output_measurement` is the measurement description of the output -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ExpressionParams { pub expression: String, @@ -37,7 +39,13 @@ pub struct ExpressionParams { } /// The `Expression` operator calculates an expression for all pixels of the input rasters bands and /// produces raster tiles of a given output type -pub type Expression = Operator; +define_operator!( + Expression, + ExpressionParams, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/expression.html" +); /// Create a parameter name from an index. /// Starts with `A`. @@ -121,10 +129,6 @@ impl RasterOperator for Expression { span_fn!(Expression); } -impl OperatorName for Expression { - const TYPE_NAME: &'static str = "Expression"; -} - pub struct InitializedExpression { name: CanonicOperatorName, result_descriptor: RasterResultDescriptor, diff --git a/operators/src/processing/interpolation/mod.rs b/operators/src/processing/interpolation/mod.rs index 9bad1ffef..0d5aab1b7 100644 --- a/operators/src/processing/interpolation/mod.rs +++ b/operators/src/processing/interpolation/mod.rs @@ -4,8 +4,9 @@ use std::sync::Arc; use crate::adapters::{ FoldTileAccu, FoldTileAccuMut, RasterSubQueryAdapter, SubQueryTileAggregator, }; +use crate::define_operator; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; @@ -24,24 +25,25 @@ use geoengine_datatypes::raster::{ NearestNeighbor, Pixel, RasterTile2D, TileInformation, TilingSpecification, }; use rayon::ThreadPool; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::{ensure, Snafu}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct InterpolationParams { pub interpolation: InterpolationMethod, pub input_resolution: InputResolution, } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type")] pub enum InputResolution { Value(SpatialResolution), Source, } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum InterpolationMethod { NearestNeighbor, @@ -57,11 +59,13 @@ pub enum InterpolationError { UnknownInputResolution, } -pub type Interpolation = Operator; - -impl OperatorName for Interpolation { - const TYPE_NAME: &'static str = "Interpolation"; -} +define_operator!( + Interpolation, + InterpolationParams, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/interpolation.html" +); #[typetag::serde] #[async_trait] diff --git a/operators/src/processing/line_simplification.rs b/operators/src/processing/line_simplification.rs index 2daf58c89..24881ff27 100644 --- a/operators/src/processing/line_simplification.rs +++ b/operators/src/processing/line_simplification.rs @@ -1,9 +1,9 @@ use crate::{ + define_operator, engine::{ CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, - Operator, OperatorName, QueryContext, QueryProcessor, SingleVectorSource, - TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, - WorkflowOperatorPath, + OperatorName, QueryContext, QueryProcessor, SingleVectorSource, TypedVectorQueryProcessor, + VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }, util::Result, }; @@ -21,6 +21,7 @@ use geoengine_datatypes::{ util::arrow::ArrowTyped, }; use rayon::prelude::{IntoParallelIterator, ParallelIterator}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -29,11 +30,13 @@ use snafu::Snafu; /// The simplification is performed on the geometry of the `Vector` and not on the data. /// The `LineSimplification` operator is only available for (multi-)lines or (multi-)polygons. /// -pub type LineSimplification = Operator; - -impl OperatorName for LineSimplification { - const TYPE_NAME: &'static str = "LineSimplification"; -} +define_operator!( + LineSimplification, + LineSimplificationParams, + SingleVectorSource, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/linesimplification.html" +); #[typetag::serde] #[async_trait] @@ -76,7 +79,7 @@ impl VectorOperator for LineSimplification { span_fn!(LineSimplification); } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct LineSimplificationParams { pub algorithm: LineSimplificationAlgorithm, @@ -85,7 +88,7 @@ pub struct LineSimplificationParams { pub epsilon: Option, } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum LineSimplificationAlgorithm { DouglasPeucker, diff --git a/operators/src/processing/neighborhood_aggregate/mod.rs b/operators/src/processing/neighborhood_aggregate/mod.rs index 10e6df017..b9f452ca4 100644 --- a/operators/src/processing/neighborhood_aggregate/mod.rs +++ b/operators/src/processing/neighborhood_aggregate/mod.rs @@ -5,8 +5,9 @@ use self::aggregate::{AggregateFunction, Neighborhood, StandardDeviation, Sum}; use self::tile_sub_query::NeighborhoodAggregateTileNeighborhood; use crate::adapters::stack_individual_aligned_raster_bands; use crate::adapters::RasterSubQueryAdapter; +use crate::define_operator; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; @@ -19,19 +20,22 @@ use geoengine_datatypes::raster::{ }; use num::Integer; use num_traits::AsPrimitive; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::{Deserialize, Serialize}; use snafu::{ensure, Snafu}; use std::marker::PhantomData; /// A neighborhood aggregate operator applies an aggregate function to each raster pixel and its surrounding. /// For each output pixel, the aggregate function is applied to an input pixel plus its neighborhood. -pub type NeighborhoodAggregate = Operator; - -impl OperatorName for NeighborhoodAggregate { - const TYPE_NAME: &'static str = "NeighborhoodAggregate"; -} - -#[derive(Debug, Serialize, Deserialize, Clone)] +define_operator!( + NeighborhoodAggregate, + NeighborhoodAggregateParams, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/neighborhoodaggregate.html" +); + +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] /// Parameters for the `NeighborhoodAggregate` operator. /// @@ -45,27 +49,117 @@ pub struct NeighborhoodAggregateParams { pub aggregate_function: AggregateFunctionParams, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum AggregateFunctionParams { Sum, StandardDeviation, } -#[derive(Debug, Serialize, Deserialize, Clone)] +/// Generates a schema for [usize; 2] where each dimension is odd. +fn odd_usize2_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Array))), + array: Some(Box::new(ArrayValidation { + items: Some(SingleOrVec::Single(Box::new(Schema::Object( + SchemaObject { + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Integer))), + format: Some("uint".to_owned()), + subschemas: Some(Box::new(SubschemaValidation { + not: Some(Box::new(Schema::Object(SchemaObject { + number: Some(Box::new(NumberValidation { + multiple_of: Some(2.0), + ..Default::default() + })), + ..Default::default() + }))), + ..Default::default() + })), + number: Some(Box::new(NumberValidation { + minimum: Some(0.0), + ..Default::default() + })), + ..Default::default() + }, + )))), + max_items: Some(2), + min_items: Some(2), + ..Default::default() + })), + ..Default::default() + }) +} + +// At the moment it is impossible to check an array has an odd item +// count using JSON Schema (see https://stackoverflow.com/a/77910644). +// Workaround using min length 1 for now. +fn weights_matrix_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Rows".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Array))), + array: Some(Box::new(ArrayValidation { + items: Some(SingleOrVec::Single(Box::new(Schema::Object( + SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Row".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Array))), + format: Some("table".to_owned()), + array: Some(Box::new(ArrayValidation { + items: Some(SingleOrVec::Single(Box::new(Schema::Object( + SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Cell".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new( + InstanceType::Number, + ))), + format: Some("double".to_owned()), + ..Default::default() + }, + )))), + min_items: Some(1), + ..Default::default() + })), + ..Default::default() + }, + )))), + min_items: Some(1), + ..Default::default() + })), + ..Default::default() + }) +} + +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(tag = "type", rename_all = "camelCase")] pub enum NeighborhoodParams { - Rectangle { dimensions: [usize; 2] }, - WeightsMatrix { weights: Vec> }, + #[schemars(title = "Rectangle")] + Rectangle { + #[schemars(schema_with = "odd_usize2_schema")] + dimensions: [usize; 2], + }, + #[schemars(title = "WeightsMatrix")] + WeightsMatrix { + #[schemars(schema_with = "weights_matrix_schema")] + weights: Vec>, + }, } impl NeighborhoodParams { fn dimensions(&self) -> GridShape2D { match self { Self::WeightsMatrix { weights } => { - let x_size = weights.len(); - let y_size = weights.first().map_or(0, Vec::len); - GridShape2D::new([x_size, y_size]) + let y_size = weights.len(); + let x_size = weights.first().map_or(0, Vec::len); + GridShape2D::new([y_size, x_size]) } Self::Rectangle { dimensions } => GridShape2D::new(*dimensions), } diff --git a/operators/src/processing/point_in_polygon.rs b/operators/src/processing/point_in_polygon.rs index 7854e04e2..03bf87d96 100644 --- a/operators/src/processing/point_in_polygon.rs +++ b/operators/src/processing/point_in_polygon.rs @@ -10,12 +10,14 @@ use geoengine_datatypes::dataset::NamedData; use geoengine_datatypes::primitives::CacheHint; use geoengine_datatypes::primitives::VectorQueryRectangle; use rayon::ThreadPool; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; use crate::adapters::FeatureCollectionChunkMerger; +use crate::define_operator; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, Operator, + CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, OperatorName, QueryContext, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; @@ -35,16 +37,18 @@ pub use wrapper::PointInPolygonTesterWithCollection; /// 1. a `MultiPointCollection` source /// 2. a `MultiPolygonCollection` source /// Then, it filters the `MultiPolygonCollection`s so that only those features are retained that are in any polygon. -pub type PointInPolygonFilter = Operator; - -impl OperatorName for PointInPolygonFilter { - const TYPE_NAME: &'static str = "PointInPolygonFilter"; -} - -#[derive(Debug, Clone, Deserialize, Serialize)] +define_operator!( + PointInPolygonFilter, + PointInPolygonFilterParams, + PointInPolygonFilterSource, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/pointinpolygon.html" +); + +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct PointInPolygonFilterParams {} -#[derive(Debug, Clone, Deserialize, Serialize)] +#[derive(Debug, Clone, Deserialize, Serialize, JsonSchema)] pub struct PointInPolygonFilterSource { pub points: Box, pub polygons: Box, diff --git a/operators/src/processing/raster_scaling.rs b/operators/src/processing/raster_scaling.rs index 8d9fcd869..4c03bc220 100644 --- a/operators/src/processing/raster_scaling.rs +++ b/operators/src/processing/raster_scaling.rs @@ -1,5 +1,6 @@ +use crate::define_operator; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, OperatorName, RasterBandDescriptor, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, }; @@ -18,11 +19,12 @@ use geoengine_datatypes::{ use num::FromPrimitive; use num_traits::AsPrimitive; use rayon::ThreadPool; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::marker::PhantomData; use std::sync::Arc; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct RasterScalingParams { slope: SlopeOffsetSelection, @@ -31,14 +33,14 @@ pub struct RasterScalingParams { scaling_mode: ScalingMode, } -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum ScalingMode { MulSlopeAddOffset, SubOffsetDivSlope, } -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type")] enum SlopeOffsetSelection { Auto, @@ -69,11 +71,13 @@ impl Default for SlopeOffsetSelection { /// /// - offset: `msg.calibration_offset` /// - slope: `msg.calibration_slope` -pub type RasterScaling = Operator; - -impl OperatorName for RasterScaling { - const TYPE_NAME: &'static str = "RasterScaling"; -} +define_operator!( + RasterScaling, + RasterScalingParams, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/rasterscaling.html" +); pub struct InitializedRasterScalingOperator { name: CanonicOperatorName, diff --git a/operators/src/processing/raster_stacker.rs b/operators/src/processing/raster_stacker.rs index 11e721aa5..bdbf51160 100644 --- a/operators/src/processing/raster_stacker.rs +++ b/operators/src/processing/raster_stacker.rs @@ -1,9 +1,9 @@ use crate::adapters::{QueryWrapper, RasterStackerAdapter, RasterStackerSource}; +use crate::define_operator; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, - MultipleRasterSources, Operator, OperatorName, QueryContext, RasterBandDescriptor, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, TypedRasterQueryProcessor, - WorkflowOperatorPath, + MultipleRasterSources, OperatorName, QueryContext, RasterBandDescriptor, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, TypedRasterQueryProcessor, WorkflowOperatorPath, }; use crate::error::{ InvalidNumberOfRasterStackerInputs, RasterInputsMustHaveSameSpatialReferenceAndDatatype, @@ -15,10 +15,11 @@ use geoengine_datatypes::primitives::{ partitions_extent, time_interval_extent, BandSelection, RasterQueryRectangle, SpatialResolution, }; use geoengine_datatypes::raster::{DynamicRasterDataType, Pixel, RasterTile2D, RenameBands}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct RasterStackerParams { pub rename_bands: RenameBands, @@ -29,11 +30,13 @@ pub struct RasterStackerParams { /// The tiles are automatically temporally aligned. /// /// All inputs must have the same data type and spatial reference. -pub type RasterStacker = Operator; - -impl OperatorName for RasterStacker { - const TYPE_NAME: &'static str = "RasterStacker"; -} +define_operator!( + RasterStacker, + RasterStackerParams, + MultipleRasterSources, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/rasterstacker.html" +); #[typetag::serde] #[async_trait] diff --git a/operators/src/processing/raster_type_conversion.rs b/operators/src/processing/raster_type_conversion.rs index 3008a5909..9011d4876 100644 --- a/operators/src/processing/raster_type_conversion.rs +++ b/operators/src/processing/raster_type_conversion.rs @@ -4,16 +4,21 @@ use geoengine_datatypes::{ primitives::{BandSelection, RasterQueryRectangle, SpatialPartition2D}, raster::{ConvertDataType, Pixel, RasterDataType, RasterTile2D}, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, Operator, - OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, - RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, WorkflowOperatorPath, -}; use crate::util::Result; +use crate::{ + define_operator, + engine::{ + CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, + Operator, OperatorName, QueryContext, QueryProcessor, RasterOperator, RasterQueryProcessor, + RasterResultDescriptor, SingleRasterSource, TypedRasterQueryProcessor, + WorkflowOperatorPath, + }, +}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct RasterTypeConversionParams { pub output_data_type: RasterDataType, @@ -21,11 +26,13 @@ pub struct RasterTypeConversionParams { /// This operator converts the type of raster data into another type. This may cause precision loss as e.g. `3.1_f32` converted to `u8` will result in `3_u8`. /// In case the value range is to small the operator will clip the values at the bounds of the data range. An example is this: The `u32` value `10000_u32` is converted to `u8`, which has a value range of 0..256. The result is `255_u8` since this is the highest value a `u8` can represent. -pub type RasterTypeConversion = Operator; - -impl OperatorName for RasterTypeConversion { - const TYPE_NAME: &'static str = "RasterTypeConversion"; -} +define_operator!( + RasterTypeConversion, + RasterTypeConversionParams, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/rastertypeconversion.html" +); pub struct InitializedRasterTypeConversionOperator { name: CanonicOperatorName, diff --git a/operators/src/processing/raster_vector_join/mod.rs b/operators/src/processing/raster_vector_join/mod.rs index 2ee37bce5..84aabdaad 100644 --- a/operators/src/processing/raster_vector_join/mod.rs +++ b/operators/src/processing/raster_vector_join/mod.rs @@ -3,6 +3,7 @@ mod aggregator; mod non_aggregated; mod util; +use crate::define_operator; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedVectorOperator, Operator, OperatorName, SingleVectorMultipleRasterSources, TypedRasterQueryProcessor, @@ -18,6 +19,7 @@ use async_trait::async_trait; use geoengine_datatypes::collections::VectorDataType; use geoengine_datatypes::primitives::FeatureDataType; use geoengine_datatypes::raster::{Pixel, RasterDataType, RenameBands}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; @@ -27,16 +29,18 @@ use self::aggregator::{ }; /// An operator that attaches raster values to vector data -pub type RasterVectorJoin = Operator; - -impl OperatorName for RasterVectorJoin { - const TYPE_NAME: &'static str = "RasterVectorJoin"; -} +define_operator!( + RasterVectorJoin, + RasterVectorJoinParams, + SingleVectorMultipleRasterSources, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/rastervectorjoin.html" +); const MAX_NUMBER_OF_RASTER_INPUTS: usize = 8; /// The parameter spec for `RasterVectorJoin` -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct RasterVectorJoinParams { /// The names of the new columns are derived from the names of the raster bands. @@ -60,7 +64,7 @@ pub struct RasterVectorJoinParams { pub temporal_aggregation_ignore_no_data: bool, } -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase", tag = "type", content = "values")] pub enum ColumnNames { Default, // use the input band name and append " (n)" to the band name for the `n`-th conflict, @@ -81,7 +85,7 @@ impl From for RenameBands { /// How to aggregate the values for the geometries inside a feature e.g. /// the mean of all the raster values corresponding to the individual /// points inside a `MultiPoint` feature. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Copy)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum FeatureAggregationMethod { First, @@ -92,7 +96,7 @@ pub enum FeatureAggregationMethod { /// If there are multiple rasters valid during the validity of a feature /// the featuer is either split into multiple (None-aggregation) or the /// values are aggreagated -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Copy)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum TemporalAggregationMethod { None, diff --git a/operators/src/processing/rasterization/mod.rs b/operators/src/processing/rasterization/mod.rs index c57559b98..2d704ffed 100644 --- a/operators/src/processing/rasterization/mod.rs +++ b/operators/src/processing/rasterization/mod.rs @@ -1,16 +1,17 @@ use crate::engine::TypedVectorQueryProcessor::MultiPoint; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, - InitializedVectorOperator, Operator, OperatorName, QueryContext, QueryProcessor, - RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - SingleVectorSource, TypedRasterQueryProcessor, TypedVectorQueryProcessor, WorkflowOperatorPath, + InitializedVectorOperator, OperatorName, QueryContext, QueryProcessor, RasterBandDescriptors, + RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleVectorSource, + TypedRasterQueryProcessor, TypedVectorQueryProcessor, WorkflowOperatorPath, }; use arrow::datatypes::ArrowNativeTypeOp; use geoengine_datatypes::primitives::{CacheHint, ColumnSelection}; +use schemars::JsonSchema; -use crate::error; use crate::processing::rasterization::GridOrDensity::Grid; use crate::util; +use crate::{define_operator, error}; use async_trait::async_trait; @@ -38,13 +39,15 @@ use crate::util::{spawn_blocking, spawn_blocking_with_thread_pool}; use typetag::serde; /// An operator that rasterizes vector data -pub type Rasterization = Operator; - -impl OperatorName for Rasterization { - const TYPE_NAME: &'static str = "Rasterization"; -} - -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +define_operator!( + Rasterization, + GridOrDensity, + SingleVectorSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/rasterization.html" +); + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub enum GridSizeMode { /// The spatial resolution is interpreted as a fixed size in coordinate units @@ -53,7 +56,7 @@ pub enum GridSizeMode { Relative, } -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] pub enum GridOrDensity { @@ -63,7 +66,7 @@ pub enum GridOrDensity { Density(DensityParams), } -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct DensityParams { /// Defines the cutoff (as percentage of maximum density) down to which a point is taken /// into account for an output pixel density value @@ -72,7 +75,7 @@ pub struct DensityParams { stddev: f64, } -#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Copy, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct GridParams { /// The size of grid cells, interpreted depending on the chosen grid size mode diff --git a/operators/src/processing/reprojection.rs b/operators/src/processing/reprojection.rs index 029bf182b..dec99ffab 100644 --- a/operators/src/processing/reprojection.rs +++ b/operators/src/processing/reprojection.rs @@ -6,10 +6,11 @@ use crate::{ fold_by_coordinate_lookup_future, FillerTileCacheExpirationStrategy, RasterSubQueryAdapter, SparseTilesFillAdapter, TileReprojectionSubQuery, }, + define_operator, engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSources, - InitializedVectorOperator, Operator, OperatorName, QueryContext, QueryProcessor, - RasterOperator, RasterQueryProcessor, RasterResultDescriptor, SingleRasterOrVectorSource, + InitializedVectorOperator, OperatorName, QueryContext, QueryProcessor, RasterOperator, + RasterQueryProcessor, RasterResultDescriptor, SingleRasterOrVectorSource, TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }, @@ -33,9 +34,10 @@ use geoengine_datatypes::{ spatial_reference::SpatialReference, util::arrow::ArrowTyped, }; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, Deserialize, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct ReprojectionParams { pub target_spatial_reference: SpatialReference, @@ -47,13 +49,13 @@ pub struct ReprojectionBounds { valid_out_bounds: SpatialPartition2D, } -pub type Reprojection = Operator; - -impl Reprojection {} - -impl OperatorName for Reprojection { - const TYPE_NAME: &'static str = "Reprojection"; -} +define_operator!( + Reprojection, + ReprojectionParams, + SingleRasterOrVectorSource, + output_type = "copyFromSource", + help_url = "https://docs.geoengine.io/operators/reprojection.html" +); pub struct InitializedVectorReprojection { name: CanonicOperatorName, diff --git a/operators/src/processing/rgb.rs b/operators/src/processing/rgb.rs index 8d644bad9..f91ccbec0 100644 --- a/operators/src/processing/rgb.rs +++ b/operators/src/processing/rgb.rs @@ -1,8 +1,9 @@ use crate::{ adapters::{QueryWrapper, RasterArrayTimeAdapter}, + define_operator, engine::{ BoxRasterQueryProcessor, CanonicOperatorName, ExecutionContext, InitializedRasterOperator, - InitializedSources, Operator, OperatorData, OperatorName, QueryContext, QueryProcessor, + InitializedSources, OperatorData, OperatorName, QueryContext, QueryProcessor, RasterBandDescriptors, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, TypedRasterQueryProcessor, WorkflowOperatorPath, }, @@ -22,16 +23,28 @@ use geoengine_datatypes::{ spatial_reference::SpatialReferenceOption, }; use num_traits::AsPrimitive; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::{ensure, Snafu}; /// The `Rgb` operator combines three raster sources into a single raster output. /// The output it of type `U32` and contains the red, green and blue values as the first, second and third byte. /// The forth byte (alpha) is always 255. -pub type Rgb = Operator; +define_operator!( + Rgb, + RgbParams, + RgbSources, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/rgb.html" +); + +// cannot use num_traits::One::one() for JsonSchema because of missing type signatures +fn one() -> f64 { + 1.0 +} /// Parameters for the `Rgb` operator. -#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Copy)] +#[derive(Debug, Serialize, Deserialize, PartialEq, Clone, Copy, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct RgbParams { /// The minimum value for the red channel. @@ -39,7 +52,7 @@ pub struct RgbParams { /// The maximum value for the red channel. pub red_max: f64, /// A scaling factor for the red channel between 0 and 1. - #[serde(default = "num_traits::One::one")] + #[serde(default = "one")] pub red_scale: f64, /// The minimum value for the red channel. @@ -47,7 +60,7 @@ pub struct RgbParams { /// The maximum value for the red channel. pub green_max: f64, /// A scaling factor for the green channel between 0 and 1. - #[serde(default = "num_traits::One::one")] + #[serde(default = "one")] pub green_scale: f64, /// The minimum value for the red channel. @@ -55,7 +68,7 @@ pub struct RgbParams { /// The maximum value for the red channel. pub blue_max: f64, /// A scaling factor for the blue channel between 0 and 1. - #[serde(default = "num_traits::One::one")] + #[serde(default = "one")] pub blue_scale: f64, } @@ -101,7 +114,7 @@ impl RgbParams { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] pub struct RgbSources { red: Box, green: Box, @@ -223,10 +236,6 @@ impl RasterOperator for Rgb { span_fn!(Rgb); } -impl OperatorName for Rgb { - const TYPE_NAME: &'static str = "Rgb"; -} - pub struct InitializedRgb { name: CanonicOperatorName, result_descriptor: RasterResultDescriptor, diff --git a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs index 17e2bb21c..3d48b8d7e 100644 --- a/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs +++ b/operators/src/processing/temporal_raster_aggregation/temporal_aggregation_operator.rs @@ -9,9 +9,10 @@ use super::first_last_subquery::{ first_tile_fold_future, last_tile_fold_future, TemporalRasterAggregationSubQueryNoDataOnly, }; use crate::adapters::stack_individual_aligned_raster_bands; +use crate::define_operator; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedSources, Operator, QueryProcessor, - RasterOperator, SingleRasterSource, WorkflowOperatorPath, + CanonicOperatorName, ExecutionContext, InitializedSources, QueryProcessor, RasterOperator, + SingleRasterSource, WorkflowOperatorPath, }; use crate::{ adapters::SubQueryTileAggregator, @@ -29,13 +30,14 @@ use geoengine_datatypes::primitives::{ use geoengine_datatypes::raster::{Pixel, RasterDataType, RasterTile2D}; use geoengine_datatypes::{primitives::TimeStep, raster::TilingSpecification}; use log::debug; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; use std::marker::PhantomData; use typetag; -#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct TemporalRasterAggregationParameters { pub aggregation: Aggregation, @@ -48,7 +50,7 @@ pub struct TemporalRasterAggregationParameters { pub output_type: Option, } -#[derive(Debug, Clone, Copy, Deserialize, Serialize)] +#[derive(Debug, Clone, Copy, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] pub enum Aggregation { @@ -68,12 +70,13 @@ pub enum Aggregation { Count { ignore_no_data: bool }, } -pub type TemporalRasterAggregation = - Operator; - -impl OperatorName for TemporalRasterAggregation { - const TYPE_NAME: &'static str = "TemporalRasterAggregation"; -} +define_operator!( + TemporalRasterAggregation, + TemporalRasterAggregationParameters, + SingleRasterSource, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/temporalrasteraggregation.html" +); #[typetag::serde] #[async_trait] diff --git a/operators/src/processing/time_shift.rs b/operators/src/processing/time_shift.rs index 4d4d79c39..8c61e1f42 100644 --- a/operators/src/processing/time_shift.rs +++ b/operators/src/processing/time_shift.rs @@ -1,10 +1,11 @@ +use crate::define_operator; use crate::engine::{ CanonicOperatorName, ExecutionContext, InitializedRasterOperator, InitializedSingleRasterOrVectorOperator, InitializedSources, InitializedVectorOperator, - Operator, OperatorName, QueryContext, RasterOperator, RasterQueryProcessor, - RasterResultDescriptor, ResultDescriptor, SingleRasterOrVectorSource, - TypedRasterQueryProcessor, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, - VectorResultDescriptor, WorkflowOperatorPath, + OperatorName, QueryContext, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, + ResultDescriptor, SingleRasterOrVectorSource, TypedRasterQueryProcessor, + TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, + WorkflowOperatorPath, }; use crate::util::Result; use async_trait::async_trait; @@ -21,17 +22,20 @@ use geoengine_datatypes::primitives::{ use geoengine_datatypes::primitives::{TimeStep, VectorQueryRectangle}; use geoengine_datatypes::raster::{Pixel, RasterTile2D}; use geoengine_datatypes::util::arrow::ArrowTyped; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::Snafu; /// Project the query rectangle to a new time interval. -pub type TimeShift = Operator; - -impl OperatorName for TimeShift { - const TYPE_NAME: &'static str = "TimeShift"; -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +define_operator!( + TimeShift, + TimeShiftParams, + SingleRasterOrVectorSource, + output_type = "copyFromSource", + help_url = "https://docs.geoengine.io/operators/timeshift.html" +); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(tag = "type", rename_all = "camelCase")] pub enum TimeShiftParams { /// Shift the query rectangle relative with a time step diff --git a/operators/src/processing/vector_join/mod.rs b/operators/src/processing/vector_join/mod.rs index 7569f5917..a350895c2 100644 --- a/operators/src/processing/vector_join/mod.rs +++ b/operators/src/processing/vector_join/mod.rs @@ -1,15 +1,16 @@ use geoengine_datatypes::dataset::NamedData; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ensure; use geoengine_datatypes::collections::VectorDataType; use crate::engine::{ - CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, Operator, + CanonicOperatorName, ExecutionContext, InitializedSources, InitializedVectorOperator, OperatorData, OperatorName, TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, WorkflowOperatorPath, }; -use crate::error; +use crate::{define_operator, error}; use crate::util::Result; use self::equi_data_join::EquiGeoToDataJoinProcessor; @@ -21,21 +22,23 @@ mod equi_data_join; mod util; /// The vector join operator requires two inputs and the join type. -pub type VectorJoin = Operator; - -impl OperatorName for VectorJoin { - const TYPE_NAME: &'static str = "VectorJoin"; -} +define_operator!( + VectorJoin, + VectorJoinParams, + VectorJoinSources, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/vectorjoin.html" +); /// A set of parameters for the `VectorJoin` -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct VectorJoinParams { #[serde(flatten)] join_type: VectorJoinType, } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct VectorJoinSources { left: Box, @@ -75,7 +78,7 @@ impl InitializedSources for VectorJoinSources { } /// Define the type of join -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(tag = "type")] pub enum VectorJoinType { /// An inner equi-join between a `GeoFeatureCollection` and a `DataCollection` diff --git a/operators/src/source/gdal_source/mod.rs b/operators/src/source/gdal_source/mod.rs index 7f018e428..8bcae1a66 100644 --- a/operators/src/source/gdal_source/mod.rs +++ b/operators/src/source/gdal_source/mod.rs @@ -1,4 +1,5 @@ use crate::adapters::{FillerTileCacheExpirationStrategy, SparseTilesFillAdapter}; +use crate::define_operator; use crate::engine::{ CanonicOperatorName, MetaData, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, }; @@ -9,7 +10,7 @@ use crate::util::TemporaryGdalThreadLocalConfigOptions; use crate::{ engine::{ InitializedRasterOperator, RasterOperator, RasterQueryProcessor, RasterResultDescriptor, - SourceOperator, TypedRasterQueryProcessor, + TypedRasterQueryProcessor, }, error::Error, util::Result, @@ -51,6 +52,7 @@ pub use loading_info::{ use log::debug; use num::FromPrimitive; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::{ensure, ResultExt}; use std::collections::HashMap; @@ -94,7 +96,7 @@ static GDAL_RETRY_EXPONENTIAL_BACKOFF_FACTOR: f64 = 2.; /// }, /// }); /// ``` -#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize, JsonSchema)] pub struct GdalSourceParameters { pub data: NamedData, } @@ -758,11 +760,12 @@ where } } -pub type GdalSource = SourceOperator; - -impl OperatorName for GdalSource { - const TYPE_NAME: &'static str = "GdalSource"; -} +define_operator!( + GdalSource, + GdalSourceParameters, + output_type = "raster", + help_url = "https://docs.geoengine.io/operators/gdalsource.html" +); #[typetag::serde] #[async_trait] diff --git a/operators/src/source/ogr_source/mod.rs b/operators/src/source/ogr_source/mod.rs index 7e2542420..9102d8593 100644 --- a/operators/src/source/ogr_source/mod.rs +++ b/operators/src/source/ogr_source/mod.rs @@ -1,6 +1,7 @@ mod dataset_iterator; use self::dataset_iterator::OgrDatasetIterator; use crate::adapters::FeatureCollectionStreamExt; +use crate::define_operator; use crate::engine::{ CanonicOperatorName, OperatorData, OperatorName, QueryProcessor, WorkflowOperatorPath, }; @@ -9,8 +10,8 @@ use crate::util::input::StringOrNumberRange; use crate::util::Result; use crate::{ engine::{ - InitializedVectorOperator, MetaData, QueryContext, SourceOperator, - TypedVectorQueryProcessor, VectorOperator, VectorQueryProcessor, VectorResultDescriptor, + InitializedVectorOperator, MetaData, QueryContext, TypedVectorQueryProcessor, + VectorOperator, VectorQueryProcessor, VectorResultDescriptor, }, error, }; @@ -40,6 +41,7 @@ use log::debug; use pin_project::pin_project; use postgres_protocol::escape::{escape_identifier, escape_literal}; use postgres_types::{FromSql, ToSql}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use snafu::ResultExt; use std::collections::{HashMap, HashSet}; @@ -54,15 +56,16 @@ use std::sync::Arc; use std::task::Poll; use tokio::sync::Mutex; -#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)] +#[derive(Clone, Debug, PartialEq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct OgrSourceParameters { pub data: NamedData, + #[schemars(title = "Attributes to select")] pub attribute_projection: Option>, pub attribute_filters: Option>, } -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, JsonSchema)] #[serde(rename_all = "camelCase")] pub struct AttributeFilter { pub attribute: String, @@ -76,11 +79,12 @@ impl OperatorData for OgrSourceParameters { } } -pub type OgrSource = SourceOperator; - -impl OperatorName for OgrSource { - const TYPE_NAME: &'static str = "OgrSource"; -} +define_operator!( + OgrSource, + OgrSourceParameters, + output_type = "vector", + help_url = "https://docs.geoengine.io/operators/ogrsource.html" +); /// - `file_name`: path to the input file /// - `layer_name`: name of the layer to load diff --git a/operators/src/util/input/raster_or_vector.rs b/operators/src/util/input/raster_or_vector.rs index c05854000..5b78f674d 100644 --- a/operators/src/util/input/raster_or_vector.rs +++ b/operators/src/util/input/raster_or_vector.rs @@ -1,9 +1,10 @@ use crate::engine::{OperatorData, RasterOperator, TypedOperator, VectorOperator}; use geoengine_datatypes::dataset::NamedData; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// It is either a `RasterOperator` or a `VectorOperator` -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(untagged)] pub enum RasterOrVectorOperator { Raster(Box), diff --git a/operators/src/util/input/string_or_number_range.rs b/operators/src/util/input/string_or_number_range.rs index 8d78bfb33..62b3ec0be 100644 --- a/operators/src/util/input/string_or_number_range.rs +++ b/operators/src/util/input/string_or_number_range.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::{convert::TryFrom, ops::RangeInclusive}; use crate::error; @@ -5,6 +6,7 @@ use crate::util::input::StringOrNumber; use crate::util::Result; use geoengine_datatypes::primitives::FeatureDataValue; use num_traits::AsPrimitive; +use schemars::{gen::SchemaGenerator, schema::Schema, JsonSchema}; use serde::de::{Error, SeqAccess, Visitor}; use serde::ser::SerializeTuple; use serde::{Deserialize, Deserializer, Serialize, Serializer}; @@ -19,6 +21,98 @@ pub enum StringOrNumberRange { Int(RangeInclusive), } +impl JsonSchema for StringOrNumberRange { + fn schema_name() -> String { + "StringOrNumberRange".to_owned() + } + + fn schema_id() -> Cow<'static, str> { + Cow::Borrowed(concat!(module_path!(), "::StringOrNumberRange")) + } + + fn json_schema(_gen: &mut SchemaGenerator) -> Schema { + use schemars::schema::*; + Schema::Object(SchemaObject { + subschemas: Some(Box::new(SubschemaValidation { + one_of: Some(vec![ + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("String range".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Array))), + array: Some(Box::new(ArrayValidation { + items: Some(SingleOrVec::Vec(vec![ + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Start inclusive".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new( + InstanceType::String, + ))), + ..Default::default() + }), + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("End inclusive".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new( + InstanceType::String, + ))), + ..Default::default() + }), + ])), + max_items: Some(2), + min_items: Some(2), + ..Default::default() + })), + ..Default::default() + }), + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Number range".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new(InstanceType::Array))), + array: Some(Box::new(ArrayValidation { + items: Some(SingleOrVec::Vec(vec![ + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("Start inclusive".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new( + InstanceType::Number, + ))), + ..Default::default() + }), + Schema::Object(SchemaObject { + metadata: Some(Box::new(Metadata { + title: Some("End inclusive".to_owned()), + ..Default::default() + })), + instance_type: Some(SingleOrVec::Single(Box::new( + InstanceType::Number, + ))), + ..Default::default() + }), + ])), + max_items: Some(2), + min_items: Some(2), + ..Default::default() + })), + ..Default::default() + }), + ]), + ..Default::default() + })), + ..Default::default() + }) + } +} + impl StringOrNumberRange { pub fn into_float_range(self) -> Result { RangeInclusive::::try_from(self).map(Into::into) diff --git a/services/Cargo.toml b/services/Cargo.toml index fa46af6b9..f46b9bc62 100644 --- a/services/Cargo.toml +++ b/services/Cargo.toml @@ -117,6 +117,7 @@ walkdir = "2.4" xgboost-rs = { version = "0.3", optional = true, features = ["use_serde"] } zip = "0.6" assert-json-diff = "2.0.2" +actix-cors = "0.7.0" [target.'cfg(target_os = "linux")'.dependencies] nix = { version = "0.27", features = ["socket"] } diff --git a/services/src/api/apidoc.rs b/services/src/api/apidoc.rs index 9969fd0cd..f73b893e0 100644 --- a/services/src/api/apidoc.rs +++ b/services/src/api/apidoc.rs @@ -106,6 +106,7 @@ use utoipa::{Modify, OpenApi}; handlers::workflows::raster_stream_websocket, handlers::workflows::register_workflow_handler, handlers::workflows::get_workflow_all_metadata_zip_handler, + handlers::workflows::get_workflow_schema_handler, handlers::datasets::delete_dataset_handler, handlers::datasets::list_datasets_handler, handlers::datasets::list_volumes_handler, diff --git a/services/src/api/handlers/workflows.rs b/services/src/api/handlers/workflows.rs index 95928de31..b68e302c9 100755 --- a/services/src/api/handlers/workflows.rs +++ b/services/src/api/handlers/workflows.rs @@ -23,7 +23,8 @@ use geoengine_datatypes::primitives::{ }; use geoengine_operators::call_on_typed_operator; use geoengine_operators::engine::{ - ExecutionContext, OperatorData, TypedOperator, TypedResultDescriptor, WorkflowOperatorPath, + build_workflow_schema, ExecutionContext, OperatorData, TypedOperator, TypedResultDescriptor, + WorkflowOperatorPath, }; use serde::{Deserialize, Serialize}; use snafu::{ResultExt, Snafu}; @@ -42,6 +43,7 @@ where // TODO: rename to plural `workflows` web::scope("/workflow") .service(web::resource("").route(web::post().to(register_workflow_handler::))) + .service(web::resource("/schema").route(web::get().to(get_workflow_schema_handler))) .service( web::scope("/{id}") .service(web::resource("").route(web::get().to(load_workflow_handler::))) @@ -650,6 +652,19 @@ async fn vector_stream_websocket( } } +/// Gets the schema of workflows +#[utoipa::path( + tag = "Workflows", + get, + path = "/workflow/schema", + responses( + (status = 200, description = "JSON Schema for workflows") + ) +)] +async fn get_workflow_schema_handler() -> impl Responder { + web::Json(build_workflow_schema()) +} + #[derive(Debug, Snafu)] #[snafu(visibility(pub(crate)))] #[snafu(module(error), context(suffix(false)))] // disables default `Snafu` suffix diff --git a/services/src/bin/main.rs b/services/src/bin/main.rs index 80724855d..e3746fa03 100644 --- a/services/src/bin/main.rs +++ b/services/src/bin/main.rs @@ -150,7 +150,6 @@ where .pretty() .with_file(false) .with_target(true) - .with_ansi(true) .with_writer(std::io::stderr) .with_filter(filter) } diff --git a/services/src/pro/api/apidoc.rs b/services/src/pro/api/apidoc.rs index 2801bb4d9..6e19b0eef 100644 --- a/services/src/pro/api/apidoc.rs +++ b/services/src/pro/api/apidoc.rs @@ -115,6 +115,7 @@ use utoipa::{Modify, OpenApi}; handlers::workflows::raster_stream_websocket, handlers::workflows::register_workflow_handler, handlers::workflows::get_workflow_all_metadata_zip_handler, + handlers::workflows::get_workflow_schema_handler, pro::api::handlers::users::anonymous_handler, pro::api::handlers::users::login_handler, pro::api::handlers::users::logout_handler, diff --git a/services/src/pro/server.rs b/services/src/pro/server.rs index 0c38f505e..0d60c7640 100644 --- a/services/src/pro/server.rs +++ b/services/src/pro/server.rs @@ -7,8 +7,9 @@ use crate::pro::api::ApiDoc; use crate::pro::contexts::ProPostgresContext; use crate::util::config::{self, get_config_element}; use crate::util::server::{ - calculate_max_blocking_threads_per_worker, configure_extractors, connection_init, - log_server_info, render_404, render_405, serve_openapi_json, CustomRootSpanBuilder, + calculate_max_blocking_threads_per_worker, configure_cors, configure_extractors, + connection_init, log_server_info, render_404, render_405, serve_openapi_json, + CustomRootSpanBuilder, }; use actix_files::Files; use actix_web::{http, middleware, web, App, FromRequest, HttpServer}; @@ -109,6 +110,7 @@ where .handler(http::StatusCode::NOT_FOUND, render_404) .handler(http::StatusCode::METHOD_NOT_ALLOWED, render_405), ) + .wrap(configure_cors()) .wrap(TracingLogger::::new()) .service(api) }) diff --git a/services/src/server.rs b/services/src/server.rs index acf688270..7dcfdb043 100644 --- a/services/src/server.rs +++ b/services/src/server.rs @@ -5,8 +5,9 @@ use crate::error::{Error, Result}; use crate::util::config; use crate::util::config::get_config_element; use crate::util::server::{ - calculate_max_blocking_threads_per_worker, configure_extractors, connection_init, - log_server_info, render_404, render_405, serve_openapi_json, CustomRootSpanBuilder, + calculate_max_blocking_threads_per_worker, configure_cors, configure_extractors, + connection_init, log_server_info, render_404, render_405, serve_openapi_json, + CustomRootSpanBuilder, }; use actix_files::Files; use actix_web::{http, middleware, web, App, HttpServer}; @@ -147,6 +148,7 @@ where .handler(http::StatusCode::NOT_FOUND, render_404) .handler(http::StatusCode::METHOD_NOT_ALLOWED, render_405), ) + .wrap(configure_cors()) .wrap(TracingLogger::::new()) .service(api) }) diff --git a/services/src/util/server.rs b/services/src/util/server.rs index da67cd592..dab5bda5d 100644 --- a/services/src/util/server.rs +++ b/services/src/util/server.rs @@ -2,6 +2,7 @@ use crate::api::model::responses::ErrorResponse; use crate::error::Result; use crate::util::config::get_config_element; +use actix_cors::Cors; use actix_http::body::{BoxBody, EitherBody, MessageBody}; use actix_http::header::{HeaderName, HeaderValue}; use actix_http::uri::PathAndQuery; @@ -435,3 +436,18 @@ impl CacheControlHeader for CacheHint { (actix_http::header::CACHE_CONTROL, value) } } + +pub(crate) fn configure_cors() -> Cors { + Cors::default() + .allowed_origin_fn(|header, _req| { + matches!( + header.to_str(), + Ok(origin) + if origin.starts_with("http://localhost") + || origin.starts_with("http://127.0.0.1") + || origin.starts_with("vscode-webview") + ) + }) + .allowed_methods(["GET", "POST", "PATCH"]) + .allowed_headers([http::header::AUTHORIZATION, http::header::CONTENT_TYPE]) +} diff --git a/test_data/provider_defs/chronicledb.json b/test_data/provider_defs/chronicledb.json new file mode 100644 index 000000000..0a629fcb6 --- /dev/null +++ b/test_data/provider_defs/chronicledb.json @@ -0,0 +1,12 @@ +{ + "type": "EdrDataProviderDefinition", + "id": "2579a45e-a2c3-4fdb-946a-d98d9d0eacc4", + "name": "ChronicleDB Demo", + "baseUrl": "", + "vectorSpec": { + "x": "geometry", + "y": null, + "start_time": "start_datetime", + "end_time": "end_datetime" + } +} \ No newline at end of file