Skip to content

Edr provider (TEST) #873

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ChristianBeilschmidt wants to merge 29 commits into
base: main
Choose a base branch
from
Open

Edr provider (TEST) #873

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
670092a
edr
1lutz Jun 2, 2023
4eca860
support geojson in edr
1lutz Jun 2, 2023
779589d
custom data provider id for edr, specify vector columns in config
1lutz Jun 3, 2023
47b5efc
can select parameter for edr raster collection
1lutz Jun 3, 2023
d6b0977
can select height, generate gdalloadinginfo based on temporal extent
1lutz Jun 6, 2023
7d6576b
download using vsicurl_streaming, specify raster metadata in config, …
1lutz Jun 6, 2023
078abc5
split long methods
1lutz Jun 27, 2023
f8f8e2b
honour negative scaley
1lutz Jun 30, 2023
9f45665
load raster info from dataset
1lutz Jul 4, 2023
acee8c5
Merge branch 'main' into edr
1lutz Jul 4, 2023
185f164
support discrete axis
1lutz Jul 31, 2023
e7b5d91
Merge branch 'main' into edr
1lutz Jul 31, 2023
6939e74
support collections with slash and no z
1lutz Aug 2, 2023
4189b47
filter empty collections
1lutz Aug 3, 2023
f72a066
load discrete vrs from config
1lutz Aug 3, 2023
d6683d1
directly show layer when no vertical extent
1lutz Aug 3, 2023
cc087d8
fix clippy
1lutz Aug 3, 2023
d38809a
use helper to load metadata in load_layer
1lutz Aug 4, 2023
bb9cc3f
Merge branch 'main' into edr
1lutz Aug 7, 2023
4c845fa
address comments
1lutz Aug 14, 2023
ef840a2
add comments in edr LayerCollectionProvider
1lutz Aug 14, 2023
655e40d
fix vector layer name
1lutz Aug 15, 2023
3118aa1
more tests for edr
1lutz Aug 16, 2023
a60b100
fix edr tests
1lutz Aug 28, 2023
6b4a66e
Merge branch 'main' into edr
1lutz Aug 28, 2023
47f8a74
include provenance and anonymize test data
1lutz Sep 5, 2023
b9c5960
Merge branch 'main' into edr
1lutz Sep 5, 2023
0e15bfe
remove typetag::serde
1lutz Sep 5, 2023
3a80eaf
add edr to TypedDataProviderDefinition
1lutz Sep 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 120 additions & 4 deletions services/src/api/model/db_types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use super::{
datatypes::{
BoundingBox2D, Breakpoint, ClassificationMeasurement, Colorizer, ContinuousMeasurement,
Coordinate2D, DateTimeParseFormat, DefaultColors, FeatureDataType, LinearGradient,
LogarithmicGradient, Measurement, MultiLineString, MultiPoint, MultiPolygon, NoGeometry,
OverUnderColors, Palette, RgbaColor, SpatialReferenceOption, TimeInstance, TimeInterval,
TimeStep, VectorDataType,
Coordinate2D, DataProviderId, DateTimeParseFormat, DefaultColors, FeatureDataType,
LinearGradient, LogarithmicGradient, Measurement, MultiLineString, MultiPoint,
MultiPolygon, NoGeometry, OverUnderColors, Palette, RgbaColor, SpatialReferenceOption,
TimeInstance, TimeInterval, TimeStep, VectorDataType,
},
operators::{
CsvHeader, FileNotFoundHandling, FormatSpecifics, GdalConfigOption,
Expand All @@ -20,12 +20,14 @@ use crate::{
datasets::{
external::{
aruna::ArunaDataProviderDefinition,
edr::{EdrDataProviderDefinition, EdrVectorSpec},
gbif::GbifDataProviderDefinition,
gfbio_abcd::GfbioAbcdDataProviderDefinition,
gfbio_collections::GfbioCollectionsDataProviderDefinition,
netcdfcf::{EbvPortalDataProviderDefinition, NetCdfCfDataProviderDefinition},
pangaea::PangaeaDataProviderDefinition,
},
listing::Provenance,
storage::MetaDataDefinition,
},
error::Error,
Expand Down Expand Up @@ -1732,6 +1734,79 @@ impl TryFrom<PangaeaDataProviderDefinitionDbType> for PangaeaDataProviderDefinit
}
}

#[derive(Debug, ToSql, FromSql)]
#[postgres(name = "EdrVectorSpec")]
pub struct EdrVectorSpecDbType {
x: String,
y: Option<String>,
time: String,
}

impl From<&EdrVectorSpec> for EdrVectorSpecDbType {
fn from(other: &EdrVectorSpec) -> Self {
Self {
x: other.x.clone(),
y: other.y.clone(),
time: other.time.clone(),
}
}
}

impl TryFrom<EdrVectorSpecDbType> for EdrVectorSpec {
type Error = Error;

fn try_from(other: EdrVectorSpecDbType) -> Result<Self, Self::Error> {
Ok(Self {
x: other.x,
y: other.y,
time: other.time,
})
}
}

#[derive(Debug, ToSql, FromSql)]
#[postgres(name = "EdrDataProviderDefinition")]
pub struct EdrDataProviderDefinitionDbType {
name: String,
id: DataProviderId,
base_url: String,
vector_spec: Option<EdrVectorSpec>,
cache_ttl: CacheTtlSeconds,
/// List of vertical reference systems with a discrete scale
discrete_vrs: Vec<String>,
provenance: Option<Vec<Provenance>>,
}

impl From<&EdrDataProviderDefinition> for EdrDataProviderDefinitionDbType {
fn from(other: &EdrDataProviderDefinition) -> Self {
Self {
name: other.name.clone(),
id: other.id,
base_url: other.base_url.clone().into(),
vector_spec: other.vector_spec.clone(),
cache_ttl: other.cache_ttl,
discrete_vrs: other.discrete_vrs.clone(),
provenance: other.provenance.clone(),
}
}
}

impl TryFrom<EdrDataProviderDefinitionDbType> for EdrDataProviderDefinition {
type Error = Error;

fn try_from(other: EdrDataProviderDefinitionDbType) -> Result<Self, Self::Error> {
Ok(Self {
name: other.name,
id: other.id,
base_url: other.base_url.as_str().try_into()?,
vector_spec: other.vector_spec,
cache_ttl: other.cache_ttl,
discrete_vrs: other.discrete_vrs,
provenance: other.provenance,
})
}
}

#[derive(Debug, ToSql, FromSql)]
#[postgres(name = "DataProviderDefinition")]
pub struct TypedDataProviderDefinitionDbType {
Expand All @@ -1742,6 +1817,7 @@ pub struct TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: Option<EbvPortalDataProviderDefinition>,
net_cdf_cf_data_provider_definition: Option<NetCdfCfDataProviderDefinition>,
pangaea_data_provider_definition: Option<PangaeaDataProviderDefinition>,
edr_data_provider_definition: Option<EdrDataProviderDefinition>,
}

impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
Expand All @@ -1756,6 +1832,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
}
}
TypedDataProviderDefinition::GbifDataProviderDefinition(data_provider_definition) => {
Expand All @@ -1767,6 +1844,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
}
}
TypedDataProviderDefinition::GfbioAbcdDataProviderDefinition(
Expand All @@ -1779,6 +1857,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
},
TypedDataProviderDefinition::GfbioCollectionsDataProviderDefinition(
data_provider_definition,
Expand All @@ -1790,6 +1869,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
},
TypedDataProviderDefinition::EbvPortalDataProviderDefinition(
data_provider_definition,
Expand All @@ -1801,6 +1881,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: Some(data_provider_definition.clone()),
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
},
TypedDataProviderDefinition::NetCdfCfDataProviderDefinition(
data_provider_definition,
Expand All @@ -1812,6 +1893,7 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: Some(data_provider_definition.clone()),
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
},
TypedDataProviderDefinition::PangaeaDataProviderDefinition(
data_provider_definition,
Expand All @@ -1823,7 +1905,20 @@ impl From<&TypedDataProviderDefinition> for TypedDataProviderDefinitionDbType {
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: Some(data_provider_definition.clone()),
edr_data_provider_definition: None,
},
TypedDataProviderDefinition::EdrDataProviderDefinition(data_provider_definition) => {
Self {
aruna_data_provider_definition: None,
gbif_data_provider_definition: None,
gfbio_abcd_data_provider_definition: None,
gfbio_collections_data_provider_definition: None,
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: Some(data_provider_definition.clone()),
}
}
}
}
}
Expand All @@ -1842,6 +1937,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(TypedDataProviderDefinition::ArunaDataProviderDefinition(
data_provider_definition,
)),
Expand All @@ -1853,6 +1949,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(TypedDataProviderDefinition::GbifDataProviderDefinition(
data_provider_definition,
)),
Expand All @@ -1864,6 +1961,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(
TypedDataProviderDefinition::GfbioAbcdDataProviderDefinition(
data_provider_definition,
Expand All @@ -1877,6 +1975,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(
TypedDataProviderDefinition::GfbioCollectionsDataProviderDefinition(
data_provider_definition,
Expand All @@ -1890,6 +1989,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: Some(data_provider_definition),
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(
TypedDataProviderDefinition::EbvPortalDataProviderDefinition(
data_provider_definition,
Expand All @@ -1903,6 +2003,7 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: Some(data_provider_definition),
pangaea_data_provider_definition: None,
edr_data_provider_definition: None,
} => Ok(TypedDataProviderDefinition::NetCdfCfDataProviderDefinition(
data_provider_definition,
)),
Expand All @@ -1914,9 +2015,22 @@ impl TryFrom<TypedDataProviderDefinitionDbType> for TypedDataProviderDefinition
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: Some(data_provider_definition),
edr_data_provider_definition: None,
} => Ok(TypedDataProviderDefinition::PangaeaDataProviderDefinition(
data_provider_definition,
)),
TypedDataProviderDefinitionDbType {
aruna_data_provider_definition: None,
gbif_data_provider_definition: None,
gfbio_abcd_data_provider_definition: None,
gfbio_collections_data_provider_definition: None,
ebv_portal_data_provider_definition: None,
net_cdf_cf_data_provider_definition: None,
pangaea_data_provider_definition: None,
edr_data_provider_definition: Some(data_provider_definition),
} => Ok(TypedDataProviderDefinition::EdrDataProviderDefinition(
data_provider_definition,
)),
_ => Err(Error::UnexpectedInvalidDbTypeConversion),
}
}
Expand Down Expand Up @@ -1999,6 +2113,8 @@ delegate_from_to_sql!(
PangaeaDataProviderDefinition,
PangaeaDataProviderDefinitionDbType
);
delegate_from_to_sql!(EdrDataProviderDefinition, EdrDataProviderDefinitionDbType);
delegate_from_to_sql!(EdrVectorSpec, EdrVectorSpecDbType);
delegate_from_to_sql!(Symbology, SymbologyDbType);
delegate_from_to_sql!(VectorResultDescriptor, VectorResultDescriptorDbType);
delegate_from_to_sql!(
Expand Down
19 changes: 18 additions & 1 deletion services/src/contexts/schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -695,6 +695,22 @@ CREATE TYPE "PangaeaDataProviderDefinition" AS (
cache_ttl int
);

CREATE TYPE "EdrVectorSpec" AS (
x text,
y text,
"time" text
);

CREATE TYPE "EdrDataProviderDefinition" AS (
"name" text,
id uuid,
base_url text,
vector_spec "EdrVectorSpec",
cache_ttl int,
discrete_vrs text [],
provenance "Provenance" []
);

CREATE TYPE "DataProviderDefinition" AS (
-- one of
aruna_data_provider_definition "ArunaDataProviderDefinition",
Expand All @@ -704,7 +720,8 @@ CREATE TYPE "DataProviderDefinition" AS (
"GfbioCollectionsDataProviderDefinition",
ebv_portal_data_provider_definition "EbvPortalDataProviderDefinition",
net_cdf_cf_data_provider_definition "NetCdfCfDataProviderDefinition",
pangaea_data_provider_definition "PangaeaDataProviderDefinition"
pangaea_data_provider_definition "PangaeaDataProviderDefinition",
edr_data_provider_definition "EdrDataProviderDefinition"
);

CREATE TABLE layer_providers (
Expand Down
Loading