Skip to content

get volume and metadata by name, add iterator for datasets #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ChristianBeilschmidt merged 9 commits into from
Jul 23, 2025
Merged

get volume and metadata by name, add iterator for datasets #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
f083850
get volume and metadata by name, add iterator for datasets
jdroenner Jul 21, 2025
58e0a47
Merge branch 'main' of github.com:geo-engine/geoengine-python into vo…
jdroenner Jul 21, 2025
acfa0f0
update tests
jdroenner Jul 22, 2025
8d28261
Merge branch 'main' of github.com:geo-engine/geoengine-python into vo…
jdroenner Jul 22, 2025
0e8c588
update test
jdroenner Jul 22, 2025
3029239
adapt tests and global exported methds
jdroenner Jul 22, 2025
b213c24
update examples
jdroenner Jul 22, 2025
5adf29e
linter reorder
jdroenner Jul 22, 2025
aec7848
add dataset_metadata_by_name to test
jdroenner Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 51 additions & 25 deletions examples/add_public_raster_dataset.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

60 changes: 43 additions & 17 deletions examples/add_public_vector_dataset.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions geoengine/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@
add_dataset,
add_or_replace_dataset_with_permissions,
dataset_info_by_name,
dataset_metadata_by_name,
delete_dataset,
list_datasets,
upload_dataframe,
volume_by_name,
volumes,
)
from .error import (
Expand Down
74 changes: 72 additions & 2 deletions geoengine/datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import tempfile
from abc import abstractmethod
from collections.abc import Iterator
from enum import Enum
from pathlib import Path
from typing import Literal, NamedTuple
Expand Down Expand Up @@ -286,7 +287,8 @@ class AddDatasetProperties:
name: str | None
display_name: str
description: str
source_operator: Literal["GdalSource", "OgrSource"] # TODO: add more operators
# TODO: add more operators
source_operator: Literal["GdalSource", "OgrSource"]
symbology: RasterSymbology | None # TODO: add vector symbology if needed
provenance: list[Provenance] | None

Expand Down Expand Up @@ -537,6 +539,20 @@ def volumes(timeout: int = 60) -> list[Volume]:
return [Volume.from_response(v) for v in response]


def volume_by_name(volume_name: str, timeout: int = 60) -> Volume | None:
"""Returns a volume with the specified name or None if none exists"""
vols = volumes(timeout)
vols = [v for v in vols if v.name == volume_name]

if len(vols) == 0:
return None

if len(vols) > 1:
raise KeyError(f"Volume name {volume_name} is not unique")

return vols[0]


def add_dataset(
data_store: Volume | UploadId,
properties: AddDatasetProperties,
Expand Down Expand Up @@ -619,7 +635,7 @@ class DatasetListOrder(Enum):
NAME_DESC = "NameDesc"


def list_datasets(
def list_datasets_page(
offset: int = 0,
limit: int = 20,
order: DatasetListOrder = DatasetListOrder.NAME_ASC,
Expand All @@ -643,6 +659,38 @@ def list_datasets(
return response


def list_datasets(
offset: int = 0,
limit: int = 200,
order: DatasetListOrder = DatasetListOrder.NAME_ASC,
name_filter: str | None = None,
timeout: int = 60,
) -> Iterator[geoengine_openapi_client.DatasetListing]:
"""List datasets"""

page_size = 20
page_count = 0

while True:
element_num = page_size * page_count

if element_num >= limit:
break

page = list_datasets_page(
element_num + offset, page_size, order=order, name_filter=name_filter, timeout=timeout
)
page_count += 1

if len(page) == 0:
break

for c, p in enumerate(page):
if element_num + c > limit:
break
yield p


def dataset_info_by_name(
dataset_name: DatasetName | str, timeout: int = 60
) -> geoengine_openapi_client.models.Dataset | None:
Expand All @@ -663,3 +711,25 @@ def dataset_info_by_name(
if isinstance(e_body, str) and "CannotLoadDataset" not in e_body:
raise e
return res


def dataset_metadata_by_name(
dataset_name: DatasetName | str, timeout: int = 60
) -> geoengine_openapi_client.models.MetaDataDefinition | None:
"""Get dataset information."""

if not isinstance(dataset_name, DatasetName):
dataset_name = DatasetName(dataset_name)

session = get_session()

with geoengine_openapi_client.ApiClient(session.configuration) as api_client:
datasets_api = geoengine_openapi_client.DatasetsApi(api_client)
res = None
try:
res = datasets_api.get_loading_info_handler(str(dataset_name), _request_timeout=timeout)
except geoengine_openapi_client.exceptions.BadRequestException as e:
e_body = e.body
if isinstance(e_body, str) and "CannotLoadDataset" not in e_body:
raise e
return res
111 changes: 63 additions & 48 deletions tests/test_datasets.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ def test_list_datasets(self):
offset=0, limit=10, order=ge.DatasetListOrder.NAME_ASC, name_filter="Natural Earth II"
)

datasets = list(datasets)

self.assertEqual(len(datasets), 3)

dataset = datasets[0]
Expand All @@ -48,7 +50,7 @@ def test_add_dataset(self):

ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin"))

volumes = ge.volumes()
volume = ge.volume_by_name("test_data")

geo_transform = ge.GeoTransform(x_min=180.0, y_max=90.0, x_pixel_size=0.1, y_pixel_size=-0.1)

Expand All @@ -71,23 +73,23 @@ def test_add_dataset(self):
result_descriptor_measurement = ge.ClassificationMeasurement(
measurement="Land Cover",
classes={
"0": "Water Bodies",
"1": "Evergreen Needleleaf Forests",
"2": "Evergreen Broadleaf Forests",
"3": "Deciduous Needleleaf Forests",
"4": "Deciduous Broadleleaf Forests",
"5": "Mixed Forests",
"6": "Closed Shrublands",
"7": "Open Shrublands",
"8": "Woody Savannas",
"9": "Savannas",
"10": "Grasslands",
"11": "Permanent Wtlands",
"12": "Croplands",
"13": "Urban and Built-Up",
"14": "Cropland-Natural Vegetation Mosaics",
"15": "Snow and Ice",
"16": "Barren or Sparsely Vegetated",
0: "Water Bodies",
1: "Evergreen Needleleaf Forests",
2: "Evergreen Broadleaf Forests",
3: "Deciduous Needleleaf Forests",
4: "Deciduous Broadleleaf Forests",
5: "Mixed Forests",
6: "Closed Shrublands",
7: "Open Shrublands",
8: "Woody Savannas",
9: "Savannas",
10: "Grasslands",
11: "Permanent Wtlands",
12: "Croplands",
13: "Urban and Built-Up",
14: "Cropland-Natural Vegetation Mosaics",
15: "Snow and Ice",
16: "Barren or Sparsely Vegetated",
},
)

Expand All @@ -105,6 +107,7 @@ def test_add_dataset(self):
"time": None,
"params": gdal_params,
"resultDescriptor": result_descriptor.to_api_dict().to_dict(),
"cacheTtl": 0,
}
)

Expand Down Expand Up @@ -138,16 +141,28 @@ def test_add_dataset(self):
],
)

metadata_for_api = geoengine_openapi_client.MetaDataDefinition(
meta_data,
)

dataset_name = ge.add_dataset(
volumes[0],
volume,
add_dataset_properties,
geoengine_openapi_client.MetaDataDefinition(
meta_data,
),
metadata_for_api,
)

self.assertEqual(dataset_name, ge.DatasetName("MCD12C1_test"))
self.assertEqual(len(ge.list_datasets(name_filter="Land Cover TEST")), 1)
self.assertEqual(len(list(ge.list_datasets(name_filter="Land Cover TEST"))), 1)

metadata_from_api = ge.dataset_metadata_by_name(dataset_name)
self.assertEqual(
metadata_from_api.actual_instance.result_descriptor, metadata_for_api.actual_instance.result_descriptor
)
self.assertTrue(
metadata_from_api.actual_instance.params.file_path.endswith(
metadata_for_api.actual_instance.params.file_path
)
)

def test_add_dataset_with_permissions(self):
"""Test `add_datset`."""
Expand All @@ -158,7 +173,7 @@ def test_add_dataset_with_permissions(self):

ge.initialize(ge_instance.address(), credentials=("admin@localhost", "adminadmin"))

volumes = ge.volumes()
volume = ge.volume_by_name("test_data")

geo_transform = ge.GeoTransform(x_min=180.0, y_max=90.0, x_pixel_size=0.1, y_pixel_size=-0.1)

Expand All @@ -181,23 +196,23 @@ def test_add_dataset_with_permissions(self):
result_descriptor_measurement = ge.ClassificationMeasurement(
measurement="Land Cover",
classes={
"0": "Water Bodies",
"1": "Evergreen Needleleaf Forests",
"2": "Evergreen Broadleaf Forests",
"3": "Deciduous Needleleaf Forests",
"4": "Deciduous Broadleleaf Forests",
"5": "Mixed Forests",
"6": "Closed Shrublands",
"7": "Open Shrublands",
"8": "Woody Savannas",
"9": "Savannas",
"10": "Grasslands",
"11": "Permanent Wtlands",
"12": "Croplands",
"13": "Urban and Built-Up",
"14": "Cropland-Natural Vegetation Mosaics",
"15": "Snow and Ice",
"16": "Barren or Sparsely Vegetated",
0: "Water Bodies",
1: "Evergreen Needleleaf Forests",
2: "Evergreen Broadleaf Forests",
3: "Deciduous Needleleaf Forests",
4: "Deciduous Broadleleaf Forests",
5: "Mixed Forests",
6: "Closed Shrublands",
7: "Open Shrublands",
8: "Woody Savannas",
9: "Savannas",
10: "Grasslands",
11: "Permanent Wtlands",
12: "Croplands",
13: "Urban and Built-Up",
14: "Cropland-Natural Vegetation Mosaics",
15: "Snow and Ice",
16: "Barren or Sparsely Vegetated",
},
)

Expand Down Expand Up @@ -244,7 +259,7 @@ def test_add_dataset_with_permissions(self):
permisions = [(REGISTERED_USER_ROLE_ID, Permission.READ)]

dataset_name = ge.add_or_replace_dataset_with_permissions(
volumes[0],
volume,
add_dataset_properties,
geoengine_openapi_client.MetaDataDefinition(
meta_data,
Expand All @@ -253,7 +268,7 @@ def test_add_dataset_with_permissions(self):
)

self.assertEqual(dataset_name, ge.DatasetName("MCD12C1_test"))
self.assertEqual(len(ge.list_datasets(name_filter="Land Cover TEST")), 1)
self.assertEqual(len(list(ge.list_datasets(name_filter="Land Cover TEST"))), 1)
dataset_info = ge.dataset_info_by_name(ge.DatasetName("MCD12C1_test"))
self.assertEqual(dataset_info.name, "MCD12C1_test")
self.assertEqual(dataset_info.description, "Land Cover")
Expand Down Expand Up @@ -292,7 +307,7 @@ def test_add_dataset_with_permissions(self):
)

dataset_name = ge.add_or_replace_dataset_with_permissions(
volumes[0],
volume,
add_dataset_properties,
geoengine_openapi_client.MetaDataDefinition(
meta_data,
Expand All @@ -301,7 +316,7 @@ def test_add_dataset_with_permissions(self):
)

self.assertEqual(dataset_name, ge.DatasetName("MCD12C1_test"))
self.assertEqual(len(ge.list_datasets(name_filter="Land Cover TEST")), 1)
self.assertEqual(len(list(ge.list_datasets(name_filter="Land Cover TEST"))), 1)
dataset_info = ge.dataset_info_by_name(ge.DatasetName("MCD12C1_test"))
self.assertEqual(dataset_info.name, "MCD12C1_test")
self.assertEqual(
Expand Down Expand Up @@ -334,7 +349,7 @@ def test_add_dataset_with_permissions(self):
)

dataset_name = ge.add_or_replace_dataset_with_permissions(
volumes[0],
volume,
add_dataset_properties,
geoengine_openapi_client.MetaDataDefinition(
meta_data,
Expand All @@ -344,8 +359,8 @@ def test_add_dataset_with_permissions(self):
)

self.assertEqual(dataset_name, ge.DatasetName("MCD12C1_test"))
self.assertEqual(len(ge.list_datasets(name_filter="Land Cover TEST")), 1)
dataset_info = ge.dataset_info_by_name(ge.DatasetName("MCD12C1_test"))
self.assertEqual(len(list(ge.list_datasets(name_filter="Land Cover TEST"))), 1)
dataset_info = ge.dataset_info_by_name(dataset_name)
self.assertEqual(dataset_info.name, "MCD12C1_test")
self.assertEqual(
dataset_info.description,
Expand Down