Skip to content

Commit 2848c68

Browse files
michaelmattigmichaelmattig
authored
Merge pull request #211 from geo-engine/ge_test
test geo engine against actual instance in unit tests
2 parents ef00121 + 2ff8af1 commit 2848c68

28 files changed

+913
-2064
lines changed

‎.github/.backend_git_ref‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
main

‎.github/workflows/ci.yml‎

Lines changed: 129 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -10,18 +10,63 @@ jobs:
1010
check:
1111
runs-on: ubuntu-22.04
1212

13+
services:
14+
postgres:
15+
image: postgis/postgis
16+
env:
17+
POSTGRES_USER: geoengine
18+
POSTGRES_PASSWORD: geoengine
19+
POSTGRES_DB: geoengine
20+
ports:
21+
- 5432:5432
22+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
23+
1324
strategy:
1425
fail-fast: false
1526
matrix:
1627
# use all supported versions from https://devguide.python.org/versions/
1728
python-version: ["3.9", "3.10", "3.11", "3.12"]
1829

30+
defaults:
31+
run:
32+
working-directory: library
33+
1934
steps:
20-
- uses: actions/checkout@v3
21-
- name: APT update
22-
run: sudo apt-get update
23-
- name: Install system dependencies
24-
run: sudo apt-get install libgeos-dev libproj-dev
35+
- name: Checkout library code
36+
uses: actions/checkout@v4
37+
with:
38+
path: library
39+
- name: Read backend version
40+
id: read-backend-version
41+
run: echo "GEOENGINE_VERSION=$(cat .github/.backend_git_ref)" >> $GITHUB_OUTPUT
42+
- name: Checkout Geo Engine code
43+
uses: actions/checkout@v4
44+
with:
45+
repository: geo-engine/geoengine
46+
ref: ${{ steps.read-backend-version.outputs.GEOENGINE_VERSION }}
47+
path: backend
48+
- name: Free Disk Space (Ubuntu)
49+
uses: jlumbroso/free-disk-space@main
50+
with:
51+
tool-cache: true
52+
android: true
53+
dotnet: true
54+
haskell: true
55+
large-packages: true
56+
docker-images: true
57+
swap-storage: true
58+
- name: Install lld & GDAL & Protobuf
59+
run: |
60+
sudo apt-get update
61+
sudo apt-get install lld libgdal-dev gdal-bin build-essential clang curl protobuf-compiler libgeos-dev libproj-dev
62+
sudo apt-get clean
63+
export C_INCLUDE_PATH=/usr/include/gdal:$C_INCLUDE_PATH
64+
export CPLUS_INCLUDE_PATH=/usr/include/gdal:$CPLUS_INCLUDE_PATH
65+
sudo ldconfig
66+
- name: Install Rustup
67+
run: |
68+
curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --profile minimal --default-toolchain none -y
69+
echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
2570
- name: Set up Python ${{ matrix.python-version }}
2671
uses: actions/setup-python@v4
2772
with:
@@ -53,12 +98,37 @@ jobs:
5398
python -m mypy tests
5499
- name: Test
55100
run: pytest
101+
env:
102+
GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
103+
GEOENGINE_TEST_BUILD_TYPE: "release"
104+
- name: Examples
105+
run: |
106+
python -m pip install -e .[examples]
107+
python test_all_notebooks.py
108+
env:
109+
GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
110+
GEOENGINE_TEST_BUILD_TYPE: "release"
56111

57112
# Checks the library using minimum version resolution
58113
# `uv` has this feature built-in, c.f. https://github.com/astral-sh/uv
59114
check-min-version:
60115
runs-on: ubuntu-22.04
61116

117+
services:
118+
postgres:
119+
image: postgis/postgis
120+
env:
121+
POSTGRES_USER: geoengine
122+
POSTGRES_PASSWORD: geoengine
123+
POSTGRES_DB: geoengine
124+
ports:
125+
- 5432:5432
126+
options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
127+
128+
defaults:
129+
run:
130+
working-directory: library
131+
62132
env:
63133
# use minimum supported versions from https://devguide.python.org/versions/
64134
python-version: "3.9"
@@ -67,11 +137,41 @@ jobs:
67137
resolution: "lowest-direct"
68138

69139
steps:
70-
- uses: actions/checkout@v3
71-
- name: APT update
72-
run: sudo apt-get update
73-
- name: Install system dependencies
74-
run: sudo apt-get install libgeos-dev libproj-dev
140+
- name: Checkout library code
141+
uses: actions/checkout@v4
142+
with:
143+
path: library
144+
- name: Read backend version
145+
id: read-backend-version
146+
run: echo "GEOENGINE_VERSION=$(cat .github/.backend_git_ref)" >> $GITHUB_OUTPUT
147+
- name: Checkout Geo Engine code
148+
uses: actions/checkout@v4
149+
with:
150+
repository: geo-engine/geoengine
151+
ref: ${{ steps.read-backend-version.outputs.GEOENGINE_VERSION }}
152+
path: backend
153+
- name: Free Disk Space (Ubuntu)
154+
uses: jlumbroso/free-disk-space@main
155+
with:
156+
tool-cache: true
157+
android: true
158+
dotnet: true
159+
haskell: true
160+
large-packages: true
161+
docker-images: true
162+
swap-storage: true
163+
- name: Install lld & GDAL & Protobuf
164+
run: |
165+
sudo apt-get update
166+
sudo apt-get install lld libgdal-dev gdal-bin build-essential clang curl protobuf-compiler libgeos-dev libproj-dev
167+
sudo apt-get clean
168+
export C_INCLUDE_PATH=/usr/include/gdal:$C_INCLUDE_PATH
169+
export CPLUS_INCLUDE_PATH=/usr/include/gdal:$CPLUS_INCLUDE_PATH
170+
sudo ldconfig
171+
- name: Install Rustup
172+
run: |
173+
curl --proto '=https' --tlsv1.2 --retry 10 --retry-connrefused -fsSL "https://sh.rustup.rs" | sh -s -- --profile minimal --default-toolchain none -y
174+
echo "${CARGO_HOME:-$HOME/.cargo}/bin" >> $GITHUB_PATH
75175
- name: Set up Python ${{ env.python-version }}
76176
uses: actions/setup-python@v4
77177
with:
@@ -84,17 +184,32 @@ jobs:
84184
uv venv
85185
source .venv/bin/activate
86186
87-
uv pip install --resolution=${{ env.resolution}} -e .
88-
uv pip install --resolution=${{ env.resolution}} -e .[dev]
187+
uv pip install --resolution=${{ env.resolution }} -e .
188+
uv pip install --resolution=${{ env.resolution }} -e .[dev]
89189
- name: Build
90190
run: |
91191
source .venv/bin/activate
92192
python -m build .
93193
- name: Install test dependencies
94194
run: |
95195
source .venv/bin/activate
96-
uv pip install --resolution=${{ env.resolution}} -e .[test]
196+
uv pip install --resolution=${{ env.resolution }} -e .[test]
97197
- name: Test
98198
run: |
99199
source .venv/bin/activate
100-
pytest
200+
pytest --cov=geoengine --cov-report=lcov
201+
env:
202+
GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
203+
GEOENGINE_TEST_BUILD_TYPE: "release"
204+
- name: Upload coverage to Coveralls
205+
uses: coverallsapp/github-action@v2
206+
with:
207+
base-path: library
208+
- name: Examples
209+
run: |
210+
source .venv/bin/activate
211+
uv pip install --resolution=${{ env.resolution }} -e .[examples]
212+
python test_all_notebooks.py
213+
env:
214+
GEOENGINE_TEST_CODE_PATH: ${{ github.workspace }}/backend
215+
GEOENGINE_TEST_BUILD_TYPE: "release"

‎README.md‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,34 @@ Run tests with:
4747
pytest
4848
```
4949

50+
#### Test instance
51+
52+
You have to set the environment variable `GEOENGINE_TEST_CODE_PATH` to the code folder of the Geo Engine instance you want to test against.
53+
Dotenv is supported, so you can create a `.env` file in the root of the project.
54+
55+
#### Coverage
56+
57+
You can check the coverage with:
58+
59+
```bash
60+
pytest --cov=geoengine
61+
```
62+
63+
### Test examples
64+
65+
You can test the examples with:
66+
67+
```bash
68+
python3 -m pip install -e .[examples]
69+
./test_all_notebooks.py
70+
```
71+
72+
Or you can test a single example with:
73+
74+
```bash
75+
./test_notebook.py examples/ndvi_ports.ipynb
76+
```
77+
5078
## Dependencies
5179

5280
Since we use `cartopy`, you need to have the following system dependencies installed.

‎check.sh‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@ function echoerr() {
1111
echo "$@" 1>&2;
1212
}
1313

14-
echoerr "Running tests"
15-
16-
pytest
17-
1814
echoerr "Check code style"
1915

2016
python3 -m pycodestyle
@@ -28,3 +24,11 @@ echoerr "Check code with type checker"
2824

2925
python3 -m mypy geoengine
3026
python3 -m mypy tests
27+
28+
echoerr "Running tests"
29+
30+
pytest
31+
32+
echoerr "Running examples"
33+
34+
python3 test_all_notebooks.py

‎examples/interactive_ml/app/Dockerfile‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
# Setup a Debian 12 based image with a Python virtualenv
2+
FROM debian:12-slim AS build
3+
RUN apt-get update && \
4+
apt-get install --no-install-suggests --no-install-recommends --yes python3-venv gcc libpython3-dev && \
5+
python3 -m venv /venv && \
6+
/venv/bin/pip install --upgrade pip setuptools wheel
7+
8+
# Install Geo Engine library and its dependencies
9+
FROM build AS build-venv
10+
COPY pyproject.toml setup.cfg setup.py /library/
11+
COPY geoengine /library/geoengine
12+
WORKDIR /library
13+
RUN /venv/bin/pip install --disable-pip-version-check .[dev,test,examples]
14+
15+
# Copy the virtualenv into a distroless image
16+
# Hint: Use the `:debug` tag to get a shell in the image
17+
FROM gcr.io/distroless/python3-debian12
18+
COPY --from=build-venv /venv /venv
19+
20+
# Copy the example notebook to run
21+
COPY examples/interactive_ml /app
22+
23+
WORKDIR /app
24+
25+
ENV GEOENGINE_INSTANCE_URL=https://zentrale.app.geoengine.io/api
26+
ENV GEOENGINE_SESSION_TOKEN=<SESSION_TOKEN>
27+
28+
EXPOSE 8866
29+
30+
ENTRYPOINT [ \
31+
"/venv/bin/python3", \
32+
"-m", \
33+
"voila", \
34+
"--no-browser", \
35+
"--Voila.ip='0.0.0.0'", \
36+
"app/Simple Random Forest Two-Class Classifier on Sentinel-2 Images.ipynb" \
37+
]

‎examples/interactive_ml/app/README.md‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
# Interactive ML App
2+
3+
This app demonstrates a complete workflow for binary classification using Sentinel-2 satellite imagery and a Random Forest classifier. The workflow includes data acquisition, preprocessing, model training, and result visualization. Initially, the environment is set up and necessary libraries are imported. The spatial and temporal bounds for the data query are then defined through a query rectangle. A workflow is created to fetch and preprocess Sentinel-2 data, which is followed by the use of a labeling tool to create training data for water and non-water classes.
4+
5+
## Local setup
6+
7+
To run the app locally, you need to install the dependencies and start the app. You can install the dependencies with:
8+
9+
```bash
10+
pip install --disable-pip-version-check -e .[dev,test,examples]
11+
12+
GEOENGINE_INSTANCE_URL=https://zentrale.app.geoengine.io/api \
13+
GEOENGINE_SESSION_TOKEN=<SESSION_TOKEN> \
14+
./examples/interactive_ml/app/app.sh
15+
```
16+
17+
The app will be available at [http://localhost:8866](http://localhost:8866).
18+
19+
## Container setup
20+
21+
To run the app in a container, you need to build the container image and start the container.
22+
You can build the container image with:
23+
24+
```bash
25+
./examples/interactive_ml/app/build.sh
26+
27+
podman run --rm \
28+
-p 8866:8866 \
29+
-e GEOENGINE_INSTANCE_URL=https://zentrale.app.geoengine.io/api \
30+
-e GEOENGINE_SESSION_TOKEN=<SESSION_TOKEN> \
31+
geoengine-interactive-ml:latest
32+
```
33+
34+
### Upload to quay.io
35+
36+
To upload the container image to quay.io, you need to log in and push the image.
37+
You can do this with:
38+
39+
```bash
40+
podman login quay.io
41+
podman push geoengine-interactive-ml:latest quay.io/geoengine/geoengine-interactive-ml:latest
42+
```

‎examples/interactive_ml/app/Simple Random Forest Two-Class Classifier on Sentinel-2 Images.ipynb‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818
"outputs": [],
1919
"source": [
2020
"import ipyvuetify as vue\n",
21+
"import IPython.display\n",
22+
"import base64\n",
2123
"from dataclasses import dataclass\n",
2224
"from collections.abc import Callable, Coroutine\n",
2325
"from enum import Enum\n",
@@ -341,6 +343,22 @@
341343
"\n"
342344
]
343345
},
346+
{
347+
"cell_type": "code",
348+
"execution_count": null,
349+
"metadata": {},
350+
"outputs": [],
351+
"source": [
352+
"with open(\"assets/favicon.ico\", \"rb\") as image_file:\n",
353+
" favicon = base64.b64encode(image_file.read()).decode()\n",
354+
"\n",
355+
"favicon_changer = IPython.display.Javascript(f'''\n",
356+
" document.querySelector(\"link[rel='icon']\").href = \"data:image/x-icon;base64,{favicon}\";\n",
357+
"''')\n",
358+
"\n",
359+
"IPython.display.display(favicon_changer)"
360+
]
361+
},
344362
{
345363
"cell_type": "code",
346364
"execution_count": null,
@@ -399,7 +417,7 @@
399417
" padding: 0 !important;\n",
400418
" margin: 0 !important;\n",
401419
" }\n",
402-
"</style>\n"
420+
"</style>"
403421
]
404422
},
405423
{
@@ -489,7 +507,7 @@
489507
" self.error_bar,\n",
490508
" vue.Html(tag='br'),\n",
491509
" ]),\n",
492-
" vue.Footer(children=[vue.Col(children=['2024 – Geo Engine GmbH'], class_='text-center')], app=True),\n",
510+
" vue.Footer(children=[vue.Col(children=['2025 – Geo Engine GmbH'], class_='text-center')], app=True),\n",
493511
" ]\n",
494512
"\n",
495513
" def register_location_button(self, callback: Callable[[Location, str, str], None]) -> None:\n",
@@ -742,7 +760,7 @@
742760
"name": "python",
743761
"nbconvert_exporter": "python",
744762
"pygments_lexer": "ipython3",
745-
"version": "3.11.3"
763+
"version": "3.10.12"
746764
}
747765
},
748766
"nbformat": 4,

‎examples/interactive_ml/app/assets/favicon.ico‎

13.2 KB
Binary file not shown.

‎examples/interactive_ml/app/build.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
4+
5+
podman build -f $SCRIPT_DIR/Dockerfile --tag geoengine-interactive-ml:latest .

0 commit comments

Comments
 (0)