Skip to content

Commit 4fad2d8

Browse files
PondiBPondiB
authored
Refactor ML validation and Torch lantern installation
Refactor ML validation and Torch lantern installation
2 parents 07d96fb + cade8c7 commit 4fad2d8

30 files changed

Lines changed: 1731 additions & 614 deletions

‎.github/workflows/docker-build-verify.yml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,8 @@ on:
1515
- "inst/**"
1616

1717
concurrency:
18-
group: docker-verify-${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
18+
group: docker-verify-${{ github.workflow }}-${{ github.event.pull_request.number
19+
|| github.sha }}
1920
cancel-in-progress: true
2021

2122
jobs:

‎docker/Dockerfile‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,9 +40,12 @@ RUN R -e "install.packages(c('devtools'), repos='https://cloud.r-project.org')"
4040
# Install basic packages first (these are dependencies for others)
4141
RUN R -e "install.packages(c('gdalcubes','plumber','useful','s2','sf','rstac','geojsonsf', 'jsonlite', 'base64enc', 'ids', 'callr' , 'sits'), repos='https://cloud.r-project.org')"
4242

43-
# Install torch's Lantern library (required for deep learning models like TempCNN)
44-
# TORCH_INSTALL=1 auto-confirms the interactive prompt
45-
RUN TORCH_INSTALL=1 R -e "torch::install_torch()"
43+
# Install torch's Lantern library (required for deep learning models like TempCNN).
44+
# TORCH_INSTALL=1 auto-confirms the interactive prompt.
45+
# The second R call fails the build immediately if install_torch() left Lantern
46+
# absent — without it, install failures are silent and only surface at runtime.
47+
RUN TORCH_INSTALL=1 R -e "torch::install_torch()" && \
48+
R -e "t <- torch::torch_tensor(1L); stopifnot(as.numeric(t) == 1L); message('Lantern OK')"
4649

4750

4851
# create directories

‎docker/plumber.R‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,32 @@
99
library(openeocraft)
1010
library(plumber)
1111

12+
# Ensure torch Lantern binary is present before any ML process tries to use it.
13+
# install_torch() is a no-op when Lantern is already installed; it only runs the
14+
# download when the binary is missing (first local run, or a broken Docker build).
15+
if (requireNamespace("torch", quietly = TRUE)) {
16+
lantern_ok <- tryCatch({
17+
torch::torch_tensor(1L)
18+
TRUE
19+
}, error = function(e) FALSE)
20+
21+
if (!lantern_ok) {
22+
message("[startup] Lantern binary not found — running install_torch()...")
23+
tryCatch(
24+
torch::install_torch(),
25+
error = function(e) {
26+
warning(
27+
"[startup] install_torch() failed: ", conditionMessage(e),
28+
"\nTorch-based processes (TempCNN, TAE, LightTAE) will not work.",
29+
call. = FALSE
30+
)
31+
}
32+
)
33+
} else {
34+
message("[startup] torch Lantern OK.")
35+
}
36+
}
37+
1238
# Increase request body size limit (100MB) to handle large payloads
1339
# such as serialized training datasets in process graphs
1440
options(plumber.maxRequestSize = 1024 * 1024 * 100)

‎docker/server.R‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
# Suppress torch's interactive "Do you want to continue?" prompt before any
2+
# package load triggers torch's .onLoad hook (via openeocraft -> sits -> torch).
3+
# Must be set here, before library() calls, for it to take effect in time.
4+
Sys.setenv(TORCH_INSTALL = "1")
5+
16
# TO-DO: show R errors into terminal
27
# Determine plumber file path - works both locally and in Docker
38
docker_path <- "/opt/dockerfiles/docker/plumber.R"

‎docker/verify-r-deps.R‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,19 @@ if (!all(ok)) {
2121
}
2222

2323
message("Docker R dependency check OK (", length(pkgs), " packages).")
24+
25+
# requireNamespace("torch") succeeds even when the Lantern C binary is absent —
26+
# it only loads the R namespace. Verify Lantern is actually present by calling
27+
# a real tensor operation, which triggers the binary load.
28+
message("Verifying torch Lantern binary loads...")
29+
tryCatch({
30+
t <- torch::torch_tensor(1L)
31+
stopifnot(as.numeric(t) == 1L)
32+
message("torch Lantern OK.")
33+
}, error = function(e) {
34+
stop(
35+
"torch Lantern binary failed to load: ", conditionMessage(e),
36+
"\nRun install_torch() to install the missing binary.",
37+
call. = FALSE
38+
)
39+
})

‎inst/demo-lps-2025/00_ml_month.ipynb‎

Lines changed: 22 additions & 161 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
},
2323
{
2424
"cell_type": "code",
25-
"execution_count": 1,
25+
"execution_count": null,
2626
"id": "0751accd",
2727
"metadata": {},
2828
"outputs": [],
@@ -32,21 +32,10 @@
3232
},
3333
{
3434
"cell_type": "code",
35-
"execution_count": 2,
35+
"execution_count": null,
3636
"id": "10662c95",
3737
"metadata": {},
38-
"outputs": [
39-
{
40-
"data": {
41-
"text/plain": [
42-
"<Connection to 'http://127.0.0.1:8000/' with BasicBearerAuth>"
43-
]
44-
},
45-
"execution_count": 2,
46-
"metadata": {},
47-
"output_type": "execute_result"
48-
}
49-
],
38+
"outputs": [],
5039
"source": [
5140
"connection = openeo.connect(\"http://127.0.0.1:8000\")\n",
5241
"connection.authenticate_basic(\"user\", \"password\")"
@@ -62,20 +51,10 @@
6251
},
6352
{
6453
"cell_type": "code",
65-
"execution_count": 3,
54+
"execution_count": null,
6655
"id": "76f35649",
6756
"metadata": {},
68-
"outputs": [
69-
{
70-
"name": "stdout",
71-
"output_type": "stream",
72-
"text": [
73-
"Available collections: ['mpc-landsat-c2-l2', 'mpc-sentinel-2-l2a', 'mpc-sentinel-1-grd', 'mpc-sentinel-1-rtc', 'aws-sentinel-2-l2a', 'aws-landsat-c2-l2', 'cdse-sentinel-2-l2a']\n",
74-
"\n",
75-
"Available processes: ['load_collection', 'mlm_class_random_forest', 'mlm_class_svm', 'mlm_class_xgboost', 'ml_class_mlp', 'mlm_class_tempcnn', 'mlm_class_tae', 'mlm_class_lighttae', 'ml_fit', 'ml_predict', 'ml_validate', 'ml_validate_kfold', 'ml_tune_grid', 'ml_tune_random', 'ml_predict_probabilities', 'ml_uncertainty_class', 'ml_smooth_class', 'ml_label_class', 'cube_regularize', 'ndvi', 'merge_cubes', 'filter_bands', 'save_result', 'load_result', 'export_cube', 'import_cube', 'export_ml_model', 'import_ml_model', 'save_ml_model', 'load_stac_ml', 'load_ml_model']\n"
76-
]
77-
}
78-
],
57+
"outputs": [],
7958
"source": [
8059
"print(\"Available collections:\", connection.list_collection_ids())\n",
8160
"print(\"\\nAvailable processes:\", [p[\"id\"] for p in connection.list_processes()])"
@@ -91,7 +70,7 @@
9170
},
9271
{
9372
"cell_type": "code",
94-
"execution_count": 4,
73+
"execution_count": null,
9574
"id": "24c76f46",
9675
"metadata": {},
9776
"outputs": [],
@@ -116,13 +95,13 @@
11695
},
11796
{
11897
"cell_type": "code",
119-
"execution_count": 5,
98+
"execution_count": null,
12099
"id": "98ebca43",
121100
"metadata": {},
122101
"outputs": [],
123102
"source": [
124103
"datacube = connection.load_collection(\n",
125-
" \"aws-sentinel-2-l2a\",\n",
104+
" \"mpc-sentinel-2-l2a\",\n",
126105
" spatial_extent=bbox,\n",
127106
" temporal_extent=temporal_extent\n",
128107
")"
@@ -138,7 +117,7 @@
138117
},
139118
{
140119
"cell_type": "code",
141-
"execution_count": 6,
120+
"execution_count": null,
142121
"id": "d94d6c9e",
143122
"metadata": {},
144123
"outputs": [],
@@ -163,7 +142,7 @@
163142
},
164143
{
165144
"cell_type": "code",
166-
"execution_count": 7,
145+
"execution_count": null,
167146
"id": "1fef4727",
168147
"metadata": {},
169148
"outputs": [],
@@ -181,7 +160,7 @@
181160
},
182161
{
183162
"cell_type": "code",
184-
"execution_count": 8,
163+
"execution_count": null,
185164
"id": "b26c07aa",
186165
"metadata": {},
187166
"outputs": [],
@@ -199,7 +178,7 @@
199178
},
200179
{
201180
"cell_type": "code",
202-
"execution_count": 9,
181+
"execution_count": null,
203182
"id": "2da13a45",
204183
"metadata": {},
205184
"outputs": [],
@@ -221,7 +200,7 @@
221200
},
222201
{
223202
"cell_type": "code",
224-
"execution_count": 10,
203+
"execution_count": null,
225204
"id": "31cc072e",
226205
"metadata": {},
227206
"outputs": [],
@@ -246,7 +225,7 @@
246225
},
247226
{
248227
"cell_type": "code",
249-
"execution_count": 11,
228+
"execution_count": null,
250229
"id": "daa43808",
251230
"metadata": {},
252231
"outputs": [],
@@ -266,42 +245,10 @@
266245
},
267246
{
268247
"cell_type": "code",
269-
"execution_count": 12,
248+
"execution_count": null,
270249
"id": "13611ba7",
271250
"metadata": {},
272-
"outputs": [
273-
{
274-
"data": {
275-
"text/html": [
276-
"\n",
277-
" <script>\n",
278-
" if (!window.customElements || !window.customElements.get('openeo-model-builder')) {\n",
279-
" var el = document.createElement('script');\n",
280-
" el.src = \"https://cdn.jsdelivr.net/npm/@openeo/vue-components@2/assets/openeo.min.js\";\n",
281-
" document.head.appendChild(el);\n",
282-
"\n",
283-
" var font = document.createElement('font');\n",
284-
" font.as = \"font\";\n",
285-
" font.type = \"font/woff2\";\n",
286-
" font.crossOrigin = true;\n",
287-
" font.href = \"https://use.fontawesome.com/releases/v5.13.0/webfonts/fa-solid-900.woff2\"\n",
288-
" document.head.appendChild(font);\n",
289-
" }\n",
290-
" </script>\n",
291-
" <openeo-model-builder>\n",
292-
" <script type=\"application/json\">{\"id\": \"4cc9ca2532094316b844d16ab271de4e\", \"explicit-zoom\": true, \"height\": \"400px\", \"value\": {\"process_graph\": {\"mlmclasstempcnn1\": {\"process_id\": \"mlm_class_tempcnn\", \"arguments\": {\"batch_size\": 64, \"cnn_dropout_rates\": [0.2, 0.2, 0.2], \"cnn_kernels\": [7, 7, 7], \"cnn_layers\": [256, 256, 256], \"dense_layer_dropout_rate\": 0.5, \"dense_layer_nodes\": 256, \"epochs\": 100, \"learning_rate\": 0.0005, \"optimizer\": \"adam\", \"seed\": 42, \"verbose\": false}}, \"mlfit1\": {\"process_id\": \"ml_fit\", \"arguments\": {\"model\": {\"from_node\": \"mlmclasstempcnn1\"}, \"target\": \"label\", \"training_set\": \"https://github.com/Open-Earth-Monitor/openeocraft/raw/main/inst/demo-paper-2025/data/samples_deforestation_rondonia.rds\"}}, \"savemlmodel1\": {\"process_id\": \"save_ml_model\", \"arguments\": {\"data\": {\"from_node\": \"mlfit1\"}, \"name\": \"tempcnn_rondonia\", \"options\": {}}, \"result\": true}}}}</script>\n",
293-
" </openeo-model-builder>\n",
294-
" "
295-
],
296-
"text/plain": [
297-
"<openeo.rest.ml_extension.MLModel at 0x114ba1750>"
298-
]
299-
},
300-
"execution_count": 12,
301-
"metadata": {},
302-
"output_type": "execute_result"
303-
}
304-
],
251+
"outputs": [],
305252
"source": [
306253
"tempcnn_model.save_ml_model(name=\"tempcnn_rondonia\")"
307254
]
@@ -318,7 +265,7 @@
318265
},
319266
{
320267
"cell_type": "code",
321-
"execution_count": 13,
268+
"execution_count": null,
322269
"id": "1fb2d7cb",
323270
"metadata": {},
324271
"outputs": [],
@@ -328,49 +275,17 @@
328275
},
329276
{
330277
"cell_type": "code",
331-
"execution_count": 14,
278+
"execution_count": null,
332279
"id": "d49d7f98",
333280
"metadata": {},
334-
"outputs": [
335-
{
336-
"data": {
337-
"text/html": [
338-
"\n",
339-
" <script>\n",
340-
" if (!window.customElements || !window.customElements.get('openeo-model-builder')) {\n",
341-
" var el = document.createElement('script');\n",
342-
" el.src = \"https://cdn.jsdelivr.net/npm/@openeo/vue-components@2/assets/openeo.min.js\";\n",
343-
" document.head.appendChild(el);\n",
344-
"\n",
345-
" var font = document.createElement('font');\n",
346-
" font.as = \"font\";\n",
347-
" font.type = \"font/woff2\";\n",
348-
" font.crossOrigin = true;\n",
349-
" font.href = \"https://use.fontawesome.com/releases/v5.13.0/webfonts/fa-solid-900.woff2\"\n",
350-
" document.head.appendChild(font);\n",
351-
" }\n",
352-
" </script>\n",
353-
" <openeo-model-builder>\n",
354-
" <script type=\"application/json\">{\"id\": \"b1eba19a6b924eea92eeeb74fecfb2f1\", \"explicit-zoom\": true, \"height\": \"400px\", \"value\": {\"process_graph\": {\"loadcollection1\": {\"process_id\": \"load_collection\", \"arguments\": {\"id\": \"aws-sentinel-2-l2a\", \"spatial_extent\": {\"west\": -63.33, \"south\": -12.03, \"east\": -62.43, \"north\": -11.13, \"crs\": 4326}, \"temporal_extent\": [\"2022-01-01\", \"2022-12-31\"]}}, \"cuberegularize1\": {\"process_id\": \"cube_regularize\", \"arguments\": {\"data\": {\"from_node\": \"loadcollection1\"}, \"period\": \"P16D\", \"resolution\": 300}}, \"ndvi1\": {\"process_id\": \"ndvi\", \"arguments\": {\"data\": {\"from_node\": \"cuberegularize1\"}, \"nir\": \"B08\", \"red\": \"B04\", \"target_band\": \"NDVI\"}}, \"mlmclasstempcnn1\": {\"process_id\": \"mlm_class_tempcnn\", \"arguments\": {\"batch_size\": 64, \"cnn_dropout_rates\": [0.2, 0.2, 0.2], \"cnn_kernels\": [7, 7, 7], \"cnn_layers\": [256, 256, 256], \"dense_layer_dropout_rate\": 0.5, \"dense_layer_nodes\": 256, \"epochs\": 100, \"learning_rate\": 0.0005, \"optimizer\": \"adam\", \"seed\": 42, \"verbose\": false}}, \"mlfit1\": {\"process_id\": \"ml_fit\", \"arguments\": {\"model\": {\"from_node\": \"mlmclasstempcnn1\"}, \"target\": \"label\", \"training_set\": \"https://github.com/Open-Earth-Monitor/openeocraft/raw/main/inst/demo-paper-2025/data/samples_deforestation_rondonia.rds\"}}, \"mlpredict1\": {\"process_id\": \"ml_predict\", \"arguments\": {\"data\": {\"from_node\": \"ndvi1\"}, \"model\": {\"from_node\": \"mlfit1\"}}}, \"saveresult1\": {\"process_id\": \"save_result\", \"arguments\": {\"data\": {\"from_node\": \"mlpredict1\"}, \"format\": \"GeoTiff\", \"options\": {}}, \"result\": true}}}}</script>\n",
355-
" </openeo-model-builder>\n",
356-
" "
357-
],
358-
"text/plain": [
359-
"<openeo.rest.result.SaveResult at 0x10a230b20>"
360-
]
361-
},
362-
"execution_count": 14,
363-
"metadata": {},
364-
"output_type": "execute_result"
365-
}
366-
],
281+
"outputs": [],
367282
"source": [
368283
"result"
369284
]
370285
},
371286
{
372287
"cell_type": "code",
373-
"execution_count": 15,
288+
"execution_count": null,
374289
"id": "e9777398",
375290
"metadata": {},
376291
"outputs": [],
@@ -383,64 +298,10 @@
383298
},
384299
{
385300
"cell_type": "code",
386-
"execution_count": 16,
301+
"execution_count": null,
387302
"id": "db973d75",
388303
"metadata": {},
389-
"outputs": [
390-
{
391-
"name": "stdout",
392-
"output_type": "stream",
393-
"text": [
394-
"0:00:00 Job '444dbf2f651d5451cb0085cd5f7bbd3c': send 'start'\n",
395-
"0:00:03 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
396-
"0:00:08 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
397-
"0:00:15 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
398-
"0:00:23 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
399-
"0:00:32 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
400-
"0:00:45 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
401-
"0:01:00 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
402-
"0:01:19 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
403-
"0:01:43 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
404-
"0:02:13 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
405-
"0:02:50 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
406-
"0:03:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
407-
"0:04:35 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
408-
"0:05:35 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
409-
"0:06:35 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
410-
"0:07:35 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
411-
"0:08:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
412-
"0:09:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
413-
"0:10:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
414-
"0:11:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
415-
"0:12:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
416-
"0:13:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
417-
"0:14:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
418-
"0:15:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
419-
"0:16:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
420-
"0:17:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
421-
"0:18:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
422-
"0:19:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
423-
"0:20:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
424-
"0:21:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
425-
"0:22:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
426-
"0:23:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
427-
"0:24:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
428-
"0:25:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': running (progress N/A)\n",
429-
"0:26:36 Job '444dbf2f651d5451cb0085cd5f7bbd3c': finished (progress N/A)\n"
430-
]
431-
},
432-
{
433-
"data": {
434-
"text/plain": [
435-
"[PosixPath('data/output_month/openeocraft_merged.tif'),\n",
436-
" PosixPath('data/output_month/job-results.json')]"
437-
]
438-
},
439-
"execution_count": 16,
440-
"metadata": {},
441-
"output_type": "execute_result"
442-
}
443-
],
304+
"outputs": [],
444305
"source": [
445306
"job.start_and_wait()\n",
446307
"results = job.get_results()\n",

0 commit comments

Comments
 (0)