Merge pull request #120 from FREVA-CLINT/revise-docker-envs

revise freva-rest docker env variables

.github/workflows/build_job.yml

@@ -119,7 +119,7 @@ jobs:
           context: freva-rest
           file: freva-rest/Dockerfile
           build-args: VERSION=${{ needs.release-type-determination.outputs.tag }}
-          platforms: linux/amd64
+          platforms: linux/amd64, linux/arm64
           push: true
           tags: |
             ghcr.io/freva-clint/freva-rest-api:${{ needs.release-type-determination.outputs.tag }}

dev-env/check-container.py

@@ -21,19 +21,23 @@ def check_container(image_name: str = "freva-rest", container_name: str = "freva
                 "--name", container_name,
                 "--net=host",
                 "-e",
-                "MONGO_USER=mongo",
+                "API_MONGO_USER=mongo",
                 "-e",
-                "MONGO_PASSWORD=secret",
+                "API_MONGO_PASSWORD=secret",
                 "-e",
-                "MONGO_HOST=localhost:27017",
+                "API_MONGO_HOST=localhost:27017",
                 "-e",
                 "API_PORT=7777",
                 "-e",
                 "API_WORKER=8",
                 "-e",
-                "MONGO_DB=search_stats",
+                "API_MONGO_DB=search_stats",
                 "-e",
-                "OIDC_URL=http://localhost:8080/realms/freva/.well-known/openid-configuration",
+                "API_OIDC_DISCOVERY_URL=http://localhost:8080/realms/freva/.well-known/openid-configuration",
+                "-e",
+                "USE_MONGODB=1",
+                "-e",
+                "USE_SOLR=1",
                 image_name,
             ],
         )

freva-rest/Dockerfile

@@ -1,5 +1,6 @@
 FROM debian:bookworm-slim AS base
 ARG VERSION
+ARG TARGETARCH
 
 LABEL org.opencontainers.image.authors="DRKZ-CLINT"
 LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/freva-nextgen/freva-rest"
@@ -11,6 +12,7 @@ RUN apt-get update && \
     python3 python3-pip python3-dev \
     gcc g++ make curl openssl pkg-config \
     python3-setuptools python3-wheel build-essential \
+    python3-numcodecs \
     gnupg wget git ca-certificates \
     default-jre bash procps \
     && rm -rf /var/lib/apt/lists/*
@@ -28,6 +30,8 @@ COPY --from=mongo:latest /usr/lib/ /usr/lib/
 COPY --from=solr:latest /opt/solr/ /opt/solr/
 COPY --from=solr:latest /opt/java/ /opt/java/
 COPY --from=solr:latest /var/solr/ /var/solr/
+
+# freva-service config files
 RUN mkdir -p /tmp/freva && \
     git clone https://github.com/FREVA-CLINT/freva-service-config.git /tmp/freva/freva-service-config && \
     mkdir -p /docker-entrypoint-initdb.d /usr/local/bin && \
@@ -37,42 +41,35 @@ RUN mkdir -p /tmp/freva && \
     cp /tmp/freva/freva-service-config/solr/daily_backup.sh /usr/local/bin/daily_backup && \
     cp /tmp/freva/freva-service-config/mongo/mongo-userdata-init.js /docker-entrypoint-initdb.d/mongo-userdata-init.js
 
-# main env
+    # main env to be used inside of container, but also can be overwritten by env in runtime
 ENV API_CONFIG=/opt/freva-rest/api_config.toml \
-    DEBUG=0 \
-    API_WORKER=8 \
     MONGO_HOME=/opt/mongodb \
     SOLR_HOME=/opt/solr_data \
     JAVA_HOME=/opt/java/openjdk \
-    SOLR_CORE=files \
-    COLUMNS=140 \
     SOLR_LOGS_DIR=/opt/solr_data/logs/solr \
     LOG4J_PROPS=/opt/solr_data/log4j2.xml \
     SOLR_PID_DIR=/opt/solr_data \
     SOLR_JETTY_HOST=0.0.0.0 \
-    PYTHONUNBUFFERED=1 \
-    CORE=files \
-    NUM_BACKUPS=7 \
     SOLR_HEAP=4g \
+    PYTHONUNBUFFERED=1 \
     PATH="/opt/mongodb/bin:/opt/java/openjdk/bin:/opt/solr/bin:$PATH" \
     GLIBC_TUNABLES=glibc.pthread.rseq=0 \
-    #ports
+    # env vars which don't get default values
+    API_SOLR_CORE=files \
     API_PORT=7777 \
-    SOLR_PORT=8983 \
-    MONGO_PORT=27017 \
-    # url
-    API_URL=http://www.example.de/ \
-    # oidc
+    API_SOLR_HOST=localhost:8983 \
+    API_MONGO_HOST=localhost:27017 \
     API_OIDC_CLIENT_ID=freva \
-    API_OIDC_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration \
-    # hosts
+    API_OIDC_DISCOVERY_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration \
     API_REDIS_HOST=redis://localhost:6379 \
     API_REDIS_SSL_CERTFILE=/certs/client-cert.pem \
     API_REDIS_SSL_KEYFILE=/certs/client-key.pem \
-    API_SOLR_HOST=localhost:8983 \
-    API_MONGO_HOST=localhost:27017 \
-    USE_MONGODB=1 \
-    USE_SOLR=1
+    API_MONGO_USER=mongo \
+    API_MONGO_PASSWORD=secret \
+    API_MONGO_DB=search_stats \
+    # disable as default
+    USE_MONGODB=0 \
+    USE_SOLR=0
 
 RUN mkdir -p /etc/mongodb /opt/app /opt/freva-rest ${SOLR_HOME} ${MONGO_HOME}/data && \
     echo "security:\n  authorization: enabled\n\
@@ -84,6 +81,8 @@ RUN /opt/solr/docker/scripts/init-var-solr && \
     /opt/solr/docker/scripts/precreate-core files && \
     find /var/solr -type d -print0 | xargs -0 chmod 0771 && \
     find /var/solr -type f -print0 | xargs -0 chmod 0661 && \
+    cp /tmp/freva/freva-service-config/solr/managed-schema.xml /var/solr/data/latest/conf/managed-schema.xml &&\
+    cp /tmp/freva/freva-service-config/solr/managed-schema.xml /var/solr/data/files/conf/managed-schema.xml &&\
     mv /var/solr ${SOLR_HOME} && \
     ln -s ${SOLR_HOME} /var/solr
 
@@ -100,14 +99,12 @@ WORKDIR /opt/freva-rest
 COPY --from=builder /opt/app/dist /opt/app/dist
 COPY src/freva_rest/api_config.toml $API_CONFIG
 COPY entrypoint.sh ./
+COPY entrypoint.sh ./
 RUN chmod +x ./entrypoint.sh && \
     python3 -m pip install /opt/app/dist/freva_rest*.whl --break-system-packages
 
-
 RUN mkdir -p /data/db && \
     mkdir -p /opt/freva-rest/mongodb/log
 
-EXPOSE $API_PORT $MONGO_PORT $SOLR_PORT
-
 USER root
 ENTRYPOINT ["./entrypoint.sh"]

freva-rest/README.md

@@ -43,6 +43,18 @@ docker pull ghcr.io/freva-clint/freva-rest:latest
 By default the container starts with the ``freva-rest-service`` command.
 See the `freva-rest-server --help` command for configure options.
 
+### Service Startup Options
+
+The container supports several startup modes on `AMD64` and `ARM64`:
+
+```console
+# Default mode - starts the freva-rest service
+docker run ghcr.io/freva-clint/freva-rest:latest
+
+# Start with custom freva-rest flags
+docker run ghcr.io/freva-clint/freva-rest:latest -p 8000
+```
+
 You can adjust the server settings by either overriding the default flags or 
 setting environment variables in the container.
 
@@ -51,6 +63,7 @@ setting environment variables in the container.
 ```ini
 # Server Configuration
 DEBUG=0                  # Start server in debug mode (1), (default: 0 -> no debug)
+API_URL=localhost:7777
 API_PORT=7777            # The port the rest service should be running on
 API_WORKER=8            # Number of multi-process workers serving the API
 API_PROXY=http://www.example.de/
@@ -60,7 +73,6 @@ API_CACHE_EXP=3600      # Expiry time in seconds of the cached data
 API_MONGO_USER=mongo
 API_MONGO_PASSWORD=secret
 API_MONGO_DB=search_stats
-API_MONGO_INITDB_DATABASE=search_stats
 API_MONGO_HOST=localhost:27017  # Host name and port should be separated by ":"
 
 # Solr Configuration
@@ -75,14 +87,14 @@ API_REDIS_SSL_CERTFILE=/certs/client-cert.pem
 API_REDIS_SSL_KEYFILE=/certs/client-key.pem
 
 # OIDC Configuration
-API_OIDC_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration
+API_OIDC_DISCOVERY_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration
 API_OIDC_CLIENT_ID=freva     #Name of the client (app) that is used to create the access tokens, defaults to freva
 API_OIDC_CLIENT_SECRET=      # Optional: Set if your OIDC instance uses a client secret
 
 # Service activation flags
 # Set to 1 to enable, 0 to disable the service
-USE_MONGODB=1  # Controls MongoDB initialization
-USE_SOLR=1     # Controls Apache Solr initialization
+USE_MONGODB=0  # Controls MongoDB initialization
+USE_SOLR=0     # Controls Apache Solr initialization
 ```
 
 ### Required Volumes
@@ -91,7 +103,7 @@ The container requires several persistent volumes that should be mounted:
 ```console
 docker run -d \
   --name freva-rest \
-  -e {mentioned envs above} \
+  -e {above envs} \ # or after saving them in .env file, call them via `--env-file .env`
   -v $(pwd)/mongodb_data:/data/db \
   -v $(pwd)/solr_data:/var/solr \
   -v $(pwd)/certs:/certs:ro \
@@ -110,4 +122,4 @@ mkdir -p {mongodb_data,solr_data,certs}
 > [!NOTE]
 > You can override the path to the default config file using the ``API_CONFIG``
          environment variable. The default location of this config file is
-         ``/opt/databrowser/api_config.toml``.
+         ``/opt/freva-rest/api_config.toml`` inside of container.

freva-rest/entrypoint.sh

@@ -9,6 +9,9 @@ MAGENTA='\033[0;35m'
 CYAN='\033[0;36m'
 NC='\033[0m'
 
+MONGO_PORT=$(echo "${API_MONGO_HOST}" | cut -d ':' -f2)
+MONGO_HOST=$(echo "${API_MONGO_HOST}" | cut -d ':' -f1)
+
 display_logo() {
     echo -e "${BLUE}"
     echo -e "${BLUE} ████████▓▒░ ████████▓▒░  ████████▓▒░ ██▓▒░  ██▓▒░  ███████▓▒░ ${NC}"
@@ -61,15 +64,15 @@ wait_for_mongo() {
 reset_mongo_user() {
     log_info "Resetting MongoDB user..."
     mongosh admin --quiet --eval "
-        if (db.getUser('${MONGO_USER}')) {
-            db.dropUser('${MONGO_USER}');
+        if (db.getUser('${API_MONGO_USER}')) {
+            db.dropUser('${API_MONGO_USER}');
             print('Existing user dropped');
         } else {
             print('User does not exist, proceeding with creation new user');
         }
         db.createUser({
-            user: '${MONGO_USER}',
-            pwd: '${MONGO_PASSWORD}',
+            user: '${API_MONGO_USER}',
+            pwd: '${API_MONGO_PASSWORD}',
             roles: [
                 { role: 'root', db: 'admin' },
                 { role: 'userAdminAnyDatabase', db: 'admin' },
@@ -88,7 +91,7 @@ verify_auth() {
     log_debug "Verifying authentication..."
     mongosh admin --quiet --eval "
         try {
-            db.auth('${MONGO_USER}', '${MONGO_PASSWORD}');
+            db.auth('${API_MONGO_USER}', '${API_MONGO_PASSWORD}');
             db.adminCommand('listDatabases');
             quit(0);
         } catch(err) {
@@ -98,14 +101,16 @@ verify_auth() {
 
 init_mongodb() {
     log_service "=== Initializing MongoDB ==="
-    if [[ -z "${MONGO_USER}" ]] || [[ -z "${MONGO_PASSWORD}" ]]; then
-        log_error "MongoDB is enabled but MONGO_USER and/or MONGO_PASSWORD are not set"
+    if [[ -z "${API_MONGO_USER}" ]] || [[ -z "${API_MONGO_PASSWORD}" ]]; then
+        log_error "MongoDB is enabled but API_MONGO_USER and/or API_MONGO_PASSWORD are not set"
         log_error "Please provide credentials via environment variables"
         exit 1
     fi
     log_info "Starting MongoDB without authentication..."
     mongod --dbpath ${MONGO_HOME}/data --port ${MONGO_PORT} --bind_ip_all --fork --logpath "$LOG_FILE" --noauth &
     wait_for_mongo
+    log_info "Initializing MongoDB userdata entrypoint..."
+    mongosh admin "/docker-entrypoint-initdb.d/mongo-userdata-init.js"
     if ! verify_auth; then
         log_warn "Authentication failed with existing credentials - resetting user..."
         reset_mongo_user
@@ -119,23 +124,52 @@ init_mongodb() {
     mongod --dbpath ${MONGO_HOME}/data --port ${MONGO_PORT} --bind_ip 0.0.0.0 --auth &
     wait_for_mongo
     log_info "Initializing MongoDB collections..."
-    mongosh --authenticationDatabase admin -u "${MONGO_USER}" -p "${MONGO_PASSWORD}" --eval "
-        if (db.getSiblingDB('${MONGO_DB}').getCollection('searches').countDocuments() == 0) {
-            db.getSiblingDB('${MONGO_DB}').createCollection('searches');
+    mongosh --authenticationDatabase admin -u "${API_MONGO_USER}" -p "${API_MONGO_PASSWORD}" --eval "
+        if (db.getSiblingDB('${API_MONGO_DB}').getCollection('searches').countDocuments() == 0) {
+            db.getSiblingDB('${API_MONGO_DB}').createCollection('searches');
         }
     "
 }
 
 init_solr() {
     log_service "=== Initializing Solr ==="
     solr start -force
-    until curl -s "http://localhost:${SOLR_PORT}/solr/admin/ping" >/dev/null 2>&1; do
+    until curl -s "http://${API_SOLR_HOST}/solr/admin/ping" >/dev/null 2>&1; do
         log_debug "Waiting for Solr to start..."
         sleep 1
     done
     log_info "Solr started successfully"
 }
 
+start_freva_service() {
+    local command=$1
+    shift || true
+
+    log_service "Starting freva-rest..."
+
+    case "${command:-}" in
+        "")
+            exec python3 -m freva_rest.cli
+            ;;
+        "sh"|"bash"|"zsh")
+            exec "${command}" "$@"
+            ;;
+        -*)
+            exec python3 -m freva_rest.cli "${command}" "$@"
+            ;;
+        "exec")
+            if [ $# -eq 0 ]; then
+                log_error "Error: 'exec' provided without a command to execute."
+                return 1
+            fi
+            exec "$@"
+            ;;
+        *)
+            exec "${command}" "$@"
+            ;;
+    esac
+}
+
 main() {
     display_logo
     log_service "Initializing services..."
@@ -151,8 +185,7 @@ main() {
     else
         log_warn "Solr service is skipped (USE_SOLR=0)"
     fi
-    log_service "Starting freva-rest..."
-    exec python3 -m freva_rest.cli "$@"
+    start_freva_service "$@"
 }
 
 main "$@"

freva-rest/src/freva_rest/__init__.py

@@ -1,7 +1,7 @@
 import os
 from pathlib import Path
 
-__version__ = "2411.1.0-dev"
+__version__ = "2412.0.0-dev2"
 __all__ = ["__version__"]
 
 REST_URL = (