Merge pull request #95 from FREVA-CLINT/all-in-one

Embed Solr and Mongo images in freva-rest image

.github/workflows/build_job.yml

@@ -116,6 +116,8 @@ jobs:
       - name: Build and push freva-rest API image
         uses: docker/build-push-action@v4
         with:
+          context: freva-rest
+          file: freva-rest/Dockerfile
           build-args: VERSION=${{ needs.release-type-determination.outputs.tag }}
           platforms: linux/amd64
           push: true

.github/workflows/ci_job.yml

@@ -89,14 +89,16 @@ jobs:
 
       - name: Set up services
         run: |
-          docker compose -f dev-env/docker-compose.yaml up -d --remove-orphans
+          docker compose -f dev-env/docker-compose.yaml up -d redis ldap keycloak --remove-orphans
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v1
 
       - name: Build freva-storage-service image
         uses: docker/build-push-action@v4
         with:
+          context: freva-rest
+          file: freva-rest/Dockerfile
           platforms: linux/amd64
           push: false
           load: true

dev-env/check-container.py

@@ -1,9 +1,6 @@
 import logging
 import subprocess
 import time
-import urllib.request
-from pathlib import Path
-import sys
 
 # Set up logging
 logging.basicConfig(
@@ -13,14 +10,15 @@
 )
 logger = logging.getLogger("container-check")
 
-def check_container(container_name: str = "freva-rest") -> None:
-    """Check if the container starts up."""
-
+
+def check_container(image_name: str = "freva-rest", container_name: str = "freva-rest") -> None:
+    """Check if the contianer starts up."""
     try:
         process = subprocess.Popen(
             [
                 "docker",
                 "run",
+                "--name", container_name,
                 "--net=host",
                 "-e",
                 "MONGO_USER=mongo",
@@ -34,22 +32,49 @@ def check_container(container_name: str = "freva-rest") -> None:
                 "API_WORKER=8",
                 "-e",
                 "MONGO_DB=search_stats",
-                container_name,
+                "-e",
+                "OIDC_URL=http://localhost:8080/realms/freva/.well-known/openid-configuration",
+                image_name,
             ],
         )
-        time.sleep(10)
+        time.sleep(20)
         if process.poll() is not None:
-            raise RuntimeError("Container died.")
+            stdout, stderr = process.communicate()
+            raise RuntimeError(f"Container died. Exit code: {process.returncode}. "
+                             f"Stdout: {stdout}, Stderr: {stderr}")
 
         logger.info("Container started successfully.")
     except Exception as error:
         logger.critical("Starting the container failed: %s", error)
         raise
     finally:
-        logger.info("Terminating container process.")
-        process.terminate()
-
-    logger.info("Container seems to work!")
+        if process and process.poll() is None:
+            try:
+                process.terminate()
+                process.wait(timeout=5)
+            except subprocess.TimeoutExpired:
+                process.kill()
+                process.wait()
+        try:
+            logger.info(f"Stopping container {container_name}")
+            stop_process = subprocess.Popen(
+                ["docker", "stop", container_name],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            stop_process.wait(timeout=15)
+
+            logger.info(f"Removing container {container_name}")
+            rm_process = subprocess.Popen(
+                ["docker", "rm", container_name],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE
+            )
+            rm_process.wait(timeout=10)
+
+            logger.info("Container cleanup completed successfully")
+        except Exception as cleanup_error:
+            logger.error("Failed to clean up container: %s", cleanup_error)
 
 
 if __name__ == "__main__":

freva-rest/Dockerfile

@@ -0,0 +1,113 @@
+FROM debian:bookworm-slim AS base
+ARG VERSION
+
+LABEL org.opencontainers.image.authors="DRKZ-CLINT"
+LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/freva-nextgen/freva-rest"
+LABEL org.opencontainers.image.version="$VERSION"
+
+# prerequisites
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+    python3 python3-pip python3-dev \
+    gcc g++ make curl openssl pkg-config \
+    python3-setuptools python3-wheel build-essential \
+    gnupg wget git ca-certificates \
+    default-jre bash procps \
+    && rm -rf /var/lib/apt/lists/*
+
+# mongodb
+COPY --from=mongo:latest /usr/bin/mongod /usr/bin/mongod
+COPY --from=mongo:latest /usr/bin/mongos /usr/bin/mongos
+COPY --from=mongo:latest /usr/bin/mongosh /usr/bin/mongosh
+COPY --from=mongo:latest /usr/bin/mongoexport /usr/bin/mongoexport
+COPY --from=mongo:latest /usr/bin/mongoimport /usr/bin/mongoimport
+COPY --from=mongo:latest /lib/ /lib/
+COPY --from=mongo:latest /usr/lib/ /usr/lib/
+
+# solr
+COPY --from=solr:latest /opt/solr/ /opt/solr/
+COPY --from=solr:latest /opt/java/ /opt/java/
+COPY --from=solr:latest /var/solr/ /var/solr/
+RUN mkdir -p /tmp/freva && \
+    git clone https://github.com/FREVA-CLINT/freva-service-config.git /tmp/freva/freva-service-config && \
+    mkdir -p /docker-entrypoint-initdb.d /usr/local/bin && \
+    cp /tmp/freva/freva-service-config/solr/managed-schema.xml /opt/solr/managed-schema.xml && \
+    cp /tmp/freva/freva-service-config/solr/create_cores.sh /docker-entrypoint-initdb.d/create_cores.sh && \
+    cp /tmp/freva/freva-service-config/solr/synonyms.txt /opt/solr/synonyms.txt && \
+    cp /tmp/freva/freva-service-config/solr/daily_backup.sh /usr/local/bin/daily_backup && \
+    cp /tmp/freva/freva-service-config/mongo/mongo-userdata-init.js /docker-entrypoint-initdb.d/mongo-userdata-init.js
+
+# main env
+ENV API_CONFIG=/opt/freva-rest/api_config.toml \
+    DEBUG=0 \
+    API_WORKER=8 \
+    MONGO_HOME=/opt/mongodb \
+    SOLR_HOME=/opt/solr_data \
+    JAVA_HOME=/opt/java/openjdk \
+    SOLR_CORE=files \
+    COLUMNS=140 \
+    SOLR_LOGS_DIR=/opt/solr_data/logs/solr \
+    LOG4J_PROPS=/opt/solr_data/log4j2.xml \
+    SOLR_PID_DIR=/opt/solr_data \
+    SOLR_JETTY_HOST=0.0.0.0 \
+    PYTHONUNBUFFERED=1 \
+    CORE=files \
+    NUM_BACKUPS=7 \
+    SOLR_HEAP=4g \
+    PATH="/opt/mongodb/bin:/opt/java/openjdk/bin:/opt/solr/bin:$PATH" \
+    GLIBC_TUNABLES=glibc.pthread.rseq=0 \
+    #ports
+    API_PORT=7777 \
+    SOLR_PORT=8983 \
+    MONGO_PORT=27017 \
+    # url
+    API_URL=http://www.example.de/ \
+    # oidc
+    OIDC_CLIENT_ID=freva \
+    OIDC_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration \
+    # hosts
+    REDIS_HOST=redis://localhost:6379 \
+    REDIS_SSL_CERTFILE=/certs/client-cert.pem \
+    REDIS_SSL_KEYFILE=/certs/client-key.pem \
+    SOLR_HOST=localhost:8983 \
+    MONGO_HOST=localhost:27017 \
+    USE_MONGODB=1 \
+    USE_SOLR=1
+
+RUN mkdir -p /etc/mongodb /opt/app /opt/freva-rest ${SOLR_HOME} ${MONGO_HOME}/data && \
+    echo "security:\n  authorization: enabled\n\
+storage:\n  dbPath: /opt/mongodb/data\n\
+net:\n  port: 27017\n  bindIp: 0.0.0.0\n" > /etc/mongodb/mongod.conf
+
+RUN /opt/solr/docker/scripts/init-var-solr && \
+    /opt/solr/docker/scripts/precreate-core latest && \
+    /opt/solr/docker/scripts/precreate-core files && \
+    find /var/solr -type d -print0 | xargs -0 chmod 0771 && \
+    find /var/solr -type f -print0 | xargs -0 chmod 0661 && \
+    mv /var/solr ${SOLR_HOME} && \
+    ln -s ${SOLR_HOME} /var/solr
+
+FROM base AS builder
+WORKDIR /opt/app
+COPY . .
+RUN python3 -m pip install --upgrade pip --break-system-packages && \
+    python3 -m pip install build --break-system-packages && \
+    python3 -m pip install . --break-system-packages && \
+    python3 -m build --sdist --wheel
+
+FROM base AS final
+WORKDIR /opt/freva-rest
+COPY --from=builder /opt/app/dist /opt/app/dist
+COPY src/freva_rest/api_config.toml $API_CONFIG
+COPY entrypoint.sh ./
+RUN chmod +x ./entrypoint.sh && \
+    python3 -m pip install /opt/app/dist/freva_rest*.whl --break-system-packages
+
+
+RUN mkdir -p /data/db && \
+    mkdir -p /opt/freva-rest/mongodb/log
+
+EXPOSE $API_PORT $MONGO_PORT $SOLR_PORT
+
+USER root
+ENTRYPOINT ["./entrypoint.sh"]

freva-rest/README.md

@@ -44,7 +44,7 @@ By default the container starts with the ``freva-rest-service`` command.
 The following default values are available on start up:
 
 ```console
-reva-rest-server --help                                                                                                                                     (python3_12)
+freva-rest-server --help
 
  Usage: freva-rest-server [OPTIONS]
 
@@ -74,41 +74,70 @@ reva-rest-server --help
 ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯
 ```
 
-You can either adjust the server settings by overriding the default flags
-listed above or setting environment variables in the container.
-
-The following environment variables can be set:
-
-- ``DEBUG``: Start server in debug mode (1), (default: 0 -> no debug).
-- ``API_PORT``: the port the rest service should be running on (default 8080).
-- ``API_WORKER``: the number of multi-process work serving the API (default: 8).
-- ``SOLR_HOST``: host name of the solr server, host name and port should be
-                 separated by a ``:``, for example ``localhost:8983``
-- ``SOLR_CORE`` : name of the solr core that contains datasets with multiple
-                  versions
-- ``MONGO_HOST``: host name of the mongodb server, where query statistics are
-                 stored. Host name and port should separated by a ``:``, for
-                 example ``localhost:27017``
-- ``MONGO_USER``: user name for the mongodb.
-- ``MONGO_PASSWORD``: password to log on to the mongodb.
-- ``MONGO_DB``: database name of the mongodb instance.
-- ``API_URL``: url of the machine that runs of the rest api
-- ``API_CACHE_EXP``: expiry time in seconds of the cached data
-- ``REDIS_HOST``: Host and port of the redis cache
-                  Host name and port should separated by a ``:``, for
-                  example ``localhost:5672``
-- ``REDIS_PASS``: Password for the redis connection.
-- ``REDIS_USER``: Username for the redis connection.
-- ``REDIS_SSL_CERTFILE``: Path to the TSL certificate file used to encrypt
-                          the redis connection.
-- ``REDIS_SSL_KEYFILE``: Path to the TSL key file used to encrypt the redis
-                         connection.
-- ``OIDC_URL``: Discovery of the open connect id service.
-- ``OIDC_CLIENT_ID``: Name of the client (app) that is used to create
-                          the access tokens, defaults to freva
-- ``OIDC_CLIENT_SECRET``: You can set a client secret, if you have
-                           configured your oidc instance to use a client secret.
-
-> ``📝`` You can override the path to the default config file using the ``API_CONFIG``
+You can adjust the server settings by either overriding the default flags or setting environment variables in the container.
+
+### Available Environment Variables
+
+```ini
+# Server Configuration
+DEBUG=0                  # Start server in debug mode (1), (default: 0 -> no debug)
+API_PORT=7777            # The port the rest service should be running on
+API_WORKER=8            # Number of multi-process workers serving the API
+API_URL=http://www.example.de/
+API_CACHE_EXP=3600      # Expiry time in seconds of the cached data
+
+# Database Configuration
+MONGO_USER=mongo
+MONGO_PASSWORD=secret
+MONGO_DB=search_stats
+MONGO_INITDB_DATABASE=search_stats
+MONGO_HOST=localhost:27017  # Host name and port should be separated by ":"
+
+# Solr Configuration
+SOLR_HOST=localhost:8983   # Host name and port should be separated by ":"
+SOLR_CORE=files           # Name of the solr core for datasets with multiple versions
+
+# Redis Configuration
+REDIS_HOST=redis://localhost:6379
+REDIS_USER=              # Username for the redis connection
+REDIS_PASS=              # Password for the redis connection
+REDIS_SSL_CERTFILE=/certs/client-cert.pem
+REDIS_SSL_KEYFILE=/certs/client-key.pem
+
+# OIDC Configuration
+OIDC_URL=http://keycloak:8080/realms/freva/.well-known/openid-configuration
+OIDC_CLIENT_ID=freva     #Name of the client (app) that is used to create the access tokens, defaults to freva
+OIDC_CLIENT_SECRET=      # Optional: Set if your OIDC instance uses a client secret
+
+# Service activation flags
+# Set to 1 to enable, 0 to disable the service
+USE_MONGODB=1  # Controls MongoDB initialization
+USE_SOLR=1     # Controls Apache Solr initialization
+```
+
+### Required Volumes
+The container requires several persistent volumes that should be mounted:
+
+```console
+docker run -d \
+  --name freva-rest \
+  -e {mentioned envs above} \
+  -v $(pwd)/mongodb_data:/data/db \
+  -v $(pwd)/solr_data:/var/solr \
+  -v $(pwd)/certs:/certs:ro \
+  -p 7777:7777 \
+  -p 27017:27017 \
+  -p 8983:8983 \
+  -p 5432:5432 \
+  ghcr.io/freva-clint/freva-rest:latest
+```
+
+Create the necessary directories before starting the container:
+```console
+mkdir -p {mongodb_data,solr_data,certs}
+```
+
+> [!NOTE]
+> You can override the path to the default config file using the ``API_CONFIG``
          environment variable. The default location of this config file is
-         ``/opt/databrowser/api_config.toml``.
+         ``/opt/databrowser/api_config.toml``.