Merge pull request #23 from FREVA-CLINT/new-layout

Restructure and client library,

.coveragerc

@@ -1,7 +1,7 @@
 # .coveragerc to control coverage.py
 [run]
 branch = True
-omit = .converage*
+omit = .converage* run_server.py
 [report]
 fail_under = 95
 # Regexes for lines to exclude from consideration
@@ -18,6 +18,7 @@ exclude_lines =
     raise AssertionError
     raise NotImplementedError
     except KeyboardInterrupt:
+    except ImportError:
     raise SystemExit
 
     # Don't complain if non-runnable code isn't run:

.github/workflows/build_job.yml

@@ -69,7 +69,7 @@ jobs:
         id: repository
         run: |
           echo "repo=$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')" >> $GITHUB_OUTPUT
-          echo "tag=$(python src/databrowser/__init__.py)" >> $GITHUB_OUTPUT
+          echo "tag=$(python freva-rest/src/freva_rest/__init__.py)" >> $GITHUB_OUTPUT
 
       -
         name: Set up QEMU
@@ -88,15 +88,15 @@ jobs:
           password: ${{ secrets.GITHUB_TOKEN }}
 
       -
-        name: Build and push databrowserAPI image
+        name: Build and push freva-rest API image
         uses: docker/build-push-action@v4
         with:
           build-args: VERSION=${{steps.repository.outputs.tag}}
           platforms: linux/amd64
           push: true
           tags: |
-            ghcr.io/${{ steps.repository.outputs.repo }}:${{ steps.repository.outputs.tag }}
-            ghcr.io/${{ steps.repository.outputs.repo }}:latest
+            ghcr.io/freva-clint/freva-rest-api:${{ steps.repository.outputs.tag }}
+            ghcr.io/freva-clint/freva-rest-api:latest
 
   bump-databrowser-version:
     runs-on: ubuntu-latest
@@ -126,7 +126,7 @@ jobs:
         id: repository
         run: |
           echo "repo=$(echo ${{ github.repository }}|tr 'A-Z' 'a-z')" >> $GITHUB_OUTPUT
-          echo "tag=$(python src/databrowser/__init__.py)" >> $GITHUB_OUTPUT
+          echo "tag=$(python freva-rest/src/freva_rest/__init__.py)" >> $GITHUB_OUTPUT
 
       - name: Installing dependencies
         run: pip install git-python packaging tomli requests
@@ -135,7 +135,7 @@ jobs:
         run: curl -Ls -o bump.py https://raw.githubusercontent.com/FREVA-CLINT/freva-deployment/versions/release.py
 
       - name: Do the release job
-        run: python3 bump.py deploy databrowser -b versions -v -s solr ${{steps.solr.outputs.tag}}
+        run: python3 bump.py deploy freva_rest -p freva-rest -v -s solr ${{steps.solr.outputs.tag}}
         env:
           GITHUB_TOKEN: ${{secrets.ACCESS_TOKEN}}
           REPO_VERSION: ${{ steps.repository.outputs.tag }}

.github/workflows/ci_job.yml

@@ -6,25 +6,65 @@ permissions:
 on: [push, pull_request, workflow_call]
 
 jobs:
-  ci:
+  lint-and-docs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4
         with:
           submodules: recursive
-      - name: Set up Python
-        uses: actions/setup-python@v2
+      - name: Set up Python "3.X"
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.X"
       - name: Set up services
         run: |
           docker-compose -f dev-env/docker-compose.yaml up -d --remove-orphans
       - name: Install dependencies
         run: |
-          python3 -m pip install tox
-      - name: Run all tests with tox
+          python3 -m pip install -e .[dev]
+        working-directory: ./freva-rest
+      - name: Adjusting PATH variable
+        run: echo "~/.local/bin" >> $GITHUB_PATH
+      - name: Starting the local server
+        run: |
+          python -m freva_rest.cli -c api_config.toml --debug --dev &
+      - name: Waiting for 10 sec
+        run: sleep 5
+      - name: Linting, type checking and docs with tox
         run: tox run-parallel --parallel-no-spinner
+
+  tests:
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: [3.9, "3.10", "3.11", "3.12"]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python-version}}
+      - name: Set up services
+        run: |
+          docker-compose -f dev-env/docker-compose.yaml up -d --remove-orphans
+      - name: Install dependencies
+        run: |
+          python3 -m pip install -e .[dev]
+        working-directory: ./freva-rest
+      - name: Adjusting PATH variable
+        run: echo "~/.local/bin" >> $GITHUB_PATH
+      - name: Starting the local server
+        run: |
+          python -m freva_rest.cli -c api_config.toml --debug --dev &
+      - name: Waiting for 10 sec
+        run: sleep 5
+      - name: Run tests
+        run: tox -e test
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
         with:
@@ -58,7 +98,7 @@ jobs:
           platforms: linux/amd64
           push: false
           load: true
-          tags: databrowser:latest
+          tags: freva-rest:latest
 
       - name: Check databrowser image
         run: python3 dev-env/check-container.py
@@ -67,7 +107,7 @@ jobs:
   dependabot:
     name: Merge PR by dependabot
     runs-on: ubuntu-latest
-    needs: [ci, build-docker-image]
+    needs: [tests, lint-and-docs, build-docker-image]
     if: github.event.pull_request.user.login == 'dependabot[bot]'
     steps:
       - name: Dependabot metadata

Dockerfile

@@ -3,29 +3,29 @@ ARG VERSION
 LABEL org.opencontainers.image.authors="DRKZ-CLINT"
 LABEL org.opencontainers.image.source="https://github.com/FREVA-CLINT/databrowserAPI"
 LABEL org.opencontainers.image.version="$VERSION"
-ENV API_CONFIG=/opt/databrowser/api_config.toml \
+ENV API_CONFIG=/opt/freva-rest/api_config.toml \
     API_PORT=8080\
     API_WORKER=8\
     DEBUG=0
 USER root
 RUN set -e &&\
     mkdir -p /opt/app &&\
     groupadd -r --gid 1000 freva &&\
-    mkdir -p /opt/databrowser &&\
+    mkdir -p /opt/freva-rest &&\
     adduser --uid 1000 --gid 1000 --gecos "Default user" \
-    --shell /bin/bash --disabled-password freva --home /opt/databrowser &&\
-    chown -R freva:freva /opt/databrowser
-COPY --chown=freva:freva src/databrowser/api_config.toml $API_CONFIG
+    --shell /bin/bash --disabled-password freva --home /opt/freva-rest &&\
+    chown -R freva:freva /opt/freva-rest
+COPY --chown=freva:freva freva-rest/src/freva_rest/api_config.toml $API_CONFIG
 FROM base as builder
-COPY . /opt/app
+COPY freva-rest /opt/app
 WORKDIR /opt/app
 RUN set -e &&\
     python3 -m ensurepip -U &&\
-    python3 -m pip install flit && flit build
+    python3 -m pip install build . && python3 -m build --sdist --wheel
 FROM base as final
 COPY --from=builder /opt/app/dist /opt/app/dist
-RUN python3 -m pip install /opt/app/dist/databrowser*.whl
-WORKDIR /opt/databrowser
+RUN python3 -m pip install /opt/app/dist/freva_rest*.whl
+WORKDIR /opt/freva-rest
 EXPOSE $API_PORT
 USER freva
-CMD ["python3", "-m", "databrowser.cli"]
+CMD ["python3", "-m", "freva_rest.cli"]

README.md

@@ -1,16 +1,15 @@
-# A REST API for the freva databrowser
+# Freva server - client structure
 
 [![License](https://img.shields.io/badge/License-BSD-purple.svg)](LICENSE)
 [![Python](https://img.shields.io/badge/python-3.12-red.svg)](https://www.python.org/downloads/release/python-312/)
 [![Docs](https://img.shields.io/badge/API-Doc-green.svg)](https://freva-clint.github.io/databrowserAPI)
 [![Tests](https://github.com/FREVA-CLINT/databrowserAPI/actions/workflows/ci_job.yml/badge.svg)](https://github.com/FREVA-CLINT/databrowserAPI/actions)
 [![Test-Coverage](https://codecov.io/github/FREVA-CLINT/databrowserAPI/branch/init/graph/badge.svg?token=dGhXxh7uP3)](https://codecov.io/github/FREVA-CLINT/databrowserAPI)
 
-The Freva Databrowser REST API is a powerful tool that enables you to search
-for climate and environmental datasets seamlessly in various programming
-languages. By generating RESTful requests, you can effortlessly access
-collections of various datasets, making it an ideal resource for climate
-scientists, researchers, and data enthusiasts.
+This repository contains the *freva-rest services* defining rest endpoints
+that make up the freva server services as well as the client
+services that provide command line interfaces and python libraries for their
+rest service counterparts.
 
 ## Table of Contents
 
@@ -30,36 +29,59 @@ git clone [email protected]:FREVA-CLINT/databrowserAPI.git
 cd databrowserAPI
 ```
 
-4. Install the API:
+3. Install flit:
+Flit is used as the build system for development purpose. Install flit
+into your current python environment:
 
 ```console
-python -m pip install -e .[dev]
+python3 -m pip install flit
+```
+
+4. Install the rest-api:
+
+```console
+pip install -e freva-rest
+```
+
+5. Install the client library
+
+```console
+pip install -e freva-client
 ```
 
 ## Development Environment
-Apache solr is needed run the system in a development environment, here we
-set up solr in a docker container using the `docker-compose` command, ensure
-you have Docker Compose installed on your system.
+Various services, such as apache solr are needed to run the rest services system
+in a development environment. Here we set up these services in a containers
+using the `docker-compose` or `podman-compose` command, ensure
+you have `docker-compose` or `podman-compose` installed on your system.
 Then, run the following command:
 
 ```console
 docker-compose -f dev-env/docker-compose.yaml up -d --remove-orphans
 ```
+
+if you use `podman-compose`:
+
+```console
+podman-compose -f dev-env/docker-compose.yaml up -d --remove-orphans
+```
+
 This will start the required services and containers to create the development
 environment. You can now develop and test the project within this environment.
 
-After solr is up and running you can start the REST server the following:
+After the containers are up and running you can start the REST server the following:
 
 ```console
-python -m databrowser.cli --config-file api_config.toml --debug --dev --port 7777
+freva-rest-server -c api_config.toml --debug --dev
 ```
 
 The ``--debug`` and ``--dev`` flag will make sure that any changes are loaded.
 You can choose any port you like. Furthermore the ``--dev`` flag will pre
-load an empty apache solr server with some data. If you don't like that
-simply do not pass the ``--dev`` flag.
+load any existing test data. If you don't like that simply do not pass the
+``--dev`` flag.
+
 
-### Testing
+## Testing
 
 Unit tests, Example notebook tests, type annotations and code style tests
 are done with [tox](https://tox.wiki/en/latest/). To run all tests, linting

dev-env/check-container.py

@@ -15,7 +15,7 @@
 logger = logging.getLogger("container-check")
 
 
-def check_container(container_name: str = "databrowser") -> None:
+def check_container(container_name: str = "freva-rest") -> None:
     """Check if the contianer starts up."""
     try:
         process = subprocess.Popen(

dev-env/envrc

@@ -0,0 +1 @@
+conda env config vars set FREVA_CONFIG=$PWD/dev-env/freva.toml

dev-env/freva.toml

@@ -0,0 +1,4 @@
+# Central freva config file
+[freva]
+project_name = "freva-dev"
+host = "localhost:8080"

docs/source/conf.py

@@ -6,14 +6,11 @@
 # -- Project information -----------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
 import os
-import sys
 from datetime import date
 
-sys.path.insert(0, os.path.abspath(os.path.join("..", "..", "src")))
-from databrowser import __version__
+from freva_client import __version__
 
-
-project = "Databrowser API"
+project = "Freva Databrowser"
 copyright = f"{date.today().year}, DKRZ"
 author = "DKRZ"
 release = __version__
@@ -22,13 +19,15 @@
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
+    "sphinx.ext.autodoc",
     "sphinx_code_tabs",
     "sphinx_copybutton",
+    "sphinx_togglebutton",
     "sphinxcontrib.httpdomain",
+    "sphinx_execute_code",
 ]
 
 html_static_path = ["_static"]
-# html_theme = "furo"
 html_theme = "pydata_sphinx_theme"
 html_logo = os.path.join(html_static_path[0], "logo.png")
 templates_path = ["_templates"]
@@ -63,8 +62,10 @@
 
 # -- Options for autosummary/autodoc output ------------------------------------
 autosummary_generate = True
-autodoc_typehints = "description"
-autodoc_member_order = "groupwise"
+# autodoc_typehints = "description"
+# autodoc_class_signature = "separated"
+# autodoc_member_order = "groupwise"
+
 
 # -- Options for autoapi -------------------------------------------------------
 autoapi_type = "python"
@@ -75,3 +76,21 @@
 
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+# -- MyST options ------------------------------------------------------------
+
+# This allows us to use ::: to denote directives, useful for admonitions
+myst_enable_extensions = ["colon_fence", "linkify", "substitution"]
+myst_heading_anchors = 2
+myst_substitutions = {"rtd": "[Read the Docs](https://readthedocs.org/)"}
+
+# ReadTheDocs has its own way of generating sitemaps, etc.
+if not os.environ.get("READTHEDOCS"):
+    extensions += ["sphinx_sitemap"]
+
+    html_baseurl = os.environ.get("SITEMAP_URL_BASE", "http://127.0.0.1:8000/")
+    sitemap_locales = [None]
+    sitemap_url_scheme = "{link}"
+
+# specifying the natural language populates some key tags
+language = "en"