diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..9bbbecd --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,25 @@ +name: Build cads-processing-api-service docker image + +on: + workflow_dispatch: + push: + branches: + - main + - automatic_package_update + tags: + - "v*" + pull_request: + branches: + - main + + + +jobs: + build_docker_image: + name: Build docker image + uses: ecmwf-projects/cads-build-farm/.github/workflows/docker-build.yaml@helm-chart-pipeline + with: + dockerfile: 'Retrieve.Dockerfile' + chart-repo: 'ecmwf/dss-retrive-api-chart' + enable-package-update: false + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/on-push.yml b/.github/workflows/on-push.yml deleted file mode 100644 index 8df94d8..0000000 --- a/.github/workflows/on-push.yml +++ /dev/null @@ -1,235 +0,0 @@ -name: on-push - -on: - push: - branches: - - main - tags: - - '*' - pull_request: - branches: - - main - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -defaults: - run: - shell: bash -l {0} - -jobs: - pre-commit: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - uses: pre-commit/action@v3.0.1 - - combine-environments: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - name: Install conda-merge - run: | - python -m pip install conda-merge - - name: Combine environments - run: | - for SUFFIX in ci integration; do - conda-merge ci/environment-$SUFFIX.yml environment.yml > ci/combined-environment-$SUFFIX.yml || exit - done - - uses: actions/upload-artifact@v4 - with: - name: combined-environments - path: ci/combined-environment-*.yml - - unit-tests: - name: unit-tests - needs: combine-environments - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.11'] - - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: combined-environments - path: ci - - name: Get current date - id: date - run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/combined-environment-ci.yml - environment-name: DEVELOP - cache-environment: true - cache-environment-key: environment-${{ steps.date.outputs.date }} - cache-downloads-key: downloads-${{ steps.date.outputs.date }} - create-args: >- - python=${{ matrix.python-version }} - - name: Install package - run: | - python -m pip install --no-deps -e . - - name: Run tests - run: | - make unit-tests COV_REPORT=xml - - type-check: - needs: [combine-environments, unit-tests] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: combined-environments - path: ci - - name: Get current date - id: date - run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/combined-environment-ci.yml - environment-name: DEVELOP - cache-environment: true - cache-environment-key: environment-${{ steps.date.outputs.date }} - cache-downloads-key: downloads-${{ steps.date.outputs.date }} - create-args: >- - python=3.11 - - name: Install package - run: | - python -m pip install --no-deps -e . - - name: Run code quality checks - run: | - make type-check - - docs-build: - needs: [combine-environments, unit-tests] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: combined-environments - path: ci - - name: Get current date - id: date - run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/combined-environment-ci.yml - environment-name: DEVELOP - cache-environment: true - cache-environment-key: environment-${{ steps.date.outputs.date }} - cache-downloads-key: downloads-${{ steps.date.outputs.date }} - create-args: >- - python=3.11 - - name: Install package - run: | - python -m pip install --no-deps -e . - - name: Build documentation - run: | - make docs-build - - integration-tests: - needs: [combine-environments, unit-tests] - if: | - success() && false - runs-on: ubuntu-latest - - strategy: - matrix: - include: - - python-version: '3.11' - extra: -integration - - steps: - - uses: actions/checkout@v4 - - uses: actions/download-artifact@v4 - with: - name: combined-environments - path: ci - - name: Get current date - id: date - run: echo "date=$(date +%Y-%m-%d)" >> "${GITHUB_OUTPUT}" - - uses: mamba-org/setup-micromamba@v1 - with: - environment-file: ci/combined-environment${{ matrix.extra }}.yml - environment-name: DEVELOP${{ matrix.extra }} - cache-environment: true - cache-environment-key: environment-${{ steps.date.outputs.date }} - cache-downloads-key: downloads-${{ steps.date.outputs.date }} - create-args: >- - python=${{ matrix.python-version }} - - name: Install package - run: | - python -m pip install --no-deps -e . - - name: Run tests - run: | - make unit-tests COV_REPORT=xml - - distribution: - runs-on: ubuntu-latest - needs: [unit-tests, type-check, docs-build, integration-tests] - if: | - always() && - needs.unit-tests.result == 'success' && - needs.type-check.result == 'success' && - needs.docs-build.result == 'success' && - (needs.integration-tests.result == 'success' || needs.integration-tests.result == 'skipped') - - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Install package - run: | - python -m pip install --upgrade pip - python -m pip install build twine - - name: Build distribution - run: | - python -m build - - name: Check wheels - run: | - cd dist || exit - python -m pip install cads_processing_api_service*.whl || exit - python -m twine check --strict * || exit - python -c "import cads_processing_api_service" || exit - cd .. - - uses: actions/upload-artifact@v4 - with: - name: distribution - path: dist - - upload-to-pypi: - runs-on: ubuntu-latest - needs: distribution - if: | - always() && false && - needs.distribution.result == 'success' && - github.event_name == 'push' && - startsWith(github.ref, 'refs/tags') - environment: - name: pypi - url: https://pypi.org/p/cads-processing-api-service - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publish - - steps: - - uses: actions/download-artifact@v4 - with: - name: distribution - path: dist - - uses: pypa/gh-action-pypi-publish@v1.9.0 - with: - verbose: true diff --git a/.gitignore b/.gitignore index 46bc2fa..de875d3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*local* # setuptools-scm version.py diff --git a/Retrieve.Dockerfile b/Retrieve.Dockerfile new file mode 100644 index 0000000..42af2a2 --- /dev/null +++ b/Retrieve.Dockerfile @@ -0,0 +1,51 @@ +FROM condaforge/miniforge3:23.11.0-0 + +ARG MODE=stable +ARG CADS_PAT + +WORKDIR /src + +COPY ./git-*-repos.py /src/ + +COPY environment.${MODE} /src/environment +COPY environment-common.yml /src/environment-common.yml + +RUN conda install -y -n base -c conda-forge gitpython typer conda-merge + +SHELL ["/bin/bash", "-c"] + +RUN set -a && source environment \ + && CADS_PAT=${CADS_PAT} python ./git-clone-repos.py --default-branch \ + cacholote \ + cads-adaptors \ + cads-broker \ + cads-catalogue \ + cads-common \ + cads-processing-api-service \ + ogc-api-processes-fastapi + +# NOTE: no environment for cads-adaptors as we only use basic features +RUN conda run -n base conda-merge \ + /src/environment-common.yml \ + /src/cacholote/environment.yml \ + /src/cads-broker/environment.yml \ + /src/cads-catalogue/environment.yml \ + /src/cads-common/environment.yml \ + /src/cads-processing-api-service/environment.yml \ + /src/ogc-api-processes-fastapi/environment.yml \ + > /src/combined-environment.yml \ + && conda env update -n base -f /src/combined-environment.yml \ + && conda clean -afy + +RUN conda run -n base pip install --no-deps \ + -e /src/cacholote \ + -e /src/cads-broker \ + -e /src/cads-catalogue \ + -e /src/cads-common \ + -e /src/cads-processing-api-service \ + -e /src/ogc-api-processes-fastapi + +# NOTE: pip install cads-adaptors mandatory dependencies +RUN conda run -n base pip install -e /src/cads-adaptors + +CMD uvicorn cads_processing_api_service.main:app --host 0.0.0.0 --log-level info diff --git a/environment.dev b/environment.dev new file mode 100644 index 0000000..f544722 --- /dev/null +++ b/environment.dev @@ -0,0 +1,18 @@ +# CADS repositories versions +CACHOLOTE_REF="main" +CADS_ADAPTORS_REF="main" +DATAPI_REF="main" +CADS_BROKER_REF="main" +CADS_CATALOGUE_REF="main" +CADS_CATALOGUE_API_SERVICE_REF="main" +CADS_COMMON_REF="main" +CADS_EXTENDED_PROFILES_REF="main" +CADS_MARS_SERVER_REF="main" +CADS_OBS_CATALOGUE_MANAGER_REF="main" +CADS_PROCESSING_API_SERVICE_REF="main" +CADS_WEBPORTAL_REF="main" +CADS_WORKER_REF="main" +OGC_API_PROCESSES_FASTAPI_REF="main" +STAC_BROWSER_REF="v3.2.0" +CDS_COMMON_REF="develop" +CDSCOMPUTE_REF="main" \ No newline at end of file diff --git a/environment.stable b/environment.stable new file mode 100644 index 0000000..f238b10 --- /dev/null +++ b/environment.stable @@ -0,0 +1,19 @@ +# CADS repositories versions +CACHOLOTE_REF="v1.7.1" +CADS_ADAPTORS_REF="v1.1.43" +DATAPI_REF="v0.4.0" +CADS_BROKER_REF="v1.9.1" +CADS_CATALOGUE_REF="v3.1.0" +CADS_CATALOGUE_API_SERVICE_REF="v1.9.2" +CADS_COMMON_REF="v0.6.0" +CADS_E2E_TESTS_REF="v0.2.0" +CADS_EXTENDED_PROFILES_REF="v1.6.3" +CADS_MARS_SERVER_REF="0.2.4" +CADS_PROCESSING_API_SERVICE_REF="v1.4.3" +CADS_WEBPORTAL_REF="v1.5.0" +CADS_WORKER_REF="v1.4.4" +OGC_API_PROCESSES_FASTAPI_REF="v1.0.2" +STAC_BROWSER_REF="v3.2.0" +CDS_COMMON_REF="0.7.15.2" +CDSCOMPUTE_REF="0.0.0.0" +PYCSW_DYNAMIC_REF="v0.5.2" \ No newline at end of file