From a1bf935bc741aa2cffd6e731e25f6e01b30d0208 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 17 Apr 2025 11:55:24 -0300 Subject: [PATCH 1/7] feat: Introduce a new local Docker env --- .gitignore | 1 + api/Dockerfile | 43 +++++++++++++++++++++++++++++++++++ api/poetry.lock | 3 ++- docker/new/docker-compose.yml | 42 ++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 api/Dockerfile create mode 100644 docker/new/docker-compose.yml diff --git a/.gitignore b/.gitignore index c639e95f4aa8..7f6f061c6f70 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ checkstyle.txt .envrc .tool-versions .elasticbeanstalk/ +.local/ # These get baked into the docker container on build src/CI_COMMIT_SHA diff --git a/api/Dockerfile b/api/Dockerfile new file mode 100644 index 000000000000..d6acf8cd24b0 --- /dev/null +++ b/api/Dockerfile @@ -0,0 +1,43 @@ +# NOTE: Why not Alpine? Because Alpine demands installation of OS software +# prior to running app code, which leads to maintenance work and a larger image +# size compared to Debian — considering Debian layers are shared across images. +FROM python:3.12 + +# Prevent Python from buffering stdout and stderr +# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUNBUFFERED +ENV PYTHONUNBUFFERED=1 + +# Prevent Python from writing .pyc files to disk +# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE +ENV PYTHONDONTWRITEBYTECODE=1 + +# Install Python packages +WORKDIR /tmp +ARG POETRY_VERSION_CONSTRAINT +ARG INSTALL_DEV_DEPENDENCIES +COPY api/pyproject.toml api/poetry.lock ./ +RUN \ + pip install -q poetry${POETRY_VERSION_CONSTRAINT:-} &&\ + poetry config virtualenvs.create false &&\ + poetry config cache-dir /var/cache/pypoetry &&\ + ### + # NOTE: Forcefully delete optional private dependencies for now since + # Poetry needs to fetch them in order to calculate the dependency tree. See + # https://github.com/python-poetry/poetry/issues/4562 + # TODO: Improve management of enterprise features + sed -ie '/tool.poetry.group.auth-controller/,+1d' pyproject.toml &&\ + sed -ie '/tool.poetry.group.saml/,+1d' pyproject.toml &&\ + sed -ie '/tool.poetry.group.ldap/,+1d' pyproject.toml &&\ + sed -ie '/tool.poetry.group.workflows/,+1d' pyproject.toml &&\ + sed -ie '/tool.poetry.group.licensing/,+1d' pyproject.toml &&\ + poetry lock &&\ + ### + poetry install --no-root $([ "$INSTALL_DEV_DEPENDENCIES" != "true" ] && echo "--without dev") &&\ + rm -rf /var/cache/pypoetry &&\ + rm -rf /tmp/* + +# Prepare the app +WORKDIR /opt/app +COPY api/ . +EXPOSE 8000 +CMD ["flagsmith", "start", "api"] diff --git a/api/poetry.lock b/api/poetry.lock index c3d59fe7765c..4fac7dbe5ebf 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -4119,6 +4119,7 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/docker/new/docker-compose.yml b/docker/new/docker-compose.yml new file mode 100644 index 000000000000..5c2b868169ab --- /dev/null +++ b/docker/new/docker-compose.yml @@ -0,0 +1,42 @@ +# Reusable app definition +x-app: &app + image: ${DOCKER_IMAGE:-local/flagsmith/flagsmith}:${DOCKER_TAG:-latest} + build: + context: ../../ + dockerfile: api/Dockerfile + args: + POETRY_VERSION_CONSTRAINT: ==2.1.2 + INSTALL_DEV_DEPENDENCIES: true + depends_on: + database: + condition: service_healthy + environment: + DEBUG: true + DATABASE_URL: postgres://postgres@database:5432/postgres + DJANGO_SETTINGS_MODULE: app.settings.local + volumes: + - ../../.local/pypoetry:/root/.config/poetry:rw # Poetry local cache + - ../../:/opt/app:rw,cached # Application directory + +services: + api: + <<: *app + command: sh -c 'cd api; flagsmith migrate && flagsmith start --reload api' + ports: + - 8000:8000 + + # TODO + # task-processor: + # <<: *app + # command: ... + + database: + image: postgres:15.5-alpine + volumes: + - ../../.local/database:/var/lib/postgresql/data:rw,cached + environment: + POSTGRES_HOST_AUTH_METHOD: trust + healthcheck: + test: pg_isready -Upostgres + interval: 1s + timeout: 30s From 51120a371b321e5b9c71801d46e4b8fe6f786ab5 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 17 Apr 2025 11:57:27 -0300 Subject: [PATCH 2/7] feat: Add an utility to run API containers easier --- bin/README.md | 26 ++++++++++++++++++++++++++ bin/run | 5 +++++ 2 files changed, 31 insertions(+) create mode 100644 bin/README.md create mode 100755 bin/run diff --git a/bin/README.md b/bin/README.md new file mode 100644 index 000000000000..4c581615b3f0 --- /dev/null +++ b/bin/README.md @@ -0,0 +1,26 @@ +# Local maintenance utilities + +This directory contains scripts to help running and maintaining the Flagsmith +API in the local environment. + +## Requirements + +- Docker + +That's it. These scripts are designed to interact with the application runtime +through ephemeral Docker containers. + +## Scripts + +### `bin/run` + +Runs any command within the `api` container. Examples: + +- `bin/run` — runs pending migrations and starts the API HTTP server in dev mode. +- `bin/run bash` — starts a bash shell in the API container. +- `bin/run python manage.py makemigrations` — runs the command in the API container. +- `bin/run flagsmith createsuperuser` — the Flagsmith CLI should be there too! +- `bin/run pytest --sw --pdb api/tests/unit/test_my_feature.py` — you get it. + +> This script is intended as a one-command option to run Flagsmith code with no +> previous setup except from installing Docker. diff --git a/bin/run b/bin/run new file mode 100755 index 000000000000..464888ea401e --- /dev/null +++ b/bin/run @@ -0,0 +1,5 @@ +#!/bin/sh +# +# Runs any command from within an application container. +# If no command is given, it runs the application's default command. +docker compose -f docker/new/docker-compose.yml run --rm $([ $# -eq 0 ] && echo --service-ports --use-aliases) api "$@" \ No newline at end of file From 5e88b5b15b3f48b8dc3d9c602a04e8b354a0f53b Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Thu, 17 Apr 2025 17:07:09 -0300 Subject: [PATCH 3/7] fix: Improve path management in local Docker --- api/run | 28 ++++++++++++++++++++++++++++ bin/README.md | 26 -------------------------- bin/run | 5 ----- docker/new/docker-compose.yml | 2 +- 4 files changed, 29 insertions(+), 32 deletions(-) create mode 100755 api/run delete mode 100644 bin/README.md delete mode 100755 bin/run diff --git a/api/run b/api/run new file mode 100755 index 000000000000..db74513a8cb5 --- /dev/null +++ b/api/run @@ -0,0 +1,28 @@ +#!/bin/sh + +# Runs any command from within an `api` container. +# If no command is given, it runs the application's default command. + +# NOTE: This script is intended as a one-command option to run Flagsmith code +# with no previous setup except from installing Docker. + +# Requirements: +# - Docker + +# Examples: +# - `./run` — runs pending migrations and starts the API HTTP server in dev mode. +# - `./run bash` — starts a bash shell in the API container. +# - `./run python manage.py makemigrations` — runs the command in the API container. +# - `./run flagsmith createsuperuser` — the Flagsmith CLI should be there too! +# - `./run pytest --sw --pdb tests/unit/test_my_feature.py` — you get it. + +SCRIPT=$(readlink -f "$0") +SCRIPT_DIR=$(dirname "$SCRIPT") +ROOT_DIR=$(cd "$SCRIPT_DIR/.."; pwd -P) +PWD=$(pwd -P) +WORKING_DIR=$(echo "$PWD" | sed "s|$ROOT_DIR||") + +docker compose -f "${ROOT_DIR}/docker/new/docker-compose.yml" run --rm \ + $([ $# -eq 0 ] && echo --service-ports --use-aliases) \ + $([ -n "${WORKING_DIR}" ] && echo --workdir /opt/app${WORKING_DIR}) \ + api "$@" diff --git a/bin/README.md b/bin/README.md deleted file mode 100644 index 4c581615b3f0..000000000000 --- a/bin/README.md +++ /dev/null @@ -1,26 +0,0 @@ -# Local maintenance utilities - -This directory contains scripts to help running and maintaining the Flagsmith -API in the local environment. - -## Requirements - -- Docker - -That's it. These scripts are designed to interact with the application runtime -through ephemeral Docker containers. - -## Scripts - -### `bin/run` - -Runs any command within the `api` container. Examples: - -- `bin/run` — runs pending migrations and starts the API HTTP server in dev mode. -- `bin/run bash` — starts a bash shell in the API container. -- `bin/run python manage.py makemigrations` — runs the command in the API container. -- `bin/run flagsmith createsuperuser` — the Flagsmith CLI should be there too! -- `bin/run pytest --sw --pdb api/tests/unit/test_my_feature.py` — you get it. - -> This script is intended as a one-command option to run Flagsmith code with no -> previous setup except from installing Docker. diff --git a/bin/run b/bin/run deleted file mode 100755 index 464888ea401e..000000000000 --- a/bin/run +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -# -# Runs any command from within an application container. -# If no command is given, it runs the application's default command. -docker compose -f docker/new/docker-compose.yml run --rm $([ $# -eq 0 ] && echo --service-ports --use-aliases) api "$@" \ No newline at end of file diff --git a/docker/new/docker-compose.yml b/docker/new/docker-compose.yml index 5c2b868169ab..2f0c3faa7710 100644 --- a/docker/new/docker-compose.yml +++ b/docker/new/docker-compose.yml @@ -21,7 +21,7 @@ x-app: &app services: api: <<: *app - command: sh -c 'cd api; flagsmith migrate && flagsmith start --reload api' + command: sh -c 'flagsmith migrate && flagsmith start --reload api' ports: - 8000:8000 From 8737ec207670cd9a04860b3f03cb9086e20a0735 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Mon, 21 Apr 2025 15:57:06 -0300 Subject: [PATCH 4/7] fix: Improve running ./run from any directory --- api/run | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/api/run b/api/run index db74513a8cb5..a5fce1c3f930 100755 --- a/api/run +++ b/api/run @@ -1,4 +1,5 @@ #!/bin/sh +set -e # Runs any command from within an `api` container. # If no command is given, it runs the application's default command. @@ -15,14 +16,22 @@ # - `./run python manage.py makemigrations` — runs the command in the API container. # - `./run flagsmith createsuperuser` — the Flagsmith CLI should be there too! # - `./run pytest --sw --pdb tests/unit/test_my_feature.py` — you get it. - SCRIPT=$(readlink -f "$0") SCRIPT_DIR=$(dirname "$SCRIPT") ROOT_DIR=$(cd "$SCRIPT_DIR/.."; pwd -P) -PWD=$(pwd -P) -WORKING_DIR=$(echo "$PWD" | sed "s|$ROOT_DIR||") -docker compose -f "${ROOT_DIR}/docker/new/docker-compose.yml" run --rm \ - $([ $# -eq 0 ] && echo --service-ports --use-aliases) \ - $([ -n "${WORKING_DIR}" ] && echo --workdir /opt/app${WORKING_DIR}) \ - api "$@" +opts="--rm" + +if [ $# -eq 0 ]; then + # Run default command, i.e. API server + opts="$opts --service-ports --use-aliases --workdir /opt/app/api" +else + PWD=$(pwd -P) + WORKING_DIR=$(echo "$PWD" | sed "s|$ROOT_DIR||") + if [ -n "${WORKING_DIR}" ]; then + opts="$opts --workdir /opt/app${WORKING_DIR}" + fi +fi + +set -x +docker compose -f "${ROOT_DIR}/docker/new/docker-compose.yml" run $opts api "$@" From 9e40ddfd75d1b86524f873506d41721cbef0bcf2 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Mon, 21 Apr 2025 20:20:36 -0300 Subject: [PATCH 5/7] fix: Revert Poetry to 2.1.1 --- api/Dockerfile | 4 ++-- api/poetry.lock | 3 +-- docker/new/docker-compose.yml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index d6acf8cd24b0..d6eff6ad72f2 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -13,11 +13,11 @@ ENV PYTHONDONTWRITEBYTECODE=1 # Install Python packages WORKDIR /tmp -ARG POETRY_VERSION_CONSTRAINT +ARG POETRY_VERSION ARG INSTALL_DEV_DEPENDENCIES COPY api/pyproject.toml api/poetry.lock ./ RUN \ - pip install -q poetry${POETRY_VERSION_CONSTRAINT:-} &&\ + pip install -q poetry==${POETRY_VERSION:-} &&\ poetry config virtualenvs.create false &&\ poetry config cache-dir /var/cache/pypoetry &&\ ### diff --git a/api/poetry.lock b/api/poetry.lock index 4fac7dbe5ebf..c3d59fe7765c 100644 --- a/api/poetry.lock +++ b/api/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. [[package]] name = "annotated-types" @@ -4119,7 +4119,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, diff --git a/docker/new/docker-compose.yml b/docker/new/docker-compose.yml index 2f0c3faa7710..5990204893e0 100644 --- a/docker/new/docker-compose.yml +++ b/docker/new/docker-compose.yml @@ -5,7 +5,7 @@ x-app: &app context: ../../ dockerfile: api/Dockerfile args: - POETRY_VERSION_CONSTRAINT: ==2.1.2 + POETRY_VERSION: 2.1.1 INSTALL_DEV_DEPENDENCIES: true depends_on: database: From abaec12458fe0c5e340bd083ad9a5c04db52df4b Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Tue, 13 May 2025 20:00:46 -0300 Subject: [PATCH 6/7] Use the provided Dockerfile --- api/Dockerfile | 43 ----------------------------------- docker/new/docker-compose.yml | 7 +++--- 2 files changed, 3 insertions(+), 47 deletions(-) delete mode 100644 api/Dockerfile diff --git a/api/Dockerfile b/api/Dockerfile deleted file mode 100644 index d6eff6ad72f2..000000000000 --- a/api/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# NOTE: Why not Alpine? Because Alpine demands installation of OS software -# prior to running app code, which leads to maintenance work and a larger image -# size compared to Debian — considering Debian layers are shared across images. -FROM python:3.12 - -# Prevent Python from buffering stdout and stderr -# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONUNBUFFERED -ENV PYTHONUNBUFFERED=1 - -# Prevent Python from writing .pyc files to disk -# https://docs.python.org/3/using/cmdline.html#envvar-PYTHONDONTWRITEBYTECODE -ENV PYTHONDONTWRITEBYTECODE=1 - -# Install Python packages -WORKDIR /tmp -ARG POETRY_VERSION -ARG INSTALL_DEV_DEPENDENCIES -COPY api/pyproject.toml api/poetry.lock ./ -RUN \ - pip install -q poetry==${POETRY_VERSION:-} &&\ - poetry config virtualenvs.create false &&\ - poetry config cache-dir /var/cache/pypoetry &&\ - ### - # NOTE: Forcefully delete optional private dependencies for now since - # Poetry needs to fetch them in order to calculate the dependency tree. See - # https://github.com/python-poetry/poetry/issues/4562 - # TODO: Improve management of enterprise features - sed -ie '/tool.poetry.group.auth-controller/,+1d' pyproject.toml &&\ - sed -ie '/tool.poetry.group.saml/,+1d' pyproject.toml &&\ - sed -ie '/tool.poetry.group.ldap/,+1d' pyproject.toml &&\ - sed -ie '/tool.poetry.group.workflows/,+1d' pyproject.toml &&\ - sed -ie '/tool.poetry.group.licensing/,+1d' pyproject.toml &&\ - poetry lock &&\ - ### - poetry install --no-root $([ "$INSTALL_DEV_DEPENDENCIES" != "true" ] && echo "--without dev") &&\ - rm -rf /var/cache/pypoetry &&\ - rm -rf /tmp/* - -# Prepare the app -WORKDIR /opt/app -COPY api/ . -EXPOSE 8000 -CMD ["flagsmith", "start", "api"] diff --git a/docker/new/docker-compose.yml b/docker/new/docker-compose.yml index 5990204893e0..d9e9ab222732 100644 --- a/docker/new/docker-compose.yml +++ b/docker/new/docker-compose.yml @@ -3,7 +3,7 @@ x-app: &app image: ${DOCKER_IMAGE:-local/flagsmith/flagsmith}:${DOCKER_TAG:-latest} build: context: ../../ - dockerfile: api/Dockerfile + target: oss-unified args: POETRY_VERSION: 2.1.1 INSTALL_DEV_DEPENDENCIES: true @@ -15,13 +15,12 @@ x-app: &app DATABASE_URL: postgres://postgres@database:5432/postgres DJANGO_SETTINGS_MODULE: app.settings.local volumes: - - ../../.local/pypoetry:/root/.config/poetry:rw # Poetry local cache - - ../../:/opt/app:rw,cached # Application directory + # - ?:/app:rw,cached # Application directory services: api: <<: *app - command: sh -c 'flagsmith migrate && flagsmith start --reload api' + command: migrate-and-serve ports: - 8000:8000 From eb3b37ddacf34ac487483d41d4f4852881309fc7 Mon Sep 17 00:00:00 2001 From: Evandro Myller Date: Tue, 13 May 2025 20:01:02 -0300 Subject: [PATCH 7/7] Run the task processor --- docker/new/docker-compose.yml | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/docker/new/docker-compose.yml b/docker/new/docker-compose.yml index d9e9ab222732..4e23121a5351 100644 --- a/docker/new/docker-compose.yml +++ b/docker/new/docker-compose.yml @@ -8,12 +8,17 @@ x-app: &app POETRY_VERSION: 2.1.1 INSTALL_DEV_DEPENDENCIES: true depends_on: - database: + default-database: + condition: service_healthy + task-processor-database: condition: service_healthy environment: + # LOG_LEVEL: DEBUG + DATABASE_URL: postgres://postgres@default-database:5432/postgres DEBUG: true - DATABASE_URL: postgres://postgres@database:5432/postgres DJANGO_SETTINGS_MODULE: app.settings.local + TASK_PROCESSOR_DATABASE_URL: postgres://postgres@task-processor-database:5432/postgres + TASK_PROCESSOR_DATABASES: task_processor volumes: # - ?:/app:rw,cached # Application directory @@ -25,14 +30,25 @@ services: - 8000:8000 # TODO - # task-processor: - # <<: *app - # command: ... + task-processor: + <<: *app + command: run-task-processor + + default-database: + image: postgres:15.5-alpine + volumes: + - ../../.local/default-database:/var/lib/postgresql/data:rw,cached + environment: + POSTGRES_HOST_AUTH_METHOD: trust + healthcheck: + test: pg_isready -Upostgres + interval: 1s + timeout: 30s - database: + task-processor-database: image: postgres:15.5-alpine volumes: - - ../../.local/database:/var/lib/postgresql/data:rw,cached + - ../../.local/task-processor-database:/var/lib/postgresql/data:rw,cached environment: POSTGRES_HOST_AUTH_METHOD: trust healthcheck: