From f5b0fc203449068b3783cc5a97b5a38ad234e738 Mon Sep 17 00:00:00 2001 From: Sage Abdullah Date: Tue, 28 Jan 2025 20:23:12 +0000 Subject: [PATCH 1/6] Added sqlite-lib service to test against specific SQLite versions. --- .env | 4 ++++ README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- compose.yml | 39 +++++++++++++++++++++++++++++++++++++-- packages.txt | 1 + 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/.env b/.env index b48d9a6..e7a0a50 100644 --- a/.env +++ b/.env @@ -5,3 +5,7 @@ MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 +SQLITE_VERSION=3.31.0 +SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_JSON1 \ + -DSQLITE_MAX_VARIABLE_NUMBER=32766" diff --git a/README.md b/README.md index c073140..d2036c8 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ $ docker compose run --rm mysql $ docker compose run --rm oracle $ docker compose run --rm postgres $ docker compose run --rm sqlite +$ docker compose run --rm sqlite-lib ``` Each of the above commands will run the test suite for a different supported @@ -204,7 +205,14 @@ The versions of various backend services can be switched by setting these enviro | `ORACLE_VERSION` | `23.5.0.0` | Version of Oracle container image to use | | `POSTGRESQL_VERSION` | `14` | Version of PostgreSQL container image to use | | `POSTGIS_VERSION` | `3.1` | Version of PostGIS extension to use | +| `SQLITE_VERSION` | `3.31.0` | Version of SQLite to compile and use | +> [!NOTE] +> +> Using a specific SQLite version requires compiling it from source. To +> customize the `CFLAGS` used for the compilation, you can set the +> `SQLITE_CFLAGS` environment variable. See the [`.env`][10] file for its +> default value. For more details, see [SQLite Versions](#SQLite-Versions). ### Python Versions @@ -229,7 +237,8 @@ restrictions with respect to the range of versions available. ### Database Versions Most database container images are pulled from [Docker Hub][2]. Oracle database -is pulled from the [Oracle Container Registry][3]. +is pulled from the [Oracle Container Registry][3]. Specific versions of SQLite +are compiled directly from the tags in the [official Git mirror][11]. You can switch the version of the database you test against by changing the appropriate environment variable. Available options and their defaults can be @@ -273,6 +282,41 @@ To determine what database versions can be used you can check the release notes for the branch of Django that you have checked out, or alternatively there is the [supported database versions][4] page on Django's Trac Wiki. +#### SQLite Versions + +SQLite is normally bundled in the Python installation using the version +available on the system where Python is compiled. We use the Python Docker image +based on Debian `bookworm`, which has SQLite 3.40.1. + +To use a different version, we compile SQLite from source and load the library +dynamically using `LD_PRELOAD`. There are a few caveats as a result: + +- Some SQLite features are only available if certain flags are set during + compilation. SQLite is known to change these flags in newer releases, such as + to enable features by default that were previously opt-in. When Python is + compiled, it inspects the system's SQLite to determine features that are + included in the `sqlite` module. A mismatch in the module and the dynamically + loaded library may result in Python failing to load, which may happen if we + use an SQLite version that is older than the system version. +- Debian and Ubuntu use a custom `CFLAGS` variable to compile their distributed + SQLite. Historically, Django's CI has only been configured with SQLite + versions that come with the operating system. If SQLite is compiled with + different flags, some tests may fail. + +We currently work around the above caveats by setting the simplest `CFLAGS` +value that allows all the tests to pass. In the future, the Django codebase may +be more robust when tested against the different SQLite configurations and these +workarounds may no longer be necessary. + +Running the tests against a specific SQLite version must be done using the +`sqlite-lib` container instead of `sqlite`. + +```console +$ docker compose run --rm sqlite-lib +``` + +This is done to avoid compiling SQLite when you are not testing against a +specific version. ### Other Versions @@ -309,3 +353,5 @@ with no promises that they'll be delivered: [7]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-unit-tests [8]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-selenium-tests [9]: https://docs.djangoproject.com/en/stable/ref/contrib/gis/testing/#geodjango-tests +[10]: .env +[11]: https://github.com/sqlite/sqlite diff --git a/compose.yml b/compose.yml index fc65ac7..baa042d 100644 --- a/compose.yml +++ b/compose.yml @@ -8,7 +8,7 @@ x-base: &base args: - PYTHON_IMPLEMENTATION=${PYTHON_IMPLEMENTATION} - PYTHON_VERSION=${PYTHON_VERSION} - additional_contexts: + additional_contexts: &additional-contexts src: ${DJANGO_PATH:-../django} volumes: - ${DJANGO_PATH:-../django}:/django/source:rw @@ -82,6 +82,36 @@ x-postgresql-base: &postgresql-base -c wal_level=minimal # 13+: -c wal_keep_size=0 +x-sqlite-lib-base: &sqlite-lib-base + image: django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION}-sqlite${SQLITE_VERSION} + pull_policy: never + build: + context: . + dockerfile_inline: | + FROM django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION} + SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] + # Only compile SQLite and set LD_PRELOAD if a version is specified. + RUN < Date: Tue, 28 Jan 2025 22:18:00 +0000 Subject: [PATCH 2/6] Consolidated sqlite-lib service into sqlite. --- .env | 2 +- README.md | 37 +++++++++++++++------------- compose.yml | 69 +++++++++++++++++++++++++---------------------------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.env b/.env index e7a0a50..c1c4223 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 -SQLITE_VERSION=3.31.0 +SQLITE_VERSION= SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE \ -DSQLITE_ENABLE_JSON1 \ -DSQLITE_MAX_VARIABLE_NUMBER=32766" diff --git a/README.md b/README.md index d2036c8..dba7dfe 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Tooling and test execution support for [Django][0] :unicorn: 3. Build the image: ```console - $ docker compose build sqlite + $ docker compose build base ``` 4. Run the tests: @@ -66,7 +66,6 @@ $ docker compose run --rm mysql $ docker compose run --rm oracle $ docker compose run --rm postgres $ docker compose run --rm sqlite -$ docker compose run --rm sqlite-lib ``` Each of the above commands will run the test suite for a different supported @@ -205,14 +204,13 @@ The versions of various backend services can be switched by setting these enviro | `ORACLE_VERSION` | `23.5.0.0` | Version of Oracle container image to use | | `POSTGRESQL_VERSION` | `14` | Version of PostgreSQL container image to use | | `POSTGIS_VERSION` | `3.1` | Version of PostGIS extension to use | -| `SQLITE_VERSION` | `3.31.0` | Version of SQLite to compile and use | +| `SQLITE_VERSION` | | Version of SQLite to compile and use | > [!NOTE] > -> Using a specific SQLite version requires compiling it from source. To -> customize the `CFLAGS` used for the compilation, you can set the -> `SQLITE_CFLAGS` environment variable. See the [`.env`][10] file for its -> default value. For more details, see [SQLite Versions](#SQLite-Versions). +> If left unspecified, the SQLite version provided by Debian will be used. +> Using a specific SQLite version requires compiling it from source. For more +> details, see [SQLite Versions](#SQLite-Versions). ### Python Versions @@ -304,19 +302,24 @@ dynamically using `LD_PRELOAD`. There are a few caveats as a result: different flags, some tests may fail. We currently work around the above caveats by setting the simplest `CFLAGS` -value that allows all the tests to pass. In the future, the Django codebase may -be more robust when tested against the different SQLite configurations and these -workarounds may no longer be necessary. +value that allows all the tests to pass. To customize the `CFLAGS` used for the +compilation, you can set the `SQLITE_CFLAGS` environment variable. See the +[`.env`][10] file for its default value. -Running the tests against a specific SQLite version must be done using the -`sqlite-lib` container instead of `sqlite`. - -```console -$ docker compose run --rm sqlite-lib +``` +SQLITE_VERSION=3.48.0 SQLITE_CFLAGS="-DSQLITE_OMIT_JSON -DSQLITE_MAX_VARIABLE_NUMBER=999" docker compose run --build --rm sqlite ``` -This is done to avoid compiling SQLite when you are not testing against a -specific version. +> [!NOTE] +> +> The `--build` argument is necessary if you've changed `SQLITE_CFLAGS` since +> the last run, as it's not part of the image tag. You can also rebuild the +> image separately by running `docker compose build sqlite`, optionally with +> `--no-cache` to ignore the cached build. + +In the future, the Django codebase may be more robust when tested against +different SQLite configurations and the `CFLAGS` workaround may no longer be +necessary. ### Other Versions diff --git a/compose.yml b/compose.yml index baa042d..c21b3e6 100644 --- a/compose.yml +++ b/compose.yml @@ -82,36 +82,6 @@ x-postgresql-base: &postgresql-base -c wal_level=minimal # 13+: -c wal_keep_size=0 -x-sqlite-lib-base: &sqlite-lib-base - image: django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION}-sqlite${SQLITE_VERSION} - pull_policy: never - build: - context: . - dockerfile_inline: | - FROM django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION} - SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] - # Only compile SQLite and set LD_PRELOAD if a version is specified. - RUN < Date: Tue, 28 Jan 2025 22:18:36 +0000 Subject: [PATCH 3/6] Fixed libsqlite3.so output path on SQLite 3.48.0+. --- compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index c21b3e6..301fb3e 100644 --- a/compose.yml +++ b/compose.yml @@ -305,7 +305,11 @@ services: cd /tmp/sqlite ./configure make - cp .libs/libsqlite3.so /tmp/ + if [ -f libsqlite3.so ]; then + cp libsqlite3.so /tmp/ + else + cp .libs/libsqlite3.so /tmp/ + fi rm -rf /tmp/sqlite fi EOF From 8ef6e51e2cbbb194cc7adcfdb05e59787abd1936 Mon Sep 17 00:00:00 2001 From: Sage Abdullah Date: Sun, 16 Feb 2025 16:09:58 +0000 Subject: [PATCH 4/6] Changed default SQLITE_CFLAGS to be single-line. --- .env | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.env b/.env index c1c4223..59df715 100644 --- a/.env +++ b/.env @@ -6,6 +6,4 @@ ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 SQLITE_VERSION= -SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE \ - -DSQLITE_ENABLE_JSON1 \ - -DSQLITE_MAX_VARIABLE_NUMBER=32766" +SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_JSON1 -DSQLITE_MAX_VARIABLE_NUMBER=32766" From 1ab68b9f1db85c8ce8e7d2d94934547cd1af493b Mon Sep 17 00:00:00 2001 From: Sage Abdullah Date: Sun, 16 Feb 2025 16:11:47 +0000 Subject: [PATCH 5/6] Refactored SQLite setup to use multi-stage build. --- Containerfile | 29 ++++++++++++++++++++++++++++- compose.yml | 34 +++++++--------------------------- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/Containerfile b/Containerfile index 8c5be34..3e5cf8a 100644 --- a/Containerfile +++ b/Containerfile @@ -2,7 +2,7 @@ ARG PYTHON_IMPLEMENTATION=python ARG PYTHON_VERSION=3.12 -FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm +FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm AS base LABEL org.opencontainers.image.authors="Django Software Foundation" LABEL org.opencontainers.image.url="https://github.com/django/django-docker-box" @@ -64,3 +64,30 @@ VOLUME /django/output VOLUME /django/source WORKDIR /django/source/tests ENTRYPOINT ["/django/entrypoint.bash"] + +FROM base AS sqlite +ARG SQLITE_VERSION +ARG SQLITE_CFLAGS +SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] +# Use cd instead of WORKDIR to allow wrapping the compilation in a single layer. +# https://github.com/hadolint/hadolint/issues/422 +# hadolint ignore=DL3003 +RUN < Date: Sun, 16 Feb 2025 16:31:07 +0000 Subject: [PATCH 6/6] Added note about running SQLite/SpatiaLite versions for SpatiaLite tests. --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dba7dfe..db26b9d 100644 --- a/README.md +++ b/README.md @@ -321,6 +321,10 @@ In the future, the Django codebase may be more robust when tested against different SQLite configurations and the `CFLAGS` workaround may no longer be necessary. +Running SpatiaLite tests against specific versions of SQLite/SpatiaLite is not +currently supported. The versions of SQLite and SpatiaLite that come with the +operating system are used for these tests. + ### Other Versions For the Memcached, Redis, and Selenium container images, the latest container @@ -341,7 +345,7 @@ with no promises that they'll be delivered: - Add support for running accessibility tooling and report generation - Support report generation during monthly runs and publish to GitHub Pages - Publish pre-built container images to the GitHub Container Registry -- Support testing against different versions of SQLite and SpatiaLite +- Support testing against different versions of SpatiaLite - Support running with Podman in addition to Docker - Support generating screenshots into `./output/screenshots/`