diff --git a/.env b/.env index b48d9a6..59df715 100644 --- a/.env +++ b/.env @@ -5,3 +5,5 @@ MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 +SQLITE_VERSION= +SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE -DSQLITE_ENABLE_JSON1 -DSQLITE_MAX_VARIABLE_NUMBER=32766" diff --git a/Containerfile b/Containerfile index 8c5be34..3e5cf8a 100644 --- a/Containerfile +++ b/Containerfile @@ -2,7 +2,7 @@ ARG PYTHON_IMPLEMENTATION=python ARG PYTHON_VERSION=3.12 -FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm +FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm AS base LABEL org.opencontainers.image.authors="Django Software Foundation" LABEL org.opencontainers.image.url="https://github.com/django/django-docker-box" @@ -64,3 +64,30 @@ VOLUME /django/output VOLUME /django/source WORKDIR /django/source/tests ENTRYPOINT ["/django/entrypoint.bash"] + +FROM base AS sqlite +ARG SQLITE_VERSION +ARG SQLITE_CFLAGS +SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] +# Use cd instead of WORKDIR to allow wrapping the compilation in a single layer. +# https://github.com/hadolint/hadolint/issues/422 +# hadolint ignore=DL3003 +RUN < [!NOTE] +> +> If left unspecified, the SQLite version provided by Debian will be used. +> Using a specific SQLite version requires compiling it from source. For more +> details, see [SQLite Versions](#SQLite-Versions). ### Python Versions @@ -229,7 +235,8 @@ restrictions with respect to the range of versions available. ### Database Versions Most database container images are pulled from [Docker Hub][2]. Oracle database -is pulled from the [Oracle Container Registry][3]. +is pulled from the [Oracle Container Registry][3]. Specific versions of SQLite +are compiled directly from the tags in the [official Git mirror][11]. You can switch the version of the database you test against by changing the appropriate environment variable. Available options and their defaults can be @@ -273,6 +280,50 @@ To determine what database versions can be used you can check the release notes for the branch of Django that you have checked out, or alternatively there is the [supported database versions][4] page on Django's Trac Wiki. +#### SQLite Versions + +SQLite is normally bundled in the Python installation using the version +available on the system where Python is compiled. We use the Python Docker image +based on Debian `bookworm`, which has SQLite 3.40.1. + +To use a different version, we compile SQLite from source and load the library +dynamically using `LD_PRELOAD`. There are a few caveats as a result: + +- Some SQLite features are only available if certain flags are set during + compilation. SQLite is known to change these flags in newer releases, such as + to enable features by default that were previously opt-in. When Python is + compiled, it inspects the system's SQLite to determine features that are + included in the `sqlite` module. A mismatch in the module and the dynamically + loaded library may result in Python failing to load, which may happen if we + use an SQLite version that is older than the system version. +- Debian and Ubuntu use a custom `CFLAGS` variable to compile their distributed + SQLite. Historically, Django's CI has only been configured with SQLite + versions that come with the operating system. If SQLite is compiled with + different flags, some tests may fail. + +We currently work around the above caveats by setting the simplest `CFLAGS` +value that allows all the tests to pass. To customize the `CFLAGS` used for the +compilation, you can set the `SQLITE_CFLAGS` environment variable. See the +[`.env`][10] file for its default value. + +``` +SQLITE_VERSION=3.48.0 SQLITE_CFLAGS="-DSQLITE_OMIT_JSON -DSQLITE_MAX_VARIABLE_NUMBER=999" docker compose run --build --rm sqlite +``` + +> [!NOTE] +> +> The `--build` argument is necessary if you've changed `SQLITE_CFLAGS` since +> the last run, as it's not part of the image tag. You can also rebuild the +> image separately by running `docker compose build sqlite`, optionally with +> `--no-cache` to ignore the cached build. + +In the future, the Django codebase may be more robust when tested against +different SQLite configurations and the `CFLAGS` workaround may no longer be +necessary. + +Running SpatiaLite tests against specific versions of SQLite/SpatiaLite is not +currently supported. The versions of SQLite and SpatiaLite that come with the +operating system are used for these tests. ### Other Versions @@ -294,7 +345,7 @@ with no promises that they'll be delivered: - Add support for running accessibility tooling and report generation - Support report generation during monthly runs and publish to GitHub Pages - Publish pre-built container images to the GitHub Container Registry -- Support testing against different versions of SQLite and SpatiaLite +- Support testing against different versions of SpatiaLite - Support running with Podman in addition to Docker - Support generating screenshots into `./output/screenshots/` @@ -309,3 +360,5 @@ with no promises that they'll be delivered: [7]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-unit-tests [8]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-selenium-tests [9]: https://docs.djangoproject.com/en/stable/ref/contrib/gis/testing/#geodjango-tests +[10]: .env +[11]: https://github.com/sqlite/sqlite diff --git a/compose.yml b/compose.yml index fc65ac7..c18d32a 100644 --- a/compose.yml +++ b/compose.yml @@ -2,9 +2,10 @@ x-base: &base image: django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION} - build: + build: &base-build context: . dockerfile: ./Containerfile + target: base args: - PYTHON_IMPLEMENTATION=${PYTHON_IMPLEMENTATION} - PYTHON_VERSION=${PYTHON_VERSION} @@ -139,6 +140,10 @@ volumes: services: + # Base service to allow building the image with `docker compose build base`. + base: + <<: *base + # Services: Databases mariadb-db: @@ -285,6 +290,17 @@ services: sqlite: <<: *base + image: "django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION}\ + -sqlite${SQLITE_VERSION}" + pull_policy: never + build: + <<: *base-build + target: sqlite + args: + - PYTHON_IMPLEMENTATION=${PYTHON_IMPLEMENTATION} + - PYTHON_VERSION=${PYTHON_VERSION} + - SQLITE_VERSION=${SQLITE_VERSION} + - SQLITE_CFLAGS=${SQLITE_CFLAGS} depends_on: <<: *depends-on-caches environment: diff --git a/packages.txt b/packages.txt index 14a4662..86f8505 100644 --- a/packages.txt +++ b/packages.txt @@ -14,4 +14,5 @@ libpq-dev libproj-dev libsqlite3-mod-spatialite pkg-config +tcl-dev unzip