Run separate web server, registry watcher and build servers in docker-compose

Nemo157 · Nemo157 · commit 62d274dd8e4d · 2023-11-18T14:56:03.000+01:00
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -38,13 +38,8 @@ jobs:
 
       - name: Launch postgres
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db
 
       - name: run database migrations
         run: cargo run -- database migrate
@@ -60,9 +55,6 @@ jobs:
             --check \
             -- --all-targets --all-features
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   build:
     runs-on: ubuntu-latest
     steps:
@@ -110,13 +102,8 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: run tests
         shell: bash
@@ -127,9 +114,6 @@ jobs:
             $f || exit 1
           done
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   GUI_test:
     runs-on: ubuntu-latest
     needs: build
@@ -145,20 +129,12 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: Run GUI tests
         run: ./dockerfiles/run-gui-tests.sh
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   build_tests:
     runs-on: ubuntu-latest
     needs: build
@@ -174,13 +150,8 @@ jobs:
 
       - name: Launch postgres and min.io
         run: |
-          cp .env.sample .env
-          mkdir -p ${DOCSRS_PREFIX}/public-html
-          docker compose up -d db s3
-          # Give the database enough time to start up
-          sleep 5
-          # Make sure the database is actually working
-          psql "${DOCSRS_DATABASE_URL}"
+          touch .docker.env
+          docker compose up --wait --wait-timeout 30 db s3
 
       - name: slow tests
         env:
@@ -193,9 +164,6 @@ jobs:
             $f --ignored --test-threads=1 || exit 1
           done
 
-      - name: Clean up the database
-        run: docker compose down --volumes
-
   fmt:
     name: Rustfmt
     runs-on: ubuntu-latest
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -9,5 +9,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Build the Docker image
-        run: docker build -t docs-rs -f dockerfiles/Dockerfile .
+      - run: docker build --target web-server -f dockerfiles/Dockerfile .
+      - run: docker build --target build-server -f dockerfiles/Dockerfile .
+      - run: docker build --target registry-watcher -f dockerfiles/Dockerfile .
+      - run: docker build --target cli -f dockerfiles/Dockerfile .
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 /ignored
 /.env
+/.docker.env
 /src/web/badge/Cargo.lock
 target
 *.css
diff --git a/README.md b/README.md
@@ -64,7 +64,7 @@ mkdir -p ignored/cratesfyi-prefix/crates.io-index
 # Builds the docs.rs binary
 cargo build
 # Start the external services.
-docker compose up -d db s3
+docker compose up --wait db s3
 # anything that doesn't run via docker-compose needs the settings defined in
 # .env. Either via `. ./.env` as below, or via any dotenv shell integration.
 . ./.env
@@ -115,21 +115,25 @@ which uses docker-compose for the web server as well.
 This will not cache dependencies - in particular, you'll have to rebuild all 400 whenever the lockfile changes -
 but makes sure that you're in a known environment so you should have fewer problems getting started.
 
-You can also use the `web` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+You'll need to `touch .docker.env` first, this file can have any environment
+variable overrides you want to use in docker containers.
+
+You can also use the `builder-a` container to run builds on systems which don't support running builds directly (mostly on Mac OS or Windows):
+
 ```sh
+# update the toolchain
+docker compose run --rm builder-a build update-toolchain
 # run a build for a single crate
-docker compose run web build crate regex 1.3.1
-# or build essential files
-docker compose run web build add-essential-files
-# rebuild the web container when you changed code.
-docker compose up -d web --build
+docker compose run --rm builder-a build crate regex 1.3.1
+# rebuild containers when you changed code.
+docker compose up --wait --build
 ```
 
-You can also run other commands like the setup above from within the container:
+You can also run other non-build commands like the setup steps above, or queueing crates for the background builders from within the `cli` container:
 
 ```sh
 docker compose run --rm cli database migrate
-docker compose run --rm cli build update-toolchain
+docker compose run --rm cli queue add regex 1.3.1
 ```
 
 Note that running tests is not supported when using pure docker-compose.
@@ -151,7 +155,7 @@ Three services are defined:
 
 #### Rebuilding Containers
 
-To rebuild the site, run `docker compose build`.
+To rebuild the site, run `docker compose --profile all build`.
 Note that docker-compose caches the build even if you change the source code,
 so this will be necessary anytime you make changes.
 
@@ -172,7 +176,7 @@ This is probably because you have `git.autocrlf` set to true,
 
 ##### I see the error `/opt/rustwide/cargo-home/bin/cargo: cannot execute binary file: Exec format error` when running builds.
 
-You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run web build [...]` workaround described above.
+You are most likely not on a Linux platform. Running builds directly is only supported on `x86_64-unknown-linux-gnu`. On other platforms you can use the `docker compose run --rm builder-a build [...]` workaround described above.
 
 See [rustwide#41](https://github.com/rust-lang/rustwide/issues/41) for more details about supporting more platforms directly.
 
@@ -200,11 +204,11 @@ cargo run -- start-web-server
 ```sh
 # Builds <CRATE_NAME> <CRATE_VERSION> and adds it into database
 # This is the main command to build and add a documentation into docs.rs.
-# For example, `docker compose run web build crate regex 1.1.6`
+# For example, `docker compose run --rm builder-a build crate regex 1.1.6`
 cargo run -- build crate <CRATE_NAME> <CRATE_VERSION>
 
-# alternatively, via the web container
-docker compose run web build crate <CRATE_NAME> <CRATE_VERSION>
+# alternatively, within docker-compose containers
+docker compose run --rm builder-a build crate <CRATE_NAME> <CRATE_VERSION>
 
 # Builds every crate on crates.io and adds them into database
 # (beware: this may take months to finish)
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,35 +1,116 @@
-version: "3"
+version: "3.4"
+
+x-healthcheck: &healthcheck-interval
+  interval: 1s
+  timeout: 1s
+  start_period: 10s
+  # TODO: https://github.com/docker/compose/issues/10461
+  # interval: 10s
+  # start_interval: 1s
+
+x-environment: &environment
+    RUST_BACKTRACE: true
+
+    DOCSRS_PREFIX: /opt/docsrs/prefix
+
+    DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
+    DOCSRS_MIN_POOL_SIZE: 2
+    DOCSRS_MAX_POOL_SIZE: 10
+
+    DOCSRS_STORAGE_BACKEND: s3
+
+    S3_ENDPOINT: http://s3:9000
+    AWS_ACCESS_KEY_ID: cratesfyi
+    AWS_SECRET_ACCESS_KEY: secret_key
+
+    DOCSRS_RENDER_THREADS: 2
+
+    DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
+    DOCSRS_DOCKER: true
+    DOCSRS_DOCKER_IMAGE: ghcr.io/rust-lang/crates-build-env/linux-micro
+    DOCSRS_BUILD_CPU_LIMIT: 2
+    DOCSRS_INCLUDE_DEFAULT_TARGETS: false
+
+x-builder: &builder
+    build:
+        context: .
+        dockerfile: ./dockerfiles/Dockerfile
+        target: build-server
+    depends_on:
+        - db
+        - s3
+    environment: *environment
+    env_file:
+        - .docker.env
+    healthcheck:
+        << : *healthcheck-interval
+        test: curl --silent --fail localhost:3000/about/metrics
+
 services:
     web:
         build:
             context: .
             dockerfile: ./dockerfiles/Dockerfile
+            target: web-server
         platform: "linux/amd64"
         depends_on:
             - db
             - s3
         ports:
-            - "3000:3000"
-        # for metrics
-        expose: ["3000"]
+            - "3000:80"
+        environment: *environment
+        env_file:
+            - .docker.env
+        healthcheck:
+            << : *healthcheck-interval
+            test: curl --silent --fail localhost:80/about/metrics
+
+    # Include the registry watcher with `docker compose --profile watch up -d`
+    registry-watcher:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: registry-watcher
+        platform: "linux/amd64"
+        depends_on:
+            - db
         volumes:
-            - "/var/run/docker.sock:/var/run/docker.sock"
-            - ".rustwide-docker:/opt/docsrs/rustwide"
             - "cratesio-index:/opt/docsrs/prefix/crates.io-index"
-        environment:
-            DOCSRS_RUSTWIDE_WORKSPACE: /opt/docsrs/rustwide
-            DOCSRS_DATABASE_URL: postgresql://cratesfyi:password@db
-            DOCSRS_STORAGE_BACKEND: s3
-            S3_ENDPOINT: http://s3:9000
-            AWS_ACCESS_KEY_ID: cratesfyi
-            AWS_SECRET_ACCESS_KEY: secret_key
+        environment: *environment
         env_file:
-            - .env
+            - .docker.env
+        profiles:
+            - watch
+            - all
         healthcheck:
-            test: ["CMD", "curl", "--silent", "--fail", "localhost:3000"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: curl --silent --fail localhost:3000/about/metrics
+
+    builder-a:
+        << : *builder
+        volumes:
+            - ".rustwide-docker/builder-a:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+
+    builder-b:
+        << : *builder
+        volumes:
+            - ".rustwide-docker/builder-b:/opt/docsrs/rustwide"
+            - "/var/run/docker.sock:/var/run/docker.sock"
+
+    cli:
+        build:
+            context: .
+            dockerfile: ./dockerfiles/Dockerfile
+            target: cli
+        depends_on:
+            - db
+            - s3
+        environment: *environment
+        env_file:
+            - .docker.env
+        profiles:
+            - all
 
     db:
         build:
@@ -44,10 +125,8 @@ services:
             # Use a non-standard port on the host to avoid conflicting with existing postgres servers
             - "127.0.0.1:15432:5432"
         healthcheck:
-            test: ["CMD", "pg_isready", "--username", "cratesfyi"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: pg_isready --username cratesfyi
 
     s3:
         image: minio/minio
@@ -65,17 +144,8 @@ services:
             MINIO_ROOT_USER: cratesfyi
             MINIO_ROOT_PASSWORD: secret_key
         healthcheck:
-            test:
-                [
-                    "CMD",
-                    "curl",
-                    "--silent",
-                    "--fail",
-                    "localhost:9000/minio/health/ready",
-                ]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: mc ready local
 
     prometheus:
         build:
@@ -84,11 +154,8 @@ services:
         ports:
             - "127.0.0.1:9090:9090"
         healthcheck:
-            test:
-                ["CMD", "curl", "--silent", "--fail", "localhost:9090/-/ready"]
-            interval: 10s
-            timeout: 5s
-            retries: 10
+            << : *healthcheck-interval
+            test: promtool check healthy
 
     gui_tests:
         build:
@@ -99,6 +166,8 @@ services:
             - "host.docker.internal:host-gateway"
         volumes:
             - "${PWD}:/build/out"
+        profiles:
+            - all
 
 volumes:
     postgres-data: {}
diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
diff --git a/dockerfiles/entrypoint.sh b/dockerfiles/entrypoint.sh

-Original file line number
+Diff line change
     && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
     && DEBIAN_FRONTEND=noninteractive apt-get install -y \
         ca-certificates \
 +        curl \
         tini \
     && rm -rf /var/lib/apt/lists/*
 +WORKDIR /srv/docsrs
++
 +# Tini is a small init binary to properly handle signals
 +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"]
 +CMD ["start-web-server", "0.0.0.0:80"]
++
 COPY --from=build /build/target/release/cratesfyi /usr/local/bin
 COPY static /srv/docsrs/static
 COPY templates /srv/docsrs/templates
 COPY vendor /srv/docsrs/vendor
 -WORKDIR /srv/docsrs
 +########################
 +#  Build server stage  #
 +########################
++
 +FROM ubuntu:22.04 AS build-server
++
 +RUN apt-get update \
 +    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
 +    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 +        ca-certificates \
 +        tini \
 +        curl \
 +        docker.io \
 +        build-essential \
 +        gcc \
 +        pkg-config \
 +        libssl-dev \
 +    && rm -rf /var/lib/apt/lists/*
++
 # Tini is a small init binary to properly handle signals
 -CMD ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "start-web-server", "0.0.0.0:80"]
 +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"]
 +CMD ["start-build-server"]
 -##################
 -#  Output stage  #
 -##################
 +COPY --from=build /build/target/release/cratesfyi /usr/local/bin
++
 +############################
 +#  Registry watcher stage  #
 +############################
 -FROM ubuntu:22.04 AS output
 +FROM ubuntu:22.04 AS registry-watcher
 -RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 -    git \
 -    libmagic1 \
 -    docker.io \
 -    ca-certificates \
 -    build-essential \
 -    gcc \
 -    pkg-config \
 -    libssl-dev
 +RUN apt-get update \
 +    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
 +    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 +        ca-certificates \
 +        tini \
 +        curl \
 +        git \
 +    && rm -rf /var/lib/apt/lists/*
++
 +# Tini is a small init binary to properly handle signals
 +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"]
 +CMD ["start-registry-watcher", "--repository-stats-updater=enabled", "--cdn-invalidator=enabled"]
++
 +COPY --from=build /build/target/release/cratesfyi /usr/local/bin
++
 +###############
 +#  CLI stage  #
 +###############
++
 +FROM ubuntu:22.04 AS cli
++
 +RUN apt-get update \
 +    && DEBIAN_FRONTEND=noninteractive apt-get upgrade -y \
 +    && DEBIAN_FRONTEND=noninteractive apt-get install -y \
 +        ca-certificates \
 +        tini \
 +    && rm -rf /var/lib/apt/lists/*
 -RUN mkdir -p /opt/docsrs/prefix
 +ENTRYPOINT ["/usr/bin/tini", "/usr/local/bin/cratesfyi", "--"]
 COPY --from=build /build/target/release/cratesfyi /usr/local/bin
 -COPY static /opt/docsrs/static
 -COPY templates /opt/docsrs/templates
 -COPY dockerfiles/entrypoint.sh /opt/docsrs/
 -COPY vendor /opt/docsrs/vendor
+-
 -WORKDIR /opt/docsrs
 -ENTRYPOINT ["/opt/docsrs/entrypoint.sh"]
 -CMD ["daemon", "--registry-watcher=disabled"]