Improvements to the integration tests framework

.github/actions/download-or-build/action.yml

@@ -2,6 +2,9 @@
 # https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository
 name: 'Download the artifact or build from source if failed'
 description: 'Fetch or, if failed, build the binaries'
+defaults:
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 inputs:
   artifact-name:
     description: 'The name of the artifact to fetch and unpack (it must contain the target.tar file)'
@@ -26,10 +29,57 @@ runs:
     # due to 'continue-on-error: true' above the conclusion/status will
     # always be success, but we can check for failure using the outcome key
     if: ${{ steps.download_binaries.outcome == 'failure' }}
-    run: cargo build
+    run: |
+      case "${{ env.build-profile }}" in
+        debug)
+          cargo build
+          ;;
+        release)
+          cargo build --release
+          ;;
+        *) exit 1
+      esac
   - name: Build dnst from keyset branch
     if: ${{ steps.download_binaries.outcome == 'failure' }}
-    run: cargo install --git https://github.com/nlnetlabs/dnst --branch keyset --root "$GITHUB_WORKSPACE/target/dnst" --locked --bin=dnst
+    run: |
+      case "${{ env.build-profile }}" in
+        debug)
+          cargo install --debug --git https://github.com/nlnetlabs/dnst --branch keyset --root "$GITHUB_WORKSPACE/target/dnst" --locked --bin=dnst
+          ;;
+        release)
+          cargo install --git https://github.com/nlnetlabs/dnst --branch keyset --root "$GITHUB_WORKSPACE/target/dnst" --locked --bin=dnst
+          ;;
+        *) exit 1
+      esac
+  - name: Add cascade to PATH
+    run: |
+      case "${{ env.build-profile }}" in
+        debug)
+          [[ -x "$GITHUB_WORKSPACE/target/debug/cascaded" ]] # Sanity check
+          echo "$GITHUB_WORKSPACE/target/debug" >> "$GITHUB_PATH"
+          ;;
+        release)
+          [[ -x "$GITHUB_WORKSPACE/target/release/cascaded" ]] # Sanity check
+          echo "$GITHUB_WORKSPACE/target/release" >> "$GITHUB_PATH"
+          ;;
+        *) exit 1
+      esac
   - name: Add dnst to PATH
     # https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands#example-of-adding-a-system-path
-    run: echo "$GITHUB_WORKSPACE/target/dnst/bin" >> "$GITHUB_PATH"
+    run: |
+      case "${{ env.build-profile }}" in
+        debug)
+          # Sanity check
+          if ! [[ -x "$GITHUB_WORKSPACE/target/dnst/debug/bin/dnst" ]]; then
+            echo "::error:: dnst binary ($GITHUB_WORKSPACE/target/dnst/debug/bin/dnst) is not executable"
+          fi
+          echo "$GITHUB_WORKSPACE/target/dnst/debug/bin" >> "$GITHUB_PATH"
+          ;;
+        release)
+          if ! [[ -x "$GITHUB_WORKSPACE/target/dnst/release/bin/dnst" ]]; then
+            echo "::error:: dnst binary ($GITHUB_WORKSPACE/target/dnst/release/bin/dnst) is not executable"
+          fi
+          echo "$GITHUB_WORKSPACE/target/dnst/release/bin" >> "$GITHUB_PATH"
+          ;;
+        *) exit 1
+      esac

.github/actions/prepare-systest-env/action.yml

@@ -2,6 +2,9 @@
 # https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository
 name: 'Prepare the system/integration test environment'
 description: 'Setup the environment, and fetch or build the binaries'
+defaults:
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 inputs:
   artifact-name:
     description: 'The name of the artifact to fetch and unpack (it must contain the target.tar file)'

.github/actions/print-logfiles/action.yml

@@ -2,15 +2,18 @@
 # https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository
 name: 'Print logfiles'
 description: 'Print the nameserver logfiles'
+defaults:
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 inputs:
   nameserver-dir:
     description: 'The base directory of the nameserver log files'
     required: false
-    default: ${{ format('{0}/nameservers', github.workspace) }}
+    default: ${{ github.workspace }}/nameservers
   cascade-dir:
     description: 'The cascade base directory containing the log files'
     required: false
-    default: ${{ format('{0}/cascade-dir', github.workspace) }}
+    default: ${{ github.workspace }}/cascade-dir
 runs:
   using: "composite"
   steps:
@@ -24,3 +27,4 @@ runs:
       cat "${{ inputs.cascade-dir }}/cascade-startup.log" >&2 || true
       echo ">>> Logfile cascade.log:" >&2
       cat "${{ inputs.cascade-dir }}//cascade.log" >&2 || true
+      # ss -tulnp >&2 || true

.github/actions/setup-and-start-cascade/action.yml

@@ -2,30 +2,25 @@
 # https://docs.github.com/en/actions/tutorials/create-actions/create-a-composite-action#creating-a-composite-action-within-the-same-repository
 name: 'Setup and start cascade'
 description: 'Setup and start the cascade daemon'
+defaults:
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 inputs:
   cascade-dir:
     description: 'The name of the artifact to fetch and unpack (it must contain the target.tar file)'
-    required: true
+    required: false
+    default: ${{ github.workspace }}/cascade-dir
 runs:
   using: "composite"
   steps:
   - name: Start cascade with a working-directory-local config
     run: |
-      # Variant 1: Just using absolute paths
-      mkdir -p cascade-dir/policies
-      cascade_dir="$PWD/cascade-dir"
-      # We need to create a custom config for cascade because the default paths are not available/permitted in this environment
-      ./target/debug/cascade template config | \
-        ./scripts/sed-config-dirs.sh "${cascade_dir}" | tee "${cascade_dir}/config.toml" >&2
-      ./target/debug/cascade template policy > "${cascade_dir}/policies/default.toml"
-      ./target/debug/cascaded --config "${cascade_dir}/config.toml" --state "${cascade_dir}/state.db" --daemonize &>"${cascade_dir}/cascade-startup.log"
+      CASCADE_DIR="${{ inputs.cascade-dir }}"
+      BUILD_PROFILE="${{ env.build-profile }}"
 
-      # Variant 2: Using cd to change directories and using relative paths (beware of Cascade --daemonize changing the current working directory to '/')
-      # mkdir -p cascade-dir/policies
-      # cd cascade-dir
-      # # We need to create a custom config for cascade because the default paths are not available/permitted in this environment
-      # ../target/debug/cascade template config | \
-      #   ../scripts/sed-config-dirs.sh > config.toml
-      # cat config.toml >&2
-      # ../target/debug/cascade template policy > policies/default.toml
-      # ../target/debug/cascaded --config config.toml --state "$GITHUB_WORKSPACE/cascade-dir/state.db" --daemonize &>cascade-startup.log
+      mkdir -p "${CASCADE_DIR}/policies"
+      # We need to create a custom config for cascade because the default paths are not available/permitted in this environment
+      cascade template config | \
+        "${GITHUB_WORKSPACE}/scripts/sed-config-dirs.sh" "${BUILD_PROFILE}" "${CASCADE_DIR}" | tee "${CASCADE_DIR}/config.toml" >&2
+      cascade template policy > "${CASCADE_DIR}/policies/default.toml"
+      cascaded --config "${CASCADE_DIR}/config.toml" --state "${CASCADE_DIR}/state.db" --daemonize &>"${CASCADE_DIR}/cascade-startup.log"

.github/actions/tests/add-zone-query/action.yml

@@ -8,27 +8,29 @@ inputs:
     required: true
 defaults:
   # see: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defaultsrunshell
-  shell: bash --noprofile --norc -eo pipefail -x {0}
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 runs:
   using: "composite"
   steps:
     - name: Prepare the system test environment
       uses: ./.github/actions/prepare-systest-env
       with:
         artifact-name: ${{ inputs.artifact-name }}
-    - run: target/debug/cascade --version
+    - run: cascade --version
     - name: Setup and start the cascade daemon
       uses: ./.github/actions/setup-and-start-cascade
     - name: Add a zone
       run: |
-        target/debug/cascade zone add --policy default --source 127.0.0.1:1055 example.test
+        cascade zone add --policy default --source 127.0.0.1:1055 example.test
     - name: Check zone status
       run: |
         timeout=10 # seconds
         start=$(date +%s)
-        until target/debug/cascade zone status example.test | grep -q "Published zone available"; do
+        until cascade zone status example.test | grep -q "Published zone available"; do
           if (($(date +%s) > (start + timeout))); then
-            echo "timeout: zone status did not report published zone available" >&2
+            cascade zone status example.test
+            echo "::error:: timeout: zone status did not report published zone available"
             exit 1
           fi
           sleep 1

.github/actions/tests/review-unsigned-zone/action.yml

@@ -8,7 +8,8 @@ inputs:
     required: true
 defaults:
   # see: https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-syntax#defaultsrunshell
-  shell: bash --noprofile --norc -eo pipefail -x {0}
+  run:
+    shell: bash --noprofile --norc -eo pipefail -x {0}
 runs:
   using: "composite"
   steps:
@@ -44,7 +45,8 @@ runs:
         start=$(date +%s)
         until target/debug/cascade zone status example.test | grep -q "Published zone available"; do
           if (($(date +%s) > (start + timeout))); then
-            echo "timeout: zone status did not report published zone available" >&2
+            cascade zone status example.test
+            echo "::error:: timeout: zone status did not report published zone available"
             exit 1
           fi
           sleep 1

.github/workflows/system-tests.yml

@@ -19,6 +19,8 @@
 #       matrix:
 #         os: [ubuntu-latest]
 #         rust: [stable] # see build job
+#     # env:
+#     #   build-profile: release # uncomment this to run this test with a release build
 #     steps:
 #     # WARNING: You MUST checkout the repository otherwise subsequent 'uses'
 #     # steps will fail to find the action.
@@ -34,7 +36,7 @@
 #     #     artifact-name: ${{ format('cascade_{0}_{1}_{2}', github.sha, matrix.os, matrix.rust) }}
 #     - name: Setup and start the cascade daemon
 #       uses: ./.github/actions/setup-and-start-cascade
-#     - run: target/debug/cascade --version
+#     - run: cascade --version
 #     ### RUN YOUR TESTS HERE
 #     # # Optional, the container gets cleaned up anyway (at least in act)
 #     # - name: Stop the setup
@@ -55,6 +57,7 @@ env:
   # Set this assignment to your choosing to set the cargo build verbosity
   CARGO_TERM_VERBOSE: ${{ github.actor != 'nektos/act' }}
   # CARGO_TERM_VERBOSE: true
+  build-profile: debug
 
 defaults:
   run:
@@ -80,7 +83,7 @@ jobs:
         compression-level: 0
   build:
     # First build the project for once for all systems to deduplicate work
-    # in the later tests
+    # in the later tests. This builds both the release and debug build.
     name: Build the project for use by the later tests
     needs: check-for-artifact-server
     runs-on: ${{ matrix.os }}
@@ -95,12 +98,18 @@ jobs:
     - name: Checkout repository
       if: needs.check-for-artifact-server.outputs.status == 'success'
       uses: actions/checkout@v4
-    - name: Build Cascade
+    - name: Build Cascade (debug)
       if: needs.check-for-artifact-server.outputs.status == 'success'
       run: cargo build
-    - name: Build dnst from keyset branch
+    - name: Build Cascade (release)
       if: needs.check-for-artifact-server.outputs.status == 'success'
-      run: cargo install --git https://github.com/nlnetlabs/dnst --branch keyset --root target/dnst --locked --bin=dnst
+      run: cargo build --release
+    - name: Build dnst from keyset branch (debug)
+      if: needs.check-for-artifact-server.outputs.status == 'success'
+      run: cargo install --debug --git https://github.com/nlnetlabs/dnst --branch keyset --root target/dnst/debug --locked --bin=dnst
+    - name: Build dnst from keyset branch (release)
+      if: needs.check-for-artifact-server.outputs.status == 'success'
+      run: cargo install --git https://github.com/nlnetlabs/dnst --branch keyset --root target/dnst/release --locked --bin=dnst
     - name: Tar built binaries to preserve permissions
       if: needs.check-for-artifact-server.outputs.status == 'success'
       run: tar -cf target.tar target
@@ -120,8 +129,28 @@ jobs:
         # Optional. Default is '6'
         compression-level: 1
 
+  test-release-version:
+    name: Example test with prepare environment and cascade --version (release)
+    runs-on: ${{ matrix.os }}
+    needs: build
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        rust: [stable] # see build job
+    env:
+      build-profile: release
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+    - name: Prepare the system test environment
+      uses: ./.github/actions/prepare-systest-env
+      with:
+        artifact-name: ${{ format('cascade_{0}_{1}_{2}', github.sha, matrix.os, matrix.rust) }}
+    - run: target/release/cascade --version
+    - run: cascade --version
+
   test-version:
-    name: Example test with prepare environment and cascade --version
+    name: Example test with prepare environment and cascade --version (debug)
     runs-on: ${{ matrix.os }}
     needs: build
 
@@ -136,10 +165,7 @@ jobs:
       uses: ./.github/actions/prepare-systest-env
       with:
         artifact-name: ${{ format('cascade_{0}_{1}_{2}', github.sha, matrix.os, matrix.rust) }}
-    - run: target/debug/cascade --version
-    # # Optional, the container gets cleaned up anyway (at least in act)
-    # - name: Stop the setup
-    #   run: scripts/manage-test-environment.sh stop
+    - run: cascade --version
 
   add-zone-query:
     name: Add a zone, query the published zone

TESTING.md

@@ -130,33 +130,13 @@ of text printed you can:
   using `unbuffer` from the `expect` package; left as an excercise for the
   user)
 
+### Miscellaneous notes
+
+- By default, tests are run using a debug build for both Cascade and dnst.
+  - This can be changed per test using the `build-profile` environment variable.
+- `cascade`, `cascaded`, and `dnst` are added to the `$PATH`.
+
 ### Example test job
 
-```yml
-  job-name:
-    name: Run tests with resolvers/nameservers
-    runs-on: ${{ matrix.os }}
-    needs: build
-    strategy:
-      matrix:
-        os: [ubuntu-latest]
-        rust: [stable] # see build job
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
-    - name: Prepare the system test environment
-      uses: ./.github/actions/prepare-systest-env
-      with:
-        artifact-name: ${{ format('cascade_{0}_{1}_{2}', github.sha, matrix.os, matrix.rust) }}
-    # - name: Only download/build the binaries without setting up the test environment
-    #   uses: ./.github/actions/download-or-build
-    #   with:
-    #     artifact-name: ${{ format('cascade_{0}_{1}_{2}', github.sha, matrix.os, matrix.rust) }}
-    - name: Setup and start the cascade daemon
-      uses: ./.github/actions/setup-and-start-cascade
-    - run: target/debug/cascade --version
-    ### RUN YOUR TESTS HERE
-    # # Optional, the container gets cleaned up anyway (at least in act)
-    # - name: Stop the setup
-    #   run: scripts/manage-test-environment.sh stop
-```
+You can find the example test job at the top of the workflow file
+`.github/workflows/system-tests.yml`.

scripts/manage-test-environment.sh

@@ -244,7 +244,7 @@ stub-zone:
 stub-zone:
   name: "example.test"
   stub-host: example.test
-  stub-addr: 127.0.0.1@${_cascade_port}
+  stub-addr: 127.0.0.1@${_nsd_port}
 
 python:
 dynlib:
@@ -409,11 +409,17 @@ function test-services() {
     log-error ">> NSD (primary) status:"
     nsd-control -c "${_nameserver_base_dir}/nsd-primary.conf" status
     log-error
+    log-error ">> NSD (primary) zonestatus example.test:"
+    nsd-control -c "${_nameserver_base_dir}/nsd-primary.conf" zonestatus example.test
+    log-error
     log-error ">> Unbound status:"
     sudo unbound-control -c "${_nameserver_base_dir}/unbound.conf" status
     log-error
     log-error ">> dig test SOA:"
     dig test SOA
+    log-error
+    log-error ">> dig @127.0.0.1 -p 1055 example.test AXFR:"
+    dig @127.0.0.1 -p 1055 example.test AXFR
   )
 }