Skip to content
ci-tests

feat: Inline buffer for captured replies #19168

Sign in to view logs

feat: Inline buffer for captured replies

feat: Inline buffer for captured replies #19168

Workflow file for this run

.github/workflows/ci.yml at 89949e0
name: ci-tests
on:
# push:
# branches: [ main ]
pull_request:
branches: [main]
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
pre-commit:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- uses: actions/setup-python@v6
with:
python-version: '3.12'
cache: 'pip'
- uses: actions/cache@v5
with:
path: ~/.cache/pre-commit
key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
- uses: pre-commit/action@v3.0.1
with:
extra_args: >-
--show-diff-on-failure --color=always
--from-ref ${{ github.event.pull_request.base.sha }}
--to-ref ${{ github.event.pull_request.head.sha }}
build:
permissions:
contents: read
id-token: write
packages: read
env:
CPP_JUNIT_DIR: ${{ github.workspace }}/test-results/junit/cpp
strategy:
matrix:
# Test of these containers
container: ["ubuntu-dev:24", "alpine-dev:latest"]
build-type: [Debug, Release]
compiler: [{ cxx: g++, c: gcc }]
# -no-pie to disable address randomization so we could symbolize stacktraces
cxx_flags: ["-Werror -no-pie"]
sanitizers: ["NoSanitizers"]
include:
- container: "alpine-dev:latest"
build-type: Debug
compiler: { cxx: clang++, c: clang }
cxx_flags: ""
sanitizers: "NoSanitizers"
- container: "ubuntu-dev:24"
build-type: Debug
compiler: { cxx: clang++, c: clang }
# https://maskray.me/blog/2023-08-25-clang-wunused-command-line-argument (search for compiler-rt)
cxx_flags: "-Wno-error=unused-command-line-argument"
sanitizers: "Sanitizers"
runs-on: ubuntu-latest
container:
image: ghcr.io/romange/${{ matrix.container }}
# Seems that docker by default prohibits running iouring syscalls
options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
volumes:
- /:/hostroot
- /mnt:/mnt
credentials:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Prepare Environment
run: |
uname -a
cmake --version
mkdir -p ${GITHUB_WORKSPACE}/build
mount
echo "===================Before freeing up space ============================================"
df -h
rm -rf /hostroot/usr/share/dotnet
rm -rf /hostroot/usr/local/share/boost
rm -rf /hostroot/usr/local/lib/android
rm -rf /hostroot/opt/ghc
echo "===================After freeing up space ============================================"
df -h
touch /mnt/foo
ls -la /mnt/foo
- name: System diagnostics
run: |
echo "ulimit is"
ulimit -s
echo "-----------------------------"
echo "disk space is:"
df -h
echo "-----------------------------"
- name: Build Dragonfly
uses: ./.github/actions/builder
with:
build-type: ${{matrix.build-type}}
c-compiler: ${{matrix.compiler.c}}
cxx-compiler: ${{matrix.compiler.cxx}}
cxx-flags: ${{matrix.cxx_flags}}
sanitizers: ${{matrix.sanitizers}}
with-aws: 'OFF'
- name: PostFail
if: failure()
run: |
echo "disk space is:"
df -h
- name: C++ Unit Tests - IoUring
run: |
cd ${GITHUB_WORKSPACE}/build
echo Run ctest -V -L DFLY
JUNIT_DIR="${CPP_JUNIT_DIR}"
CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
GTEST_IOURING_DIR="${JUNIT_DIR}/gtest/iouring"
GTEST_MANUAL_DIR="${JUNIT_DIR}/gtest/iouring-manual"
mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_IOURING_DIR}" "${GTEST_MANUAL_DIR}"
run_manual_gtest() {
GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" timeout 5m "$@"
}
GTEST_OUTPUT="xml:${GTEST_IOURING_DIR}/" \
GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \
FLAGS_fiber_safety_margin=4096 timeout 20m ctest -V -L DFLY -E allocation_tracker_test \
--output-junit "${CTEST_JUNIT_DIR}/iouring.xml"
# Run allocation tracker test separately without alsologtostderr because it generates a TON of logs.
GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" FLAGS_fiber_safety_margin=4096 \
timeout 5m ./allocation_tracker_test
run_manual_gtest ./dragonfly_test
run_manual_gtest ./json_family_test --jsonpathv2=false
run_manual_gtest ./tiered_storage_test --vmodule=db_slice=2 --logtostderr
run_manual_gtest ./search_test --use_numeric_range_tree=false
run_manual_gtest ./search_family_test --use_numeric_range_tree=false
- name: C++ Unit Tests - Epoll
run: |
cd ${GITHUB_WORKSPACE}/build
JUNIT_DIR="${CPP_JUNIT_DIR}"
CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
GTEST_EPOLL_DIR="${JUNIT_DIR}/gtest/epoll"
GTEST_MANUAL_DIR="${JUNIT_DIR}/gtest/epoll-manual"
mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_EPOLL_DIR}" "${GTEST_MANUAL_DIR}"
# Create a rule that automatically prints stacktrace upon segfault
cat > ./init.gdb <<EOF
catch signal SIGSEGV
command
bt
end
EOF
GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" \
gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll
GTEST_OUTPUT="xml:${GTEST_EPOLL_DIR}/" \
GLOG_alsologtostderr=1 FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 \
timeout 20m ctest -V -L DFLY -E allocation_tracker_test \
--output-junit "${CTEST_JUNIT_DIR}/epoll.xml"
GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" FLAGS_fiber_safety_margin=4096 \
FLAGS_force_epoll=true timeout 5m ./allocation_tracker_test
- name: C++ Unit Tests - IoUring with cluster mode
run: |
cd ${GITHUB_WORKSPACE}/build
JUNIT_DIR="${CPP_JUNIT_DIR}"
CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
GTEST_CLUSTER_DIR="${JUNIT_DIR}/gtest/cluster"
mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_CLUSTER_DIR}"
GTEST_OUTPUT="xml:${GTEST_CLUSTER_DIR}/" FLAGS_fiber_safety_margin=4096 \
FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY \
--output-junit "${CTEST_JUNIT_DIR}/cluster.xml"
- name: C++ Unit Tests - IoUring with cluster mode and FLAGS_lock_on_hashtags
run: |
cd ${GITHUB_WORKSPACE}/build
JUNIT_DIR="${CPP_JUNIT_DIR}"
CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
GTEST_CLUSTER_DIR="${JUNIT_DIR}/gtest/cluster-hashtags"
mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_CLUSTER_DIR}"
GTEST_OUTPUT="xml:${GTEST_CLUSTER_DIR}/" FLAGS_fiber_safety_margin=4096 \
FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY \
--output-junit "${CTEST_JUNIT_DIR}/cluster-hashtags.xml"
- name: Upload unit logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: unit_logs
path: /tmp/*INFO*
- name: Run regression tests
if: matrix.container == 'ubuntu-dev:24' && matrix.sanitizers == 'NoSanitizers'
uses: ./.github/actions/regression-tests
with:
dfly-executable: dragonfly
run-only-on-ubuntu-latest: true
build-folder-name: build
# Non-release build will not run tests marked as opt_only
# "not empty" string is needed for release build because pytest command can not get empty string for filter
filter: ${{ matrix.build-type == 'Release' && 'not debug_only' || 'not opt_only' }}
junit-s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
aws-role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
azure-container: ${{ secrets.AZURE_REGTEST_CONTAINER }}
azure-storage-connection-string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
- name: Upload regression logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: regression_logs
path: /tmp/failed/*
- name: Prepare JUnit artifact name
if: always()
run: |
NAME="junit-ci-${{ matrix.container }}-${{ matrix.build-type }}-${{ matrix.compiler.cxx }}-${{ matrix.sanitizers }}"
SAFE_NAME=$("${GITHUB_WORKSPACE}/.github/scripts/sanitize-token.sh" "${NAME}")
echo "CI_JUNIT_ARTIFACT=${SAFE_NAME}" >> "${GITHUB_ENV}"
- name: Upload C++ JUnit test results
if: always()
uses: actions/upload-artifact@v7
with:
name: ${{ env.CI_JUNIT_ARTIFACT }}
# Scope to cpp/ only: regression XMLs are uploaded by the regression action itself.
path: ${{ github.workspace }}/test-results/junit/cpp/**/*.xml
if-no-files-found: ignore
- name: Refresh AWS credentials for C++ JUnit upload
if: always() && github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
uses: aws-actions/configure-aws-credentials@v6
with:
role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
aws-region: us-east-1
unset-current-credentials: true
- name: Upload C++ JUnit results to S3
if: always() && github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
env:
S3_REGTEST_BUCKET: ${{ secrets.S3_REGTEST_BUCKET }}
run: |
JUNIT_DIR="${CPP_JUNIT_DIR}"
MATRIX_NAME="container-${{ matrix.container }}-build-${{ matrix.build-type }}-compiler-${{ matrix.compiler.cxx }}-sanitizers-${{ matrix.sanitizers }}"
"${GITHUB_WORKSPACE}/.github/scripts/upload-junit-to-s3.sh" \
"${JUNIT_DIR}" "${S3_REGTEST_BUCKET}" "cpp" "${MATRIX_NAME}" "C++ JUnit"
- name: Upload dragonfly binary for cgroup test
if: matrix.container == 'ubuntu-dev:24' && matrix.build-type == 'Release' && matrix.sanitizers == 'NoSanitizers' && matrix.compiler.cxx == 'g++'
uses: actions/upload-artifact@v7
with:
name: dragonfly-binary
path: ${{ github.workspace }}/build/dragonfly
retention-days: 1
cgroup-thread-detection:
runs-on: ubuntu-latest
needs: [build]
steps:
- uses: actions/checkout@v6
- uses: actions/download-artifact@v8
with:
name: dragonfly-binary
path: build
- name: Install redis-tools
run: sudo apt-get update && sudo apt-get install -y redis-tools
- name: Prepare cgroup test image
run: |
chmod +x build/dragonfly
# Verify the binary's runtime dependencies and install them in a local image.
# The binary is built in ubuntu-dev:24 and links dynamically against Boost,
# which is not present in the minimal ubuntu:24.04 image.
#
# Note: GitHub Actions runners are hosted on Azure. We rewrite the apt sources
# to use Azure's internal Ubuntu mirror to prevent CI failures caused by
# rate-limited or corrupted public Canonical mirrors.
echo "Runtime deps of dragonfly binary:"
ldd build/dragonfly
docker build -t df-cgroup-test - <<'EOF'
FROM ubuntu:24.04
RUN sed -i -E 's@http://(archive|security)\.ubuntu\.com@http://azure.archive.ubuntu.com@g' /etc/apt/sources.list.d/ubuntu.sources && \
apt-get update -qq && \
apt-get install -y --no-install-recommends libboost-context1.83.0 libssl3 && \
rm -rf /var/lib/apt/lists/*
EOF
- name: Test cgroup CPU auto-detection
run: |
wait_for_server() {
local port=$1
local cid=$2
for i in $(seq 1 30); do
sleep 1
redis-cli -p "$port" PING 2>/dev/null | grep -q PONG && return 0
done
echo "ERROR: dragonfly on port $port did not start within 30s"
docker logs "$cid" 2>&1 || true
docker rm -f "$cid" 2>/dev/null || true
return 1
}
# Scenario 1: --cpus=2, no flag -> should auto-detect 2 threads
CID=$(docker run --rm -d --cpus=2 --security-opt seccomp=unconfined \
-v "$PWD/build/dragonfly:/dragonfly" -p 16380:6379 df-cgroup-test \
/dragonfly --port 6379 --maxmemory 1G --dbfilename "" --noversion_check)
wait_for_server 16380 "$CID"
THREADS=$(redis-cli -p 16380 INFO server | grep -oP 'thread_count:\K\d+')
docker rm -f "$CID" 2>/dev/null || true
echo "Scenario 1: expected 2, got $THREADS"
[ "$THREADS" -eq 2 ]
# Scenario 2: --cpus=2 + --proactor_threads=4 -> flag must win
CID=$(docker run --rm -d --cpus=2 --security-opt seccomp=unconfined \
-v "$PWD/build/dragonfly:/dragonfly" -p 16381:6379 df-cgroup-test \
/dragonfly --port 6379 --proactor_threads 4 --maxmemory 1G --dbfilename "" --noversion_check)
wait_for_server 16381 "$CID"
THREADS=$(redis-cli -p 16381 INFO server | grep -oP 'thread_count:\K\d+')
docker rm -f "$CID" 2>/dev/null || true
echo "Scenario 2: expected 4, got $THREADS"
[ "$THREADS" -eq 4 ]
lint-test-chart:
runs-on: ubuntu-latest
needs: [build]
steps:
- uses: actions/checkout@v6
- uses: ./.github/actions/lint-test-chart
large-tests-arm:
runs-on: CI-LARGE-ARM
permissions:
id-token: write
contents: read
container:
image: ghcr.io/romange/ubuntu-dev:24
options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
volumes:
- /var/crash:/var/crash
- /:/hostroot
- /mnt:/mnt
steps:
- uses: actions/checkout@v6
with:
submodules: true
- name: Print environment info
run: |
cat /proc/cpuinfo
ulimit -a
env
lsblk -l
- name: Build Dragonfly
uses: ./.github/actions/builder
with:
build-type: Release
targets: 'dragonfly'
- name: Authenticate to AWS
if: github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)
uses: aws-actions/configure-aws-credentials@v6
with:
role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
aws-region: us-east-1
- name: Run large tests on ARM
uses: ./.github/actions/regression-tests
with:
dfly-executable: dragonfly
gspace-secret: ${{ secrets.GSPACES_BOT_DF_BUILD }}
build-folder-name: build
run-only-on-ubuntu-latest: true
filter: large
s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
junit-s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
aws-role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
azure-container: ${{ secrets.AZURE_REGTEST_CONTAINER }}
azure-storage-connection-string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}
- name: Upload logs on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: large-tests-arm-logs
path: /tmp/failed/*