fix(replication): bound RESP client parser against unbounded allocation #20497

Workflow file for this run

	name: ci-tests

	on:
	# On push to main: build only (all test steps + heavy jobs below are gated to pull_request)
	# to warm the shared ccache on refs/heads/main, so a new PR's first build starts warm.
	push:
	branches: [main]
	pull_request:
	branches: [main]
	workflow_dispatch:

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: true

	env:
	GTEST_BRIEF: 1

	jobs:
	# Do not rename this job name without updating any matching required status check in repository settings.
	pre-commit:
	if: github.event_name == 'pull_request'
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v7
	with:
	fetch-depth: 0
	- uses: actions/setup-python@v7
	with:
	python-version: '3.12'
	cache: 'pip'
	- uses: actions/cache@v6
	with:
	path: ~/.cache/pre-commit
	key: pre-commit-${{ runner.os }}-${{ hashFiles('.pre-commit-config.yaml') }}
	- uses: pre-commit/action@v3.0.1
	with:
	extra_args: >-
	--show-diff-on-failure --color=always
	--from-ref ${{ github.event.pull_request.base.sha }}
	--to-ref ${{ github.event.pull_request.head.sha }}
	- name: Check fuzz command coverage
	run: python3 fuzz/check_fuzz_coverage.py
	build:
	permissions:
	contents: read
	id-token: write
	packages: read
	actions: write # To allow deleting cache

	env:
	CPP_JUNIT_DIR: ${{ github.workspace }}/test-results/junit/cpp
	CPP_DASHBOARD_DIR: ${{ github.workspace }}/test-results/dashboard/cpp

	strategy:
	matrix:
	# This matrix also runs on push to main (build-only) to warm the ccache that PR builds restore.
	# Test of these containers
	container: ["ubuntu-dev:24", "alpine-dev:latest"]
	build-type: [Debug, Release]
	compiler: [{ cxx: g++, c: gcc }]
	# -no-pie to disable address randomization so we could symbolize stacktraces
	cxx_flags: ["-Werror -no-pie"]
	sanitizers: ["NoSanitizers"]
	include:
	- container: "alpine-dev:latest"
	build-type: Debug
	compiler: { cxx: clang++, c: clang }
	cxx_flags: ""
	sanitizers: "NoSanitizers"
	- container: "ubuntu-dev:24"
	build-type: Debug
	compiler: { cxx: clang++, c: clang }
	# https://maskray.me/blog/2023-08-25-clang-wunused-command-line-argument (search for compiler-rt)
	cxx_flags: "-Wno-error=unused-command-line-argument"
	sanitizers: "Sanitizers"
	# Sanitizer builds produce larger objects, so give this job a bigger ccache.
	ccache-max-size: "2G"

	runs-on: ubuntu-latest
	container:
	image: ghcr.io/romange/${{ matrix.container }}
	# Seems that docker by default prohibits running iouring syscalls
	options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
	volumes:
	- /:/hostroot
	- /mnt:/mnt
	credentials:
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GITHUB_TOKEN }}

	steps:
	- uses: actions/checkout@v7
	with:
	submodules: true

	- name: Prepare Environment
	run: \|
	uname -a
	cmake --version
	mkdir -p ${GITHUB_WORKSPACE}/build
	mount

	echo "===================Before freeing up space ============================================"
	df -h
	rm -rf /hostroot/usr/share/dotnet
	rm -rf /hostroot/usr/local/share/boost
	rm -rf /hostroot/usr/local/lib/android
	rm -rf /hostroot/opt/ghc
	echo "===================After freeing up space ============================================"
	df -h
	touch /mnt/foo
	ls -la /mnt/foo

	- name: System diagnostics
	run: \|
	echo "ulimit is"
	ulimit -s
	echo "-----------------------------"
	echo "disk space is:"
	df -h
	echo "-----------------------------"

	- name: Test dependency-cache manifest
	if: matrix.container == 'ubuntu-dev:24' && matrix.build-type == 'Debug' && matrix.compiler.cxx == 'g++' && matrix.sanitizers == 'NoSanitizers'
	# run only once for both pull requests and pushes.
	run: bash .github/actions/builder/scripts/test_deps_cache_manifest.sh

	- name: Build Dragonfly
	uses: ./.github/actions/builder
	with:
	enable-ccache: 'true'
	ccache-save: 'true'
	enable-deps-cache: 'true'
	build-type: ${{matrix.build-type}}
	cache-key-suffix: ${{ matrix.container }}
	ccache-max-size: ${{ matrix.ccache-max-size \|\| '1G' }}
	c-compiler: ${{matrix.compiler.c}}
	cxx-compiler: ${{matrix.compiler.cxx}}
	cxx-flags: ${{matrix.cxx_flags}}
	sanitizers: ${{matrix.sanitizers}}
	with-aws: 'OFF'

	- name: PostFail
	if: failure()
	run: \|
	echo "disk space is:"
	df -h

	- name: C++ Unit Tests - IoUring
	if: github.event_name != 'push'
	run: \|
	cd ${GITHUB_WORKSPACE}/build
	echo Run ctest -V -L DFLY
	JUNIT_DIR="${CPP_JUNIT_DIR}"
	CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
	GTEST_IOURING_DIR="${JUNIT_DIR}/gtest/iouring"
	GTEST_MANUAL_DIR="${JUNIT_DIR}/gtest/iouring-manual"
	mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_IOURING_DIR}" "${GTEST_MANUAL_DIR}"

	run_manual_gtest() {
	GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" timeout 5m "$@"
	}

	GTEST_OUTPUT="xml:${GTEST_IOURING_DIR}/" \
	GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \
	FLAGS_fiber_safety_margin=4096 timeout 20m ctest -V -L DFLY -E allocation_tracker_test \
	--output-junit "${CTEST_JUNIT_DIR}/iouring.xml"

	# Run allocation tracker test separately without alsologtostderr because it generates a TON of logs.
	GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" FLAGS_fiber_safety_margin=4096 \
	timeout 5m ./allocation_tracker_test

	run_manual_gtest ./dragonfly_test
	run_manual_gtest ./json_family_test --jsonpathv2=false
	run_manual_gtest ./tiered_storage_test --vmodule=db_slice=2 --logtostderr
	run_manual_gtest ./search_test --use_numeric_range_tree=false
	run_manual_gtest ./search_family_test --use_numeric_range_tree=false


	- name: C++ Unit Tests - Epoll
	if: github.event_name != 'push'
	run: \|
	cd ${GITHUB_WORKSPACE}/build
	JUNIT_DIR="${CPP_JUNIT_DIR}"
	CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
	GTEST_EPOLL_DIR="${JUNIT_DIR}/gtest/epoll"
	GTEST_MANUAL_DIR="${JUNIT_DIR}/gtest/epoll-manual"
	mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_EPOLL_DIR}" "${GTEST_MANUAL_DIR}"

	# Create a rule that automatically prints stacktrace upon segfault
	cat > ./init.gdb <<EOF
	catch signal SIGSEGV
	command
	bt
	end
	EOF

	GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" \
	gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll
	GTEST_OUTPUT="xml:${GTEST_EPOLL_DIR}/" \
	GLOG_alsologtostderr=1 FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 \
	timeout 20m ctest -V -L DFLY -E allocation_tracker_test \
	--output-junit "${CTEST_JUNIT_DIR}/epoll.xml"

	GTEST_OUTPUT="xml:${GTEST_MANUAL_DIR}/" FLAGS_fiber_safety_margin=4096 \
	FLAGS_force_epoll=true timeout 5m ./allocation_tracker_test

	- name: C++ Unit Tests - IoUring with cluster mode
	if: github.event_name != 'push'
	run: \|
	cd ${GITHUB_WORKSPACE}/build
	JUNIT_DIR="${CPP_JUNIT_DIR}"
	CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
	GTEST_CLUSTER_DIR="${JUNIT_DIR}/gtest/cluster"
	mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_CLUSTER_DIR}"
	GTEST_OUTPUT="xml:${GTEST_CLUSTER_DIR}/" FLAGS_fiber_safety_margin=4096 \
	FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY \
	--output-junit "${CTEST_JUNIT_DIR}/cluster.xml"

	- name: C++ Unit Tests - IoUring with cluster mode and FLAGS_lock_on_hashtags
	if: github.event_name != 'push'
	run: \|
	cd ${GITHUB_WORKSPACE}/build
	JUNIT_DIR="${CPP_JUNIT_DIR}"
	CTEST_JUNIT_DIR="${JUNIT_DIR}/ctest"
	GTEST_CLUSTER_DIR="${JUNIT_DIR}/gtest/cluster-hashtags"
	mkdir -p "${CTEST_JUNIT_DIR}" "${GTEST_CLUSTER_DIR}"
	GTEST_OUTPUT="xml:${GTEST_CLUSTER_DIR}/" FLAGS_fiber_safety_margin=4096 \
	FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY \
	--output-junit "${CTEST_JUNIT_DIR}/cluster-hashtags.xml"

	- name: Upload unit logs on failure
	if: failure() && github.event_name != 'push'
	uses: actions/upload-artifact@v7
	with:
	name: unit_logs
	path: /tmp/INFO

	- name: Run regression tests
	if: matrix.container == 'ubuntu-dev:24' && matrix.sanitizers == 'NoSanitizers' && github.event_name != 'push'
	uses: ./.github/actions/regression-tests
	with:
	dfly-executable: dragonfly
	run-only-on-ubuntu-latest: true
	build-folder-name: build
	# Non-release build will not run tests marked as opt_only
	# "not empty" string is needed for release build because pytest command can not get empty string for filter
	filter: ${{ matrix.build-type == 'Release' && 'not debug_only' \|\| 'not opt_only' }}
	junit-s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
	aws-role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
	azure-container: ${{ secrets.AZURE_REGTEST_CONTAINER }}
	azure-storage-connection-string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}

	- name: Upload regression logs on failure
	if: failure() && github.event_name != 'push'
	uses: actions/upload-artifact@v7
	with:
	name: regression_logs
	path: /tmp/failed/*

	- name: Run Valkey TCL tests
	# Same gating as the regression tests above: runs the upstream Valkey TCL suites
	# (unit/type/stream[-cgroups], unit/pubsub) in external-server mode against the just-built
	# binary, on both Debug and Release. continue-on-error keeps it a canary for now — flip to a
	# blocking step once proven stable in CI. See tests/dragonfly/valkey_tcl/README.md.
	if: matrix.container == 'ubuntu-dev:24' && matrix.sanitizers == 'NoSanitizers' && github.event_name != 'push'
	continue-on-error: true
	timeout-minutes: 20
	run: \|
	# tcl runs the harness; redis-tools + iproute2 (ss) for the fail-closed readiness probe.
	apt-get update && apt-get install -y --no-install-recommends tcl redis-tools iproute2
	cd ${GITHUB_WORKSPACE}/tests/dragonfly/valkey_tcl
	./sync-valkey-tcl-tests.sh
	./run-stream-tests.sh ${GITHUB_WORKSPACE}/build/dragonfly 6399

	- name: Prepare JUnit artifact name
	if: always() && github.event_name != 'push'
	run: \|
	NAME="junit-ci-${{ matrix.container }}-${{ matrix.build-type }}-${{ matrix.compiler.cxx }}-${{ matrix.sanitizers }}"
	MATRIX_NAME="container-${{ matrix.container }}-build-${{ matrix.build-type }}-compiler-${{ matrix.compiler.cxx }}-sanitizers-${{ matrix.sanitizers }}"
	SAFE_NAME=$("${GITHUB_WORKSPACE}/.github/scripts/sanitize-token.sh" "${NAME}")
	{
	echo "CI_JUNIT_ARTIFACT=${SAFE_NAME}"
	echo "CI_JUNIT_UPLOAD_SUFFIX=${MATRIX_NAME}"
	} >> "${GITHUB_ENV}"

	- name: Generate C++ GTest dashboard summary
	if: always() && github.event_name != 'push'
	run: \|
	mkdir -p "${CPP_DASHBOARD_DIR}"

	python3 "${GITHUB_WORKSPACE}/.github/scripts/gtest-junit-to-dashboard-json.py" \
	"${CPP_JUNIT_DIR}/gtest" "${CPP_DASHBOARD_DIR}/gtest-summary.json" \
	--workflow "${GITHUB_WORKFLOW}" \
	--run-id "${GITHUB_RUN_ID}" \
	--run-attempt "${GITHUB_RUN_ATTEMPT}" \
	--job "${GITHUB_JOB}" \
	--variant "${CI_JUNIT_UPLOAD_SUFFIX}"

	- name: Upload C++ JUnit test results
	if: always() && github.event_name != 'push'
	uses: actions/upload-artifact@v7
	with:
	name: ${{ env.CI_JUNIT_ARTIFACT }}
	path: \|
	${{ github.workspace }}/test-results/junit/cpp/ctest/*/.xml
	${{ github.workspace }}/test-results/dashboard/cpp/*/.json
	if-no-files-found: ignore

	- name: Refresh AWS credentials for C++ JUnit upload
	if: always() && github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository) && github.event_name != 'push'
	uses: aws-actions/configure-aws-credentials@v6
	with:
	role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
	aws-region: us-east-1
	unset-current-credentials: true

	- name: Upload C++ JUnit results to S3
	if: always() && github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository) && github.event_name != 'push'
	env:
	S3_REGTEST_BUCKET: ${{ secrets.S3_REGTEST_BUCKET }}
	run: \|
	JUNIT_DIR="${CPP_JUNIT_DIR}"
	MATRIX_NAME="${CI_JUNIT_UPLOAD_SUFFIX}"

	"${GITHUB_WORKSPACE}/.github/scripts/upload-test-results-to-s3.sh" \
	dir junit "${JUNIT_DIR}" "${S3_REGTEST_BUCKET}" "cpp" "${MATRIX_NAME}" \
	"C++ JUnit" ".xml" "gtest//*.xml"

	"${GITHUB_WORKSPACE}/.github/scripts/upload-test-results-to-s3.sh" \
	file dashboard "${CPP_DASHBOARD_DIR}/gtest-summary.json" "${S3_REGTEST_BUCKET}" "cpp" \
	"${MATRIX_NAME}" "C++ GTest dashboard JSON"

	- name: Upload dragonfly binary for cgroup test
	if: matrix.container == 'ubuntu-dev:24' && matrix.build-type == 'Release' && matrix.sanitizers == 'NoSanitizers' && matrix.compiler.cxx == 'g++' && github.event_name != 'push'
	uses: actions/upload-artifact@v7
	with:
	name: dragonfly-binary
	path: ${{ github.workspace }}/build/dragonfly
	retention-days: 1

	cgroup-thread-detection:
	runs-on: ubuntu-latest
	needs: [build]
	if: github.event_name != 'push'
	steps:
	- uses: actions/checkout@v7

	- uses: actions/download-artifact@v8
	with:
	name: dragonfly-binary
	path: build

	- name: Install redis-tools
	run: sudo apt-get update && sudo apt-get install -y redis-tools

	- name: Prepare cgroup test image
	run: \|
	chmod +x build/dragonfly
	# Verify the binary's runtime dependencies and install them in a local image.
	# The binary is built in ubuntu-dev:24 and links dynamically against Boost,
	# which is not present in the minimal ubuntu:24.04 image.
	#
	# Note: GitHub Actions runners are hosted on Azure. We rewrite the apt sources
	# to use Azure's internal Ubuntu mirror to prevent CI failures caused by
	# rate-limited or corrupted public Canonical mirrors.
	echo "Runtime deps of dragonfly binary:"
	ldd build/dragonfly
	docker build -t df-cgroup-test - <<'EOF'
	FROM ubuntu:24.04
	RUN sed -i -E 's@http://(archive\|security)\.ubuntu\.com@http://azure.archive.ubuntu.com@g' /etc/apt/sources.list.d/ubuntu.sources && \
	apt-get update -qq && \
	apt-get install -y --no-install-recommends libboost-context1.83.0 libssl3 && \
	rm -rf /var/lib/apt/lists/*
	EOF

	- name: Test cgroup CPU auto-detection
	run: \|
	wait_for_server() {
	local port=$1
	local cid=$2
	for i in $(seq 1 30); do
	sleep 1
	redis-cli -p "$port" PING 2>/dev/null \| grep -q PONG && return 0
	done
	echo "ERROR: dragonfly on port $port did not start within 30s"
	docker logs "$cid" 2>&1 \|\| true
	docker rm -f "$cid" 2>/dev/null \|\| true
	return 1
	}

	# Scenario 1: --cpus=2, no flag -> should auto-detect 2 threads
	CID=$(docker run --rm -d --cpus=2 --security-opt seccomp=unconfined \
	-v "$PWD/build/dragonfly:/dragonfly" -p 16380:6379 df-cgroup-test \
	/dragonfly --port 6379 --maxmemory 1G --dbfilename "" --noversion_check)

	wait_for_server 16380 "$CID"

	THREADS=$(redis-cli -p 16380 INFO server \| grep -oP 'thread_count:\K\d+')
	docker rm -f "$CID" 2>/dev/null \|\| true
	echo "Scenario 1: expected 2, got $THREADS"
	[ "$THREADS" -eq 2 ]

	# Scenario 2: --cpus=2 + --proactor_threads=4 -> flag must win
	CID=$(docker run --rm -d --cpus=2 --security-opt seccomp=unconfined \
	-v "$PWD/build/dragonfly:/dragonfly" -p 16381:6379 df-cgroup-test \
	/dragonfly --port 6379 --proactor_threads 4 --maxmemory 1G --dbfilename "" --noversion_check)

	wait_for_server 16381 "$CID"

	THREADS=$(redis-cli -p 16381 INFO server \| grep -oP 'thread_count:\K\d+')
	docker rm -f "$CID" 2>/dev/null \|\| true
	echo "Scenario 2: expected 4, got $THREADS"
	[ "$THREADS" -eq 4 ]

	lint-test-chart:
	runs-on: ubuntu-latest
	needs: [build]
	if: github.event_name != 'push'
	steps:
	- uses: actions/checkout@v7
	- uses: ./.github/actions/lint-test-chart

	list-caches:
	if: always() && github.event_name != 'push'
	needs: [build]
	runs-on: ubuntu-latest
	permissions:
	actions: read
	steps:
	- name: List all Actions caches
	env:
	GH_TOKEN: ${{ github.token }}
	run: gh cache list -R ${{ github.repository }} --limit 100 --sort size_in_bytes --order desc \|\| true

	large-tests-arm:
	runs-on: CI-LARGE-ARM
	# Skipped on push-to-main (warm) because its build cache (df_common_bin / Linux-ARM64-Release-
	# gcc-g++) is already warmed on main by the scheduled regression-tests job.
	if: github.event_name != 'push'

	permissions:
	id-token: write
	contents: read
	actions: write # To allow deleting cache

	container:
	image: ghcr.io/romange/ubuntu-dev:24
	options: --security-opt seccomp=unconfined --sysctl "net.ipv6.conf.all.disable_ipv6=0"
	volumes:
	- /var/crash:/var/crash
	- /:/hostroot
	- /mnt:/mnt

	steps:
	- uses: actions/checkout@v7
	with:
	submodules: true

	- name: Print environment info
	run: \|
	cat /proc/cpuinfo
	ulimit -a
	env
	lsblk -l

	- name: Build Dragonfly
	uses: ./.github/actions/builder
	with:
	enable-ccache: 'true'
	ccache-save: 'true'
	enable-deps-cache: 'true'
	build-type: Release
	cache-key-suffix: 'df_common_bin'
	c-compiler: gcc
	cxx-compiler: g++
	targets: 'dragonfly'

	- name: Authenticate to AWS
	if: github.repository_owner == 'dragonflydb' && github.actor != 'dependabot[bot]' && (github.event_name != 'pull_request' \|\| github.event.pull_request.head.repo.full_name == github.repository)
	uses: aws-actions/configure-aws-credentials@v6
	with:
	role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
	aws-region: us-east-1

	- name: Run large tests on ARM
	uses: ./.github/actions/regression-tests
	with:
	dfly-executable: dragonfly
	gspace-secret: ${{ secrets.GSPACES_BOT_DF_BUILD }}
	build-folder-name: build
	run-only-on-ubuntu-latest: true
	filter: large
	s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
	junit-s3-bucket: ${{ secrets.S3_REGTEST_BUCKET }}
	aws-role-to-assume: ${{ secrets.AWS_CI_S3_ROLE_ARN }}
	azure-container: ${{ secrets.AZURE_REGTEST_CONTAINER }}
	azure-storage-connection-string: ${{ secrets.AZURE_STORAGE_CONNECTION_STRING }}

	- name: Upload logs on failure
	if: failure()
	uses: actions/upload-artifact@v7
	with:
	name: large-tests-arm-logs
	path: /tmp/failed/*

Navigation Menu

Uh oh!

fix(replication): bound RESP client parser against unbounded allocation #20497

Workflow file

fix(replication): bound RESP client parser against unbounded allocation #20497

Uh oh!

Workflow file for this run