Restore RBAC objects across different user_directories types #2621
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build | |
| on: | |
| pull_request: | |
| branches: | |
| - "*" | |
| push: | |
| branches: | |
| - master | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.27" | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| # - name: Cache golang | |
| # id: cache-golang | |
| # uses: actions/cache@v5 | |
| # with: | |
| # path: | | |
| # ~/go/pkg/mod | |
| # ~/.cache/go-build | |
| # key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - name: Install golang dependencies | |
| run: go mod download -x | |
| # if: | | |
| # steps.cache-golang.outputs.cache-hit != 'true' | |
| - name: Build clickhouse-backup binary | |
| id: make-race | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| run: | | |
| make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup | |
| make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips | |
| make build-race build-race-fips config test | |
| - name: Report unittest coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| file: _coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: unittest-${{ matrix.clickhouse }} | |
| - name: Extract GCS credentials | |
| id: secrets | |
| uses: ./.github/actions/extract-gcs-creds | |
| with: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: build-artifacts | |
| path: | | |
| build/linux/amd64/clickhouse-backup | |
| build/linux/arm64/clickhouse-backup | |
| build/linux/amd64/clickhouse-backup-fips | |
| build/linux/arm64/clickhouse-backup-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: | | |
| clickhouse-backup/clickhouse-backup-race | |
| clickhouse-backup/clickhouse-backup-race-fips | |
| if-no-files-found: error | |
| retention-days: 1 | |
| outputs: | |
| GCS_TESTS: ${{ steps.secrets.outputs.GCS_TESTS }} | |
| testflows: | |
| needs: build | |
| name: Testflows | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| golang-version: | |
| - "1.27" | |
| clickhouse: | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| - '25.3' | |
| - '25.8' | |
| - '26.3' | |
| - '26.8' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Install python venv | |
| run: | | |
| set -x | |
| (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv) | |
| python3 -m venv ~/venv/qa | |
| - name: Cache python | |
| uses: actions/cache@v5 | |
| id: cache-python | |
| with: | |
| path: ~/venv/qa | |
| key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }} | |
| - name: Install python dependencies | |
| run: | | |
| set -x | |
| ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt | |
| if: | | |
| steps.cache-python.outputs.cache-hit != 'true' | |
| - name: Running TestFlows tests | |
| # Backstop for hangs outside the per-suite timeout in run.sh; killing | |
| # the step (not the job) lets the `if: always()` log upload still run | |
| timeout-minutes: 30 | |
| env: | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_ENDPOINT: ${{ secrets.QA_AWS_ENDPOINT }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_GCS_CRED_JSON: ${{ secrets.QA_GCS_CRED_JSON }} | |
| RUN_PARALLEL: 4 | |
| DEBUG: 1 | |
| NO_COLORS: 1 | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| # don't change it to avoid not working CI/CD | |
| RUN_TESTS: "*" | |
| run: | | |
| set -xe | |
| export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| source ~/venv/qa/bin/activate | |
| ./test/testflows/run.sh | |
| - name: Report testflows coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| file: test/testflows/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: testflows-${{ matrix.clickhouse }} | |
| # todo possible failures https://github.com/actions/upload-artifact/issues/270 | |
| - name: Fix testflows _instances log permissions | |
| if: always() | |
| run: | | |
| set -xe | |
| if [ -d "test/testflows/clickhouse_backup/_instances" ]; then | |
| sudo chown -R "$(id -u)":"$(id -g)" test/testflows/clickhouse_backup/_instances || true | |
| chmod -R u+rwX test/testflows/clickhouse_backup/_instances || true | |
| fi | |
| - name: Upload testflows logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: testflows-logs-and-reports-${{ matrix.clickhouse }}-${{ github.run_id }} | |
| path: | | |
| test/testflows/*.log | |
| test/testflows/*.log.txt | |
| test/testflows/_logs_/**/*.log | |
| test/testflows/clickhouse_backup/_instances/**/*.log | |
| test/testflows/*.html | |
| retention-days: 7 | |
| testflows_fips: | |
| needs: | |
| - build | |
| name: Testflows FIPS | |
| runs-on: ubuntu-22.04 | |
| strategy: | |
| matrix: | |
| golang-version: | |
| - "1.27" | |
| clickhouse: | |
| - "fips" | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Install python venv | |
| run: | | |
| set -x | |
| (dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv) | |
| python3 -m venv ~/venv/qa | |
| - name: Cache python | |
| uses: actions/cache@v5 | |
| id: cache-python | |
| with: | |
| path: ~/venv/qa | |
| key: clickhouse-backup-python-${{ hashFiles('test/testflows/requirements.txt','.github/workflows/*.yaml') }} | |
| - name: Install python dependencies | |
| run: | | |
| set -x | |
| ~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt | |
| if: | | |
| steps.cache-python.outputs.cache-hit != 'true' | |
| - name: Running FIPS TestFlows tests | |
| # Backstop for hangs outside the per-suite timeout in run.sh; killing | |
| # the step (not the job) lets the `if: always()` log upload still run | |
| timeout-minutes: 30 | |
| env: | |
| CLICKHOUSE_IMAGE: altinity/clickhouse-server | |
| CLICKHOUSE_VERSION: 25.3.8.30001.altinityfips | |
| RUN_PARALLEL: 1 | |
| DEBUG: 1 | |
| NO_COLORS: 1 | |
| # Keep in sync with @Name("FIPS 140-3 Compatibility") in tests/fips_140_3.py | |
| RUN_TESTS: "/clickhouse backup/FIPS 140-3 Compatibility*" | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| run: | | |
| set -xe | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| source ~/venv/qa/bin/activate | |
| export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup | |
| ./test/testflows/run.sh | |
| - name: Report FIPS testflows coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| file: test/testflows/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: testflows-${{ matrix.clickhouse }} | |
| - name: Fix FIPS log permissions for artifact upload | |
| if: always() | |
| run: | | |
| set -xe | |
| if [ -d "test/testflows/clickhouse_backup/_instances" ]; then | |
| sudo chown -R "$(id -u)":"$(id -g)" test/testflows/clickhouse_backup/_instances || true | |
| chmod -R u+rwX test/testflows/clickhouse_backup/_instances || true | |
| fi | |
| - name: Upload FIPS testflows logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: fips-testflows-logs-and-reports-${{ github.run_id }} | |
| path: | | |
| test/testflows/*.log | |
| test/testflows/*.log.txt | |
| test/testflows/_logs_/**/*.log | |
| test/testflows/clickhouse_backup/_instances/**/*.log | |
| test/testflows/*.html | |
| retention-days: 7 | |
| test: | |
| needs: build | |
| name: Test | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| golang-version: | |
| - "1.27" | |
| clickhouse: | |
| - '1.1.54394' | |
| - '19.17' | |
| - '20.3' | |
| - '20.8' | |
| - '21.3' | |
| - '21.8' | |
| - '22.3' | |
| - '22.8' | |
| - '23.3' | |
| - '23.8' | |
| - '24.3' | |
| - '24.8' | |
| - '25.3' | |
| - '25.8' | |
| - '26.3' | |
| - '26.8' | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^${{ matrix.golang-version }}' | |
| # - name: Cache golang | |
| # id: cache-golang | |
| # uses: actions/cache@v5 | |
| # with: | |
| # path: | | |
| # ~/go/pkg/mod | |
| # ~/.cache/go-build | |
| # key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }} | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-test-artifacts | |
| path: ./clickhouse-backup/ | |
| - name: Extract GCS credentials | |
| id: secrets | |
| uses: ./.github/actions/extract-gcs-creds | |
| with: | |
| VAULT_PASSWORD: ${{ secrets.VAULT_PASSWORD }} | |
| - name: Running integration tests | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| CLICKHOUSE_VERSION: ${{ matrix.clickhouse }} | |
| RUN_PARALLEL: 4 | |
| # LOG_LEVEL: "debug" | |
| # TEST_LOG_LEVEL: "debug" | |
| # options for advanced debug CI/CD | |
| # RUN_TESTS: "^TestSkipDisk$" | |
| # CLICKHOUSE_DEBUG: "true" | |
| # GCS_DEBUG: "true" | |
| # SFTP_DEBUG: "true" | |
| # AZBLOB_DEBUG: "true" | |
| # FTP_DEBUG: "true" | |
| # S3_DEBUG: "true" | |
| CGO_ENABLED: 1 | |
| GCS_TESTS: ${{ needs.build.outputs.GCS_TESTS }} | |
| KEEPER_TLS_ENABLED: 1 | |
| RUN_ADVANCED_TESTS: 1 | |
| AZURE_TESTS: 1 | |
| # need for FIPS | |
| QA_AWS_ACCESS_KEY: ${{ secrets.QA_AWS_ACCESS_KEY }} | |
| QA_AWS_SECRET_KEY: ${{ secrets.QA_AWS_SECRET_KEY }} | |
| QA_AWS_BUCKET: ${{ secrets.QA_AWS_BUCKET }} | |
| QA_AWS_REGION: ${{ secrets.QA_AWS_REGION }} | |
| # need for TestRestoreCloud* (restore ClickHouse Cloud backup) | |
| QA_AWS_CLOUD_ENDPOINT: ${{ secrets.QA_AWS_CLOUD_ENDPOINT }} | |
| QA_AWS_CLOUD_BUCKET: ${{ secrets.QA_AWS_CLOUD_BUCKET }} | |
| QA_AWS_CLOUD_REGION: ${{ secrets.QA_AWS_CLOUD_REGION }} | |
| QA_AWS_CLOUD_ROLE_ARN: ${{ secrets.QA_AWS_CLOUD_ROLE_ARN }} | |
| QA_AWS_CLOUD_ACCESS_KEY: ${{ secrets.QA_AWS_CLOUD_ACCESS_KEY }} | |
| QA_AWS_CLOUD_SECRET_KEY: ${{ secrets.QA_AWS_CLOUD_SECRET_KEY }} | |
| QA_AZBLOB_ACCOUNT_NAME: ${{ secrets.QA_AZBLOB_ACCOUNT_NAME }} | |
| QA_AZBLOB_ACCOUNT_KEY: ${{ secrets.QA_AZBLOB_ACCOUNT_KEY }} | |
| QA_AZBLOB_CONTAINER: ${{ secrets.QA_AZBLOB_CONTAINER }} | |
| # need for GCP over S3 | |
| QA_GCS_OVER_S3_ACCESS_KEY: ${{ secrets.QA_GCS_OVER_S3_ACCESS_KEY }} | |
| QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }} | |
| QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }} | |
| # alibabacloud.com over S3 | |
| QA_ALIBABA_ACCESS_KEY: ${{ secrets.QA_ALIBABA_ACCESS_KEY }} | |
| QA_ALIBABA_SECRET_KEY: ${{ secrets.QA_ALIBABA_SECRET_KEY }} | |
| # tencentcloud | |
| QA_TENCENT_SECRET_ID: ${{ secrets.QA_TENCENT_SECRET_ID }} | |
| QA_TENCENT_SECRET_KEY: ${{ secrets.QA_TENCENT_SECRET_KEY }} | |
| run: | | |
| set -xe | |
| echo "RUN_PARALLEL=${RUN_PARALLEL}" | |
| echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}" | |
| echo "GCS_TESTS=${GCS_TESTS}" | |
| GCS_ENCRYPTION_KEY=$(openssl rand -base64 32) | |
| export GCS_ENCRYPTION_KEY | |
| chmod +x $(pwd)/clickhouse-backup/clickhouse-backup* | |
| if [[ "${CLICKHOUSE_VERSION}" =~ ^2[1-9]+ || "${CLICKHOUSE_VERSION}" == "head" ]]; then | |
| export CLICKHOUSE_IMAGE=clickhouse/clickhouse-server | |
| else | |
| export CLICKHOUSE_IMAGE=yandex/clickhouse-server | |
| fi | |
| export CUR_DIR="$(pwd)/test/integration" | |
| export CLICKHOUSE_BACKUP_BIN="$(pwd)/clickhouse-backup/clickhouse-backup-race" | |
| set +e | |
| go test -count=1 -parallel "${RUN_PARALLEL}" -race -timeout "${TEST_TIMEOUT:-180m}" -failfast -tags=integration -shuffle=$(date +%Y%m%d) -run "${RUN_TESTS:-.+}" -v "./test/integration/..." | |
| TEST_FAILED=$? | |
| set -e | |
| if [[ "0" != "${TEST_FAILED}" ]]; then | |
| exit 1 | |
| fi | |
| - name: Format integration coverage | |
| env: | |
| GOROOT: ${{ env.GOROOT_1_26_X64 }} | |
| run: | | |
| sudo chmod -Rv a+rw test/integration/_coverage_/ | |
| ls -la test/integration/_coverage_ | |
| go tool covdata textfmt -i test/integration/_coverage_/ -o test/integration/_coverage_/coverage.out | |
| - name: Report integration coverage | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| file: test/integration/_coverage_/coverage.out | |
| parallel: true | |
| format: golang | |
| flag-name: integration-${{ matrix.clickhouse }} | |
| coverage: | |
| needs: | |
| - test | |
| - testflows | |
| - testflows_fips | |
| name: coverage | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Coveralls Finished | |
| uses: coverallsapp/github-action@v2 | |
| with: | |
| fail-on-error: false | |
| base-path: ./ | |
| parallel-finished: true | |
| docker: | |
| needs: | |
| - test | |
| - testflows | |
| - testflows_fips | |
| name: Docker | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout project | |
| uses: actions/checkout@v6 | |
| - name: Setup golang | |
| id: setup-go | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '^1.26' | |
| - uses: actions/download-artifact@v7 | |
| with: | |
| name: build-artifacts | |
| path: ./build/linux/ | |
| - name: Extract DOCKER_TAG version | |
| id: docker_tag | |
| run: | | |
| DOCKER_TAG=${GITHUB_REF##*/} | |
| export DOCKER_TAG=${DOCKER_TAG##*\\} | |
| echo "docker_tag=${DOCKER_TAG:-dev}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v4 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v4 | |
| - name: Verify FIPS 140-3 compatibility before push | |
| run: | | |
| set -x | |
| # Build the exact image_fips target locally for the runner arch and load | |
| # it, so we inspect/run the same image that is pushed below. VERSION is | |
| # intentionally omitted: it only sets an image label and does not affect | |
| # the FIPS checks. | |
| docker buildx build --progress=plain --platform=linux/amd64 --tag=clickhouse-backup-fips-verify:local --target=image_fips --pull --load . | |
| bash .github/scripts/verify_fips_image.sh \ | |
| --image clickhouse-backup-fips-verify:local \ | |
| --binary build/linux/amd64/clickhouse-backup-fips | |
| - name: Building docker image | |
| env: | |
| DOCKER_REPO: ${{ secrets.DOCKER_REPO }} | |
| DOCKER_IMAGE: ${{ secrets.DOCKER_IMAGE }} | |
| DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }} | |
| DOCKER_USER: ${{ secrets.DOCKER_USER }} | |
| DOCKER_REGISTRY: ${{ secrets.DOCKER_REGISTRY }} | |
| DOCKER_TAG: ${{ steps.docker_tag.outputs.docker_tag }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| if [[ "${DOCKER_TOKEN}" != "" ]]; then | |
| export DOCKER_REGISTRY=${DOCKER_REGISTRY:-docker.io} | |
| echo ${DOCKER_TOKEN} | docker login -u ${DOCKER_USER} --password-stdin ${DOCKER_REGISTRY} | |
| export VERSION=$(git describe --always --tags --abbrev=0 2>/dev/null | sed 's/^v//;s/\([^-]*-g\)/c\1/;s/-/./g') | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=${DOCKER_REGISTRY}/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . | |
| # Publish to GitHub Container Registry | |
| echo ${GITHUB_TOKEN} | docker login -u $GITHUB_ACTOR --password-stdin ghcr.io | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_short --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}-full:${DOCKER_TAG} --build-arg VERSION=$VERSION --target=image_full --pull --push . | |
| docker buildx build --progress=plain --platform=linux/amd64,linux/arm64 --tag=ghcr.io/${DOCKER_REPO}/${DOCKER_IMAGE}:${DOCKER_TAG}-fips --build-arg VERSION=$VERSION --target=image_fips --pull --push . | |
| fi | |