fix(openapi): collapse dotted operationIds into valid endpoint names … #53794
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Seed Snapshot Tests | |
| on: | |
| push: | |
| branches: | |
| - main | |
| # Note: pull request trigger will be removed once repository_dispatch trigger is verified | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: | |
| - main | |
| repository_dispatch: | |
| types: [seed-test-metrics] | |
| workflow_call: | |
| workflow_dispatch: | |
| # On PRs: previous runs are cancelled when new commits are pushed. | |
| # On main: the running job is protected, but a pending run can still be | |
| # cancelled if a third push arrives (GitHub allows only 1 pending per group). | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| env: | |
| DO_NOT_TRACK: "1" | |
| TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} | |
| TURBO_TEAM: "buildwithfern" | |
| TURBO_NO_UPDATE_NOTIFIER: "1" | |
| TURBO_DAEMON: "false" | |
| jobs: | |
| setup: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| outputs: | |
| post-metrics: ${{ steps.set-options.outputs.post-metrics }} | |
| steps: | |
| - name: Set Options | |
| id: set-options | |
| shell: bash | |
| run: | | |
| echo "GitHub Event: ${{ github.event_name }}" | |
| echo "GitHub Event Action: ${{ github.event.action }}" | |
| if [[ "${{ github.event_name }}" == "repository_dispatch" && "${{ github.event.action }}" == "seed-test-metrics" ]]; then | |
| echo "post-metrics=true" >> $GITHUB_OUTPUT | |
| echo "Set to collect metrics on this run" | |
| else | |
| echo "post-metrics=false" >> $GITHUB_OUTPUT | |
| echo "Will not collect metrics on this run" | |
| fi | |
| check-orphaned-seed-folders: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [get-all-test-matrices] | |
| if: ${{ needs.get-all-test-matrices.outputs.seed-infrastructure-changed == 'true' || github.event_name == 'workflow_dispatch' }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Check for orphaned seed folders (PRs only) | |
| if: ${{ github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| shell: bash | |
| env: | |
| FORCE_COLOR: "2" | |
| run: | | |
| pnpm seed:local clean --dry-run | |
| - name: Auto-clean orphaned seed folders (main only) | |
| if: ${{ github.event_name != 'pull_request' }} | |
| shell: bash | |
| env: | |
| FORCE_COLOR: "2" | |
| run: | | |
| pnpm seed:local clean | |
| if [ -n "$(git status --porcelain seed/)" ]; then | |
| git config --global user.name 'github-actions[bot]' | |
| git config --global user.email 'github-actions[bot]@users.noreply.github.com' | |
| git add seed/ | |
| git commit -m "chore(seed): auto-clean orphaned seed folders" | |
| git push | |
| else | |
| echo "No orphaned seed folders found." | |
| fi | |
| get-all-test-matrices: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: [setup] | |
| outputs: | |
| seed-infrastructure-changed: ${{ steps.detect-affected.outputs.seed-infrastructure-changed }} | |
| ruby-sdk-v2: ${{ steps.create-dynamic-matrix.outputs.ruby-sdk-v2-test-matrix }} | |
| pydantic: ${{ steps.create-dynamic-matrix.outputs.pydantic-test-matrix }} | |
| python-sdk: ${{ steps.create-dynamic-matrix.outputs.python-sdk-test-matrix }} | |
| openapi: ${{ steps.create-dynamic-matrix.outputs.openapi-test-matrix }} | |
| java-sdk: ${{ steps.create-dynamic-matrix.outputs.java-sdk-test-matrix }} | |
| java-model: ${{ steps.create-dynamic-matrix.outputs.java-model-test-matrix }} | |
| ts-sdk: ${{ steps.create-dynamic-matrix.outputs.ts-sdk-test-matrix }} | |
| go-model: ${{ steps.create-dynamic-matrix.outputs.go-model-test-matrix }} | |
| go-sdk: ${{ steps.create-dynamic-matrix.outputs.go-sdk-test-matrix }} | |
| csharp-model: ${{ steps.create-dynamic-matrix.outputs.csharp-model-test-matrix }} | |
| csharp-sdk: ${{ steps.create-dynamic-matrix.outputs.csharp-sdk-test-matrix }} | |
| php-model: ${{ steps.create-dynamic-matrix.outputs.php-model-test-matrix }} | |
| php-sdk: ${{ steps.create-dynamic-matrix.outputs.php-sdk-test-matrix }} | |
| swift-sdk: ${{ steps.create-dynamic-matrix.outputs.swift-sdk-test-matrix }} | |
| rust-model: ${{ steps.create-dynamic-matrix.outputs.rust-model-test-matrix }} | |
| rust-sdk: ${{ steps.create-dynamic-matrix.outputs.rust-sdk-test-matrix }} | |
| cli: ${{ steps.create-dynamic-matrix.outputs.cli-test-matrix }} | |
| benchmark-generators: ${{ steps.create-dynamic-matrix.outputs.benchmark-generators }} | |
| benchmark-docs: ${{ steps.create-dynamic-matrix.outputs.benchmark-docs }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| with: | |
| # fetch-depth: 2 ensures the merge commit's parents are available. | |
| # For PRs, HEAD is a merge commit; HEAD~1 is the base branch tip. | |
| # Without this, HEAD^1/HEAD^2 are not resolvable in a shallow clone | |
| # and the affected detection falls back to diffing against the | |
| # (potentially stale) event payload SHA. | |
| fetch-depth: 2 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Build Seed CLI | |
| shell: bash | |
| env: | |
| FORCE_COLOR: "2" | |
| run: pnpm seed:build | |
| - name: Detect affected generators and fixtures | |
| id: detect-affected | |
| shell: bash | |
| env: | |
| FORCE_COLOR: "2" | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # Determine base ref for affected detection. | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| # CI checks out pull/N/merge — a merge commit whose first parent | |
| # is the exact main tip used to create the merge ref. Using HEAD^1 | |
| # instead of base.sha avoids a race where main advances between the | |
| # webhook event and the merge ref computation, which caused unrelated | |
| # generators to run (e.g. go-model on PHP-only PRs). | |
| # NOTE: fetch-depth: 2 (above) ensures HEAD^1/HEAD^2 are available. | |
| if git rev-parse --verify HEAD^2 >/dev/null 2>&1; then | |
| BASE_REF=$(git rev-parse HEAD^1) | |
| echo "Using merge commit first parent: $BASE_REF" | |
| else | |
| # Fallback: not a merge commit (shouldn't happen for PRs, but be safe) | |
| BASE_REF="${{ github.event.pull_request.base.sha }}" | |
| echo "Using PR base SHA (fallback): $BASE_REF" | |
| git fetch --no-tags --depth=1 origin "$BASE_REF" 2>/dev/null || true | |
| fi | |
| else | |
| # For push/dispatch/workflow_call: use last successful run SHA | |
| LAST_SUCCESS_SHA=$(gh api \ | |
| "/repos/${{ github.repository }}/actions/workflows/seed.yml/runs?branch=main&status=success&per_page=1" \ | |
| --jq '.workflow_runs[0].head_sha // empty' 2>/dev/null || echo "") | |
| if [[ -n "$LAST_SUCCESS_SHA" ]]; then | |
| BASE_REF="$LAST_SUCCESS_SHA" | |
| echo "Using last successful run SHA: $BASE_REF" | |
| # Fetch just the target commit | |
| git fetch --no-tags --depth=1 origin "$LAST_SUCCESS_SHA" 2>/dev/null || true | |
| else | |
| BASE_REF="origin/main" | |
| echo "No successful run found, using origin/main" | |
| fi | |
| fi | |
| # Force run everything for workflow_dispatch (manual trigger) and metrics collection | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then | |
| echo "Manual dispatch — forcing all generators and fixtures to run." | |
| AFFECTED_JSON='{"allGeneratorsAffected":true,"allFixturesAffected":true,"generators":[],"generatorsWithAllFixtures":[],"fixtures":[],"summary":["Forced: workflow_dispatch event."]}' | |
| elif [[ "${{ needs.setup.outputs.post-metrics }}" == "true" ]]; then | |
| echo "Metrics collection mode — forcing all generators and fixtures to run." | |
| AFFECTED_JSON='{"allGeneratorsAffected":true,"allFixturesAffected":true,"generators":[],"generatorsWithAllFixtures":[],"fixtures":[],"summary":["Forced: metrics collection mode."]}' | |
| else | |
| # Run affected detection (falls back to "run everything" on failure) | |
| AFFECTED_JSON=$(node --enable-source-maps packages/seed/dist/cli.cjs affected --json --base-ref "$BASE_REF" 2>/dev/null || echo '{"allGeneratorsAffected":true,"allFixturesAffected":true,"generators":[],"generatorsWithAllFixtures":[],"fixtures":[],"summary":["Affected detection failed — running everything as fallback."]}') | |
| fi | |
| echo "=== Affected Detection Result ===" | |
| echo "$AFFECTED_JSON" | jq . | |
| echo "=================================" | |
| # Save to temp file for the matrix step | |
| echo "$AFFECTED_JSON" > /tmp/affected.json | |
| # Determine if seed infrastructure changed (for check-orphaned-seed-folders) | |
| ALL_GENERATORS=$(echo "$AFFECTED_JSON" | jq -r '.allGeneratorsAffected') | |
| ALL_FIXTURES=$(echo "$AFFECTED_JSON" | jq -r '.allFixturesAffected') | |
| if [[ "$ALL_GENERATORS" == "true" && "$ALL_FIXTURES" == "true" ]]; then | |
| echo "seed-infrastructure-changed=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "seed-infrastructure-changed=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Create Dynamic Matrix | |
| id: create-dynamic-matrix | |
| shell: bash | |
| run: | | |
| # List of active (non-disabled) generators, sourced from the seed CLI so | |
| # that this list cannot drift when a workspace is disabled via its | |
| # `seed.yml` (`disabled: true`). | |
| GENERATORS=$(node --enable-source-maps packages/seed/dist/cli.cjs list-generators) | |
| echo "Active generators: $GENERATORS" | |
| # Read affected detection result | |
| AFFECTED_JSON=$(cat /tmp/affected.json) | |
| ALL_GENERATORS_AFFECTED=$(echo "$AFFECTED_JSON" | jq -r '.allGeneratorsAffected') | |
| ALL_FIXTURES_AFFECTED=$(echo "$AFFECTED_JSON" | jq -r '.allFixturesAffected') | |
| # Process each generator and set its output | |
| for GEN in $GENERATORS; do | |
| # Check if this generator is affected | |
| IS_AFFECTED=false | |
| if [[ "$ALL_GENERATORS_AFFECTED" == "true" ]]; then | |
| IS_AFFECTED=true | |
| else | |
| if echo "$AFFECTED_JSON" | jq -e --arg gen "$GEN" '.generators | index($gen)' > /dev/null 2>&1; then | |
| IS_AFFECTED=true | |
| fi | |
| fi | |
| if [[ "$IS_AFFECTED" != "true" ]]; then | |
| echo "Skipping $GEN (not affected)" | |
| echo "${GEN}-test-matrix=" >> $GITHUB_OUTPUT | |
| continue | |
| fi | |
| # Get the full fixture matrix | |
| MATRIX=$(node --enable-source-maps packages/seed/dist/cli.cjs list-test-fixtures --generator "$GEN" --groups auto) | |
| if [ -z "$MATRIX" ] || [ "$MATRIX" = "null" ] || [ "$MATRIX" = "undefined" ]; then | |
| echo "Error: Failed to create matrix for $GEN" | |
| exit 1 | |
| fi | |
| # Check if this generator needs all fixtures or only specific ones | |
| NEEDS_ALL_FIXTURES=false | |
| if [[ "$ALL_FIXTURES_AFFECTED" == "true" ]]; then | |
| NEEDS_ALL_FIXTURES=true | |
| else | |
| if echo "$AFFECTED_JSON" | jq -e --arg gen "$GEN" '.generatorsWithAllFixtures | index($gen)' > /dev/null 2>&1; then | |
| NEEDS_ALL_FIXTURES=true | |
| fi | |
| fi | |
| if [[ "$NEEDS_ALL_FIXTURES" == "true" ]]; then | |
| # Generator needs all fixtures — use full matrix | |
| echo "${GEN}-test-matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "Set output for $GEN: ALL fixtures" | |
| else | |
| # Generator needs only specific changed fixtures — filter the matrix | |
| FILTERED_MATRIX=$(echo "$MATRIX" | jq -c --argjson affected "$(echo "$AFFECTED_JSON" | jq -c '.fixtures')" ' | |
| [.[] | .fixtures = [.fixtures[] | | |
| (if contains(":") then split(":")[0] else . end) as $name | | |
| select(any($affected[]; . == $name)) | |
| ] | select(.fixtures | length > 0)] | |
| ') | |
| if [[ "$FILTERED_MATRIX" == "[]" || -z "$FILTERED_MATRIX" ]]; then | |
| echo "No affected fixtures for $GEN after filtering" | |
| echo "${GEN}-test-matrix=" >> $GITHUB_OUTPUT | |
| else | |
| echo "${GEN}-test-matrix=$FILTERED_MATRIX" >> $GITHUB_OUTPUT | |
| echo "Set output for $GEN: filtered fixtures" | |
| fi | |
| fi | |
| done | |
| # Build list of affected SDK generators for benchmarking. | |
| # When ALL_GENERATORS_AFFECTED is true (e.g. core/shared package changes), | |
| # all 9 SDK generators will be benchmarked. | |
| SDK_GENERATORS="ts-sdk python-sdk java-sdk go-sdk csharp-sdk php-sdk ruby-sdk-v2 swift-sdk rust-sdk" | |
| BENCHMARK_LIST="[]" | |
| for GEN in $SDK_GENERATORS; do | |
| IS_AFFECTED=false | |
| if [[ "$ALL_GENERATORS_AFFECTED" == "true" ]]; then | |
| IS_AFFECTED=true | |
| else | |
| if echo "$AFFECTED_JSON" | jq -e --arg gen "$GEN" '.generators | index($gen)' > /dev/null 2>&1; then | |
| IS_AFFECTED=true | |
| fi | |
| fi | |
| if [[ "$IS_AFFECTED" == "true" ]]; then | |
| BENCHMARK_LIST=$(echo "$BENCHMARK_LIST" | jq -c --arg gen "$GEN" '. + [{"generator": $gen}]') | |
| fi | |
| done | |
| if [[ "$BENCHMARK_LIST" == "[]" ]]; then | |
| echo "benchmark-generators=" >> $GITHUB_OUTPUT | |
| echo "No SDK generators affected for benchmarking" | |
| else | |
| echo "benchmark-generators=$BENCHMARK_LIST" >> $GITHUB_OUTPUT | |
| echo "Benchmark generators: $BENCHMARK_LIST" | |
| fi | |
| # Detect if docs-related code changed for docs benchmarking. | |
| # Docs benchmarks run when core/shared packages change (ALL_GENERATORS_AFFECTED) | |
| # or when docs-specific packages are modified. | |
| DOCS_AFFECTED=false | |
| if [[ "$ALL_GENERATORS_AFFECTED" == "true" ]]; then | |
| DOCS_AFFECTED=true | |
| else | |
| DOCS_PATHS="packages/cli/docs-resolver packages/cli/remote-workspace-runner packages/cli/docs-validator packages/cli/configuration-loader packages/cli/cli/src/commands/generate/generateDocsWorkspace benchmarks/fern/docs.yml benchmarks/fern/pages/template.md benchmarks/fern/generate-fixture-pages.sh benchmarks/fern/generate-fixture-versions.sh benchmarks/fern/api-versions.txt .github/actions/run-docs-benchmark" | |
| MERGE_BASE=$(git merge-base HEAD origin/main 2>/dev/null || echo "HEAD~1") | |
| CHANGED_FILES=$(git diff --name-only "$MERGE_BASE" HEAD 2>/dev/null || echo "") | |
| for DPATH in $DOCS_PATHS; do | |
| if echo "$CHANGED_FILES" | grep -q "^${DPATH}"; then | |
| DOCS_AFFECTED=true | |
| break | |
| fi | |
| done | |
| fi | |
| if [[ "$DOCS_AFFECTED" == "true" ]]; then | |
| echo "benchmark-docs=true" >> $GITHUB_OUTPUT | |
| echo "Docs benchmark: affected" | |
| else | |
| echo "benchmark-docs=" >> $GITHUB_OUTPUT | |
| echo "Docs benchmark: not affected" | |
| fi | |
| ruby-sdk-v2: | |
| runs-on: Seed | |
| timeout-minutes: 20 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.ruby-sdk-v2 != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.ruby-sdk-v2) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: ruby-sdk-v2 | |
| generator-path: generators/ruby-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| determinism-exclude-paths: '["seed/*/.mock/*", "seed/**/Gemfile.lock"]' | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| pydantic: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.pydantic != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.pydantic) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: pydantic | |
| generator-path: generators/python generators/python-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| python-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 30 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.python-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.python-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: python-sdk | |
| generator-path: generators/python generators/python-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| determinism-exclude-paths: '["seed/*/.mock/*", "seed/**/poetry.lock"]' | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| openapi: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.openapi != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.openapi) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: openapi | |
| generator-path: generators/openapi | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| java-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 30 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.java-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.java-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: java-sdk | |
| generator-path: generators/java generators/java-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| java-model: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.java-model != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.java-model) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: java-model | |
| generator-path: generators/java generators/java-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| ts-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.ts-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJson(needs.get-all-test-matrices.outputs.ts-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: ts-sdk | |
| generator-path: generators/typescript generators/typescript-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| determinism-exclude-paths: '["seed/*/.mock/*", "seed/ts-sdk/**/pnpm-lock.yaml"]' | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| go-model: | |
| runs-on: Seed | |
| timeout-minutes: 20 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.go-model != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.go-model) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: go-model | |
| generator-path: generators/go generators/go-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| go-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 20 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.go-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.go-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: go-sdk | |
| generator-path: generators/go generators/go-v2 | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| csharp-model: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.csharp-model != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.csharp-model) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: csharp-model | |
| generator-path: generators/csharp | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| csharp-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 20 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.csharp-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.csharp-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: csharp-sdk | |
| generator-path: generators/csharp | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| php-model: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.php-model != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.php-model) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: php-model | |
| generator-path: generators/php | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| php-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.php-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.php-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: php-sdk | |
| generator-path: generators/php | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| swift-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.swift-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.swift-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: swift-sdk | |
| generator-path: generators/swift | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| rust-model: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| # if: >- | |
| # ${{ | |
| # (needs.get-all-test-matrices.outputs.rust-model != '' | |
| # && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| # }} | |
| if: false # generator not actively supported | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.rust-model) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: rust-model | |
| generator-path: generators/rust | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| rust-sdk: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.rust-sdk != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| max-parallel: 15 # Limit the number of runners for this job | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.rust-sdk) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: rust-sdk | |
| generator-path: generators/rust | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| cli: | |
| runs-on: Seed | |
| timeout-minutes: 15 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| (needs.get-all-test-matrices.outputs.cli != '' | |
| && github.event.pull_request.draft == false) || github.event_name == 'workflow_dispatch' | |
| }} | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 15 | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.cli) }} | |
| steps: | |
| - name: Checkout action file | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/run-seed-process | |
| - name: Run All Seed Tests | |
| uses: ./.github/actions/run-seed-process | |
| with: | |
| sdk-name: cli | |
| generator-path: generators/cli | |
| fixtures-to-run: ${{toJson(matrix.fixtures)}} | |
| job-index: ${{ strategy.job-index }} | |
| collect-metrics: ${{ needs.setup.outputs.post-metrics }} | |
| dockerhub-username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| dockerhub-token: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| # SDK Generation Benchmarks — runs affected SDK generators against large public OAS specs. | |
| # Uses --skip-scripts to measure generator-only time (no npm install/build/test noise). | |
| # Baseline timings come from the nightly benchmark-baseline.yml workflow (via artifacts API). | |
| # | |
| # Specs are fetched once in benchmark-setup and shared via artifact to avoid redundant downloads. | |
| benchmark-setup: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| github.event_name == 'pull_request' | |
| && (needs.get-all-test-matrices.outputs.benchmark-generators != '' || needs.get-all-test-matrices.outputs.benchmark-docs != '') | |
| && github.event.pull_request.draft == false | |
| }} | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: | | |
| .github/actions/fetch-benchmark-specs | |
| benchmarks/fern/apis | |
| benchmarks/fern/api-versions.txt | |
| - name: Fetch benchmark OpenAPI specs | |
| uses: ./.github/actions/fetch-benchmark-specs | |
| - name: Upload specs as shared artifact | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: benchmark-specs | |
| path: benchmarks/fern/apis/*/openapi.json | |
| retention-days: 1 | |
| benchmark: | |
| runs-on: Seed | |
| timeout-minutes: 60 | |
| needs: [get-all-test-matrices, benchmark-setup] | |
| if: >- | |
| ${{ | |
| github.event_name == 'pull_request' | |
| && needs.get-all-test-matrices.outputs.benchmark-generators != '' | |
| && github.event.pull_request.draft == false | |
| }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJSON(needs.get-all-test-matrices.outputs.benchmark-generators) }} | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v6 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Log in to Docker Hub | |
| if: ${{ env.DOCKERHUB_USERNAME != '' }} | |
| uses: docker/login-action@v4 | |
| env: | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME_PUBLIC_READONLY }} | |
| password: ${{ secrets.DOCKER_PASSWORD_PUBLIC_READONLY }} | |
| - name: Clear stale Docker image cache | |
| run: rm -rf /tmp/docker-cache | |
| - name: Cache Docker images | |
| id: docker-cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: /tmp/docker-cache | |
| key: ${{ runner.os }}-docker-${{ matrix.generator }}-${{ hashFiles('**/Dockerfile*') }} | |
| restore-keys: | | |
| ${{ runner.os }}-docker-${{ matrix.generator }}- | |
| - name: Load cached Docker images | |
| run: | | |
| for f in /tmp/docker-cache/*.tar; do | |
| [ -f "$f" ] && docker load -i "$f" || true | |
| done | |
| - name: Record pre-existing Docker images | |
| run: docker images -q | sort -u > /tmp/docker-images-before.txt | |
| - name: Download shared specs | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: benchmark-specs | |
| path: benchmarks/fern/apis | |
| - name: Run benchmark (PR branch, generator-only) | |
| uses: ./.github/actions/run-benchmark | |
| with: | |
| generator: ${{ matrix.generator }} | |
| skip-scripts: "true" | |
| - name: Save Docker images for cache | |
| if: always() | |
| run: | | |
| if [ ! -f /tmp/docker-images-before.txt ]; then | |
| echo "Skipping cache save: docker-images-before.txt not found" | |
| exit 0 | |
| fi | |
| mkdir -p /tmp/docker-cache | |
| comm -13 /tmp/docker-images-before.txt <(docker images -q | sort -u) | while read -r id; do | |
| img=$(docker inspect --format '{{index .RepoTags 0}}' "$id" 2>/dev/null || true) | |
| [ -z "$img" ] && continue | |
| fname=$(echo "$img" | tr '/:' '_') | |
| docker save "$img" -o "/tmp/docker-cache/${fname}.tar" 2>/dev/null || true | |
| done | |
| - name: Upload PR results | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: benchmark-pr-${{ matrix.generator }} | |
| path: benchmark-results/ | |
| retention-days: 5 | |
| benchmark-report: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [benchmark] | |
| if: >- | |
| ${{ | |
| always() && !cancelled() | |
| && github.event_name == 'pull_request' | |
| && needs.benchmark.result != 'skipped' | |
| }} | |
| permissions: | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: .github/scripts | |
| - name: Download all PR benchmark artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: benchmark-pr-* | |
| path: benchmark-pr-results | |
| merge-multiple: true | |
| - name: Download baseline from recent nightly artifacts | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| shell: bash | |
| run: | | |
| mkdir -p benchmark-baseline-results/history | |
| BASELINE_RUNS_TO_FETCH=5 | |
| # Fetch the last N completed runs of benchmark-baseline.yml on main. | |
| # Use status=completed (not status=success) so we include runs where | |
| # some generators failed but others produced valid artifacts. A single | |
| # failing E2E job marks the whole workflow as "failure", which would | |
| # hide valid baseline data from every other generator. | |
| RUNS=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark-baseline.yml/runs?status=completed&per_page=20&branch=main" \ | |
| --jq '[.workflow_runs[] | select(.conclusion == "success" or .conclusion == "failure")] | .['":${BASELINE_RUNS_TO_FETCH}"']' 2>/dev/null || echo "[]") | |
| RUN_COUNT=$(echo "$RUNS" | jq 'length') | |
| if [ "$RUN_COUNT" -eq 0 ]; then | |
| echo "::warning::No completed nightly baseline runs found. Report will show N/A for main timings." | |
| echo "Trigger the benchmark-baseline workflow manually to generate initial baselines." | |
| exit 0 | |
| fi | |
| echo "Found ${RUN_COUNT} nightly baseline run(s)" | |
| # Save metadata from the latest run for the report footer | |
| LATEST_DATE=$(echo "$RUNS" | jq -r '.[0].created_at') | |
| LATEST_SHA=$(echo "$RUNS" | jq -r '.[0].head_sha') | |
| LATEST_RUN_ID=$(echo "$RUNS" | jq -r '.[0].id') | |
| echo "{\"latest_timestamp\":\"${LATEST_DATE}\",\"sha\":\"${LATEST_SHA}\",\"run_id\":\"${LATEST_RUN_ID}\"}" \ | |
| > benchmark-baseline-results/_metadata.json | |
| # Download artifacts from each run into history/<run-id>/ | |
| for i in $(seq 0 $(( RUN_COUNT - 1 ))); do | |
| RUN_ID=$(echo "$RUNS" | jq -r ".[$i].id") | |
| RUN_DATE=$(echo "$RUNS" | jq -r ".[$i].created_at") | |
| RUN_DIR="benchmark-baseline-results/history/${RUN_ID}" | |
| mkdir -p "${RUN_DIR}/e2e" | |
| echo "Downloading baseline from run ${RUN_ID} (${RUN_DATE})..." | |
| # Download generator-only baseline artifacts | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --pattern "benchmark-baseline-*" --dir "/tmp/baseline-gen-${RUN_ID}/" 2>/dev/null || true | |
| for f in /tmp/baseline-gen-${RUN_ID}/*/*.jsonl; do | |
| [ -f "$f" ] && cp "$f" "${RUN_DIR}/" | |
| done | |
| # Download E2E baseline artifacts | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --pattern "benchmark-e2e-*" --dir "/tmp/baseline-e2e-${RUN_ID}/" 2>/dev/null || true | |
| for f in /tmp/baseline-e2e-${RUN_ID}/*/*.jsonl; do | |
| [ -f "$f" ] && cp "$f" "${RUN_DIR}/e2e/" | |
| done | |
| # Remove empty run directories (check both generator and E2E artifacts) | |
| if ! ls "${RUN_DIR}"/*.jsonl 1>/dev/null 2>&1 && ! ls "${RUN_DIR}"/e2e/*.jsonl 1>/dev/null 2>&1; then | |
| echo "::warning::No artifacts found for run ${RUN_ID}, skipping" | |
| rm -rf "${RUN_DIR}" | |
| fi | |
| done | |
| # Check if we got any results | |
| HISTORY_COUNT=$(ls -d benchmark-baseline-results/history/*/ 2>/dev/null | wc -l) | |
| if [ "$HISTORY_COUNT" -eq 0 ]; then | |
| echo "::warning::No baseline artifacts found across ${RUN_COUNT} runs. Artifacts may have expired." | |
| echo "Trigger the benchmark-baseline workflow to generate fresh baselines." | |
| rm -f benchmark-baseline-results/_metadata.json | |
| rm -rf benchmark-baseline-results/history | |
| else | |
| echo "Baseline artifacts downloaded from ${HISTORY_COUNT} nightly run(s):" | |
| for d in benchmark-baseline-results/history/*/; do | |
| echo " $(basename "$d"): $(ls "$d"/*.jsonl 2>/dev/null | wc -l) generator(s), $(ls "$d"/e2e/*.jsonl 2>/dev/null | wc -l) E2E" | |
| done | |
| fi | |
| - name: Generate benchmark report | |
| id: report | |
| shell: bash | |
| run: | | |
| # Extract baseline timestamp for the report footer | |
| BASELINE_TS="" | |
| if [ -f benchmark-baseline-results/_metadata.json ]; then | |
| BASELINE_TS=$(cat benchmark-baseline-results/_metadata.json | jq -r '.latest_timestamp // .timestamp // empty') | |
| fi | |
| export BASELINE_TIMESTAMP="$BASELINE_TS" | |
| REPORT=$(bash .github/scripts/format-benchmark-report.sh \ | |
| benchmark-pr-results \ | |
| benchmark-baseline-results) | |
| # Write to file for the comment action (printf avoids echo mangling special chars) | |
| printf '%s\n' "$REPORT" > benchmark-comment.md | |
| echo "Generated benchmark report" | |
| - name: Post or update PR comment | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPO: ${{ github.repository }} | |
| shell: bash | |
| run: | | |
| SENTINEL="<!-- fern-sdk-benchmark-results -->" | |
| BODY=$(cat benchmark-comment.md) | |
| # Search existing comments for the sentinel marker | |
| COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ | |
| --paginate --jq '.[] | select(.body | contains("'"${SENTINEL}"'")) | .id' \ | |
| | head -n1) | |
| if [ -n "$COMMENT_ID" ]; then | |
| echo "Updating existing benchmark comment ${COMMENT_ID}" | |
| gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" \ | |
| --method PATCH \ | |
| --input <(jq -n --arg body "$BODY" '{"body": $body}') | |
| else | |
| echo "Creating new benchmark comment" | |
| gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ | |
| --method POST \ | |
| --input <(jq -n --arg body "$BODY" '{"body": $body}') | |
| fi | |
| # PostHog step runs after the PR comment is posted so that comment generation | |
| # is never affected by PostHog failures. Fire-and-forget: all errors suppressed. | |
| - name: Send PR benchmark events to PostHog | |
| shell: bash | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| PR_BRANCH: ${{ github.head_ref }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PR_SHA: ${{ github.event.pull_request.head.sha }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [ -z "$POSTHOG_API_KEY" ]; then | |
| echo "::notice::POSTHOG_API_KEY not set, skipping PR benchmark metrics" | |
| exit 0 | |
| fi | |
| TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| SENT=0 | |
| FAILED=0 | |
| for f in benchmark-pr-results/*.jsonl; do | |
| [ -f "$f" ] || continue | |
| while IFS= read -r line; do | |
| gen=$(echo "$line" | jq -r '.generator') | |
| spec=$(echo "$line" | jq -r '.spec') | |
| dur=$(echo "$line" | jq -r '.duration_seconds') | |
| ec=$(echo "$line" | jq -r '.exit_code // 0') | |
| skip=$(echo "$line" | jq -r '.skipped // false') | |
| payload=$(jq -n \ | |
| --arg api_key "$POSTHOG_API_KEY" \ | |
| --arg event "benchmark-pr-result" \ | |
| --arg distinct_id "fern-ci-benchmark" \ | |
| --arg timestamp "$TIMESTAMP" \ | |
| --arg generator "$gen" \ | |
| --arg spec "$spec" \ | |
| --argjson duration_seconds "$dur" \ | |
| --argjson exit_code "$ec" \ | |
| --argjson skipped "$skip" \ | |
| --arg mode "generator-only" \ | |
| --arg branch "$PR_BRANCH" \ | |
| --argjson pr_number "$PR_NUMBER" \ | |
| --arg commit_sha "$PR_SHA" \ | |
| --arg run_id "$RUN_ID" \ | |
| --arg run_url "$RUN_URL" \ | |
| '{ | |
| api_key: $api_key, | |
| event: $event, | |
| distinct_id: $distinct_id, | |
| timestamp: $timestamp, | |
| properties: { | |
| generator: $generator, | |
| spec: $spec, | |
| duration_seconds: $duration_seconds, | |
| exit_code: $exit_code, | |
| skipped: $skipped, | |
| mode: $mode, | |
| branch: $branch, | |
| pr_number: $pr_number, | |
| commit_sha: $commit_sha, | |
| run_id: $run_id, | |
| run_url: $run_url | |
| } | |
| }') | |
| if ! curl -sf --max-time 10 -X POST https://us.i.posthog.com/capture/ \ | |
| -H "Content-Type: application/json" \ | |
| -d "$payload" 2>/dev/null; then | |
| FAILED=$((FAILED + 1)) | |
| else | |
| SENT=$((SENT + 1)) | |
| fi | |
| done < "$f" | |
| done | |
| echo "PostHog PR metrics: ${SENT} sent, ${FAILED} failed" | |
| # --------------------------------------------------------------------------- | |
| # Docs Generation Benchmark (PR) | |
| # --------------------------------------------------------------------------- | |
| # Runs the real `fern generate --docs --preview` command against the benchmark | |
| # fixture and measures wall-clock time, same philosophy as SDK benchmarks. | |
| docs-benchmark: | |
| runs-on: Seed | |
| timeout-minutes: 45 | |
| needs: [get-all-test-matrices, benchmark-setup] | |
| if: >- | |
| ${{ | |
| github.event_name == 'pull_request' | |
| && needs.get-all-test-matrices.outputs.benchmark-docs != '' | |
| && github.event.pull_request.draft == false | |
| }} | |
| steps: | |
| - name: Checkout PR branch | |
| uses: actions/checkout@v6 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Download shared specs | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: benchmark-specs | |
| path: benchmarks/fern/apis | |
| - name: Run docs benchmark (PR branch) | |
| uses: ./.github/actions/run-docs-benchmark | |
| with: | |
| fern-token: ${{ secrets.FERN_FERN_TOKEN }} | |
| - name: Upload PR docs benchmark results | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| # Must not match the `benchmark-pr-*` pattern used by the SDK | |
| # benchmark report download, or docs jsonl rows (which have no | |
| # .generator field) leak into the SDK benchmark table. | |
| name: benchmark-docs-pr | |
| path: benchmark-results/ | |
| retention-days: 5 | |
| docs-benchmark-report: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [docs-benchmark] | |
| if: >- | |
| ${{ | |
| always() && !cancelled() | |
| && github.event_name == 'pull_request' | |
| && needs.docs-benchmark.result != 'skipped' | |
| }} | |
| permissions: | |
| pull-requests: write | |
| actions: read | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| with: | |
| sparse-checkout: .github/scripts | |
| - name: Download PR docs benchmark artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| pattern: benchmark-docs-pr | |
| path: docs-pr-results | |
| merge-multiple: true | |
| - name: Download docs baseline from recent nightly artifacts | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| shell: bash | |
| run: | | |
| mkdir -p docs-baseline-results/history | |
| BASELINE_RUNS_TO_FETCH=5 | |
| # Fetch the last N completed runs of benchmark-baseline.yml on main. | |
| # Use status=completed (not status=success) — see SDK benchmark step | |
| # comment for rationale. | |
| RUNS=$(gh api "repos/${{ github.repository }}/actions/workflows/benchmark-baseline.yml/runs?status=completed&per_page=20&branch=main" \ | |
| --jq '[.workflow_runs[] | select(.conclusion == "success" or .conclusion == "failure")] | .['":${BASELINE_RUNS_TO_FETCH}"']' 2>/dev/null || echo "[]") | |
| RUN_COUNT=$(echo "$RUNS" | jq 'length') | |
| if [ "$RUN_COUNT" -eq 0 ]; then | |
| echo "::warning::No completed nightly baseline runs found. Report will show N/A for main timings." | |
| exit 0 | |
| fi | |
| echo "Found ${RUN_COUNT} nightly baseline run(s)" | |
| # Save metadata from the latest run for the report footer | |
| LATEST_DATE=$(echo "$RUNS" | jq -r '.[0].created_at') | |
| LATEST_RUN_ID=$(echo "$RUNS" | jq -r '.[0].id') | |
| echo "{\"latest_timestamp\":\"${LATEST_DATE}\",\"run_id\":\"${LATEST_RUN_ID}\"}" \ | |
| > docs-baseline-results/_metadata.json | |
| # Download docs baseline artifacts from each run into history/<run-id>/ | |
| for i in $(seq 0 $(( RUN_COUNT - 1 ))); do | |
| RUN_ID=$(echo "$RUNS" | jq -r ".[$i].id") | |
| RUN_DATE=$(echo "$RUNS" | jq -r ".[$i].created_at") | |
| RUN_DIR="docs-baseline-results/history/${RUN_ID}" | |
| mkdir -p "${RUN_DIR}" | |
| echo "Downloading docs baseline from run ${RUN_ID} (${RUN_DATE})..." | |
| gh run download "$RUN_ID" --repo "${{ github.repository }}" --pattern "benchmark-docs-baseline" --dir "/tmp/docs-baseline-${RUN_ID}/" 2>/dev/null || true | |
| for f in /tmp/docs-baseline-${RUN_ID}/*/*.jsonl /tmp/docs-baseline-${RUN_ID}/*.jsonl; do | |
| [ -f "$f" ] && cp "$f" "${RUN_DIR}/" | |
| done | |
| # Remove empty run directories | |
| if ! ls "${RUN_DIR}"/*.jsonl 1>/dev/null 2>&1; then | |
| echo "::warning::No docs artifacts found for run ${RUN_ID}, skipping" | |
| rm -rf "${RUN_DIR}" | |
| fi | |
| done | |
| # Check if we got any results | |
| HISTORY_COUNT=$(ls -d docs-baseline-results/history/*/ 2>/dev/null | wc -l) | |
| if [ "$HISTORY_COUNT" -eq 0 ]; then | |
| echo "::warning::No docs baseline artifacts found across ${RUN_COUNT} runs." | |
| rm -f docs-baseline-results/_metadata.json | |
| rm -rf docs-baseline-results/history | |
| else | |
| echo "Docs baseline artifacts downloaded from ${HISTORY_COUNT} nightly run(s)" | |
| fi | |
| - name: Generate docs benchmark report | |
| id: report | |
| shell: bash | |
| run: | | |
| BASELINE_TS="" | |
| if [ -f docs-baseline-results/_metadata.json ]; then | |
| BASELINE_TS=$(cat docs-baseline-results/_metadata.json | jq -r '.latest_timestamp // empty') | |
| fi | |
| export BASELINE_TIMESTAMP="$BASELINE_TS" | |
| REPORT=$(bash .github/scripts/format-docs-benchmark-report.sh \ | |
| docs-pr-results \ | |
| docs-baseline-results) | |
| printf '%s\n' "$REPORT" > docs-benchmark-comment.md | |
| echo "Generated docs benchmark report" | |
| - name: Post or update PR comment | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REPO: ${{ github.repository }} | |
| shell: bash | |
| run: | | |
| SENTINEL="<!-- fern-docs-benchmark-results -->" | |
| BODY=$(cat docs-benchmark-comment.md) | |
| # Search existing comments for the sentinel marker | |
| COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ | |
| --paginate --jq '.[] | select(.body | contains("'"${SENTINEL}"'")) | .id' \ | |
| | head -n1) | |
| if [ -n "$COMMENT_ID" ]; then | |
| echo "Updating existing docs benchmark comment ${COMMENT_ID}" | |
| gh api "repos/${REPO}/issues/comments/${COMMENT_ID}" \ | |
| --method PATCH \ | |
| --input <(jq -n --arg body "$BODY" '{"body": $body}') | |
| else | |
| echo "Creating new docs benchmark comment" | |
| gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" \ | |
| --method POST \ | |
| --input <(jq -n --arg body "$BODY" '{"body": $body}') | |
| fi | |
| - name: Send PR docs benchmark events to PostHog | |
| shell: bash | |
| env: | |
| POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }} | |
| PR_BRANCH: ${{ github.head_ref }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| PR_SHA: ${{ github.event.pull_request.head.sha }} | |
| RUN_ID: ${{ github.run_id }} | |
| RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| run: | | |
| if [ -z "$POSTHOG_API_KEY" ]; then | |
| echo "::notice::POSTHOG_API_KEY not set, skipping PR docs benchmark metrics" | |
| exit 0 | |
| fi | |
| TIMESTAMP=$(date -u +%Y-%m-%dT%H:%M:%SZ) | |
| SENT=0 | |
| FAILED=0 | |
| for f in docs-pr-results/*.jsonl; do | |
| [ -f "$f" ] || continue | |
| while IFS= read -r line; do | |
| spec=$(echo "$line" | jq -r '.spec') | |
| dur=$(echo "$line" | jq -r '.duration_seconds') | |
| ec=$(echo "$line" | jq -r '.exit_code // 0') | |
| skip=$(echo "$line" | jq -r '.skipped // false') | |
| payload=$(jq -n \ | |
| --arg api_key "$POSTHOG_API_KEY" \ | |
| --arg event "benchmark-pr-result" \ | |
| --arg distinct_id "fern-ci-benchmark" \ | |
| --arg timestamp "$TIMESTAMP" \ | |
| --arg generator "docs" \ | |
| --arg spec "$spec" \ | |
| --argjson duration_seconds "$dur" \ | |
| --argjson exit_code "$ec" \ | |
| --argjson skipped "$skip" \ | |
| --arg mode "docs" \ | |
| --arg branch "$PR_BRANCH" \ | |
| --argjson pr_number "$PR_NUMBER" \ | |
| --arg commit_sha "$PR_SHA" \ | |
| --arg run_id "$RUN_ID" \ | |
| --arg run_url "$RUN_URL" \ | |
| '{ | |
| api_key: $api_key, | |
| event: $event, | |
| distinct_id: $distinct_id, | |
| timestamp: $timestamp, | |
| properties: { | |
| generator: $generator, | |
| spec: $spec, | |
| duration_seconds: $duration_seconds, | |
| exit_code: $exit_code, | |
| skipped: $skipped, | |
| mode: $mode, | |
| branch: $branch, | |
| pr_number: $pr_number, | |
| commit_sha: $commit_sha, | |
| run_id: $run_id, | |
| run_url: $run_url | |
| } | |
| }') | |
| if ! curl -sf --max-time 10 -X POST https://us.i.posthog.com/capture/ \ | |
| -H "Content-Type: application/json" \ | |
| -d "$payload" 2>/dev/null; then | |
| FAILED=$((FAILED + 1)) | |
| else | |
| SENT=$((SENT + 1)) | |
| fi | |
| done < "$f" | |
| done | |
| echo "PostHog PR docs metrics: ${SENT} sent, ${FAILED} failed" | |
| # Round up all tests into job with defined name for PR merge requirements | |
| seed-test-results: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| needs: | |
| [ | |
| ruby-sdk-v2, | |
| pydantic, | |
| python-sdk, | |
| openapi, | |
| java-sdk, | |
| java-model, | |
| ts-sdk, | |
| go-model, | |
| go-sdk, | |
| csharp-model, | |
| csharp-sdk, | |
| php-model, | |
| php-sdk, | |
| swift-sdk, | |
| rust-model, | |
| rust-sdk, | |
| cli | |
| ] | |
| if: >- | |
| ${{ | |
| always() && !cancelled() | |
| }} | |
| strategy: | |
| fail-fast: false # Let all tests run, even if some fail | |
| matrix: | |
| job-name: | |
| [ | |
| ruby-sdk-v2, | |
| pydantic, | |
| python-sdk, | |
| openapi, | |
| java-sdk, | |
| java-model, | |
| ts-sdk, | |
| go-model, | |
| go-sdk, | |
| csharp-model, | |
| csharp-sdk, | |
| php-model, | |
| php-sdk, | |
| swift-sdk, | |
| rust-model, | |
| rust-sdk, | |
| cli | |
| ] | |
| outputs: | |
| ruby-sdk-v2-passing: ${{ steps.report-results.outputs.ruby-sdk-v2-passing }} | |
| pydantic-passing: ${{ steps.report-results.outputs.pydantic-passing }} | |
| python-sdk-passing: ${{ steps.report-results.outputs.python-sdk-passing }} | |
| openapi-passing: ${{ steps.report-results.outputs.openapi-passing }} | |
| java-sdk-passing: ${{ steps.report-results.outputs.java-sdk-passing }} | |
| java-model-passing: ${{ steps.report-results.outputs.java-model-passing }} | |
| ts-sdk-passing: ${{ steps.report-results.outputs.ts-sdk-passing }} | |
| go-model-passing: ${{ steps.report-results.outputs.go-model-passing }} | |
| go-sdk-passing: ${{ steps.report-results.outputs.go-sdk-passing }} | |
| csharp-model-passing: ${{ steps.report-results.outputs.csharp-model-passing }} | |
| csharp-sdk-passing: ${{ steps.report-results.outputs.csharp-sdk-passing }} | |
| php-model-passing: ${{ steps.report-results.outputs.php-model-passing }} | |
| php-sdk-passing: ${{ steps.report-results.outputs.php-sdk-passing }} | |
| swift-sdk-passing: ${{ steps.report-results.outputs.swift-sdk-passing }} | |
| rust-model-passing: ${{ steps.report-results.outputs.rust-model-passing }} | |
| rust-sdk-passing: ${{ steps.report-results.outputs.rust-sdk-passing }} | |
| cli-passing: ${{ steps.report-results.outputs.cli-passing }} | |
| steps: | |
| - name: Report ${{ matrix.job-name }} seed test results | |
| id: report-results | |
| run: | | |
| RESULT="${{ fromJSON(toJSON(needs))[matrix.job-name].result }}" | |
| # Set output for passing status | |
| if [ "$RESULT" = "success" ] || [ "$RESULT" = "skipped" ]; then | |
| echo "${{ matrix.job-name }}-passing=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "${{ matrix.job-name }}-passing=false" >> $GITHUB_OUTPUT | |
| fi | |
| # Log results and fail if required (for PR requirements) | |
| if [ "$RESULT" = "success" ]; then | |
| echo "${{ matrix.job-name }} seed tests are all passing" | |
| elif [ "$RESULT" = "skipped" ]; then | |
| echo "No changes detected for ${{ matrix.job-name }} seed tests. Passing." | |
| else | |
| echo "${{ matrix.job-name }} seed tests had changes and were not successful" | |
| echo "Result of parallelized tests: $RESULT" | |
| exit 1 | |
| fi | |
| collect-allowed-failures: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [setup, get-all-test-matrices] | |
| if: >- | |
| ${{ | |
| always() && !cancelled() && needs.setup.outputs.post-metrics == 'true' | |
| }} | |
| strategy: | |
| fail-fast: false # Let all collections run, even if some fail | |
| matrix: | |
| seed-generator-alias: | |
| [ | |
| ruby-sdk-v2, | |
| pydantic, | |
| python-sdk, | |
| openapi, | |
| java-sdk, | |
| java-model, | |
| ts-sdk, | |
| go-model, | |
| go-sdk, | |
| csharp-model, | |
| csharp-sdk, | |
| php-model, | |
| php-sdk, | |
| swift-sdk, | |
| rust-model, | |
| rust-sdk, | |
| cli | |
| ] | |
| outputs: | |
| ruby-sdk-v2-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.ruby-sdk-v2-total-fixtures-count }} | |
| ruby-sdk-v2-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.ruby-sdk-v2-allowed-failures }} | |
| ruby-sdk-v2-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.ruby-sdk-v2-allowed-failures-count }} | |
| pydantic-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.pydantic-total-fixtures-count }} | |
| pydantic-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.pydantic-allowed-failures }} | |
| pydantic-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.pydantic-allowed-failures-count }} | |
| python-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.python-sdk-total-fixtures-count }} | |
| python-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.python-sdk-allowed-failures }} | |
| python-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.python-sdk-allowed-failures-count }} | |
| openapi-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.openapi-total-fixtures-count }} | |
| openapi-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.openapi-allowed-failures }} | |
| openapi-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.openapi-allowed-failures-count }} | |
| java-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.java-sdk-total-fixtures-count }} | |
| java-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.java-sdk-allowed-failures }} | |
| java-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.java-sdk-allowed-failures-count }} | |
| java-model-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.java-model-total-fixtures-count }} | |
| java-model-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.java-model-allowed-failures }} | |
| java-model-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.java-model-allowed-failures-count }} | |
| ts-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.ts-sdk-total-fixtures-count }} | |
| ts-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.ts-sdk-allowed-failures }} | |
| ts-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.ts-sdk-allowed-failures-count }} | |
| go-model-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.go-model-total-fixtures-count }} | |
| go-model-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.go-model-allowed-failures }} | |
| go-model-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.go-model-allowed-failures-count }} | |
| go-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.go-sdk-total-fixtures-count }} | |
| go-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.go-sdk-allowed-failures }} | |
| go-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.go-sdk-allowed-failures-count }} | |
| csharp-model-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.csharp-model-total-fixtures-count }} | |
| csharp-model-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.csharp-model-allowed-failures }} | |
| csharp-model-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.csharp-model-allowed-failures-count }} | |
| csharp-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.csharp-sdk-total-fixtures-count }} | |
| csharp-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.csharp-sdk-allowed-failures }} | |
| csharp-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.csharp-sdk-allowed-failures-count }} | |
| php-model-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.php-model-total-fixtures-count }} | |
| php-model-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.php-model-allowed-failures }} | |
| php-model-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.php-model-allowed-failures-count }} | |
| php-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.php-sdk-total-fixtures-count }} | |
| php-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.php-sdk-allowed-failures }} | |
| php-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.php-sdk-allowed-failures-count }} | |
| swift-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.swift-sdk-total-fixtures-count }} | |
| swift-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.swift-sdk-allowed-failures }} | |
| swift-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.swift-sdk-allowed-failures-count }} | |
| rust-model-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.rust-model-total-fixtures-count }} | |
| rust-model-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.rust-model-allowed-failures }} | |
| rust-model-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.rust-model-allowed-failures-count }} | |
| rust-sdk-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.rust-sdk-total-fixtures-count }} | |
| rust-sdk-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.rust-sdk-allowed-failures }} | |
| rust-sdk-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.rust-sdk-allowed-failures-count }} | |
| cli-total-fixtures-count: ${{ steps.extract-all-fixture-info.outputs.cli-total-fixtures-count }} | |
| cli-allowed-failures: ${{ steps.extract-all-fixture-info.outputs.cli-allowed-failures }} | |
| cli-allowed-failures-count: ${{ steps.extract-all-fixture-info.outputs.cli-allowed-failures-count }} | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v6 | |
| - name: Install | |
| uses: ./.github/actions/install | |
| - name: Build Seed CLI | |
| shell: bash | |
| env: | |
| FORCE_COLOR: "2" | |
| run: pnpm seed:build | |
| - name: Extract All Fixture Info | |
| shell: bash | |
| id: extract-all-fixture-info | |
| run: | | |
| # Query fixtures dynamically using the seed CLI | |
| FIXTURES_JSON=$(pnpm seed list-test-fixtures --generator ${{ matrix.seed-generator-alias }}) | |
| FIXTURE_COUNT=$(echo "$FIXTURES_JSON" | jq '.generators["${{ matrix.seed-generator-alias }}"] | length') | |
| # Output for debugging | |
| echo "Found $FIXTURE_COUNT total fixtures" | |
| # Set defaults for allowed failures data | |
| ALLOWED_FAILURES="[]" | |
| ALLOWED_FAILURES_COUNT=0 | |
| # Read allowed failures from yml file as JSON data, if it exists | |
| if [[ $(yq 'has("allowedFailures")' seed/${{ matrix.seed-generator-alias }}/seed.yml) == "true" ]]; then | |
| # Parameter exists, extract allowed failures as JSON data from file | |
| ALLOWED_FAILURES=$(yq -o json '.allowedFailures' seed/${{ matrix.seed-generator-alias }}/seed.yml) | |
| ALLOWED_FAILURES_COUNT=$(echo "$ALLOWED_FAILURES" | jq 'length') | |
| fi | |
| # Output for debugging | |
| echo "Found $ALLOWED_FAILURES_COUNT total allowed failures" | |
| echo "$ALLOWED_FAILURES" | jq . | |
| # Set outputs | |
| echo "${{ matrix.seed-generator-alias }}-total-fixtures-count=$FIXTURE_COUNT" >> $GITHUB_OUTPUT | |
| echo "${{ matrix.seed-generator-alias }}-allowed-failures=$(echo "$ALLOWED_FAILURES" | jq -c .)" >> $GITHUB_OUTPUT | |
| echo "${{ matrix.seed-generator-alias }}-allowed-failures-count=$ALLOWED_FAILURES_COUNT" >> $GITHUB_OUTPUT | |
| post-metrics: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: | |
| [ | |
| setup, | |
| seed-test-results, | |
| ruby-sdk-v2, | |
| pydantic, | |
| python-sdk, | |
| openapi, | |
| java-sdk, | |
| java-model, | |
| ts-sdk, | |
| go-model, | |
| go-sdk, | |
| csharp-model, | |
| csharp-sdk, | |
| php-model, | |
| php-sdk, | |
| swift-sdk, | |
| rust-model, | |
| rust-sdk, | |
| cli, | |
| collect-allowed-failures | |
| ] | |
| if: ${{ always() && !cancelled() && needs.setup.outputs.post-metrics == 'true' }} | |
| strategy: | |
| fail-fast: false # Let all posts run, even if some fail | |
| matrix: | |
| seed-generator-alias: | |
| [ | |
| ruby-sdk-v2, | |
| pydantic, | |
| python-sdk, | |
| openapi, | |
| java-sdk, | |
| java-model, | |
| ts-sdk, | |
| go-model, | |
| go-sdk, | |
| csharp-model, | |
| csharp-sdk, | |
| php-model, | |
| php-sdk, | |
| swift-sdk, | |
| rust-model, | |
| rust-sdk, | |
| cli | |
| ] | |
| steps: | |
| - name: Create Metrics Artifacts Info | |
| shell: bash | |
| id: artifacts-info | |
| run: | | |
| DIRECTORY=seed-test-time-artifacts | |
| echo "metrics_path=$DIRECTORY" >> $GITHUB_OUTPUT | |
| echo "metrics_pattern=seed-metrics-${{ matrix.seed-generator-alias }}-*.json" >> $GITHUB_OUTPUT | |
| if [ ! -d "$DIRECTORY" ]; then | |
| echo "'$DIRECTORY' does NOT exist, creating it now." | |
| mkdir $DIRECTORY | |
| else | |
| echo "WARNING: Directory '$DIRECTORY' already exists. Not an immediate error but could cause a problem downstream." | |
| fi | |
| - name: Download Seed Test Time Artifacts | |
| uses: actions/download-artifact@v7 | |
| with: | |
| path: ${{ steps.artifacts-info.outputs.metrics_path }} | |
| pattern: ${{ steps.artifacts-info.outputs.metrics_pattern }} | |
| merge-multiple: true | |
| - name: Get Number of Metrics Artifacts | |
| shell: bash | |
| id: get-number-of-metric-artifacts | |
| run: | | |
| METRICS_PATH="${{ steps.artifacts-info.outputs.metrics_path }}" | |
| METRICS_PATTERN="${{ steps.artifacts-info.outputs.metrics_pattern }}" | |
| echo "Looking for files in: $METRICS_PATH" | |
| echo "Using pattern: $METRICS_PATTERN" | |
| # Check for artifacts | |
| file_count=$(find "./$METRICS_PATH" -type f -name "$METRICS_PATTERN" | wc -l) | |
| if [ "$file_count" -gt 0 ]; then | |
| echo "Found $file_count artifacts for seed metrics" | |
| else | |
| echo "Found 0 artifacts for seed metrics" | |
| exit 1 | |
| fi | |
| - name: Display Content of Metrics Artifacts Folder | |
| run: | | |
| METRICS_PATH="${{ steps.artifacts-info.outputs.metrics_path }}" | |
| echo "Content of '$METRICS_PATH' folder:" | |
| ls -R $METRICS_PATH/ | |
| - name: Parse Seed Test Metrics | |
| id: parse-seed-test-metrics | |
| run: | | |
| METRICS_PATH="${{ steps.artifacts-info.outputs.metrics_path }}" | |
| METRICS_PATTERN="${{ steps.artifacts-info.outputs.metrics_pattern }}" | |
| # Combine all JSON files into an array and find min start time and max end time | |
| min_start=$(jq -r '.startTimeSeconds' ./$METRICS_PATH/$METRICS_PATTERN | sort | head -n 1) | |
| max_end=$(jq -r '.endTimeSeconds' ./$METRICS_PATH/$METRICS_PATTERN | sort | tail -n 1) | |
| # Calculate duration in seconds | |
| duration_seconds=$((max_end - min_start)) | |
| echo "Earliest start time: $min_start" | |
| echo "Latest end time: $max_end" | |
| echo "Duration: $duration_seconds" | |
| echo "duration_seconds=$duration_seconds" >> $GITHUB_OUTPUT | |
| # Determine if all tests passing. Note: seed-generator-alias needs to match CI job name for this parsing to work | |
| all_tests_passing="${{ needs.seed-test-results.outputs[format('{0}-passing', matrix.seed-generator-alias)] }}" | |
| # If empty for any reason, default to false | |
| if [ -z "$all_tests_passing" ]; then | |
| all_tests_passing="false" | |
| fi | |
| echo "all_tests_passing=$all_tests_passing" >> $GITHUB_OUTPUT | |
| # Determine if all of seed generation is deterministic for a given generator. One reporting as a failure means the collective is a failure. | |
| all_seed_files_deterministic=$(find "./$METRICS_PATH" -name "$METRICS_PATTERN" -exec jq -r '.deterministic' {} \; | grep -v -q "true" && echo "false" || echo "true") | |
| # If empty for any reason, default to false | |
| if [ -z "$all_seed_files_deterministic" ]; then | |
| all_seed_files_deterministic="false" | |
| fi | |
| echo "all_seed_files_deterministic=$all_seed_files_deterministic" >> $GITHUB_OUTPUT | |
| - name: Post to PostHog | |
| # https://github.com/PostHog/posthog-github-action | |
| uses: PostHog/posthog-github-action@v1 | |
| with: | |
| posthog-token: ${{ secrets.POSTHOG_API_KEY }} | |
| event: "gh-actions-seed-test-metrics-event" | |
| properties: | | |
| { | |
| "event_type": "${{ github.event_name }}", | |
| "branch": "${{ github.ref_name }}", | |
| "seed_generator_alias": "${{ matrix.seed-generator-alias }}", | |
| "run_url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", | |
| "duration_seconds": ${{ steps.parse-seed-test-metrics.outputs.duration_seconds }}, | |
| "all_tests_passing": "${{ steps.parse-seed-test-metrics.outputs.all_tests_passing }}", | |
| "all_seed_files_deterministic": "${{ steps.parse-seed-test-metrics.outputs.all_seed_files_deterministic }}", | |
| "total_fixture_count": ${{ needs.collect-allowed-failures.outputs[format('{0}-total-fixtures-count', matrix.seed-generator-alias)] }}, | |
| "allowed_failures": ${{ needs.collect-allowed-failures.outputs[format('{0}-allowed-failures', matrix.seed-generator-alias)] }}, | |
| "allowed_failures_count": ${{ needs.collect-allowed-failures.outputs[format('{0}-allowed-failures-count', matrix.seed-generator-alias)] }} | |
| } |