memory-soak.yml

name: Memory Soak Test

# Nightly long-session memory SLO regression guard (issue #554).
# Runs the 60-minute round-robin soak test against a self-hosted macOS runner
# that has a booted iOS simulator. On failure the sentinel job posts a comment
# on the tracking issue and, after 7 consecutive failures, opens a labelled
# regression issue.

on:
  schedule:
    - cron: '0 3 * * *'  # 03:00 UTC daily
  workflow_dispatch: {}

permissions:
  contents: read
  issues: write

jobs:
  soak:
    runs-on: [self-hosted, macOS]
    timeout-minutes: 90
    env:
      OPENSAFARI_RUN_SOAK: '1'

    steps:
      - uses: actions/checkout@v4

      - name: Print runner info
        run: |
          sw_vers
          xcodebuild -version
          echo "Xcode select: $(xcode-select -p)"
          node --version
          npm --version

      - uses: actions/setup-node@v4
        with:
          node-version: '20'
          cache: 'npm'

      - name: Install dependencies
        run: npm ci

      - name: Build
        run: npm run build

      - name: Boot simulator
        run: |
          DEVICE_NAME="iPhone 16"

          # Reuse an already-booted device if one exists.
          BOOTED=$(xcrun simctl list devices booted -j \
            | python3 -c "
          import json,sys
          data=json.load(sys.stdin)
          udids=[d['udid'] for devs in data['devices'].values() for d in devs if d.get('state')=='Booted']
          print(udids[0] if udids else '')
          " 2>/dev/null || echo "")

          if [ -n "$BOOTED" ]; then
            DEVICE="$BOOTED"
            echo "Reusing booted simulator: $DEVICE"
          else
            # Find an available device matching the target name.
            DEVICE=$(xcrun simctl list devices available -j \
              | python3 -c "
          import json,sys
          data=json.load(sys.stdin)
          udids=[d['udid'] for devs in data['devices'].values() for d in devs if d.get('name')=='${DEVICE_NAME}']
          print(udids[0] if udids else '')
              " 2>/dev/null || echo "")

            if [ -z "$DEVICE" ]; then
              echo "No '${DEVICE_NAME}' device found; creating one."
              DEVICE=$(xcrun simctl create "SoakTest" "${DEVICE_NAME}" 2>/dev/null || echo "")
            fi

            if [ -z "$DEVICE" ]; then
              echo "ERROR: Could not find or create simulator." >&2
              exit 1
            fi

            xcrun simctl boot "$DEVICE" 2>/dev/null || true
            xcrun simctl bootstatus "$DEVICE" -b
          fi

          echo "DEVICE_UDID=$DEVICE" >> "$GITHUB_ENV"
          xcrun simctl list devices booted

      - name: Run soak test
        # testPathIgnorePatterns in jest.config.js excludes tests/soak/;
        # override with --testPathIgnorePatterns=/node_modules/ so the test
        # actually executes here (same pattern as sim-hid-sentinel workflow).
        run: |
          npx jest tests/soak/ \
            --testPathIgnorePatterns=/node_modules/ \
            --testTimeout=3700000 \
            --forceExit \
            2>&1 | tee soak.log
        timeout-minutes: 75

      - name: Upload RSS baseline and logs
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: rss-baseline-${{ github.run_number }}
          path: |
            soak.log
            tests/soak/output/
          retention-days: 90
          if-no-files-found: warn

      - name: Shutdown simulator
        if: always()
        run: xcrun simctl shutdown "$DEVICE_UDID" 2>/dev/null || true

  sentinel:
    needs: soak
    if: failure()
    runs-on: [self-hosted, macOS]

    steps:
      - uses: actions/checkout@v4

      - name: Check consecutive failures and open issue
        uses: actions/github-script@v7
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            // Check the last 7 completed runs of this workflow for consecutive failures.
            const runs = await github.rest.actions.listWorkflowRuns({
              owner: context.repo.owner,
              repo: context.repo.repo,
              workflow_id: 'memory-soak.yml',
              per_page: 7,
              status: 'completed',
            });

            let consecutiveFailures = 0;
            for (const r of runs.data.workflow_runs) {
              if (r.conclusion === 'failure') consecutiveFailures++;
              else break;
            }

            if (consecutiveFailures >= 7) {
              // Open a labelled regression issue if none is already open.
              const existing = await github.rest.issues.listForRepo({
                owner: context.repo.owner,
                repo: context.repo.repo,
                labels: 'memory-regression',
                state: 'open',
              });

              if (existing.data.length === 0) {
                await github.rest.issues.create({
                  owner: context.repo.owner,
                  repo: context.repo.repo,
                  title: 'Memory regression detected — 7 consecutive soak test failures',
                  body: [
                    'The nightly memory soak test has failed 7 times in a row.',
                    '',
                    `See: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/workflows/memory-soak.yml`,
                    '',
                    '## Triage checklist',
                    '',
                    '- Download the `rss-baseline-*` artifact from the latest failing run.',
                    '- Identify which backend tier correlates with the RSS spike (Tier 0 Flutter VM, Tier 1 SimHID, Tier 1.5 AX press, Tier 3 WebKit).',
                    '- Check `tests/soak/README.md` for interpretation guidance.',
                    '- Re-run locally with `OPENSAFARI_RUN_SOAK=1 npx jest tests/soak/ --testTimeout=3700000`.',
                    '',
                    'Ref: #554',
                  ].join('\n'),
                  labels: ['memory-regression', 'reliability'],
                });
              }
            }

            // Always post a failure comment on the tracking issue.
            await github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: 554,
              body: [
                `Memory soak test failed — run #${context.runNumber}`,
                '',
                `See: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
                '',
                `Consecutive failures in last 7 runs: ${consecutiveFailures}`,
              ].join('\n'),
            });