test: Add comprehensive sc-type testing suite and setup scripts #12
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ChatSpatial Test Suite | |
| on: | |
| push: | |
| branches: [ main, develop ] | |
| pull_request: | |
| branches: [ main ] | |
| schedule: | |
| # Run tests daily at 2 AM UTC | |
| - cron: '0 2 * * *' | |
| jobs: | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, windows-latest] | |
| python-version: ['3.9', '3.10', '3.11'] | |
| exclude: | |
| # Reduce CI load - test fewer combinations on Windows | |
| - os: windows-latest | |
| python-version: '3.9' | |
| - os: windows-latest | |
| python-version: '3.11' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Set up Node.js (for MCP Inspector) | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - name: Cache Python dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install system dependencies (Ubuntu) | |
| if: matrix.os == 'ubuntu-latest' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y libhdf5-dev pkg-config | |
| - name: Install system dependencies (macOS) | |
| if: matrix.os == 'macos-latest' | |
| run: | | |
| brew install hdf5 pkg-config | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install pytest pytest-asyncio pytest-cov aiohttp | |
| - name: Install MCP Inspector | |
| run: | | |
| npm install -g @modelcontextprotocol/inspector | |
| - name: Lint with flake8 | |
| run: | | |
| pip install flake8 | |
| # Stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # Exit-zero treats all errors as warnings | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| - name: Type check with mypy | |
| continue-on-error: true # Don't fail CI on type errors for now | |
| run: | | |
| pip install mypy | |
| mypy chatspatial --ignore-missing-imports || true | |
| - name: Run Unit Tests | |
| run: | | |
| python run_tests.py --suite unit --verbose --ci | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| - name: Run Tool Tests | |
| run: | | |
| python run_tests.py --suite tool --verbose --ci | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| - name: Run Workflow Tests | |
| run: | | |
| python run_tests.py --suite workflow --verbose --ci | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| - name: Run E2E Tests (Ubuntu only) | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' | |
| run: | | |
| python run_tests.py --suite e2e --verbose | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| # E2E tests don't fail CI since they may be flaky | |
| continue-on-error: true | |
| - name: Generate Test Report | |
| if: always() | |
| run: | | |
| python run_tests.py --suite all --report test-report-${{ matrix.os }}-py${{ matrix.python-version }}.json | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| continue-on-error: true | |
| - name: Upload Test Report | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-report-${{ matrix.os }}-py${{ matrix.python-version }} | |
| path: test-report-*.json | |
| retention-days: 30 | |
| - name: Upload Coverage to Codecov | |
| if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| file: ./coverage.xml | |
| flags: unittests | |
| name: codecov-umbrella | |
| fail_ci_if_error: false | |
| performance-test: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: github.event_name == 'schedule' || github.event_name == 'push' && github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install pytest pytest-asyncio pytest-benchmark | |
| - name: Run Performance Tests | |
| run: | | |
| python run_tests.py --suite performance --include-slow --verbose --report performance-report.json | |
| env: | |
| PYTHONPATH: ${{ github.workspace }} | |
| continue-on-error: true | |
| - name: Upload Performance Report | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: performance-report | |
| path: performance-report.json | |
| retention-days: 90 | |
| security-scan: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| pip install bandit safety | |
| - name: Run Bandit security scan | |
| run: | | |
| bandit -r chatspatial -f json -o bandit-report.json || true | |
| - name: Run Safety check | |
| run: | | |
| safety check --json --output safety-report.json || true | |
| - name: Upload Security Reports | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: security-reports | |
| path: | | |
| bandit-report.json | |
| safety-report.json | |
| retention-days: 30 | |
| integration-test: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| if: github.event_name == 'pull_request' || github.ref == 'refs/heads/main' | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '18' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| npm install -g @modelcontextprotocol/inspector | |
| - name: Run Integration Tests with Real MCP Inspector | |
| run: | | |
| # Start server in background | |
| timeout 300 npx @modelcontextprotocol/inspector -- python -m chatspatial & | |
| SERVER_PID=$! | |
| # Wait for server to start | |
| sleep 10 | |
| # Run basic connectivity test | |
| curl -f http://localhost:6277/health || echo "Health check failed" | |
| # Stop server | |
| kill $SERVER_PID || true | |
| env: | |
| DANGEROUSLY_OMIT_AUTH: "true" | |
| PYTHONPATH: ${{ github.workspace }} | |
| continue-on-error: true | |
| notify: | |
| runs-on: ubuntu-latest | |
| needs: [test, performance-test, security-scan, integration-test] | |
| if: always() && (github.event_name == 'push' && github.ref == 'refs/heads/main') | |
| steps: | |
| - name: Notify on Success | |
| if: needs.test.result == 'success' | |
| run: | | |
| echo "✅ All tests passed on main branch!" | |
| # Add notification logic here (Slack, email, etc.) | |
| - name: Notify on Failure | |
| if: needs.test.result == 'failure' | |
| run: | | |
| echo "❌ Tests failed on main branch!" | |
| # Add notification logic here (Slack, email, etc.) | |
| exit 1 |