Skip to content

Commit 9f20ec0

Browse files
rdimitrovrdimitrov
authored
Group workflows by trigger type and initial commit for the integration tests workflow (#611)
* Group workflows by trigger type Signed-off-by: Radoslav Dimitrov <[email protected]>
1 parent c88deda commit 9f20ec0

15 files changed

+393
-40
lines changed

Diff for: ‎.github/workflows/ci.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
name: CI
1+
# This workflow builds, tests and lints the project
2+
name: Build, Test & Lint
23

34
on:
4-
push:
5-
branches: [ main ]
6-
pull_request:
7-
branches: [ main ]
5+
workflow_call:
6+
workflow_dispatch:
87

98
jobs:
109
test:
10+
name: Run
1111
runs-on: ubuntu-latest
1212
strategy:
1313
matrix:

Diff for: ‎.github/workflows/image-build.yml

+41-10
Original file line numberDiff line numberDiff line change
@@ -1,29 +1,48 @@
1-
# Attempt to build (but not push) the Docker image on Pull Requests
2-
name: Image build
1+
# This workflow build (but does not push) the OCI image
2+
name: OCI Image - Build
3+
34
on:
4-
pull_request:
5-
branches:
6-
- main
7-
paths-ignore:
8-
- "**.md"
9-
- "docs/**"
10-
- "static/**"
11-
- "LICENSE"
5+
workflow_call:
6+
inputs:
7+
artifact-name:
8+
description: 'The name of the artifact to upload'
9+
required: true
10+
type: string
11+
1212
permissions:
1313
contents: read
14+
1415
jobs:
1516
docker-image:
1617
name: Check docker image build
1718
runs-on: ubuntu-latest
19+
env:
20+
IMAGE_NAME: stacklok/codegate
21+
IMAGE_TAG: dev
1822
steps:
1923
- name: Checkout
2024
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
2125
- name: Set up Docker Buildx
2226
uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3
27+
- name: Download artifact
28+
id: download-artifact
29+
uses: dawidd6/action-download-artifact@80620a5d27ce0ae443b965134db88467fc607b43 # v7
30+
with:
31+
github_token: ${{ github.token }}
32+
workflow: ".github/workflows/import_packages.yml"
33+
workflow_conclusion: success
34+
name: sqlite_data
35+
name_is_regexp: true
36+
skip_unpack: false
37+
if_no_artifact_found: ignore
2338
- name: Fetch latest release URL
2439
id: fetch_release_url
2540
run: |
2641
echo "LATEST_RELEASE=$(curl -s "https://api.github.com/repos/stacklok/codegate-ui/releases/latest" | jq -r '.zipball_url')" >> $GITHUB_ENV
42+
- name: Download git lfs dependencies
43+
run: |
44+
git lfs install
45+
git lfs pull
2746
- name: Test build on x86
2847
id: docker_build
2948
uses: docker/build-push-action@67a2d409c0a876cbe6b11854e3e25193efe4e62d # v5
@@ -37,3 +56,15 @@ jobs:
3756
cache-to: type=gha,mode=max
3857
build-args: |
3958
LATEST_RELEASE=${{ env.LATEST_RELEASE }}
59+
tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
60+
- name: Save Docker image as a tarball
61+
run: |
62+
# List all images
63+
docker images
64+
# Save the image as a tarball
65+
docker save -o image.tar ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
66+
- name: Upload Docker image artifact
67+
uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4
68+
with:
69+
name: ${{ inputs.artifact-name }}
70+
path: image.tar

Diff for: ‎.github/workflows/image-publish.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
name: Publish Docker Image
1+
# This workflow builds and publishes the OCI image
2+
name: OCI Image - Publish
3+
24
on:
3-
release:
4-
types:
5-
- published
5+
workflow_call:
66
workflow_dispatch:
77

88
jobs:

Diff for: ‎.github/workflows/import_packages.yml

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
# This workflow syncs the vector database
12
name: Sync vector DB
23

34
on:

Diff for: ‎.github/workflows/integration-tests.yml

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
# This workflow will run the integration tests for the project
2+
name: Tests - Integration
3+
4+
on:
5+
workflow_call:
6+
inputs:
7+
artifact-name:
8+
description: 'The name of the artifact to download'
9+
required: true
10+
type: string
11+
secrets:
12+
copilot-key:
13+
description: 'The Copilot key to use for integration tests'
14+
required: true
15+
16+
jobs:
17+
integration-tests:
18+
name: Run
19+
runs-on: ubuntu-latest
20+
strategy:
21+
matrix:
22+
python-version: [ "3.12" ]
23+
env:
24+
CONTAINER_NAME: "codegate"
25+
CERT_FILE: "/app/codegate_volume/certs/ca.crt"
26+
steps:
27+
- name: Checkout
28+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
29+
with:
30+
lfs: true
31+
32+
- name: Ensure file permissions for mounted volume
33+
run: |
34+
mkdir -p ./codegate_volume/certs ./codegate_volume/models ./codegate_volume/db
35+
chmod -R 777 ./codegate_volume
36+
37+
- name: Download Docker image artifact
38+
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4
39+
with:
40+
name: ${{ inputs.artifact-name }}
41+
42+
- name: Load Docker image
43+
run: |
44+
docker load -i image.tar
45+
echo "Loaded image:"
46+
docker images
47+
48+
- name: Run container from the loaded image
49+
run: |
50+
# Get the image name
51+
DOCKER_IMAGE=$(docker images --format "{{.Repository}}:{{.Tag}}" | head -n 1)
52+
echo "Running container from image: $DOCKER_IMAGE"
53+
54+
# Run the container
55+
docker run --name $CONTAINER_NAME -d -p 8989:8989 -p 9090:9090 \
56+
-p 8990:8990 \
57+
-v "$(pwd)"/codegate_volume:/app/codegate_volume \
58+
-e CODEGATE_APP_LOG_LEVEL=DEBUG \
59+
--restart unless-stopped $DOCKER_IMAGE
60+
61+
# Confirm the container started
62+
echo "Container started:"
63+
docker ps
64+
65+
# Verify container is running with correct ports
66+
docker ps -f name=$CONTAINER_NAME
67+
68+
# Check mount configuration
69+
docker inspect $CONTAINER_NAME -f '{{ json .Mounts }}' | jq
70+
71+
- name: Ensure certificates are available in the container
72+
timeout-minutes: 4
73+
run: |
74+
# Wait for the cert file to be available in the container
75+
while true; do
76+
echo "Checking for $CERT_FILE in container $CONTAINER_NAME..."
77+
78+
if docker exec "$CONTAINER_NAME" test -f "$CERT_FILE"; then
79+
echo "Cert file found: $CERT_FILE"
80+
break
81+
else
82+
echo "Cert file not found. Retrying in 5 seconds..."
83+
sleep 5
84+
fi
85+
done
86+
87+
# Verify volume contents are accessible
88+
docker exec $CONTAINER_NAME ls -la /app/codegate_volume
89+
90+
# Print the container logs we got so far
91+
docker logs $CONTAINER_NAME
92+
93+
- name: Install the CodeGate certificate
94+
run: |
95+
docker cp codegate:/app/codegate_volume/certs/ca.crt ./codegate.crt
96+
sudo cp ./codegate.crt /usr/local/share/ca-certificates/codegate.crt
97+
sudo update-ca-certificates
98+
99+
- name: Set up Python ${{ matrix.python-version }}
100+
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5
101+
with:
102+
python-version: ${{ matrix.python-version }}
103+
104+
- name: Install Poetry
105+
uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
106+
with:
107+
version: 2.0.1
108+
virtualenvs-create: true
109+
virtualenvs-in-project: true
110+
111+
- name: Load cached venv
112+
id: cached-poetry-dependencies
113+
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4
114+
with:
115+
path: .venv
116+
key: venv-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
117+
118+
- name: Install dependencies
119+
run: poetry install --with dev
120+
121+
- name: Run integration tests - Copilot
122+
env:
123+
CODEGATE_PROVIDERS: "copilot"
124+
CA_CERT_FILE: "/home/runner/work/codegate/codegate/codegate_volume/certs/ca.crt"
125+
ENV_COPILOT_KEY: ${{ secrets.copilot-key }}
126+
run: |
127+
poetry run python tests/integration/integration_tests.py
128+
129+
- name: Print the container logs (useful for debugging)
130+
if: always()
131+
run: |
132+
docker logs $CONTAINER_NAME
133+
echo "Models contents:"
134+
ls -la codegate_volume/models
135+
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/models
136+
echo "Certs contents:"
137+
ls -la codegate_volume/certs
138+
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/certs
139+
echo "DB contents:"
140+
ls -la codegate_volume/db
141+
docker exec $CONTAINER_NAME ls -la /app/codegate_volume/db

Diff for: ‎.github/workflows/openapi.yml

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
1+
# This workflow generates the OpenAPI docs
12
name: Generate OpenAPI Documentation
23

34
on:
5+
workflow_call:
46
workflow_dispatch:
5-
push:
6-
branches:
7-
- main
87

98
jobs:
109
generate_openapi:

Diff for: ‎.github/workflows/run-on-pr.yml

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# These set of workflows run on every pull request
2+
name: Run - pull_request
3+
4+
on:
5+
workflow_dispatch:
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
security:
11+
name: Security Checks
12+
uses: ./.github/workflows/security.yml
13+
ci:
14+
name: Build, Test & Lint
15+
uses: ./.github/workflows/ci.yml
16+
image-build:
17+
name: OCI Image - Build
18+
needs: [ci] # No need to build the image if the CI fails
19+
uses: ./.github/workflows/image-build.yml
20+
with:
21+
artifact-name: "codegate-image"
22+
integration-tests:
23+
name: Integration Tests
24+
needs: [ci, image-build] # We need the image available in order to run the integration tests
25+
uses: ./.github/workflows/integration-tests.yml
26+
with:
27+
artifact-name: "codegate-image"
28+
secrets:
29+
copilot-key: ${{ secrets.INTEGRATION_TESTS_COPILOT_KEY }}

Diff for: ‎.github/workflows/run-on-push.yml

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# These set of workflows run on every push to the main branch
2+
name: Run - push_to_main
3+
4+
on:
5+
workflow_dispatch:
6+
push:
7+
branches: [ main ]
8+
9+
jobs:
10+
security:
11+
name: Security Checks
12+
uses: ./.github/workflows/security.yml
13+
ci:
14+
name: Build, Test & Lint
15+
uses: ./.github/workflows/ci.yml
16+
openapi:
17+
name: Generate the OpenAPI docs
18+
needs: [ci]
19+
uses: ./.github/workflows/openapi.yml

Diff for: ‎.github/workflows/run-on-release-published.yml

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# These set of workflows run on every published release
2+
name: Run - release_published
3+
4+
on:
5+
workflow_dispatch:
6+
release:
7+
types:
8+
- published
9+
10+
jobs:
11+
image-publish:
12+
name: OCI Image - Publish
13+
uses: ./.github/workflows/image-publish.yml

Diff for: ‎.github/workflows/security.yml

+2-5
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,14 @@
11
name: Security
22

33
on:
4-
pull_request:
5-
push:
6-
branches:
7-
- main
4+
workflow_call:
85
schedule:
96
- cron: '0 0 * * *'
107

118
jobs:
129
dependencies:
1310
runs-on: ubuntu-latest
14-
name: Dependencies & Secrets Scan
11+
name: Dependencies & Secrets Scan via Trivy
1512
steps:
1613
- name: Checkout
1714
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

Diff for: ‎README.md

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,10 @@
33
<img alt="CodeGate logo" src="./static/codegate-logo-dark.svg" width="800px" style="max-width: 100%;">
44
</picture>
55

6-
[![CI](https://github.com/stacklok/codegate/actions/workflows/ci.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/ci.yml) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK)
6+
---
7+
[![CI](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml/badge.svg)](https://github.com/stacklok/codegate/actions/workflows/run-on-push.yml) | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache2.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK)
78

9+
---
810
## Introduction
911

1012
<img src="./assets/codegate.gif" style="width: 70%; height: 70%;" />

Diff for: ‎codegate_volume/models/all-minilm-L6-v2-q5_k_m.gguf

20.7 MB
Binary file not shown.

0 commit comments

Comments
 (0)