Skip to content

Commit 311332d

Browse files
committed
Add GitHub Actions health check workflow
Introduces a new GitHub Actions workflow to periodically check service health and version information for specified endpoints. The workflow includes: - Scheduled runs every 5 minutes - Checks version information - Monitors service status - Raises an error if any services are down TMP: test Test it
1 parent 176f50a commit 311332d

File tree

2 files changed

+154
-0
lines changed

2 files changed

+154
-0
lines changed

.github/workflows/health_check.yml

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
name: Health Check
2+
3+
on:
4+
# Run the workflow test on push events
5+
push:
6+
# Run the main workflow on workflow_dispatch or schedule
7+
workflow_dispatch:
8+
schedule:
9+
# Every 5 minutes
10+
- cron: '*/5 * * * *'
11+
12+
jobs:
13+
health_check:
14+
runs-on: ubuntu-latest
15+
strategy:
16+
fail-fast: false
17+
matrix:
18+
environment: ${{fromJson(github.event_name == 'push' && '["test"]' || '["dev","stage","prod"]')}}
19+
20+
steps:
21+
- uses: actions/checkout@v4
22+
23+
- name: Set up Python
24+
uses: actions/setup-python@v5
25+
with:
26+
python-version: '3.11'
27+
cache: 'pip'
28+
29+
- name: Install dependencies
30+
run: |
31+
python -m pip install --upgrade pip
32+
pip install requests
33+
34+
- name: Run Health Checks
35+
shell: bash
36+
run: |
37+
set -ue
38+
39+
environment="${{ matrix.environment }}"
40+
output_file="out.json"
41+
./scripts/health_check.py --env $environment --verbose --output $output_file
42+
43+
version=$(cat $output_file | jq -r '.version')
44+
monitors=$(cat $output_file | jq -r '.monitors')
45+
46+
echo "Version: $version"
47+
echo "Monitors: $monitors"
48+
49+
if [ "$version" = "null" ] || [ "$monitors" = "null" ]; then
50+
echo "Environment $environment is not reachable"
51+
exit 1
52+
fi
53+
54+
message=""
55+
56+
data=$(echo $monitors | jq -r 'to_entries[] | select(.value.state == false) | .key')
57+
for monitor in $data; do
58+
message="$message\n- $monitor: $(echo $monitors | jq -r ".[\"$monitor\"].status")"
59+
done
60+
61+
echo "Environment: $environment"
62+
echo "Message:"
63+
echo $message
64+
65+
66+
67+

scripts/health_check.py

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/usr/bin/env python3
2+
3+
import argparse
4+
import json
5+
from enum import Enum
6+
7+
import requests
8+
9+
10+
ENV_ENUM = Enum(
11+
'ENV',
12+
[
13+
('dev', 'https://addons-dev.allizom.org'),
14+
('stage', 'https://addons.allizom.org'),
15+
('prod', 'https://addons.mozilla.org'),
16+
# TODO: maybe we could use the local environmnet here
17+
('test', ''),
18+
],
19+
)
20+
21+
22+
class Fetcher:
23+
def __init__(self, env: ENV_ENUM, verbose: bool = False):
24+
self.environment = ENV_ENUM[env]
25+
self.verbose = verbose
26+
27+
def _fetch(self, path: str) -> dict[str, str] | None:
28+
url = f'{self.environment.value}/{path}'
29+
if self.verbose:
30+
print(f'Requesting {url} for {self.environment.name}')
31+
32+
try:
33+
response = requests.get(url)
34+
response.raise_for_status()
35+
try:
36+
data = response.json()
37+
except json.JSONDecodeError as e:
38+
if self.verbose:
39+
print(f'Error decoding JSON for {url}: {e}')
40+
41+
except requests.exceptions.HTTPError as e:
42+
if self.verbose:
43+
print(f'Error fetching {url}: {e}')
44+
45+
if self.verbose and data is not None:
46+
print(json.dumps(data, indent=2))
47+
48+
return data
49+
50+
def version(self):
51+
if self.environment.name == 'test':
52+
return {}
53+
return self._fetch('__version__')
54+
55+
def monitors(self):
56+
if self.environment.name == 'test':
57+
return {
58+
'up': {'state': True},
59+
'down': {'state': False, 'status': 'something is wrong'},
60+
}
61+
return self._fetch('services/monitor.json')
62+
63+
64+
def main(env: ENV_ENUM, verbose: bool = False, output: str | None = None):
65+
fetcher = Fetcher(env, verbose)
66+
67+
version_data = fetcher.version()
68+
monitors_data = fetcher.monitors()
69+
70+
if output:
71+
with open(output, 'w') as f:
72+
json.dump({'version': version_data, 'monitors': monitors_data}, f, indent=2)
73+
elif monitors_data is not None:
74+
if any(monitor['state'] is False for monitor in monitors_data.values()):
75+
raise ValueError(f'Some monitors are failing {monitors_data}')
76+
77+
78+
if __name__ == '__main__':
79+
args = argparse.ArgumentParser()
80+
args.add_argument(
81+
'--env', type=str, choices=list(ENV_ENUM.__members__.keys()), required=True
82+
)
83+
args.add_argument('--verbose', action='store_true')
84+
args.add_argument('--output', type=str, required=False)
85+
args = args.parse_args()
86+
87+
main(args.env, args.verbose, args.output)

0 commit comments

Comments
 (0)