Add GitHub Actions health check workflow

Introduces a new GitHub Actions workflow to periodically check service health and version information for specified endpoints. The workflow includes:
- Scheduled runs every 5 minutes
- Checks version information
- Monitors service status
- Raises an error if any services are down

Author: KevinMind

.github/workflows/health_check.yml

@@ -0,0 +1,31 @@
+name: Health Check
+
+on:
+  # Run the workflow test on push events
+  push:
+  # Run the main workflow on workflow_dispatch or schedule
+  workflow_dispatch:
+  schedule:
+    # Every 5 minutes
+    - cron: '*/5 * * * *'
+
+jobs:
+  health_check:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ${{fromJson(github.event_name == 'push' && '["local"]' || '["dev","stage","prod"]')}}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ./.github/actions/run-docker
+        with:
+          target: development
+          version: local
+          logs: true
+          run: ./scripts/health_check.py --env ${{ matrix.environment }} --verbose
+
+
+

docker/nginx/addons.conf

@@ -49,7 +49,7 @@ server {
         try_files $uri @olympia;
     }
 
-    location /__version__ {
+    location ~ ^/(__version__|__heartbeat__|__healthcheck__)(\?.*)?$ {
         try_files $uri @olympia;
     }
 

scripts/health_check.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+from enum import Enum
+import time
+
+import requests
+
+
+ENV_ENUM = Enum(
+    'ENV',
+    [
+        ('dev', 'https://addons-dev.allizom.org'),
+        ('stage', 'https://addons.allizom.org'),
+        ('prod', 'https://addons.mozilla.org'),
+        # For local environments hit the nginx container as set in docker-compose.yml
+        ('local', 'http://nginx'),
+    ],
+)
+
+
+class Fetcher:
+    def __init__(self, env: ENV_ENUM, verbose: bool = False):
+        self.environment = ENV_ENUM[env]
+        self.verbose = verbose
+
+    def _fetch(self, path: str) -> dict[str, str] | None:
+        url = f'{self.environment.value}/{path}'
+        if self.verbose:
+            print(f'Requesting {url} for {self.environment.name}')
+
+        data = None
+        # We return 500 if any of the monitors are failing.
+        # So instead of raising, we should try to form valid JSON
+        # and determine if we should raise later based on the json values.
+        try:
+            response = requests.get(url, allow_redirects=False)
+            data = response.json()
+        except (requests.exceptions.HTTPError, json.JSONDecodeError) as e:
+            if self.verbose:
+                print({
+                    'error': e,
+                    'data': data,
+                    'response': response,
+                })
+
+        if self.verbose and data is not None:
+            print(json.dumps(data, indent=2))
+
+        return data
+
+    def version(self):
+        return self._fetch('__version__')
+
+    def healthcheck(self):
+        return self._fetch('__healthcheck__?verbose=true')
+
+
+def main(env: ENV_ENUM, verbose: bool = False):
+    fetcher = Fetcher(env, verbose)
+
+    version_data = fetcher.version()
+    healthcheck_data = fetcher.healthcheck()
+
+    if version_data is None:
+        raise ValueError('Error fetching version data')
+
+    if healthcheck_data is None:
+        raise ValueError('Error fetching healthcheck data')
+
+    if healthcheck_data is not None:
+        if any(monitor['state'] is False for monitor in healthcheck_data.values()):
+            raise ValueError(f'Some monitors are failing {healthcheck_data.keys()}')
+
+
+if __name__ == '__main__':
+    args = argparse.ArgumentParser()
+    args.add_argument(
+        '--env', type=str, choices=list(ENV_ENUM.__members__.keys()), required=True
+    )
+    args.add_argument('--verbose', action='store_true')
+    args.add_argument('--retries', type=int, default=3)
+    args = args.parse_args()
+
+    attempt = 1
+
+    while attempt <= args.retries:
+        try:
+            main(args.env, args.verbose)
+            break
+        except Exception as e:
+            print(f'Error: {e}')
+            time.sleep(2 ** attempt)
+            attempt += 1

settings.py

@@ -204,3 +204,5 @@ def insert_debug_toolbar_middleware(middlewares):
             'static_url_prefix': 'bundle',
         }
     }
+
+MEMCACHE_MIN_SERVER_COUNT = 1

src/olympia/amo/management/__init__.py

@@ -165,7 +165,11 @@ def handle(self, *args, **options):
             'test': '',
             'uploads': '',
         },
-        'uploads': '',
+        'uploads': {
+            'addon_icons': '',
+            'previews': '',
+            'userpics': '',
+        },
     },
 }
 

src/olympia/amo/management/commands/monitors.py

@@ -22,29 +22,39 @@ def add_arguments(self, parser):
             default=5,
             help='Number of attempts to check the services',
         )
+        parser.add_argument(
+            '--verbose',
+            action='store_true',
+            help='Verbose output',
+        )
 
     def handle(self, *args, **options):
         attempts = options.get('attempts')
         services = options.get('services')
-
+        verbose = options.get('verbose')
         self.logger.info(f'services: {services}')
 
         if not services:
             raise CommandError('No services specified')
 
-        failing_services = services.copy()
-
         current = 0
 
+        services_to_check = set(services.copy())
+
         while current < attempts:
             current += 1
-            self.logger.info(f'Checking services {services} for the {current} time')
-            status_summary = monitors.execute_checks(services)
-            failing_services = [
-                service
-                for service, result in status_summary.items()
-                if result['state'] is False
-            ]
+            self.logger.info(
+                f'Checking services {services_to_check} for the {current} time'
+            )
+            status_summary = monitors.execute_checks(list(services_to_check), verbose)
+
+            failing_services = {}
+
+            for service, result in status_summary.items():
+                if result['state'] is True and service in services_to_check:
+                    services_to_check.remove(service)
+                else:
+                    failing_services[service] = result
 
             if len(failing_services) > 0:
                 self.logger.error('Some services are failing: %s', failing_services)
@@ -54,7 +64,7 @@ def handle(self, *args, **options):
             else:
                 break
 
-        if len(failing_services) > 0:
-            raise CommandError(f'Some services are failing: {failing_services}')
+        if len(services_to_check) > 0:
+            raise CommandError(f'Some services are failing: {list(services_to_check)}')
         else:
             self.logger.info(f'All services are healthy {services}')

src/olympia/amo/monitors.py

@@ -22,13 +22,15 @@
 monitor_log = olympia.core.logger.getLogger('z.monitor')
 
 
-def execute_checks(checks: list[str]):
+def execute_checks(checks: list[str], verbose: bool = False):
     status_summary = {}
     for check in checks:
         with statsd.timer('monitor.%s' % check):
-            status, _ = globals()[check]()
+            status, results = globals()[check]()
         # state is a string. If it is empty, that means everything is fine.
         status_summary[check] = {'state': not status, 'status': status}
+        if verbose:
+            status_summary[check]['results'] = results
     return status_summary
 
 
@@ -40,7 +42,7 @@ def localdev_web():
     """
     status = ''
     try:
-        response = requests.get('http://127.0.0.1:8002/__version__')
+        response = requests.get('http://nginx/__version__')
         response.raise_for_status()
     except Exception as e:
         status = f'Failed to ping web: {e}'
@@ -94,9 +96,11 @@ def memcache():
                 s.close()
 
             memcache_results.append((ip, port, result))
-        if not using_twemproxy and len(memcache_results) < 2:
-            status = ('2+ memcache servers are required. %s available') % len(
-                memcache_results
+        expected_count = settings.MEMCACHE_MIN_SERVER_COUNT
+        if not using_twemproxy and len(memcache_results) < expected_count:
+            status = ('%d+ memcache servers are required. %d available') % (
+                expected_count,
+                len(memcache_results),
             )
             monitor_log.warning(status)
 
@@ -285,6 +289,10 @@ def database():
 
 
 def remotesettings():
+    # TODO: We should be able to check remote settings
+    # connectivity if the credentials are set.
+    if settings.ENV in ('local', 'test'):
+        return '', None
     # check Remote Settings connectivity.
     # Since the blocklist filter task is performed by
     # a worker, and since workers have different network

src/olympia/amo/tests/test_commands.py

@@ -519,7 +519,9 @@ def walk_keys(root, dir_dict):
             os.path.join(settings.STORAGE_ROOT, 'shared_storage/tmp/preview'),
             os.path.join(settings.STORAGE_ROOT, 'shared_storage/tmp/test'),
             os.path.join(settings.STORAGE_ROOT, 'shared_storage/tmp/uploads'),
-            os.path.join(settings.STORAGE_ROOT, 'shared_storage/uploads'),
+            os.path.join(settings.STORAGE_ROOT, 'shared_storage/uploads/addon_icons'),
+            os.path.join(settings.STORAGE_ROOT, 'shared_storage/uploads/previews'),
+            os.path.join(settings.STORAGE_ROOT, 'shared_storage/uploads/userpics'),
         ]
 
         for key in keys:
@@ -723,7 +725,7 @@ def test_monitors_fails_after_specified_attempts(self):
     def test_monitors_succeeds_after_specified_attempts(self):
         succeed_after = 3
 
-        def mock_handler(services):
+        def mock_handler(services, _verbose):
             state = self.mock_execute_checks.call_count >= succeed_after
             return {service: {'state': state} for service in services}
 
@@ -733,7 +735,7 @@ def mock_handler(services):
     def test_monitors_succeeds_after_all_services_are_healthy(self):
         succeed_after = 3
 
-        def mock_handler(_):
+        def mock_handler(_a, _b):
             state = self.mock_execute_checks.call_count >= succeed_after
             return {
                 'database': {'state': state},

src/olympia/amo/tests/test_monitor.py

@@ -110,6 +110,7 @@ def test_remotesettings_success(self):
         obtained, _ = monitors.remotesettings()
         assert obtained == ''
 
+    @override_settings(ENV='production')
     def test_remotesettings_bad_credentials(self):
         responses.add(
             responses.GET,
@@ -126,6 +127,7 @@ def test_remotesettings_bad_credentials(self):
         obtained, _ = monitors.remotesettings()
         assert 'Invalid credentials' in obtained
 
+    @override_settings(ENV='production')
     def test_remotesettings_fail(self):
         responses.add(
             responses.GET,
@@ -164,7 +166,7 @@ def test_cinder_fail(self):
     def test_localdev_web_fail(self):
         responses.add(
             responses.GET,
-            'http://127.0.0.1:8002/__version__',
+            'http://nginx/__version__',
             status=500,
         )
         status, _ = monitors.localdev_web()
@@ -173,7 +175,7 @@ def test_localdev_web_fail(self):
     def test_localdev_web_success(self):
         responses.add(
             responses.GET,
-            'http://127.0.0.1:8002/__version__',
+            'http://nginx/__version__',
             status=200,
         )
         status, _ = monitors.localdev_web()

src/olympia/amo/urls.py

@@ -34,6 +34,7 @@
     re_path(r'^services/', include(shared_services_api_patterns)),
     re_path(r'^__version__$', views.version, name='version.json'),
     re_path(r'^__heartbeat__$', views.front_heartbeat, name='amo.front_heartbeat'),
+    re_path(r'^__healthcheck__$', views.healthcheck, name='amo.healthcheck'),
     re_path(
         r'^opensearch\.xml$',
         TemplateView.as_view(