feat: moved system status data from db to redis

src/statistic/work_load.py

@@ -1,6 +1,7 @@
 """
 generate workload statistics
 """
+
 import logging
 import os
 import sys
@@ -10,20 +11,23 @@
 import psutil
 
 import config
-from storage.db_interface_stats import StatsUpdateDbInterface
+from storage.redis_status_interface import RedisStatusInterface
 from version import __VERSION__
 
 
 class WorkLoadStatistic:
     def __init__(self, component):
         self.component = component
-        self.db = StatsUpdateDbInterface()
+        self.status = RedisStatusInterface()
         self.platform_information = self._get_platform_information()
         logging.debug(f'{self.component}: Online')
 
     def shutdown(self):
         logging.debug(f'{self.component}: shutting down -> set offline message')
-        self.db.update_statistic(self.component, {'status': 'offline', 'last_update': time()})
+        self.status.set_component_status(
+            self.component,
+            {'name': self.component, 'status': 'offline', 'last_update': time()},
+        )
 
     def update(self, unpacking_workload=None, analysis_workload=None, compare_workload=None):
         stats = {
@@ -39,9 +43,10 @@ def update(self, unpacking_workload=None, analysis_workload=None, compare_worklo
             stats['analysis'] = analysis_workload
         if compare_workload:
             stats['compare'] = compare_workload
-        self.db.update_statistic(self.component, stats)
+        self.status.set_component_status(self.component, stats)
 
-    def _get_system_information(self):
+    @staticmethod
+    def _get_system_information():
         memory_usage = psutil.virtual_memory()
         try:
             disk_usage = psutil.disk_usage(config.backend.firmware_file_storage_directory)

src/storage/redis_status_interface.py

@@ -1,14 +1,35 @@
+from __future__ import annotations
+
 import json
 
 from storage.redis_interface import RedisInterface
 
 ANALYSIS_STATUS_REDIS_KEY = '__fact_analysis_status__'
+COMPONENT_STATUS_REDIS_KEYS = {
+    'frontend': '__fact_frontend_status__',
+    'backend': '__fact_backend_status__',
+    'database': '__fact_database_status__',
+}
 
 
 class RedisStatusInterface:
     def __init__(self):
         self.redis = RedisInterface()
 
+    def set_component_status(self, component: str, status: dict):
+        status['_id'] = component  # for backwards compatibility
+        if not (key := COMPONENT_STATUS_REDIS_KEYS.get(component)):
+            raise ValueError(f'Unknown component {component}')
+        self.redis.set(key, json.dumps(status))
+
+    def get_component_status(self, component: str) -> dict | None:
+        if not (key := COMPONENT_STATUS_REDIS_KEYS.get(component)):
+            raise ValueError(f'Unknown component {component}')
+        try:
+            return json.loads(self.redis.get(key, delete=False))
+        except TypeError:
+            return None
+
     def set_analysis_status(self, status: dict):
         self.redis.set(ANALYSIS_STATUS_REDIS_KEY, json.dumps(status))
 

src/test/integration/statistic/test_work_load.py

@@ -5,11 +5,12 @@
 
 from statistic.work_load import WorkLoadStatistic
 from storage.db_interface_stats import StatsDbViewer
+from storage.redis_status_interface import RedisStatusInterface
 
 
 @pytest.fixture
 def workload_stat():
-    workload_stat = WorkLoadStatistic(component='test')
+    workload_stat = WorkLoadStatistic(component='frontend')
     yield workload_stat
     workload_stat.shutdown()
 
@@ -20,10 +21,11 @@ def stats_db():
 
 
 @pytest.mark.usefixtures('database_interfaces')
-def test_update_workload_statistic(workload_stat, stats_db):
+def test_update_workload_statistic(workload_stat):
     workload_stat.update()
-    result = stats_db.get_statistic('test')
-    assert result['name'] == 'test', 'name not set'
+    status = RedisStatusInterface()
+    result = status.get_component_status('frontend')
+    assert result['name'] == 'frontend', 'name not set'
     assert isclose(time(), result['last_update'], abs_tol=0.1), 'timestamp not valid'
     assert isinstance(result['platform'], dict), 'platform is not a dict'
     assert isinstance(result['system'], dict), 'system is not a dict'

src/test/integration/storage/test_redis_status_interface.py

@@ -13,8 +13,15 @@ def status_interface():
     interface.redis.redis.flushdb()
 
 
-def test_redis_status_interface(status_interface):
+def test_analysis_status(status_interface):
     assert status_interface.get_analysis_status() == EMPTY_RESULT
 
     status_interface.set_analysis_status(TEST_RESULT)
     assert status_interface.get_analysis_status() == TEST_RESULT
+
+
+def test_component_status(status_interface):
+    assert status_interface.get_component_status('frontend') is None
+
+    status_interface.set_component_status('frontend', TEST_RESULT)
+    assert status_interface.get_component_status('frontend') == TEST_RESULT

src/test/unit/conftest.py

@@ -117,6 +117,9 @@ def set_analysis_status(self, status: dict):
     def get_analysis_status(self):
         return self._status
 
+    def get_component_status(self, component):
+        return {'name': component, 'status': 'foo'}
+
 
 class WebInterfaceUnitTestConfig(BaseModel):
     """A class configuring the :py:func:`web_frontend` fixture."""

src/test/unit/web_interface/rest/test_rest_status.py

@@ -1,27 +1,27 @@
 import pytest
 
-from test.common_helper import CommonDatabaseMock
+from test.unit.conftest import StatusInterfaceMock
 
 BACKEND_STATS = {'system': {'cpu_percentage': 13.37}, 'analysis': {'current_analyses': [None, None]}}
 
 
-class StatisticDbViewerMock(CommonDatabaseMock):
+class ComponentStatusMock(StatusInterfaceMock):
     down = None
 
-    def get_statistic(self, identifier):
-        return None if self.down or identifier != 'backend' else BACKEND_STATS
+    def get_component_status(self, component):
+        return None if self.down or component != 'backend' else BACKEND_STATS
 
 
-@pytest.mark.WebInterfaceUnitTestConfig(database_mock_class=StatisticDbViewerMock)
-class TestRestFirmware:
+@pytest.mark.WebInterfaceUnitTestConfig(status_mock_class=ComponentStatusMock)
+class TestRestStatus:
     def test_empty_uid(self, test_client):
-        StatisticDbViewerMock.down = False
+        ComponentStatusMock.down = False
         result = test_client.get('/rest/status').json
 
         assert result['status'] == 0
         assert result['system_status'] == {'backend': BACKEND_STATS, 'database': None, 'frontend': None}
 
     def test_empty_result(self, test_client):
-        StatisticDbViewerMock.down = True
+        ComponentStatusMock.down = True
         result = test_client.get('/rest/status').json
         assert 'Cannot get FACT component status' in result['error_message']

src/test/unit/web_interface/test_app_ajax_routes.py

@@ -70,10 +70,12 @@ def test_ajax_get_system_stats_error(self, test_client):
         assert result['number_of_running_analyses'] == 'n/a'
 
     def test_ajax_system_health(self, test_client):
-        DbMock.get_stats_list = lambda *_: [{'foo': 'bar'}]
         result = test_client.get('/ajax/system_health').json
+        assert 'analysisStatus' in result
+        assert result['analysisStatus'] == {'current_analyses': {}, 'recently_finished_analyses': {}}
         assert 'systemHealth' in result
-        assert result['systemHealth'] == [{'foo': 'bar'}]
+        assert len(result['systemHealth']) == 3
+        assert all(isinstance(d, dict) for d in result['systemHealth'])
 
     def test_ajax_get_hex_preview(self, test_client):
         DbMock.peek_in_binary = lambda *_: b'foobar'

src/web_interface/components/ajax_routes.py

@@ -143,8 +143,9 @@ def get_system_stats(self):
     @roles_accepted(*PRIVILEGES['status'])
     @AppRoute('/ajax/system_health', GET)
     def get_system_health_update(self):
+        components = ('backend', 'frontend', 'database')
         return {
-            'systemHealth': self.db.stats_viewer.get_stats_list('backend', 'frontend', 'database'),
+            'systemHealth': [status for c in components if (status := self.status.get_component_status(c))],
             'analysisStatus': self.status.get_analysis_status(),
         }
 

src/web_interface/rest/rest_status.py

@@ -22,7 +22,7 @@ def get(self):
         components = ['frontend', 'database', 'backend']
         status = {}
         for component in components:
-            status[component] = self.db.stats_viewer.get_statistic(component)
+            status[component] = self.status.get_component_status(component)
 
         plugins = self.intercom.get_available_analysis_plugins()
 

src/web_interface/static/js/system_health.js

@@ -5,7 +5,7 @@ async function getSystemHealthData() {
 
 async function updateSystemHealth() {
     getSystemHealthData().then(data => data.systemHealth.map(entry => {
-        const statusElement = document.getElementById(`${entry._id}-status`);
+        const statusElement = document.getElementById(`${entry.name}-status`);
         statusElement.innerText = entry.status;
         if (entry.status === "offline") {
             statusElement.classList.add('text-danger');
@@ -14,13 +14,13 @@ async function updateSystemHealth() {
         }
         statusElement.classList.remove('text-danger');
         statusElement.classList.add('text-success');
-        document.getElementById(`${entry._id}-os`).innerText = entry.platform.os;
-        document.getElementById(`${entry._id}-python`).innerText = entry.platform.python;
-        document.getElementById(`${entry._id}-version`).innerText = entry.platform.fact_version;
-        document.getElementById(`${entry._id}-cpu`).innerText = `${entry.system.cpu_cores} cores (${entry.system.virtual_cpu_cores} threads) @ ${entry.system.cpu_percentage}%`;
-        updateProgressBarElement(`${entry._id}-memory`, entry.system.memory_percent, entry.system.memory_used, entry.system.memory_total);
-        updateProgressBarElement(`${entry._id}-disk`, entry.system.disk_percent, entry.system.disk_used, entry.system.disk_total);
-        if (entry._id === "backend") {
+        document.getElementById(`${entry.name}-os`).innerText = entry.platform.os;
+        document.getElementById(`${entry.name}-python`).innerText = entry.platform.python;
+        document.getElementById(`${entry.name}-version`).innerText = entry.platform.fact_version;
+        document.getElementById(`${entry.name}-cpu`).innerText = `${entry.system.cpu_cores} cores (${entry.system.virtual_cpu_cores} threads) @ ${entry.system.cpu_percentage}%`;
+        updateProgressBarElement(`${entry.name}-memory`, entry.system.memory_percent, entry.system.memory_used, entry.system.memory_total);
+        updateProgressBarElement(`${entry.name}-disk`, entry.system.disk_percent, entry.system.disk_used, entry.system.disk_total);
+        if (entry.name === "backend") {
             const queueElement = document.getElementById("backend-unpacking-queue");
             if (entry.unpacking.unpacking_queue > 500) {
                 queueElement.classList.add("text-warning");