diff --git a/src/statistic/work_load.py b/src/statistic/work_load.py index e82c3a2bd..5d5e12082 100644 --- a/src/statistic/work_load.py +++ b/src/statistic/work_load.py @@ -1,6 +1,7 @@ """ generate workload statistics """ + import logging import os import sys @@ -10,20 +11,23 @@ import psutil import config -from storage.db_interface_stats import StatsUpdateDbInterface +from storage.redis_status_interface import RedisStatusInterface from version import __VERSION__ class WorkLoadStatistic: def __init__(self, component): self.component = component - self.db = StatsUpdateDbInterface() + self.status = RedisStatusInterface() self.platform_information = self._get_platform_information() logging.debug(f'{self.component}: Online') def shutdown(self): logging.debug(f'{self.component}: shutting down -> set offline message') - self.db.update_statistic(self.component, {'status': 'offline', 'last_update': time()}) + self.status.set_component_status( + self.component, + {'name': self.component, 'status': 'offline', 'last_update': time()}, + ) def update(self, unpacking_workload=None, analysis_workload=None, compare_workload=None): stats = { @@ -39,9 +43,10 @@ def update(self, unpacking_workload=None, analysis_workload=None, compare_worklo stats['analysis'] = analysis_workload if compare_workload: stats['compare'] = compare_workload - self.db.update_statistic(self.component, stats) + self.status.set_component_status(self.component, stats) - def _get_system_information(self): + @staticmethod + def _get_system_information(): memory_usage = psutil.virtual_memory() try: disk_usage = psutil.disk_usage(config.backend.firmware_file_storage_directory) diff --git a/src/storage/redis_status_interface.py b/src/storage/redis_status_interface.py index d738ab35f..ac1eb3dc9 100644 --- a/src/storage/redis_status_interface.py +++ b/src/storage/redis_status_interface.py @@ -1,14 +1,35 @@ +from __future__ import annotations + import json from storage.redis_interface import RedisInterface ANALYSIS_STATUS_REDIS_KEY = '__fact_analysis_status__' +COMPONENT_STATUS_REDIS_KEYS = { + 'frontend': '__fact_frontend_status__', + 'backend': '__fact_backend_status__', + 'database': '__fact_database_status__', +} class RedisStatusInterface: def __init__(self): self.redis = RedisInterface() + def set_component_status(self, component: str, status: dict): + status['_id'] = component # for backwards compatibility + if not (key := COMPONENT_STATUS_REDIS_KEYS.get(component)): + raise ValueError(f'Unknown component {component}') + self.redis.set(key, json.dumps(status)) + + def get_component_status(self, component: str) -> dict | None: + if not (key := COMPONENT_STATUS_REDIS_KEYS.get(component)): + raise ValueError(f'Unknown component {component}') + try: + return json.loads(self.redis.get(key, delete=False)) + except TypeError: + return None + def set_analysis_status(self, status: dict): self.redis.set(ANALYSIS_STATUS_REDIS_KEY, json.dumps(status)) diff --git a/src/test/integration/statistic/test_work_load.py b/src/test/integration/statistic/test_work_load.py index 41f8c47b5..217d7623a 100644 --- a/src/test/integration/statistic/test_work_load.py +++ b/src/test/integration/statistic/test_work_load.py @@ -5,11 +5,12 @@ from statistic.work_load import WorkLoadStatistic from storage.db_interface_stats import StatsDbViewer +from storage.redis_status_interface import RedisStatusInterface @pytest.fixture def workload_stat(): - workload_stat = WorkLoadStatistic(component='test') + workload_stat = WorkLoadStatistic(component='frontend') yield workload_stat workload_stat.shutdown() @@ -20,10 +21,11 @@ def stats_db(): @pytest.mark.usefixtures('database_interfaces') -def test_update_workload_statistic(workload_stat, stats_db): +def test_update_workload_statistic(workload_stat): workload_stat.update() - result = stats_db.get_statistic('test') - assert result['name'] == 'test', 'name not set' + status = RedisStatusInterface() + result = status.get_component_status('frontend') + assert result['name'] == 'frontend', 'name not set' assert isclose(time(), result['last_update'], abs_tol=0.1), 'timestamp not valid' assert isinstance(result['platform'], dict), 'platform is not a dict' assert isinstance(result['system'], dict), 'system is not a dict' diff --git a/src/test/integration/storage/test_redis_status_interface.py b/src/test/integration/storage/test_redis_status_interface.py index 12bc3bbdd..cba8a6115 100644 --- a/src/test/integration/storage/test_redis_status_interface.py +++ b/src/test/integration/storage/test_redis_status_interface.py @@ -13,8 +13,15 @@ def status_interface(): interface.redis.redis.flushdb() -def test_redis_status_interface(status_interface): +def test_analysis_status(status_interface): assert status_interface.get_analysis_status() == EMPTY_RESULT status_interface.set_analysis_status(TEST_RESULT) assert status_interface.get_analysis_status() == TEST_RESULT + + +def test_component_status(status_interface): + assert status_interface.get_component_status('frontend') is None + + status_interface.set_component_status('frontend', TEST_RESULT) + assert status_interface.get_component_status('frontend') == TEST_RESULT diff --git a/src/test/unit/conftest.py b/src/test/unit/conftest.py index ae87d02b7..d941719dd 100644 --- a/src/test/unit/conftest.py +++ b/src/test/unit/conftest.py @@ -117,6 +117,9 @@ def set_analysis_status(self, status: dict): def get_analysis_status(self): return self._status + def get_component_status(self, component): + return {'name': component, 'status': 'foo'} + class WebInterfaceUnitTestConfig(BaseModel): """A class configuring the :py:func:`web_frontend` fixture.""" diff --git a/src/test/unit/web_interface/rest/test_rest_status.py b/src/test/unit/web_interface/rest/test_rest_status.py index 4f1ec9ddf..7a03a34e0 100644 --- a/src/test/unit/web_interface/rest/test_rest_status.py +++ b/src/test/unit/web_interface/rest/test_rest_status.py @@ -1,27 +1,27 @@ import pytest -from test.common_helper import CommonDatabaseMock +from test.unit.conftest import StatusInterfaceMock BACKEND_STATS = {'system': {'cpu_percentage': 13.37}, 'analysis': {'current_analyses': [None, None]}} -class StatisticDbViewerMock(CommonDatabaseMock): +class ComponentStatusMock(StatusInterfaceMock): down = None - def get_statistic(self, identifier): - return None if self.down or identifier != 'backend' else BACKEND_STATS + def get_component_status(self, component): + return None if self.down or component != 'backend' else BACKEND_STATS -@pytest.mark.WebInterfaceUnitTestConfig(database_mock_class=StatisticDbViewerMock) -class TestRestFirmware: +@pytest.mark.WebInterfaceUnitTestConfig(status_mock_class=ComponentStatusMock) +class TestRestStatus: def test_empty_uid(self, test_client): - StatisticDbViewerMock.down = False + ComponentStatusMock.down = False result = test_client.get('/rest/status').json assert result['status'] == 0 assert result['system_status'] == {'backend': BACKEND_STATS, 'database': None, 'frontend': None} def test_empty_result(self, test_client): - StatisticDbViewerMock.down = True + ComponentStatusMock.down = True result = test_client.get('/rest/status').json assert 'Cannot get FACT component status' in result['error_message'] diff --git a/src/test/unit/web_interface/test_app_ajax_routes.py b/src/test/unit/web_interface/test_app_ajax_routes.py index c11e193f1..9c14aa15c 100644 --- a/src/test/unit/web_interface/test_app_ajax_routes.py +++ b/src/test/unit/web_interface/test_app_ajax_routes.py @@ -70,10 +70,12 @@ def test_ajax_get_system_stats_error(self, test_client): assert result['number_of_running_analyses'] == 'n/a' def test_ajax_system_health(self, test_client): - DbMock.get_stats_list = lambda *_: [{'foo': 'bar'}] result = test_client.get('/ajax/system_health').json + assert 'analysisStatus' in result + assert result['analysisStatus'] == {'current_analyses': {}, 'recently_finished_analyses': {}} assert 'systemHealth' in result - assert result['systemHealth'] == [{'foo': 'bar'}] + assert len(result['systemHealth']) == 3 + assert all(isinstance(d, dict) for d in result['systemHealth']) def test_ajax_get_hex_preview(self, test_client): DbMock.peek_in_binary = lambda *_: b'foobar' diff --git a/src/web_interface/components/ajax_routes.py b/src/web_interface/components/ajax_routes.py index a497c7ee7..be0303588 100644 --- a/src/web_interface/components/ajax_routes.py +++ b/src/web_interface/components/ajax_routes.py @@ -143,8 +143,9 @@ def get_system_stats(self): @roles_accepted(*PRIVILEGES['status']) @AppRoute('/ajax/system_health', GET) def get_system_health_update(self): + components = ('backend', 'frontend', 'database') return { - 'systemHealth': self.db.stats_viewer.get_stats_list('backend', 'frontend', 'database'), + 'systemHealth': [status for c in components if (status := self.status.get_component_status(c))], 'analysisStatus': self.status.get_analysis_status(), } diff --git a/src/web_interface/rest/rest_status.py b/src/web_interface/rest/rest_status.py index 59e388b2b..b97d09b56 100644 --- a/src/web_interface/rest/rest_status.py +++ b/src/web_interface/rest/rest_status.py @@ -22,7 +22,7 @@ def get(self): components = ['frontend', 'database', 'backend'] status = {} for component in components: - status[component] = self.db.stats_viewer.get_statistic(component) + status[component] = self.status.get_component_status(component) plugins = self.intercom.get_available_analysis_plugins() diff --git a/src/web_interface/static/js/system_health.js b/src/web_interface/static/js/system_health.js index babc029cf..55d010c4a 100644 --- a/src/web_interface/static/js/system_health.js +++ b/src/web_interface/static/js/system_health.js @@ -5,7 +5,7 @@ async function getSystemHealthData() { async function updateSystemHealth() { getSystemHealthData().then(data => data.systemHealth.map(entry => { - const statusElement = document.getElementById(`${entry._id}-status`); + const statusElement = document.getElementById(`${entry.name}-status`); statusElement.innerText = entry.status; if (entry.status === "offline") { statusElement.classList.add('text-danger'); @@ -14,13 +14,13 @@ async function updateSystemHealth() { } statusElement.classList.remove('text-danger'); statusElement.classList.add('text-success'); - document.getElementById(`${entry._id}-os`).innerText = entry.platform.os; - document.getElementById(`${entry._id}-python`).innerText = entry.platform.python; - document.getElementById(`${entry._id}-version`).innerText = entry.platform.fact_version; - document.getElementById(`${entry._id}-cpu`).innerText = `${entry.system.cpu_cores} cores (${entry.system.virtual_cpu_cores} threads) @ ${entry.system.cpu_percentage}%`; - updateProgressBarElement(`${entry._id}-memory`, entry.system.memory_percent, entry.system.memory_used, entry.system.memory_total); - updateProgressBarElement(`${entry._id}-disk`, entry.system.disk_percent, entry.system.disk_used, entry.system.disk_total); - if (entry._id === "backend") { + document.getElementById(`${entry.name}-os`).innerText = entry.platform.os; + document.getElementById(`${entry.name}-python`).innerText = entry.platform.python; + document.getElementById(`${entry.name}-version`).innerText = entry.platform.fact_version; + document.getElementById(`${entry.name}-cpu`).innerText = `${entry.system.cpu_cores} cores (${entry.system.virtual_cpu_cores} threads) @ ${entry.system.cpu_percentage}%`; + updateProgressBarElement(`${entry.name}-memory`, entry.system.memory_percent, entry.system.memory_used, entry.system.memory_total); + updateProgressBarElement(`${entry.name}-disk`, entry.system.disk_percent, entry.system.disk_used, entry.system.disk_total); + if (entry.name === "backend") { const queueElement = document.getElementById("backend-unpacking-queue"); if (entry.unpacking.unpacking_queue > 500) { queueElement.classList.add("text-warning");