diff --git a/src/storage/db_interface_common.py b/src/storage/db_interface_common.py index 2f6228223..ecc0dab09 100644 --- a/src/storage/db_interface_common.py +++ b/src/storage/db_interface_common.py @@ -214,7 +214,7 @@ def get_file_tree_path(self, uid: str, root_uid: str | None = None) -> list[list return self.get_file_tree_path_for_uid_list([uid], root_uid=root_uid).get(uid, []) def get_file_tree_path_for_uid_list( - self, uid_list: list[str], root_uid: str | None = None + self, uid_list: Iterable[str], root_uid: str | None = None ) -> dict[str, list[list[str]]]: """ Generate all file paths for a list of UIDs `uid_list`. A path is a list of UIDs representing the path from root diff --git a/src/storage/db_interface_frontend.py b/src/storage/db_interface_frontend.py index 78dd506ee..9613ec2ea 100644 --- a/src/storage/db_interface_frontend.py +++ b/src/storage/db_interface_frontend.py @@ -3,7 +3,7 @@ import os.path import re from pathlib import Path -from typing import Any, NamedTuple, Optional +from typing import Any, Iterable, NamedTuple, Optional from sqlalchemy import Column, Integer, cast, func, or_, select @@ -75,26 +75,52 @@ def _get_hid_fo(self, uid: str, root_uid: str | None = None) -> str | None: # --- "nice list" --- - def get_data_for_nice_list(self, uid_list: list[str], root_uid: str | None) -> list[dict]: + def _get_some_path_for_uid_list(self, uid_list: Iterable[str], root_uid: str | None = None) -> dict[str, str]: with self.get_read_only_session() as session: - mime_dict = self._get_mime_types_for_uid_list(session, uid_list) - query = select(FileObjectEntry.uid, FileObjectEntry.size, FileObjectEntry.file_name).filter( - FileObjectEntry.uid.in_(uid_list) + query = ( + select(VirtualFilePath.file_uid, VirtualFilePath.file_path) + .filter(VirtualFilePath.file_uid.in_(uid_list)) + .distinct(VirtualFilePath.file_uid) # only one path per file + ) + if root_uid: # if root_uid is set, only return paths contained in that FW + query = query.join(fw_files_table, VirtualFilePath.file_uid == fw_files_table.c.file_uid).filter( + fw_files_table.c.root_uid == root_uid + ) + return {uid: path for uid, path in session.execute(query)} # noqa: C416 # dict() does not work here + + def get_data_for_nice_list( + self, uid_list: Iterable[str], root_uid: str | None, include_vfp: bool = True + ) -> list[dict]: + with self.get_read_only_session() as session: + query = ( + select( + FileObjectEntry.uid, + FileObjectEntry.size, + FileObjectEntry.file_name, + AnalysisEntry.result.op('->>')('mime'), + ) + .filter(FileObjectEntry.uid.in_(uid_list)) + .outerjoin(AnalysisEntry, AnalysisEntry.uid == FileObjectEntry.uid) + .filter(AnalysisEntry.plugin == 'file_type') ) - file_tree_data = self.get_file_tree_path_for_uid_list(uid_list, root_uid=root_uid) + path_dict = self._get_some_path_for_uid_list(uid_list) + nice_list_data = [ { 'uid': uid, 'size': size, - 'file_name': file_name, - 'mime-type': mime_dict.get(uid, 'file-type-plugin/not-run-yet'), - 'current_virtual_path': file_tree_data.get(uid, [[file_name]]), # "orphan fallback" - # FixMe: if orphaned files (i.e. relation FW -> file exists, but there is no path from FW to the - # file through included files) exist, they break the "file tree path" + 'file_name': path_dict.get(uid, file_name), + 'mime-type': mime or 'N/A', } - for uid, size, file_name in session.execute(query) + for uid, size, file_name, mime in session.execute(query) ] - self._replace_uids_in_nice_list(nice_list_data, root_uid) + if include_vfp: + file_tree_data = self.get_file_tree_path_for_uid_list(uid_list, root_uid=root_uid) + for dict_ in nice_list_data: + dict_['current_virtual_path'] = file_tree_data.get(dict_['uid'], [[dict_['file_name']]]) + # FixMe: default = "orphan fallback": if orphaned files exist (i.e. relation FW -> file exists, but + # there is no path from FW to the file through included files), they break the "file tree path" + self._replace_uids_in_nice_list(nice_list_data, root_uid) return nice_list_data def _replace_uids_in_nice_list(self, nice_list_data: list[dict], root_uid: str): diff --git a/src/test/common_helper.py b/src/test/common_helper.py index 8bf93d494..3adf5d28f 100644 --- a/src/test/common_helper.py +++ b/src/test/common_helper.py @@ -208,7 +208,7 @@ def uid_list_exists(self, uid_list): def all_uids_found_in_database(self, uid_list): return True - def get_data_for_nice_list(self, input_data, root_uid): + def get_data_for_nice_list(self, input_data, root_uid, include_vfp=None): return [NICE_LIST_DATA] @staticmethod diff --git a/src/test/integration/storage/test_db_interface_frontend.py b/src/test/integration/storage/test_db_interface_frontend.py index 57df5dae9..6b1880a31 100644 --- a/src/test/integration/storage/test_db_interface_frontend.py +++ b/src/test/integration/storage/test_db_interface_frontend.py @@ -90,6 +90,10 @@ def test_get_data_for_nice_list(frontend_db, backend_db): assert nice_list_data[0]['current_virtual_path'][0] == expected_hid, 'UID should be replaced with HID' assert nice_list_data[1]['current_virtual_path'][0] == f'{expected_hid} | /file/path' + nice_list_data = frontend_db.get_data_for_nice_list(uid_list, uid_list[0], include_vfp=False) + assert len(nice_list_data) == 2 + assert sorted(nice_list_data[0].keys()) == ['file_name', 'mime-type', 'size', 'uid'] + def test_get_device_class_list(frontend_db, backend_db): insert_test_fw(backend_db, 'fw1', device_class='class1') diff --git a/src/web_interface/components/ajax_routes.py b/src/web_interface/components/ajax_routes.py index a497c7ee7..755314ce8 100644 --- a/src/web_interface/components/ajax_routes.py +++ b/src/web_interface/components/ajax_routes.py @@ -118,13 +118,16 @@ def ajax_get_hex_preview(self, uid: str, offset: int, length: int) -> str: def ajax_get_summary(self, uid, selected_analysis): with get_shared_session(self.db.frontend) as frontend_db: firmware = frontend_db.get_object(uid, analysis_filter=selected_analysis) - summary_of_included_files = frontend_db.get_summary(firmware, selected_analysis) + summary_of_included_files = frontend_db.get_summary(firmware, selected_analysis) or {} + all_uids = {uid for uid_list in summary_of_included_files.values() for uid in uid_list} + nice_list_data = self.db.frontend.get_data_for_nice_list(all_uids, uid, False) root_uid = uid if isinstance(firmware, Firmware) else frontend_db.get_root_uid(uid) return render_template( 'summary.html', summary_of_included_files=summary_of_included_files, root_uid=root_uid, selected_analysis=selected_analysis, + nice_list_data=nice_list_data, ) @roles_accepted(*PRIVILEGES['status']) diff --git a/src/web_interface/components/jinja_filter.py b/src/web_interface/components/jinja_filter.py index 11fa4b2cf..9c3b1fa7f 100644 --- a/src/web_interface/components/jinja_filter.py +++ b/src/web_interface/components/jinja_filter.py @@ -70,12 +70,23 @@ def _filter_replace_uid_with_hid_link(self, input_data, root_uid=None): content = content.replace(uid, f'{hid}') return content - def _filter_nice_uid_list(self, uids, root_uid=None, selected_analysis=None, filename_only=False): + def _filter_nice_uid_list( + self, + uids, + root_uid=None, + selected_analysis=None, + filename_only=False, + nice_list_data: list[dict] | None = None, + ): root_uid = none_to_none(root_uid) if not is_list_of_uids(uids): return uids - analyzed_uids = self.db.frontend.get_data_for_nice_list(uids, root_uid) + if nice_list_data: + # if multiple "nice lists" are rendered at once, nice_list_data should be prefetched for all files + analyzed_uids = [d for d in nice_list_data if d['uid'] in uids] + else: + analyzed_uids = self.db.frontend.get_data_for_nice_list(uids, root_uid, include_vfp=not filename_only) number_of_unanalyzed_files = len(uids) - len(analyzed_uids) first_item = analyzed_uids.pop(0) diff --git a/src/web_interface/templates/generic_view/file_object_macros.html b/src/web_interface/templates/generic_view/file_object_macros.html index c11c2f90a..921b40954 100644 --- a/src/web_interface/templates/generic_view/file_object_macros.html +++ b/src/web_interface/templates/generic_view/file_object_macros.html @@ -1,7 +1,7 @@ {% macro file_information_span(file_object, root_uid, selected_analysis=None, filename_only=False) %} {% set selected_analysis_url = selected_analysis + '/' if selected_analysis else '' %} - {% if filename_only %} + {% if filename_only or "current_virtual_path" not in file_object %}
{{ file_object.file_name | safe }}
{% else %}
{{ file_object.current_virtual_path[0] | safe }}
diff --git a/src/web_interface/templates/summary.html b/src/web_interface/templates/summary.html index 34643e3da..cf6574b19 100644 --- a/src/web_interface/templates/summary.html +++ b/src/web_interface/templates/summary.html @@ -17,7 +17,7 @@ {{ item | wordwrap(width=60, break_long_words=True) }} - {{ summary_of_included_files[item] | nice_uid_list(root_uid=root_uid, selected_analysis=selected_analysis) | safe }} + {{ summary_of_included_files[item] | nice_uid_list(root_uid=root_uid, selected_analysis=selected_analysis, nice_list_data=nice_list_data) | safe }} {% endfor %}