From 4d1a825353f9450c8fbe7e194990a5eaaab0c304 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Mon, 27 Jan 2025 15:58:05 +0100 Subject: [PATCH 01/58] refactor: Introduce generator class to handle global flow --- src/opossum_lib/cli.py | 58 +++------------------ src/opossum_lib/opossum_generator.py | 78 ++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 52 deletions(-) create mode 100644 src/opossum_lib/opossum_generator.py diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 53b2521c..0687a7ac 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -5,19 +5,9 @@ # SPDX-License-Identifier: Apache-2.0 -import logging -import sys -from pathlib import Path - import click -from opossum_lib.opossum.file_generation import OpossumFileWriter -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.read_opossum_file import read_opossum_file -from opossum_lib.scancode.convert_scancode_to_opossum import ( - convert_scancode_file_to_opossum, -) -from opossum_lib.spdx.convert_to_opossum import convert_spdx_to_opossum_information +from opossum_lib.opossum_generator import OpossumGenerator @click.group() @@ -73,49 +63,13 @@ def generate( - ScanCode - Opossum """ - validate_input_and_exit_on_error(spdx_files, scancode_json_files, opossum_files) - opossum_file_content = convert_after_valid_input( - spdx_files, scancode_json_files, opossum_files - ) - - if not outfile.endswith(".opossum"): - outfile += ".opossum" - - if Path.is_file(Path(outfile)): - logging.warning(f"{outfile} already exists and will be overwritten.") - - OpossumFileWriter.write_opossum_information_to_file( - opossum_file_content, Path(outfile) + OpossumGenerator.generate( + spdx_files=spdx_files, + opossum_files=opossum_files, + scancode_json_files=scancode_json_files, + outfile=outfile, ) -def validate_input_and_exit_on_error( - spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] -) -> None: - total_number_of_files = ( - len(spdx_files) + len(scancode_json_files) + len(opossum_files) - ) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) - - -def convert_after_valid_input( - spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] -) -> OpossumFileContent: - if len(spdx_files) == 1: - spdx_input_file = spdx_files[0] - return convert_spdx_to_opossum_information(spdx_input_file) - elif len(scancode_json_files) == 1: - scancode_json_input_file = scancode_json_files[0] - return convert_scancode_file_to_opossum(scancode_json_input_file) - else: - opossum_input_file = opossum_files[0] - return read_opossum_file(opossum_input_file) - - if __name__ == "__main__": opossum_file() diff --git a/src/opossum_lib/opossum_generator.py b/src/opossum_lib/opossum_generator.py new file mode 100644 index 00000000..67b07ed1 --- /dev/null +++ b/src/opossum_lib/opossum_generator.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import sys +from pathlib import Path + +from opossum_lib.opossum.file_generation import OpossumFileWriter +from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.opossum.read_opossum_file import read_opossum_file +from opossum_lib.scancode.convert_scancode_to_opossum import ( + convert_scancode_file_to_opossum, +) +from opossum_lib.spdx.convert_to_opossum import convert_spdx_to_opossum_information + + +class OpossumGenerator: + @staticmethod + def generate( + spdx_files: list[str], + scancode_json_files: list[str], + opossum_files: list[str], + outfile: str, + ) -> None: + """ + Generate an Opossum file from various other file formats. + + \b + Currently supported input formats: + - SPDX + - ScanCode + - Opossum + """ + OpossumGenerator.validate_input_and_exit_on_error( + spdx_files, scancode_json_files, opossum_files + ) + opossum_file_content = OpossumGenerator.convert_after_valid_input( + spdx_files, scancode_json_files, opossum_files + ) + + if not outfile.endswith(".opossum"): + outfile += ".opossum" + + if Path.is_file(Path(outfile)): + logging.warning(f"{outfile} already exists and will be overwritten.") + + OpossumFileWriter.write_opossum_information_to_file( + opossum_file_content, Path(outfile) + ) + + @staticmethod + def validate_input_and_exit_on_error( + spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] + ) -> None: + total_number_of_files = ( + len(spdx_files) + len(scancode_json_files) + len(opossum_files) + ) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) + + @staticmethod + def convert_after_valid_input( + spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] + ) -> OpossumFileContent: + if len(spdx_files) == 1: + spdx_input_file = spdx_files[0] + return convert_spdx_to_opossum_information(spdx_input_file) + elif len(scancode_json_files) == 1: + scancode_json_input_file = scancode_json_files[0] + return convert_scancode_file_to_opossum(scancode_json_input_file) + else: + opossum_input_file = opossum_files[0] + return read_opossum_file(opossum_input_file) From fee2d18346f41ae4320018442a375c1c3d4babb2 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Mon, 27 Jan 2025 16:06:52 +0100 Subject: [PATCH 02/58] refactor: Introduce argument class --- src/opossum_lib/cli.py | 12 +++-- src/opossum_lib/opossum_generator.py | 79 +++++++++++++++------------- 2 files changed, 49 insertions(+), 42 deletions(-) diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 0687a7ac..eeb59933 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -7,7 +7,7 @@ import click -from opossum_lib.opossum_generator import OpossumGenerator +from opossum_lib.opossum_generator import OpossumGenerationArguments, OpossumGenerator @click.group() @@ -64,10 +64,12 @@ def generate( - Opossum """ OpossumGenerator.generate( - spdx_files=spdx_files, - opossum_files=opossum_files, - scancode_json_files=scancode_json_files, - outfile=outfile, + opossum_generation_arguments=OpossumGenerationArguments( + spdx_files=spdx_files, + opossum_files=opossum_files, + scancode_json_files=scancode_json_files, + outfile=outfile, + ) ) diff --git a/src/opossum_lib/opossum_generator.py b/src/opossum_lib/opossum_generator.py index 67b07ed1..cea52df6 100644 --- a/src/opossum_lib/opossum_generator.py +++ b/src/opossum_lib/opossum_generator.py @@ -6,6 +6,8 @@ import sys from pathlib import Path +from pydantic import BaseModel + from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.opossum.read_opossum_file import read_opossum_file @@ -15,14 +17,36 @@ from opossum_lib.spdx.convert_to_opossum import convert_spdx_to_opossum_information +class OpossumGenerationArguments(BaseModel): + spdx_files: list[str] + scancode_json_files: list[str] + opossum_files: list[str] + outfile: str + + def validate_input_and_exit_on_error(self) -> None: + total_number_of_files = ( + len(self.spdx_files) + + len(self.scancode_json_files) + + len(self.opossum_files) + ) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) + + def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: + if not self.outfile.endswith(".opossum"): + self.outfile += ".opossum" + + if Path.is_file(Path(self.outfile)): + logging.warning(f"{self.outfile} already exists and will be overwritten.") + + class OpossumGenerator: @staticmethod - def generate( - spdx_files: list[str], - scancode_json_files: list[str], - opossum_files: list[str], - outfile: str, - ) -> None: + def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: """ Generate an Opossum file from various other file formats. @@ -32,47 +56,28 @@ def generate( - ScanCode - Opossum """ - OpossumGenerator.validate_input_and_exit_on_error( - spdx_files, scancode_json_files, opossum_files - ) + opossum_generation_arguments.validate_input_and_exit_on_error() opossum_file_content = OpossumGenerator.convert_after_valid_input( - spdx_files, scancode_json_files, opossum_files + opossum_generation_arguments ) - - if not outfile.endswith(".opossum"): - outfile += ".opossum" - - if Path.is_file(Path(outfile)): - logging.warning(f"{outfile} already exists and will be overwritten.") + opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() OpossumFileWriter.write_opossum_information_to_file( - opossum_file_content, Path(outfile) - ) - - @staticmethod - def validate_input_and_exit_on_error( - spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] - ) -> None: - total_number_of_files = ( - len(spdx_files) + len(scancode_json_files) + len(opossum_files) + opossum_file_content, Path(opossum_generation_arguments.outfile) ) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) @staticmethod def convert_after_valid_input( - spdx_files: list[str], scancode_json_files: list[str], opossum_files: list[str] + opossum_generation_arguments: OpossumGenerationArguments, ) -> OpossumFileContent: - if len(spdx_files) == 1: - spdx_input_file = spdx_files[0] + if len(opossum_generation_arguments.spdx_files) == 1: + spdx_input_file = opossum_generation_arguments.spdx_files[0] return convert_spdx_to_opossum_information(spdx_input_file) - elif len(scancode_json_files) == 1: - scancode_json_input_file = scancode_json_files[0] + elif len(opossum_generation_arguments.scancode_json_files) == 1: + scancode_json_input_file = opossum_generation_arguments.scancode_json_files[ + 0 + ] return convert_scancode_file_to_opossum(scancode_json_input_file) else: - opossum_input_file = opossum_files[0] + opossum_input_file = opossum_generation_arguments.opossum_files[0] return read_opossum_file(opossum_input_file) From 3a118fd871fc273cafca1b50aa364b6b7f6a0f42 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Mon, 27 Jan 2025 16:20:31 +0100 Subject: [PATCH 03/58] refactor: Remove unnecessary comment --- src/opossum_lib/opossum_generator.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/opossum_lib/opossum_generator.py b/src/opossum_lib/opossum_generator.py index cea52df6..cdafec3e 100644 --- a/src/opossum_lib/opossum_generator.py +++ b/src/opossum_lib/opossum_generator.py @@ -47,15 +47,6 @@ def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: class OpossumGenerator: @staticmethod def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: - """ - Generate an Opossum file from various other file formats. - - \b - Currently supported input formats: - - SPDX - - ScanCode - - Opossum - """ opossum_generation_arguments.validate_input_and_exit_on_error() opossum_file_content = OpossumGenerator.convert_after_valid_input( opossum_generation_arguments From 168ddc1d4b413ea7d3180e7c0a5f73f3ee33a078 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Mon, 27 Jan 2025 16:22:45 +0100 Subject: [PATCH 04/58] refactor: Introduce core package --- src/opossum_lib/cli.py | 5 ++++- src/opossum_lib/core/__init__.py | 0 src/opossum_lib/{ => core}/opossum_generator.py | 0 src/opossum_lib/{ => core}/opossum_model.py | 0 .../opossum/opossum_file_to_opossum_converter.py | 12 ++++++------ .../scancode/convert_scancode_to_opossum.py | 2 +- tests/core/__init__.py | 0 tests/{ => core}/test_opossum_model.py | 0 .../external_attribution_source_provider.py | 2 +- tests/opossum_model_generators/metadata_provider.py | 2 +- tests/opossum_model_generators/opossum_provider.py | 2 +- tests/opossum_model_generators/package_provider.py | 2 +- tests/opossum_model_generators/resource_provider.py | 2 +- .../scan_results_provider.py | 2 +- .../opossum_model_generators/source_info_provider.py | 2 +- .../test_convert_scancode_to_opossum.py | 2 +- 16 files changed, 19 insertions(+), 16 deletions(-) create mode 100644 src/opossum_lib/core/__init__.py rename src/opossum_lib/{ => core}/opossum_generator.py (100%) rename src/opossum_lib/{ => core}/opossum_model.py (100%) create mode 100644 tests/core/__init__.py rename tests/{ => core}/test_opossum_model.py (100%) diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index eeb59933..4aa04264 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -7,7 +7,10 @@ import click -from opossum_lib.opossum_generator import OpossumGenerationArguments, OpossumGenerator +from opossum_lib.core.opossum_generator import ( + OpossumGenerationArguments, + OpossumGenerator, +) @click.group() diff --git a/src/opossum_lib/core/__init__.py b/src/opossum_lib/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py similarity index 100% rename from src/opossum_lib/opossum_generator.py rename to src/opossum_lib/core/opossum_generator.py diff --git a/src/opossum_lib/opossum_model.py b/src/opossum_lib/core/opossum_model.py similarity index 100% rename from src/opossum_lib/opossum_model.py rename to src/opossum_lib/core/opossum_model.py diff --git a/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py b/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py index 6f9a10d1..97a82b35 100644 --- a/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py @@ -6,12 +6,7 @@ from pathlib import PurePath import opossum_lib.opossum.opossum_file -from opossum_lib.opossum.opossum_file import ( - ExternalAttributionSource as FileExternalAttributionSource, -) -from opossum_lib.opossum.opossum_file import OpossumInformation -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum_model import ( +from opossum_lib.core.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, @@ -25,6 +20,11 @@ SourceInfo, _convert_path_to_str, ) +from opossum_lib.opossum.opossum_file import ( + ExternalAttributionSource as FileExternalAttributionSource, +) +from opossum_lib.opossum.opossum_file import OpossumInformation +from opossum_lib.opossum.opossum_file_content import OpossumFileContent class OpossumFileToOpossumConverter: diff --git a/src/opossum_lib/scancode/convert_scancode_to_opossum.py b/src/opossum_lib/scancode/convert_scancode_to_opossum.py index 7f6e2dc8..62599376 100644 --- a/src/opossum_lib/scancode/convert_scancode_to_opossum.py +++ b/src/opossum_lib/scancode/convert_scancode_to_opossum.py @@ -9,7 +9,7 @@ import uuid from pathlib import PurePath -import opossum_lib.opossum_model as opossum_model +import opossum_lib.core.opossum_model as opossum_model from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME from opossum_lib.scancode.model import File, FileType, Header, ScanCodeData diff --git a/tests/core/__init__.py b/tests/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_opossum_model.py b/tests/core/test_opossum_model.py similarity index 100% rename from tests/test_opossum_model.py rename to tests/core/test_opossum_model.py diff --git a/tests/opossum_model_generators/external_attribution_source_provider.py b/tests/opossum_model_generators/external_attribution_source_provider.py index 5856e1e7..0ff8f184 100644 --- a/tests/opossum_model_generators/external_attribution_source_provider.py +++ b/tests/opossum_model_generators/external_attribution_source_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.opossum_model import ExternalAttributionSource +from opossum_lib.core.opossum_model import ExternalAttributionSource from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/metadata_provider.py b/tests/opossum_model_generators/metadata_provider.py index 49050795..51b97714 100644 --- a/tests/opossum_model_generators/metadata_provider.py +++ b/tests/opossum_model_generators/metadata_provider.py @@ -8,7 +8,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.opossum_model import Metadata +from opossum_lib.core.opossum_model import Metadata from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index 0151cdaa..77e2d461 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -5,8 +5,8 @@ from faker.providers import BaseProvider +from opossum_lib.core.opossum_model import Opossum, ScanResults from opossum_lib.opossum.output_model import OpossumOutputFile -from opossum_lib.opossum_model import Opossum, ScanResults from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, diff --git a/tests/opossum_model_generators/package_provider.py b/tests/opossum_model_generators/package_provider.py index 6e232163..0566bff5 100644 --- a/tests/opossum_model_generators/package_provider.py +++ b/tests/opossum_model_generators/package_provider.py @@ -10,7 +10,7 @@ from faker.providers.misc.en_US import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.opossum_model import OpossumPackage, SourceInfo +from opossum_lib.core.opossum_model import OpossumPackage, SourceInfo from tests.opossum_model_generators.source_info_provider import SourceInfoProvider from tests.util.generator_helpers import entry_or_none, random_list diff --git a/tests/opossum_model_generators/resource_provider.py b/tests/opossum_model_generators/resource_provider.py index dffae127..067de70f 100644 --- a/tests/opossum_model_generators/resource_provider.py +++ b/tests/opossum_model_generators/resource_provider.py @@ -8,7 +8,7 @@ from faker.providers.file import Provider as FileProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.opossum_model import OpossumPackage, Resource, ResourceType +from opossum_lib.core.opossum_model import OpossumPackage, Resource, ResourceType from tests.opossum_model_generators.package_provider import PackageProvider from tests.util.generator_helpers import random_list diff --git a/tests/opossum_model_generators/scan_results_provider.py b/tests/opossum_model_generators/scan_results_provider.py index f62781fb..ba08d319 100644 --- a/tests/opossum_model_generators/scan_results_provider.py +++ b/tests/opossum_model_generators/scan_results_provider.py @@ -8,7 +8,7 @@ from faker.providers.file.en_US import Provider as FileProvider from faker.providers.misc.en_US import Provider as MiscProvider -from opossum_lib.opossum_model import ( +from opossum_lib.core.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/opossum_model_generators/source_info_provider.py b/tests/opossum_model_generators/source_info_provider.py index abd8f17f..a67f96ad 100644 --- a/tests/opossum_model_generators/source_info_provider.py +++ b/tests/opossum_model_generators/source_info_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.opossum_model import SourceInfo +from opossum_lib.core.opossum_model import SourceInfo from tests.util.generator_helpers import entry_or_none diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/test_scancode/test_convert_scancode_to_opossum.py index dcfe7318..91044de1 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/test_scancode/test_convert_scancode_to_opossum.py @@ -6,7 +6,7 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.opossum_model import Resource +from opossum_lib.core.opossum_model import Resource from opossum_lib.scancode.convert_scancode_to_opossum import ( convert_scancode_to_opossum, ) From 1ade466968d5a8119809f29d54dec46983d4e34c Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Mon, 27 Jan 2025 16:36:37 +0100 Subject: [PATCH 05/58] refactor: Introduce file list --- src/opossum_lib/core/input_file.py | 17 ++++++ .../core/opossum_generation_arguments.py | 50 +++++++++++++++++ src/opossum_lib/core/opossum_generator.py | 55 ++++--------------- 3 files changed, 79 insertions(+), 43 deletions(-) create mode 100644 src/opossum_lib/core/input_file.py create mode 100644 src/opossum_lib/core/opossum_generation_arguments.py diff --git a/src/opossum_lib/core/input_file.py b/src/opossum_lib/core/input_file.py new file mode 100644 index 00000000..ca2cdfc8 --- /dev/null +++ b/src/opossum_lib/core/input_file.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 +from enum import Enum, auto + +from pydantic import BaseModel + + +class FileType(Enum): + SPDX = (auto(),) + OPOSSUM = (auto(),) + SCAN_CODE = (auto(),) + + +class InputFile(BaseModel): + path: str + type: FileType diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py new file mode 100644 index 00000000..0344c962 --- /dev/null +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import sys +from pathlib import Path + +from pydantic import BaseModel + +from opossum_lib.core.input_file import FileType, InputFile + + +class OpossumGenerationArguments(BaseModel): + spdx_files: list[str] + scancode_json_files: list[str] + opossum_files: list[str] + outfile: str + + def validate_input_and_exit_on_error(self) -> None: + total_number_of_files = ( + len(self.spdx_files) + + len(self.scancode_json_files) + + len(self.opossum_files) + ) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) + + def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: + if not self.outfile.endswith(".opossum"): + self.outfile += ".opossum" + + if Path.is_file(Path(self.outfile)): + logging.warning(f"{self.outfile} already exists and will be overwritten.") + + def generate_input_file_list(self) -> list[InputFile]: + result = [] + result += [ + InputFile(path=path, type=FileType.SCAN_CODE) + for path in self.scancode_json_files + ] + result += [ + InputFile(path=path, type=FileType.OPOSSUM) for path in self.opossum_files + ] + result += [InputFile(path=path, type=FileType.SPDX) for path in self.spdx_files] + return result diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index cdafec3e..c7609cba 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -2,12 +2,10 @@ # # SPDX-License-Identifier: Apache-2.0 -import logging -import sys from pathlib import Path -from pydantic import BaseModel - +from opossum_lib.core.input_file import FileType, InputFile +from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.opossum.read_opossum_file import read_opossum_file @@ -17,39 +15,13 @@ from opossum_lib.spdx.convert_to_opossum import convert_spdx_to_opossum_information -class OpossumGenerationArguments(BaseModel): - spdx_files: list[str] - scancode_json_files: list[str] - opossum_files: list[str] - outfile: str - - def validate_input_and_exit_on_error(self) -> None: - total_number_of_files = ( - len(self.spdx_files) - + len(self.scancode_json_files) - + len(self.opossum_files) - ) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) - - def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: - if not self.outfile.endswith(".opossum"): - self.outfile += ".opossum" - - if Path.is_file(Path(self.outfile)): - logging.warning(f"{self.outfile} already exists and will be overwritten.") - - class OpossumGenerator: @staticmethod def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: opossum_generation_arguments.validate_input_and_exit_on_error() + input_files = opossum_generation_arguments.generate_input_file_list() opossum_file_content = OpossumGenerator.convert_after_valid_input( - opossum_generation_arguments + input_files ) opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() @@ -59,16 +31,13 @@ def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: @staticmethod def convert_after_valid_input( - opossum_generation_arguments: OpossumGenerationArguments, + input_file_list: list[InputFile], ) -> OpossumFileContent: - if len(opossum_generation_arguments.spdx_files) == 1: - spdx_input_file = opossum_generation_arguments.spdx_files[0] - return convert_spdx_to_opossum_information(spdx_input_file) - elif len(opossum_generation_arguments.scancode_json_files) == 1: - scancode_json_input_file = opossum_generation_arguments.scancode_json_files[ - 0 - ] - return convert_scancode_file_to_opossum(scancode_json_input_file) + input_file = input_file_list[0] + + if input_file.type == FileType.SPDX: + return convert_spdx_to_opossum_information(input_file.path) + elif input_file.type == FileType.SCAN_CODE: + return convert_scancode_file_to_opossum(input_file.path) else: - opossum_input_file = opossum_generation_arguments.opossum_files[0] - return read_opossum_file(opossum_input_file) + return read_opossum_file(input_file.path) From 0441c6034ee4f7024196b4269b1c39441955b777 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 07:31:59 +0100 Subject: [PATCH 06/58] refactor: Make internal method "private" --- src/opossum_lib/core/opossum_generator.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index c7609cba..b3cd9819 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -20,9 +20,7 @@ class OpossumGenerator: def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: opossum_generation_arguments.validate_input_and_exit_on_error() input_files = opossum_generation_arguments.generate_input_file_list() - opossum_file_content = OpossumGenerator.convert_after_valid_input( - input_files - ) + opossum_file_content = OpossumGenerator._convert_after_valid_input(input_files) opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() OpossumFileWriter.write_opossum_information_to_file( @@ -30,7 +28,7 @@ def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: ) @staticmethod - def convert_after_valid_input( + def _convert_after_valid_input( input_file_list: list[InputFile], ) -> OpossumFileContent: input_file = input_file_list[0] From 2c734e61c420021e795e92529af73e91c5825fd5 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 07:43:13 +0100 Subject: [PATCH 07/58] refactor: Make scancode converter publish to the internal model --- src/opossum_lib/core/opossum_generator.py | 7 +++- .../scancode/convert_scancode_to_opossum.py | 41 +++++++++++-------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index b3cd9819..f94d87f2 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -20,9 +20,9 @@ class OpossumGenerator: def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: opossum_generation_arguments.validate_input_and_exit_on_error() input_files = opossum_generation_arguments.generate_input_file_list() - opossum_file_content = OpossumGenerator._convert_after_valid_input(input_files) opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() + opossum_file_content = OpossumGenerator._convert_after_valid_input(input_files) OpossumFileWriter.write_opossum_information_to_file( opossum_file_content, Path(opossum_generation_arguments.outfile) ) @@ -35,7 +35,10 @@ def _convert_after_valid_input( if input_file.type == FileType.SPDX: return convert_spdx_to_opossum_information(input_file.path) + elif input_file.type == FileType.SCAN_CODE: - return convert_scancode_file_to_opossum(input_file.path) + return convert_scancode_file_to_opossum( + input_file.path + ).to_opossum_file_format() else: return read_opossum_file(input_file.path) diff --git a/src/opossum_lib/scancode/convert_scancode_to_opossum.py b/src/opossum_lib/scancode/convert_scancode_to_opossum.py index 62599376..bc9226dd 100644 --- a/src/opossum_lib/scancode/convert_scancode_to_opossum.py +++ b/src/opossum_lib/scancode/convert_scancode_to_opossum.py @@ -9,32 +9,39 @@ import uuid from pathlib import PurePath -import opossum_lib.core.opossum_model as opossum_model -from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.core.opossum_model import ( + Metadata, + Opossum, + OpossumPackage, + Resource, + ResourceType, + ScanResults, + SourceInfo, +) from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME from opossum_lib.scancode.model import File, FileType, Header, ScanCodeData -def convert_scancode_file_to_opossum(filename: str) -> OpossumFileContent: +def convert_scancode_file_to_opossum(filename: str) -> Opossum: logging.info(f"Converting scancode to opossum {filename}") scancode_data = load_scancode_json(filename) - return convert_scancode_to_opossum(scancode_data).to_opossum_file_format() + return convert_scancode_to_opossum(scancode_data) -def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> opossum_model.Opossum: +def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> Opossum: resources = extract_opossum_resources(scancode_data) scancode_header = extract_scancode_header(scancode_data) - metadata = opossum_model.Metadata( + metadata = Metadata( project_id=str(uuid.uuid4()), file_creation_date=scancode_header.end_timestamp, project_title="ScanCode file", ) - return opossum_model.Opossum( - scan_results=opossum_model.ScanResults( + return Opossum( + scan_results=ScanResults( metadata=metadata, resources=resources, ) @@ -66,10 +73,10 @@ def extract_scancode_header(scancode_data: ScanCodeData) -> Header: def extract_opossum_resources( scancode_data: ScanCodeData, -) -> list[opossum_model.Resource]: - temp_root = opossum_model.Resource(path=PurePath("")) +) -> list[Resource]: + temp_root = Resource(path=PurePath("")) for file in scancode_data.files: - resource = opossum_model.Resource( + resource = Resource( path=PurePath(file.path), attributions=get_attribution_info(file), type=convert_resource_type(file.type), @@ -79,18 +86,18 @@ def extract_opossum_resources( return list(temp_root.children.values()) -def convert_resource_type(file_type: FileType) -> opossum_model.ResourceType: +def convert_resource_type(file_type: FileType) -> ResourceType: if file_type == FileType.FILE: - return opossum_model.ResourceType.FILE + return ResourceType.FILE else: - return opossum_model.ResourceType.FOLDER + return ResourceType.FOLDER -def get_attribution_info(file: File) -> list[opossum_model.OpossumPackage]: +def get_attribution_info(file: File) -> list[OpossumPackage]: if file.type == FileType.DIRECTORY: return [] copyright = "\n".join(c.copyright for c in file.copyrights) - source_info = opossum_model.SourceInfo(name=SCANCODE_SOURCE_NAME) + source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) attribution_infos = [] for license_detection in file.license_detections: @@ -98,7 +105,7 @@ def get_attribution_info(file: File) -> list[opossum_model.OpossumPackage]: max_score = max(m.score for m in license_detection.matches) attribution_confidence = int(max_score) - package = opossum_model.OpossumPackage( + package = OpossumPackage( source=source_info, license_name=license_name, attribution_confidence=attribution_confidence, From e471c659ac5035674d3e305ba4c10e76ab7063c8 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 07:45:49 +0100 Subject: [PATCH 08/58] refactor: Make opossum converter publish to the internal model --- src/opossum_lib/core/opossum_generator.py | 2 +- src/opossum_lib/opossum/read_opossum_file.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index f94d87f2..e7c21556 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -41,4 +41,4 @@ def _convert_after_valid_input( input_file.path ).to_opossum_file_format() else: - return read_opossum_file(input_file.path) + return read_opossum_file(input_file.path).to_opossum_file_format() diff --git a/src/opossum_lib/opossum/read_opossum_file.py b/src/opossum_lib/opossum/read_opossum_file.py index 21b3f19b..bf87bea2 100644 --- a/src/opossum_lib/opossum/read_opossum_file.py +++ b/src/opossum_lib/opossum/read_opossum_file.py @@ -1,15 +1,13 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 - +from opossum_lib.core.opossum_model import Opossum from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) -def read_opossum_file(filename: str) -> OpossumFileContent: +def read_opossum_file(filename: str) -> Opossum: opossum_input_file = OpossumFileContent.from_file(file_name=filename) - opossum = OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) - - return opossum.to_opossum_file_format() + return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) From dd7e45606be2e426192206ae3a6ed22c349e28cb Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 07:51:27 +0100 Subject: [PATCH 09/58] refactor: Extract reading into the internal format --- src/opossum_lib/core/opossum_generator.py | 15 ++++++++++----- tests/test_opossum/test_read_opossum_file.py | 4 ++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index e7c21556..60f3e32b 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -6,6 +6,7 @@ from opossum_lib.core.input_file import FileType, InputFile from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments +from opossum_lib.core.opossum_model import Opossum from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.opossum.read_opossum_file import read_opossum_file @@ -35,10 +36,14 @@ def _convert_after_valid_input( if input_file.type == FileType.SPDX: return convert_spdx_to_opossum_information(input_file.path) - - elif input_file.type == FileType.SCAN_CODE: - return convert_scancode_file_to_opossum( - input_file.path + else: + return OpossumGenerator._read_to_internal_format( + input_file ).to_opossum_file_format() + + @staticmethod + def _read_to_internal_format(input_file: InputFile) -> Opossum: + if input_file.type == FileType.SCAN_CODE: + return convert_scancode_file_to_opossum(input_file.path) else: - return read_opossum_file(input_file.path).to_opossum_file_format() + return read_opossum_file(input_file.path) diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/test_opossum/test_read_opossum_file.py index 2586f06f..882c655e 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/test_opossum/test_read_opossum_file.py @@ -26,5 +26,5 @@ def test_read_opossum_file_containing_output_json() -> None: result = read_opossum_file(str(input_path)) assert result is not None - assert result.input_file is not None - assert result.output_file is not None + assert result.scan_results is not None + assert result.review_results is not None From 08b8a67bd972023faa0c9caaad20d0895402ee9b Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 10:05:19 +0100 Subject: [PATCH 10/58] refactor: Define interface and implement for opossum file --- src/opossum_lib/cli.py | 2 +- src/opossum_lib/core/input_format_reader.py | 13 ++++++++++ src/opossum_lib/core/opossum_generator.py | 24 ++++++++++--------- .../opossum/opossum_format_reader.py | 16 +++++++++++++ src/opossum_lib/opossum/read_opossum_file.py | 13 ---------- tests/test_opossum/test_read_opossum_file.py | 8 ++++--- 6 files changed, 48 insertions(+), 28 deletions(-) create mode 100644 src/opossum_lib/core/input_format_reader.py create mode 100644 src/opossum_lib/opossum/opossum_format_reader.py delete mode 100644 src/opossum_lib/opossum/read_opossum_file.py diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 4aa04264..0e90060f 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -66,7 +66,7 @@ def generate( - ScanCode - Opossum """ - OpossumGenerator.generate( + OpossumGenerator().generate( opossum_generation_arguments=OpossumGenerationArguments( spdx_files=spdx_files, opossum_files=opossum_files, diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/input_format_reader.py new file mode 100644 index 00000000..74ac58ec --- /dev/null +++ b/src/opossum_lib/core/input_format_reader.py @@ -0,0 +1,13 @@ +from abc import abstractmethod +from typing import Protocol + +from opossum_lib.core.input_file import FileType +from opossum_lib.core.opossum_model import Opossum + + +class InputFormatReader(Protocol): + @abstractmethod + def can_handle(self, file_type: FileType) -> bool: ... + + @abstractmethod + def read(self, path: str) -> Opossum: ... diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 60f3e32b..0d93f437 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -9,7 +9,7 @@ from opossum_lib.core.opossum_model import Opossum from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.read_opossum_file import read_opossum_file +from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader from opossum_lib.scancode.convert_scancode_to_opossum import ( convert_scancode_file_to_opossum, ) @@ -17,19 +17,22 @@ class OpossumGenerator: - @staticmethod - def generate(opossum_generation_arguments: OpossumGenerationArguments) -> None: + opossum_format_reader: OpossumFormatReader = OpossumFormatReader() + + def generate( + self, opossum_generation_arguments: OpossumGenerationArguments + ) -> None: opossum_generation_arguments.validate_input_and_exit_on_error() input_files = opossum_generation_arguments.generate_input_file_list() opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() - opossum_file_content = OpossumGenerator._convert_after_valid_input(input_files) + opossum_file_content = self._convert_after_valid_input(input_files) OpossumFileWriter.write_opossum_information_to_file( opossum_file_content, Path(opossum_generation_arguments.outfile) ) - @staticmethod def _convert_after_valid_input( + self, input_file_list: list[InputFile], ) -> OpossumFileContent: input_file = input_file_list[0] @@ -37,13 +40,12 @@ def _convert_after_valid_input( if input_file.type == FileType.SPDX: return convert_spdx_to_opossum_information(input_file.path) else: - return OpossumGenerator._read_to_internal_format( - input_file - ).to_opossum_file_format() + return self._read_to_internal_format(input_file).to_opossum_file_format() - @staticmethod - def _read_to_internal_format(input_file: InputFile) -> Opossum: + def _read_to_internal_format(self, input_file: InputFile) -> Opossum: if input_file.type == FileType.SCAN_CODE: return convert_scancode_file_to_opossum(input_file.path) + elif self.opossum_format_reader.can_handle(input_file.type): + return self.opossum_format_reader.read(input_file.path) else: - return read_opossum_file(input_file.path) + raise NotImplementedError(f"Unsupported file type: {input_file.type}") diff --git a/src/opossum_lib/opossum/opossum_format_reader.py b/src/opossum_lib/opossum/opossum_format_reader.py new file mode 100644 index 00000000..b154b207 --- /dev/null +++ b/src/opossum_lib/opossum/opossum_format_reader.py @@ -0,0 +1,16 @@ +from opossum_lib.core.input_file import FileType +from opossum_lib.core.input_format_reader import InputFormatReader +from opossum_lib.core.opossum_model import Opossum +from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.opossum.opossum_file_to_opossum_converter import ( + OpossumFileToOpossumConverter, +) + + +class OpossumFormatReader(InputFormatReader): + def can_handle(self, file_type: FileType) -> bool: + return file_type == FileType.OPOSSUM + + def read(self, path: str) -> Opossum: + opossum_input_file = OpossumFileContent.from_file(file_name=path) + return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/opossum/read_opossum_file.py b/src/opossum_lib/opossum/read_opossum_file.py deleted file mode 100644 index bf87bea2..00000000 --- a/src/opossum_lib/opossum/read_opossum_file.py +++ /dev/null @@ -1,13 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from opossum_lib.core.opossum_model import Opossum -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.opossum_file_to_opossum_converter import ( - OpossumFileToOpossumConverter, -) - - -def read_opossum_file(filename: str) -> Opossum: - opossum_input_file = OpossumFileContent.from_file(file_name=filename) - return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/test_opossum/test_read_opossum_file.py index 882c655e..d6fd6055 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/test_opossum/test_read_opossum_file.py @@ -6,24 +6,26 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.opossum.read_opossum_file import read_opossum_file +from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader TEST_DATA_DIRECTORY = Path(__file__).resolve().parent.parent / "data" def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: input_path = TEST_DATA_DIRECTORY / "opossum_input_corrupt.opossum" + opossum_format_reader = OpossumFormatReader() with pytest.raises(SystemExit) as system_exit: - read_opossum_file(str(input_path)) + opossum_format_reader.read(str(input_path)) assert system_exit.value.code == 1 assert "is corrupt and does not contain 'input.json'" in caplog.messages[0] def test_read_opossum_file_containing_output_json() -> None: input_path = TEST_DATA_DIRECTORY / "opossum_input_with_result.opossum" + opossum_format_reader = OpossumFormatReader() - result = read_opossum_file(str(input_path)) + result = opossum_format_reader.read(str(input_path)) assert result is not None assert result.scan_results is not None From 95b552323cb0678ef40494cbe5ce389a4948e3f1 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 10:20:31 +0100 Subject: [PATCH 11/58] refactor: Implement for interface for scancode file Note: Added a converter class for scancode to Opossum to allow for individual testing --- src/opossum_lib/core/input_format_reader.py | 4 + src/opossum_lib/core/opossum_generator.py | 9 +- .../opossum/opossum_format_reader.py | 4 + .../scancode/convert_scancode_to_opossum.py | 116 ------------------ .../scancode_data_to_opossum_converter.py | 96 +++++++++++++++ .../scancode/scancode_format_reader.py | 45 +++++++ .../test_convert_scancode_to_opossum.py | 14 ++- .../test_get_attribution_info.py | 10 +- 8 files changed, 167 insertions(+), 131 deletions(-) delete mode 100644 src/opossum_lib/scancode/convert_scancode_to_opossum.py create mode 100644 src/opossum_lib/scancode/scancode_data_to_opossum_converter.py create mode 100644 src/opossum_lib/scancode/scancode_format_reader.py diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/input_format_reader.py index 74ac58ec..90b06ef4 100644 --- a/src/opossum_lib/core/input_format_reader.py +++ b/src/opossum_lib/core/input_format_reader.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + from abc import abstractmethod from typing import Protocol diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 0d93f437..e245ed84 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -10,14 +10,13 @@ from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader -from opossum_lib.scancode.convert_scancode_to_opossum import ( - convert_scancode_file_to_opossum, -) +from opossum_lib.scancode.scancode_format_reader import ScancodeFormatReader from opossum_lib.spdx.convert_to_opossum import convert_spdx_to_opossum_information class OpossumGenerator: opossum_format_reader: OpossumFormatReader = OpossumFormatReader() + scancode_format_reader: ScancodeFormatReader = ScancodeFormatReader() def generate( self, opossum_generation_arguments: OpossumGenerationArguments @@ -43,8 +42,8 @@ def _convert_after_valid_input( return self._read_to_internal_format(input_file).to_opossum_file_format() def _read_to_internal_format(self, input_file: InputFile) -> Opossum: - if input_file.type == FileType.SCAN_CODE: - return convert_scancode_file_to_opossum(input_file.path) + if self.scancode_format_reader.can_handle(input_file.type): + return self.scancode_format_reader.read(input_file.path) elif self.opossum_format_reader.can_handle(input_file.type): return self.opossum_format_reader.read(input_file.path) else: diff --git a/src/opossum_lib/opossum/opossum_format_reader.py b/src/opossum_lib/opossum/opossum_format_reader.py index b154b207..f0515aac 100644 --- a/src/opossum_lib/opossum/opossum_format_reader.py +++ b/src/opossum_lib/opossum/opossum_format_reader.py @@ -1,3 +1,7 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + from opossum_lib.core.input_file import FileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum diff --git a/src/opossum_lib/scancode/convert_scancode_to_opossum.py b/src/opossum_lib/scancode/convert_scancode_to_opossum.py deleted file mode 100644 index bc9226dd..00000000 --- a/src/opossum_lib/scancode/convert_scancode_to_opossum.py +++ /dev/null @@ -1,116 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import json -import logging -import sys -import uuid -from pathlib import PurePath - -from opossum_lib.core.opossum_model import ( - Metadata, - Opossum, - OpossumPackage, - Resource, - ResourceType, - ScanResults, - SourceInfo, -) -from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.scancode.model import File, FileType, Header, ScanCodeData - - -def convert_scancode_file_to_opossum(filename: str) -> Opossum: - logging.info(f"Converting scancode to opossum {filename}") - - scancode_data = load_scancode_json(filename) - - return convert_scancode_to_opossum(scancode_data) - - -def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> Opossum: - resources = extract_opossum_resources(scancode_data) - - scancode_header = extract_scancode_header(scancode_data) - metadata = Metadata( - project_id=str(uuid.uuid4()), - file_creation_date=scancode_header.end_timestamp, - project_title="ScanCode file", - ) - - return Opossum( - scan_results=ScanResults( - metadata=metadata, - resources=resources, - ) - ) - - -def load_scancode_json(filename: str) -> ScanCodeData: - try: - with open(filename) as inp: - json_data = json.load(inp) - except json.JSONDecodeError as e: - logging.error(f"Error decoding json for file {filename}. Message: {e.msg}") - sys.exit(1) - except UnicodeDecodeError: - logging.error(f"Error decoding json for file {filename}.") - sys.exit(1) - - scancode_data = ScanCodeData.model_validate(json_data) - - return scancode_data - - -def extract_scancode_header(scancode_data: ScanCodeData) -> Header: - if len(scancode_data.headers) != 1: - logging.error("Headers of ScanCode file are invalid.") - sys.exit(1) - return scancode_data.headers[0] - - -def extract_opossum_resources( - scancode_data: ScanCodeData, -) -> list[Resource]: - temp_root = Resource(path=PurePath("")) - for file in scancode_data.files: - resource = Resource( - path=PurePath(file.path), - attributions=get_attribution_info(file), - type=convert_resource_type(file.type), - ) - temp_root.add_resource(resource) - - return list(temp_root.children.values()) - - -def convert_resource_type(file_type: FileType) -> ResourceType: - if file_type == FileType.FILE: - return ResourceType.FILE - else: - return ResourceType.FOLDER - - -def get_attribution_info(file: File) -> list[OpossumPackage]: - if file.type == FileType.DIRECTORY: - return [] - copyright = "\n".join(c.copyright for c in file.copyrights) - source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) - - attribution_infos = [] - for license_detection in file.license_detections: - license_name = license_detection.license_expression_spdx - max_score = max(m.score for m in license_detection.matches) - attribution_confidence = int(max_score) - - package = OpossumPackage( - source=source_info, - license_name=license_name, - attribution_confidence=attribution_confidence, - copyright=copyright, - ) - attribution_infos.append(package) - - return attribution_infos diff --git a/src/opossum_lib/scancode/scancode_data_to_opossum_converter.py b/src/opossum_lib/scancode/scancode_data_to_opossum_converter.py new file mode 100644 index 00000000..97f64528 --- /dev/null +++ b/src/opossum_lib/scancode/scancode_data_to_opossum_converter.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import sys +import uuid +from pathlib import PurePath + +from opossum_lib.core.opossum_model import ( + Metadata, + Opossum, + OpossumPackage, + Resource, + ResourceType, + ScanResults, + SourceInfo, +) +from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME +from opossum_lib.scancode.model import File, FileType, Header, ScanCodeData + + +class ScancodeDataToOpossumConverter: + @staticmethod + def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> Opossum: + resources = ScancodeDataToOpossumConverter.extract_opossum_resources( + scancode_data + ) + + scancode_header = ScancodeDataToOpossumConverter.extract_scancode_header( + scancode_data + ) + metadata = Metadata( + project_id=str(uuid.uuid4()), + file_creation_date=scancode_header.end_timestamp, + project_title="ScanCode file", + ) + + return Opossum( + scan_results=ScanResults( + metadata=metadata, + resources=resources, + ) + ) + + @staticmethod + def extract_scancode_header(scancode_data: ScanCodeData) -> Header: + if len(scancode_data.headers) != 1: + logging.error("Headers of ScanCode file are invalid.") + sys.exit(1) + return scancode_data.headers[0] + + @staticmethod + def extract_opossum_resources( + scancode_data: ScanCodeData, + ) -> list[Resource]: + temp_root = Resource(path=PurePath("")) + for file in scancode_data.files: + resource = Resource( + path=PurePath(file.path), + attributions=ScancodeDataToOpossumConverter.get_attribution_info(file), + type=ScancodeDataToOpossumConverter.convert_resource_type(file.type), + ) + temp_root.add_resource(resource) + + return list(temp_root.children.values()) + + @staticmethod + def convert_resource_type(file_type: FileType) -> ResourceType: + if file_type == FileType.FILE: + return ResourceType.FILE + else: + return ResourceType.FOLDER + + @staticmethod + def get_attribution_info(file: File) -> list[OpossumPackage]: + if file.type == FileType.DIRECTORY: + return [] + copyright = "\n".join(c.copyright for c in file.copyrights) + source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) + + attribution_infos = [] + for license_detection in file.license_detections: + license_name = license_detection.license_expression_spdx + max_score = max(m.score for m in license_detection.matches) + attribution_confidence = int(max_score) + + package = OpossumPackage( + source=source_info, + license_name=license_name, + attribution_confidence=attribution_confidence, + copyright=copyright, + ) + attribution_infos.append(package) + + return attribution_infos diff --git a/src/opossum_lib/scancode/scancode_format_reader.py b/src/opossum_lib/scancode/scancode_format_reader.py new file mode 100644 index 00000000..c098489e --- /dev/null +++ b/src/opossum_lib/scancode/scancode_format_reader.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging +import sys + +from opossum_lib.core.input_file import FileType +from opossum_lib.core.input_format_reader import InputFormatReader +from opossum_lib.core.opossum_model import ( + Opossum, +) +from opossum_lib.scancode.model import ScanCodeData +from opossum_lib.scancode.scancode_data_to_opossum_converter import ( + ScancodeDataToOpossumConverter, +) + + +class ScancodeFormatReader(InputFormatReader): + def can_handle(self, file_type: FileType) -> bool: + return file_type == FileType.SCAN_CODE + + def read(self, path: str) -> Opossum: + logging.info(f"Converting scancode to opossum {path}") + + scancode_data = ScancodeFormatReader.load_scancode_json(path) + + return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + + @staticmethod + def load_scancode_json(filename: str) -> ScanCodeData: + try: + with open(filename) as inp: + json_data = json.load(inp) + except json.JSONDecodeError as e: + logging.error(f"Error decoding json for file {filename}. Message: {e.msg}") + sys.exit(1) + except UnicodeDecodeError: + logging.error(f"Error decoding json for file {filename}.") + sys.exit(1) + + scancode_data = ScanCodeData.model_validate(json_data) + + return scancode_data diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/test_scancode/test_convert_scancode_to_opossum.py index 91044de1..44290a66 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/test_scancode/test_convert_scancode_to_opossum.py @@ -7,8 +7,8 @@ from _pytest.logging import LogCaptureFixture from opossum_lib.core.opossum_model import Resource -from opossum_lib.scancode.convert_scancode_to_opossum import ( - convert_scancode_to_opossum, +from opossum_lib.scancode.scancode_data_to_opossum_converter import ( + ScancodeDataToOpossumConverter, ) from tests.test_setup.scancode_faker_setup import ScanCodeFaker @@ -17,7 +17,7 @@ def test_extract_scancode_header_produces_expected_result( scancode_faker: ScanCodeFaker, ) -> None: scancode_data = scancode_faker.scancode_data() - opossum = convert_scancode_to_opossum( + opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum( scancode_data, ) metadata = opossum.scan_results.metadata @@ -32,7 +32,7 @@ def test_extract_scancode_header_errors_with_missing_header( scancode_data = scancode_faker.scancode_data(headers=[]) with pytest.raises(SystemExit): - convert_scancode_to_opossum(scancode_data) + ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) assert "header" in caplog.messages[0].lower() @@ -45,7 +45,7 @@ def test_extract_scancode_header_error_with_multiple_headers( scancode_data = scancode_faker.scancode_data(headers=[header1, header2]) with pytest.raises(SystemExit): - convert_scancode_to_opossum(scancode_data) + ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) assert "header" in caplog.messages[0].lower() @@ -64,7 +64,9 @@ def test_convert_scancode_produces_expected_result( scancode_faker: ScanCodeFaker, ) -> None: scancode_data = scancode_faker.scancode_data() - opossum_data = convert_scancode_to_opossum(scancode_data) + opossum_data = ScancodeDataToOpossumConverter.convert_scancode_to_opossum( + scancode_data + ) assert opossum_data.review_results is None scan_results = opossum_data.scan_results diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index cf1f8476..d5101f55 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -5,14 +5,16 @@ from opossum_lib.opossum.opossum_file import OpossumPackage, SourceInfo from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.scancode.convert_scancode_to_opossum import convert_scancode_to_opossum +from opossum_lib.scancode.scancode_data_to_opossum_converter import ( + ScancodeDataToOpossumConverter, +) from tests.test_setup.scancode_faker_setup import ScanCodeFaker def test_get_attribution_info_directory(scancode_faker: ScanCodeFaker) -> None: folder = scancode_faker.single_folder(path="some/single/folder") scancode_data = scancode_faker.scancode_data(files=[folder]) - opossum = convert_scancode_to_opossum(scancode_data) + opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) assert len(opossum.scan_results.resources) == 1 assert opossum.scan_results.resources[0].attributions == [] @@ -22,7 +24,7 @@ def test_get_attribution_info_from_file_without_detections( ) -> None: file = scancode_faker.single_file(path="some/single/file", license_detections=[]) scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = convert_scancode_to_opossum(scancode_data) + opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) assert len(opossum.scan_results.resources) == 1 assert opossum.scan_results.resources[0].attributions == [] @@ -63,7 +65,7 @@ def test_get_attribution_info_file_multiple(scancode_faker: ScanCodeFaker) -> No copyrights=[copyright1, copyright2, copyright3], ) scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = convert_scancode_to_opossum(scancode_data) + opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) attributions = ( opossum.to_opossum_file_format().input_file.external_attributions.values() ) From 57adfde7031b21c2ae9187e6d8d4782b0a37be4f Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 10:37:58 +0100 Subject: [PATCH 12/58] fix: remove spdx remainings --- src/opossum_lib/core/opossum_generation_arguments.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 96f9edc3..53f248ce 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -12,17 +12,12 @@ class OpossumGenerationArguments(BaseModel): - spdx_files: list[str] scancode_json_files: list[str] opossum_files: list[str] outfile: str def validate_input_and_exit_on_error(self) -> None: - total_number_of_files = ( - len(self.spdx_files) - + len(self.scancode_json_files) - + len(self.opossum_files) - ) + total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) if total_number_of_files == 0: logging.warning("No input provided. Exiting.") sys.exit(1) From 86ea7cf06df144de7f36e8a356a33be8dea8708e Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 10:50:38 +0100 Subject: [PATCH 13/58] refactor: introduce can handle pattern --- src/opossum_lib/core/input_reader.py | 22 +++++++++++++++ src/opossum_lib/core/opossum_generator.py | 33 +++++----------------- src/opossum_lib/opossum/file_generation.py | 4 +-- tests/test_cli.py | 8 ++---- tests/test_opossum/test_file_generation.py | 8 ++---- 5 files changed, 34 insertions(+), 41 deletions(-) create mode 100644 src/opossum_lib/core/input_reader.py diff --git a/src/opossum_lib/core/input_reader.py b/src/opossum_lib/core/input_reader.py new file mode 100644 index 00000000..d036575d --- /dev/null +++ b/src/opossum_lib/core/input_reader.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +from opossum_lib.core.input_file import InputFile +from opossum_lib.core.input_format_reader import InputFormatReader +from opossum_lib.core.opossum_model import Opossum +from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader +from opossum_lib.scancode.scancode_format_reader import ScancodeFormatReader + + +class InputReader: + input_format_readers: list[InputFormatReader] = [ + OpossumFormatReader(), + ScancodeFormatReader(), + ] + + def read(self, input_file: InputFile) -> Opossum: + for input_format_reader in self.input_format_readers: + if input_format_reader.can_handle(input_file.type): + return input_format_reader.read(input_file.path) + raise NotImplementedError(f"Unsupported file type: {input_file.type}") diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index a93bbc51..2e5b9349 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -4,43 +4,24 @@ from pathlib import Path -from opossum_lib.core.input_file import InputFile +from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments -from opossum_lib.core.opossum_model import Opossum from opossum_lib.opossum.file_generation import OpossumFileWriter -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader -from opossum_lib.scancode.scancode_format_reader import ScancodeFormatReader class OpossumGenerator: - opossum_format_reader: OpossumFormatReader = OpossumFormatReader() - scancode_format_reader: ScancodeFormatReader = ScancodeFormatReader() + input_reader: InputReader = InputReader() def generate( self, opossum_generation_arguments: OpossumGenerationArguments ) -> None: opossum_generation_arguments.validate_input_and_exit_on_error() input_files = opossum_generation_arguments.generate_input_file_list() - opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() - opossum_file_content = self._convert_after_valid_input(input_files) - OpossumFileWriter.write_opossum_information_to_file( + opossum = self.input_reader.read(input_files[0]) + + opossum_file_content = opossum.to_opossum_file_format() + opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() + OpossumFileWriter.write( opossum_file_content, Path(opossum_generation_arguments.outfile) ) - - def _convert_after_valid_input( - self, - input_file_list: list[InputFile], - ) -> OpossumFileContent: - input_file = input_file_list[0] - - return self._read_to_internal_format(input_file).to_opossum_file_format() - - def _read_to_internal_format(self, input_file: InputFile) -> Opossum: - if self.scancode_format_reader.can_handle(input_file.type): - return self.scancode_format_reader.read(input_file.path) - elif self.opossum_format_reader.can_handle(input_file.type): - return self.opossum_format_reader.read(input_file.path) - else: - raise NotImplementedError(f"Unsupported file type: {input_file.type}") diff --git a/src/opossum_lib/opossum/file_generation.py b/src/opossum_lib/opossum/file_generation.py index 59092bc2..b832e81f 100644 --- a/src/opossum_lib/opossum/file_generation.py +++ b/src/opossum_lib/opossum/file_generation.py @@ -14,9 +14,7 @@ class OpossumFileWriter: @staticmethod - def write_opossum_information_to_file( - opossum_file_content: OpossumFileContent, file_path: Path - ) -> None: + def write(opossum_file_content: OpossumFileContent, file_path: Path) -> None: with ZipFile( file_path, "w", compression=ZIP_DEFLATED, compresslevel=COMPRESSION_LEVEL ) as zip_file: diff --git a/tests/test_cli.py b/tests/test_cli.py index d6870d2e..018295cf 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -177,9 +177,7 @@ def test_cli_no_output_file_provided(opossum_file_faker: OpossumFileFaker) -> No with runner.isolated_filesystem(): file_path = "input.opossum" opossum_file = opossum_file_faker.opossum_file_content() - OpossumFileWriter.write_opossum_information_to_file( - opossum_file, Path(file_path) - ) + OpossumFileWriter.write(opossum_file, Path(file_path)) result = runner.invoke( generate, "--opossum " + file_path, @@ -197,9 +195,7 @@ def test_cli_warning_if_outfile_already_exists( with runner.isolated_filesystem(): file_path = "input.opossum" opossum_file = opossum_file_faker.opossum_file_content() - OpossumFileWriter.write_opossum_information_to_file( - opossum_file, Path(file_path) - ) + OpossumFileWriter.write(opossum_file, Path(file_path)) with open("output.opossum", "w") as f: f.write("") result = runner.invoke( diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index 8ae81dd6..1ea1d8fa 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -19,9 +19,7 @@ def test_only_input_information_available_writes_only_input_information( ) output_path = tmp_path / "output.opossum" - OpossumFileWriter.write_opossum_information_to_file( - opossum_file_content, output_path - ) + OpossumFileWriter.write(opossum_file_content, output_path) with ZipFile(output_path, "r") as zip_file: assert zip_file.namelist() == [INPUT_JSON_NAME] @@ -33,9 +31,7 @@ def test_input_and_output_information_available_writes_both( opossum_file_content = opossum_file_faker.opossum_file_content() output_path = tmp_path / "output.opossum" - OpossumFileWriter.write_opossum_information_to_file( - opossum_file_content, output_path - ) + OpossumFileWriter.write(opossum_file_content, output_path) with ZipFile(output_path, "r") as zip_file: assert INPUT_JSON_NAME in zip_file.namelist() From 5ab963524badd01a240484ad71b177759cf6334f Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 10:52:15 +0100 Subject: [PATCH 14/58] refactor: rename --- src/opossum_lib/core/opossum_generation_arguments.py | 4 ++-- src/opossum_lib/core/opossum_generator.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 53f248ce..02013608 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -16,7 +16,7 @@ class OpossumGenerationArguments(BaseModel): opossum_files: list[str] outfile: str - def validate_input_and_exit_on_error(self) -> None: + def validate_and_exit_on_error(self) -> None: total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) if total_number_of_files == 0: logging.warning("No input provided. Exiting.") @@ -32,7 +32,7 @@ def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: if Path.is_file(Path(self.outfile)): logging.warning(f"{self.outfile} already exists and will be overwritten.") - def generate_input_file_list(self) -> list[InputFile]: + def input_files(self) -> list[InputFile]: result = [] result += [ InputFile(path=path, type=FileType.SCAN_CODE) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 2e5b9349..0206f0e8 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -15,8 +15,8 @@ class OpossumGenerator: def generate( self, opossum_generation_arguments: OpossumGenerationArguments ) -> None: - opossum_generation_arguments.validate_input_and_exit_on_error() - input_files = opossum_generation_arguments.generate_input_file_list() + opossum_generation_arguments.validate_and_exit_on_error() + input_files = opossum_generation_arguments.input_files() opossum = self.input_reader.read(input_files[0]) From ed40898a409afbeb4a83b6aea7b42be90b80baf1 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 11:04:02 +0100 Subject: [PATCH 15/58] refactor: use Path instead of strings for file paths --- src/opossum_lib/cli.py | 10 ++++------ src/opossum_lib/core/input_file.py | 3 ++- src/opossum_lib/core/input_format_reader.py | 3 ++- src/opossum_lib/core/opossum_generation_arguments.py | 10 +++++----- src/opossum_lib/core/opossum_generator.py | 3 +-- src/opossum_lib/opossum/opossum_file_content.py | 9 +++++---- src/opossum_lib/opossum/opossum_format_reader.py | 5 +++-- src/opossum_lib/scancode/scancode_format_reader.py | 11 ++++++----- tests/test_opossum/test_read_opossum_file.py | 4 ++-- 9 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 7c22ffb8..3cb13916 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -1,10 +1,9 @@ #!/usr/bin/env python3 +from pathlib import Path # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 - - import click from opossum_lib.core.opossum_generator import ( @@ -44,9 +43,9 @@ def opossum_file() -> None: 'If appropriate, the extension ".opossum" will be appended.', ) def generate( - scancode_json_files: list[str], - opossum_files: list[str], - outfile: str, + scancode_json_files: list[Path], + opossum_files: list[Path], + outfile: Path, ) -> None: """ Generate an Opossum file from various other file formats. @@ -58,7 +57,6 @@ def generate( """ OpossumGenerator().generate( opossum_generation_arguments=OpossumGenerationArguments( - spdx_files=[], opossum_files=opossum_files, scancode_json_files=scancode_json_files, outfile=outfile, diff --git a/src/opossum_lib/core/input_file.py b/src/opossum_lib/core/input_file.py index f17a4452..f0275552 100644 --- a/src/opossum_lib/core/input_file.py +++ b/src/opossum_lib/core/input_file.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 from enum import Enum, auto +from pathlib import Path from pydantic import BaseModel @@ -12,5 +13,5 @@ class FileType(Enum): class InputFile(BaseModel): - path: str + path: Path type: FileType diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/input_format_reader.py index 90b06ef4..50872a86 100644 --- a/src/opossum_lib/core/input_format_reader.py +++ b/src/opossum_lib/core/input_format_reader.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from abc import abstractmethod +from pathlib import Path from typing import Protocol from opossum_lib.core.input_file import FileType @@ -14,4 +15,4 @@ class InputFormatReader(Protocol): def can_handle(self, file_type: FileType) -> bool: ... @abstractmethod - def read(self, path: str) -> Opossum: ... + def read(self, path: Path) -> Opossum: ... diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 02013608..895d582c 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -12,9 +12,9 @@ class OpossumGenerationArguments(BaseModel): - scancode_json_files: list[str] - opossum_files: list[str] - outfile: str + scancode_json_files: list[Path] + opossum_files: list[Path] + outfile: Path def validate_and_exit_on_error(self) -> None: total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) @@ -26,8 +26,8 @@ def validate_and_exit_on_error(self) -> None: sys.exit(1) def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: - if not self.outfile.endswith(".opossum"): - self.outfile += ".opossum" + if self.outfile.suffix != ".opossum": + self.outfile = self.outfile.with_suffix(".opossum") if Path.is_file(Path(self.outfile)): logging.warning(f"{self.outfile} already exists and will be overwritten.") diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 0206f0e8..2154769e 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -2,7 +2,6 @@ # # SPDX-License-Identifier: Apache-2.0 -from pathlib import Path from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments @@ -23,5 +22,5 @@ def generate( opossum_file_content = opossum.to_opossum_file_format() opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() OpossumFileWriter.write( - opossum_file_content, Path(opossum_generation_arguments.outfile) + opossum_file_content, opossum_generation_arguments.outfile ) diff --git a/src/opossum_lib/opossum/opossum_file_content.py b/src/opossum_lib/opossum/opossum_file_content.py index 900553ff..b951c7d8 100644 --- a/src/opossum_lib/opossum/opossum_file_content.py +++ b/src/opossum_lib/opossum/opossum_file_content.py @@ -6,6 +6,7 @@ import json import logging import sys +from pathlib import Path from zipfile import ZipFile from pydantic import BaseModel, TypeAdapter @@ -20,12 +21,12 @@ class OpossumFileContent(BaseModel): output_file: OpossumOutputFile | None = None @staticmethod - def from_file(file_name: str) -> OpossumFileContent: - logging.info(f"Converting opossum to opossum {file_name}") + def from_file(path: Path) -> OpossumFileContent: + logging.info(f"Converting opossum to opossum {path}") try: with ( - ZipFile(file_name, "r") as input_zip_file, + ZipFile(path, "r") as input_zip_file, ): OpossumFileContent._validate_zip_file_contents(input_zip_file) input_file = OpossumFileContent._read_input_json_from_zip_file( @@ -38,7 +39,7 @@ def from_file(file_name: str) -> OpossumFileContent: ), ) except Exception as e: - print(f"Error reading file {file_name}: {e}") + print(f"Error reading file {path}: {e}") sys.exit(1) @staticmethod diff --git a/src/opossum_lib/opossum/opossum_format_reader.py b/src/opossum_lib/opossum/opossum_format_reader.py index f0515aac..a1e0245f 100644 --- a/src/opossum_lib/opossum/opossum_format_reader.py +++ b/src/opossum_lib/opossum/opossum_format_reader.py @@ -1,6 +1,7 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 +from pathlib import Path from opossum_lib.core.input_file import FileType from opossum_lib.core.input_format_reader import InputFormatReader @@ -15,6 +16,6 @@ class OpossumFormatReader(InputFormatReader): def can_handle(self, file_type: FileType) -> bool: return file_type == FileType.OPOSSUM - def read(self, path: str) -> Opossum: - opossum_input_file = OpossumFileContent.from_file(file_name=path) + def read(self, path: Path) -> Opossum: + opossum_input_file = OpossumFileContent.from_file(path=path) return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/scancode/scancode_format_reader.py b/src/opossum_lib/scancode/scancode_format_reader.py index c098489e..c2ef19c5 100644 --- a/src/opossum_lib/scancode/scancode_format_reader.py +++ b/src/opossum_lib/scancode/scancode_format_reader.py @@ -5,6 +5,7 @@ import json import logging import sys +from pathlib import Path from opossum_lib.core.input_file import FileType from opossum_lib.core.input_format_reader import InputFormatReader @@ -21,7 +22,7 @@ class ScancodeFormatReader(InputFormatReader): def can_handle(self, file_type: FileType) -> bool: return file_type == FileType.SCAN_CODE - def read(self, path: str) -> Opossum: + def read(self, path: Path) -> Opossum: logging.info(f"Converting scancode to opossum {path}") scancode_data = ScancodeFormatReader.load_scancode_json(path) @@ -29,15 +30,15 @@ def read(self, path: str) -> Opossum: return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) @staticmethod - def load_scancode_json(filename: str) -> ScanCodeData: + def load_scancode_json(path: Path) -> ScanCodeData: try: - with open(filename) as inp: + with open(path) as inp: json_data = json.load(inp) except json.JSONDecodeError as e: - logging.error(f"Error decoding json for file {filename}. Message: {e.msg}") + logging.error(f"Error decoding json for file {path}. Message: {e.msg}") sys.exit(1) except UnicodeDecodeError: - logging.error(f"Error decoding json for file {filename}.") + logging.error(f"Error decoding json for file {path}.") sys.exit(1) scancode_data = ScanCodeData.model_validate(json_data) diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/test_opossum/test_read_opossum_file.py index d6fd6055..c5911a76 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/test_opossum/test_read_opossum_file.py @@ -16,7 +16,7 @@ def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> opossum_format_reader = OpossumFormatReader() with pytest.raises(SystemExit) as system_exit: - opossum_format_reader.read(str(input_path)) + opossum_format_reader.read(input_path) assert system_exit.value.code == 1 assert "is corrupt and does not contain 'input.json'" in caplog.messages[0] @@ -25,7 +25,7 @@ def test_read_opossum_file_containing_output_json() -> None: input_path = TEST_DATA_DIRECTORY / "opossum_input_with_result.opossum" opossum_format_reader = OpossumFormatReader() - result = opossum_format_reader.read(str(input_path)) + result = opossum_format_reader.read(input_path) assert result is not None assert result.scan_results is not None From 1051de718d405bfb7fe1bd81b079be9b7f18eb53 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 11:22:03 +0100 Subject: [PATCH 16/58] refactor: rename --- src/opossum_lib/core/opossum_generator.py | 2 +- .../opossum/{file_generation.py => opossum_file_writer.py} | 0 tests/test_cli.py | 2 +- tests/test_opossum/test_file_generation.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/opossum_lib/opossum/{file_generation.py => opossum_file_writer.py} (100%) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 2154769e..451ed8f1 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -5,7 +5,7 @@ from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments -from opossum_lib.opossum.file_generation import OpossumFileWriter +from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter class OpossumGenerator: diff --git a/src/opossum_lib/opossum/file_generation.py b/src/opossum_lib/opossum/opossum_file_writer.py similarity index 100% rename from src/opossum_lib/opossum/file_generation.py rename to src/opossum_lib/opossum/opossum_file_writer.py diff --git a/tests/test_cli.py b/tests/test_cli.py index 018295cf..554e7ac1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -12,8 +12,8 @@ from opossum_lib.cli import generate from opossum_lib.opossum.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME -from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file import OpossumPackage +from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index 1ea1d8fa..6f614e4d 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -6,8 +6,8 @@ from zipfile import ZipFile from opossum_lib.opossum.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME -from opossum_lib.opossum.file_generation import OpossumFileWriter from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker From 854e16ed3087fd28c2be8e452c27f5a1eaced28e Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 11:38:02 +0100 Subject: [PATCH 17/58] refactor: remove not used merging functionality --- src/opossum_lib/opossum/merger.py | 140 ---------------- src/opossum_lib/opossum/opossum_file.py | 23 +-- tests/test_opossum/test_merge.py | 210 ------------------------ 3 files changed, 1 insertion(+), 372 deletions(-) delete mode 100644 src/opossum_lib/opossum/merger.py delete mode 100644 tests/test_opossum/test_merge.py diff --git a/src/opossum_lib/opossum/merger.py b/src/opossum_lib/opossum/merger.py deleted file mode 100644 index ec6e16d7..00000000 --- a/src/opossum_lib/opossum/merger.py +++ /dev/null @@ -1,140 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from typing import Any - -from opossum_lib.opossum.opossum_file import ( - OpossumInformation, - OpossumPackageIdentifier, - Resource, - ResourcePath, - ResourceType, - convert_resource_in_file_to_resource, -) - - -def merge_opossum_information( - elements_to_merge: list[OpossumInformation], -) -> OpossumInformation: - expanded_opossum_information = [ - expand_opossum_package_identifier(opossum_information) - for opossum_information in elements_to_merge - ] - return OpossumInformation( - metadata=expanded_opossum_information[0].metadata, - resources=_merge_resources( - [ - convert_resource_in_file_to_resource(opossum_information.resources) - for opossum_information in expanded_opossum_information - ] - ).convert_to_file_resource(), - external_attributions=_merge_dicts_without_duplicates( - [ - opossum_information.external_attributions - for opossum_information in expanded_opossum_information - ] - ), - resources_to_attributions=_merge_resources_to_attributions( - [ - opossum_information.resources_to_attributions - for opossum_information in expanded_opossum_information - ] - ), - attribution_breakpoints=_merge_attribution_breakpoints( - [ - opossum_information.attribution_breakpoints - for opossum_information in expanded_opossum_information - ] - ), - external_attribution_sources=_merge_dicts_without_duplicates( - [ - opossum_information.external_attribution_sources - for opossum_information in expanded_opossum_information - ] - ), - ) - - -def expand_opossum_package_identifier( - opossum_information: OpossumInformation, -) -> OpossumInformation: - """IDs for the attributions should be unique per OpossumInformation. - To prevent possible duplicates we add the projectId of the - OpossumInformation to the IDs as a prefix.""" - prefix = opossum_information.metadata.project_id - extended_resources_to_attributions = dict() - for ( - resource_path, - identifiers, - ) in opossum_information.resources_to_attributions.items(): - extended_resources_to_attributions[resource_path] = [ - prefix + "-" + identifier for identifier in identifiers - ] - extended_external_attributions = dict() - for ( - identifier, - external_attribution, - ) in opossum_information.external_attributions.items(): - extended_external_attributions[prefix + "-" + identifier] = external_attribution - - return OpossumInformation( - metadata=opossum_information.metadata, - resources=opossum_information.resources, - external_attributions=extended_external_attributions, - resources_to_attributions=extended_resources_to_attributions, - attribution_breakpoints=opossum_information.attribution_breakpoints, - external_attribution_sources=opossum_information.external_attribution_sources, - ) - - -def _merge_resources(resources: list[Resource]) -> Resource: - merged_resource = Resource(type=ResourceType.TOP_LEVEL) - for resource in resources: - for path in resource.get_paths_of_all_leaf_nodes_with_types(): - merged_resource = merged_resource.add_path(path) - return merged_resource - - -def _merge_resources_to_attributions( - resources_to_attributions: list[dict[ResourcePath, list[OpossumPackageIdentifier]]], -) -> dict[ResourcePath, list[OpossumPackageIdentifier]]: - merged_resources_to_attributions: dict[ - ResourcePath, list[OpossumPackageIdentifier] - ] = dict() - for resource_to_attribution in resources_to_attributions: - for resource_path, identifiers in resource_to_attribution.items(): - identifiers_merged = merged_resources_to_attributions.get(resource_path, []) - identifiers_merged.extend( - [idf for idf in identifiers if idf not in identifiers_merged] - ) - merged_resources_to_attributions[resource_path] = identifiers_merged - - return merged_resources_to_attributions - - -def _merge_attribution_breakpoints( - attribution_breakpoints_to_merge: list[list[str]], -) -> list[str]: - merged_attribution_breakpoints = [] - for attribution_breakpoints in attribution_breakpoints_to_merge: - merged_attribution_breakpoints.extend( - [ - attribution_breakpoint - for attribution_breakpoint in attribution_breakpoints - if attribution_breakpoint not in merged_attribution_breakpoints - ] - ) - return merged_attribution_breakpoints - - -def _merge_dicts_without_duplicates(dicts: list[dict[str, Any]]) -> dict[str, Any]: - merged_dict: dict[str, Any] = dict() - for single_dict in dicts: - for key, value in single_dict.items(): - if key in merged_dict and merged_dict.get(key) != value: - raise TypeError( - "Couldn't merge and deduplicate: " - "Values for identical keys don't match." - ) - merged_dict.update({key: value}) - return merged_dict diff --git a/src/opossum_lib/opossum/opossum_file.py b/src/opossum_lib/opossum/opossum_file.py index bffd41ff..513bbe9f 100644 --- a/src/opossum_lib/opossum/opossum_file.py +++ b/src/opossum_lib/opossum/opossum_file.py @@ -6,7 +6,7 @@ from copy import deepcopy from dataclasses import field from enum import Enum, auto -from typing import Literal, cast +from typing import Literal from pydantic import BaseModel, ConfigDict, model_serializer from pydantic.alias_generators import to_camel @@ -189,24 +189,3 @@ class ExternalAttributionSource(CamelBaseModel): name: str priority: int is_relevant_for_preferred: bool | None = None - - -def _build_resource_tree(resource: ResourceInFile) -> Resource: - if isinstance(resource, int): - return Resource(type=ResourceType.FILE) - else: - result = Resource(type=ResourceType.FOLDER) - for name, child_resource in resource.items(): - result.children[name] = _build_resource_tree(child_resource) - return result - - -def convert_resource_in_file_to_resource(resource: ResourceInFile) -> Resource: - root_node = Resource(type=ResourceType.TOP_LEVEL) - - if isinstance(resource, dict): - dict_resource = cast(dict[str, ResourceInFile], resource) - for name, child_resource in dict_resource.items(): - root_node.children[name] = _build_resource_tree(child_resource) - - return root_node diff --git a/tests/test_opossum/test_merge.py b/tests/test_opossum/test_merge.py deleted file mode 100644 index 941b16cd..00000000 --- a/tests/test_opossum/test_merge.py +++ /dev/null @@ -1,210 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from unittest import mock - -import pytest - -from opossum_lib.opossum.merger import ( - _merge_dicts_without_duplicates, - _merge_resources, - _merge_resources_to_attributions, - expand_opossum_package_identifier, - merge_opossum_information, -) -from opossum_lib.opossum.opossum_file import ( - Metadata, - OpossumInformation, - OpossumPackage, - OpossumPackageIdentifier, - Resource, - ResourcePath, - ResourceType, - SourceInfo, -) - - -def test_merge_opossum_information() -> None: - opossum_package = OpossumPackage(source=SourceInfo(name="source")) - opossum_information = OpossumInformation( - metadata=Metadata( - project_id="project-id", - file_creation_date="30-05-2023", - project_title="test data", - ), - resources={"A": {"B": {}}}, - external_attributions={"SPDXRef-Package": opossum_package}, - resources_to_attributions={"/A/B/": ["SPDXRef-Package"]}, - ) - - opossum_information_2 = OpossumInformation( - metadata=Metadata( - project_id="test-data-id", - file_creation_date="29-05-2023", - project_title="second test data", - ), - resources={"A": {"D": {"C": 1}}}, - external_attributions={"SPDXRef-File": opossum_package}, - resources_to_attributions={"/A/D/C": ["SPDXRef-File"]}, - ) - - merged_information = merge_opossum_information( - [opossum_information, opossum_information_2] - ) - - assert merged_information.metadata == opossum_information.metadata - assert merged_information.resources == { - "A": { - "B": {}, - "D": {"C": 1}, - } - } - assert merged_information.external_attributions == { - "project-id-SPDXRef-Package": opossum_package, - "test-data-id-SPDXRef-File": opossum_package, - } - assert merged_information.resources_to_attributions == { - "/A/B/": ["project-id-SPDXRef-Package"], - "/A/D/C": ["test-data-id-SPDXRef-File"], - } - - -def test_merge_resources() -> None: - list_of_paths_with_resource_types = [ - [("A", ResourceType.FOLDER)], - [ - ("A", ResourceType.FOLDER), - ("B", ResourceType.FOLDER), - ("C", ResourceType.FILE), - ], - [("A", ResourceType.FOLDER), ("D", ResourceType.FILE)], - ] - - resource = Resource(type=ResourceType.TOP_LEVEL) - for path in list_of_paths_with_resource_types: - resource = resource.add_path(path) - - list_of_paths_with_resource_type = [ - [("A", ResourceType.FOLDER)], - [ - ("A", ResourceType.FOLDER), - ("B", ResourceType.FOLDER), - ("C", ResourceType.FILE), - ], - [("A", ResourceType.FOLDER), ("D", ResourceType.FILE)], - [ - ("C", ResourceType.FOLDER), - ("D", ResourceType.FOLDER), - ("E", ResourceType.FOLDER), - ], - ] - resource2 = Resource(type=ResourceType.TOP_LEVEL) - for path in list_of_paths_with_resource_type: - resource2 = resource2.add_path(path) - - resources = [resource, resource2] - merged_resource = _merge_resources(resources) - - assert merged_resource == Resource( - type=ResourceType.TOP_LEVEL, - children={ - "A": Resource( - type=ResourceType.FOLDER, - children={ - "B": Resource( - type=ResourceType.FOLDER, - children={"C": Resource(type=ResourceType.FILE)}, - ), - "D": Resource(type=ResourceType.FILE), - }, - ), - "C": Resource( - type=ResourceType.FOLDER, - children={ - "D": Resource( - type=ResourceType.FOLDER, - children={"E": Resource(type=ResourceType.FOLDER)}, - ) - }, - ), - }, - ) - - -@pytest.mark.parametrize( - "resources_to_attributions, expected_resources_to_attributions", - [ - ( - [ - {"resources/Path": ["identifier-A", "identifier-B"]}, - {"resources/Path": ["identifier-C"]}, - {"resources/Path/different": ["identifier-A"]}, - ], - { - "resources/Path": ["identifier-A", "identifier-B", "identifier-C"], - "resources/Path/different": ["identifier-A"], - }, - ), - ( - [{"resources/Path": ["uuid_1"]}, {"resources/Path": ["uuid_1", "uuid_2"]}], - { - "resources/Path": ["uuid_1", "uuid_2"], - }, - ), - ], -) -def test_merge_resources_to_attributions( - resources_to_attributions: list[dict[ResourcePath, list[OpossumPackageIdentifier]]], - expected_resources_to_attributions: dict[ - ResourcePath, list[OpossumPackageIdentifier] - ], -) -> None: - merged_resources_to_attributions = _merge_resources_to_attributions( - resources_to_attributions - ) - assert merged_resources_to_attributions == expected_resources_to_attributions - - -@mock.patch("opossum_lib.opossum.opossum_file.OpossumPackage", autospec=True) -def test_merge_dicts_without_duplicates(opossum_package: OpossumPackage) -> None: - dicts = [{"A": opossum_package}, {"B": opossum_package}] - merged_dict = _merge_dicts_without_duplicates(dicts) - - assert merged_dict == {"A": opossum_package, "B": opossum_package} - - -@mock.patch("opossum_lib.opossum.opossum_file.SourceInfo", autospec=True) -def test_merge_dicts_without_duplicates_type_error( - source_info: SourceInfo, -) -> None: - dicts = [ - {"A": OpossumPackage(source=source_info, comment="test package 1")}, - {"A": OpossumPackage(source=source_info, comment="test package 2")}, - ] - with pytest.raises(TypeError): - _merge_dicts_without_duplicates(dicts) - - -def test_expand_opossum_package_identifier() -> None: - opossum_package = OpossumPackage(source=SourceInfo(name="source-info")) - opossum_information_expanded = expand_opossum_package_identifier( - OpossumInformation( - metadata=Metadata( - project_id="project-id", - file_creation_date="2022-03-02", - project_title="project title", - ), - resources={}, - external_attributions={"SPDXRef-Package": opossum_package}, - resources_to_attributions={"/path/to/resource": ["SPDXRef-Package"]}, - attribution_breakpoints=[], - external_attribution_sources={}, - ) - ) - - assert opossum_information_expanded.resources_to_attributions == { - "/path/to/resource": ["project-id-SPDXRef-Package"] - } - assert opossum_information_expanded.external_attributions == { - "project-id-SPDXRef-Package": opossum_package - } From fd3a28547d41dbe6b9d9634c6dc2d22610376e8e Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 11:42:59 +0100 Subject: [PATCH 18/58] refactor: use common generator helpers for testing --- .../generators/generate_scancode_file.py | 2 +- tests/test_scancode/generators/helpers.py | 26 ------------------- tests/util/generator_helpers.py | 3 ++- 3 files changed, 3 insertions(+), 28 deletions(-) delete mode 100644 tests/test_scancode/generators/helpers.py diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/test_scancode/generators/generate_scancode_file.py index ddcf0075..9cc092d7 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/test_scancode/generators/generate_scancode_file.py @@ -37,7 +37,7 @@ SystemEnvironment, Url, ) -from tests.test_scancode.generators.helpers import entry_or_none, random_list +from tests.util.generator_helpers import entry_or_none, random_list type TempPathTree = dict[str, TempPathTree | None] diff --git a/tests/test_scancode/generators/helpers.py b/tests/test_scancode/generators/helpers.py deleted file mode 100644 index 6161cf5c..00000000 --- a/tests/test_scancode/generators/helpers.py +++ /dev/null @@ -1,26 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from collections.abc import Callable - -from faker.providers import BaseProvider -from faker.providers.misc import Provider as MiscProvider - - -def entry_or_none[T]( - faker: MiscProvider, entry: T, chance_of_getting_entry: int = 50 -) -> T | None: - if faker.boolean(chance_of_getting_entry): - return entry - else: - return None - - -def random_list[T]( - faker: BaseProvider, - entry_generator: Callable[[], T], - min_number_of_entries: int = 0, - max_number_of_entries: int = 3, -) -> list[T]: - number_of_entries = faker.random_int(min_number_of_entries, max_number_of_entries) - return [entry_generator() for _ in range(number_of_entries)] diff --git a/tests/util/generator_helpers.py b/tests/util/generator_helpers.py index 4e9ddddc..903163bb 100644 --- a/tests/util/generator_helpers.py +++ b/tests/util/generator_helpers.py @@ -20,6 +20,7 @@ def random_list[T]( faker: BaseProvider, entry_generator: Callable[[], T], max_number_of_entries: int = 3, + min_number_of_entries: int = 1, ) -> list[T]: - number_of_entries = faker.random_int(1, max_number_of_entries) + number_of_entries = faker.random_int(min_number_of_entries, max_number_of_entries) return [entry_generator() for _ in range(number_of_entries)] From b1d0e2d58f55452d59abcf5fd0d3cd994bcbb94a Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 12:19:36 +0100 Subject: [PATCH 19/58] feat: introduce import linter Goal: The different file formats should not depend on each other --- .github/workflows/lint_and_run_tests.yml | 3 ++ .importlinter | 18 +++++++++ pyproject.toml | 4 +- uv.lock | 49 ++++++++++++++++++++++-- 4 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 .importlinter diff --git a/.github/workflows/lint_and_run_tests.yml b/.github/workflows/lint_and_run_tests.yml index dd913041..7c0b4206 100644 --- a/.github/workflows/lint_and_run_tests.yml +++ b/.github/workflows/lint_and_run_tests.yml @@ -48,6 +48,9 @@ jobs: - name: Run mypy run: uv run python -m mypy src/ tests/ + - name: Run import linter + run: uv run lint-imports + test: runs-on: ${{ matrix.os }} if: | diff --git a/.importlinter b/.importlinter new file mode 100644 index 00000000..1c2be2da --- /dev/null +++ b/.importlinter @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +[importlinter] +root_package = opossum_lib +# Optional: +include_external_packages = True +exclude_type_checking_imports = True + +[importlinter:contract:file-formats-independent] +name = The different file formats should be independent +type = independence +modules = + opossum_lib.opossum + opossum_lib.scancode +ignore_imports = + ** -> opossum_lib.core.opossum_model diff --git a/pyproject.toml b/pyproject.toml index 90ee51c1..fdb6fb76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,6 @@ dependencies = [ "click>=8.1.8,<9", "pydantic>=2.10.6", "pyinstaller>=6.11.1", - "faker>=35.0.0", ] [project.urls] @@ -22,11 +21,12 @@ Repository = "https://github.com/opossum-tool/opossum-file" opossum-file = "opossum_lib.cli:opossum_file" [dependency-groups] -test = ["pytest>=8.3.4,<9"] +test = ["pytest>=8.3.4,<9", "faker>=35.0.0",] dev = [ "mypy>=1.14.1,<2", "pre-commit>=4.1.0,<5", "ruff>=0.9.3", + "import-linter>=2.1", ] [tool.uv] diff --git a/uv.lock b/uv.lock index 002d57a2..975ca4b8 100644 --- a/uv.lock +++ b/uv.lock @@ -80,6 +80,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b9/f8/feced7779d755758a52d1f6635d990b8d98dc0a29fa568bbe0625f18fdf3/filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", size = 16163 }, ] +[[package]] +name = "grimp" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bd/4f919930f4b9c89a6d6905e4e8e4f11aa2489e385064501ff191c7b7d9cb/grimp-3.5.tar.gz", hash = "sha256:dc9560aed1d82222b361fe5f312a775b15a96b9237670f3a9fb20f44d30d5762", size = 833830 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/02/33f4b4bc6d22b5ff7672ce5c67464f4dcee6e86cf3a072cdc9b1d46e5178/grimp-3.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:448dba63f938d0e13e6121559749816e3b2644202c912cc308e7608c6034737a", size = 351545 }, + { url = "https://files.pythonhosted.org/packages/78/e0/44bd64767763160ea0862ad756b88dfd375060b5636aaeb79c10861f09ae/grimp-3.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:17113aba41269d0ee91512c96eeb850c7c668440c6a8e0bfc94d17762184b293", size = 342780 }, + { url = "https://files.pythonhosted.org/packages/55/e6/b0fc52ab1ee04c80f1709eaa2e99468a8981d0a23dcbc7c2c6e781da22cc/grimp-3.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e6a55614945c319d1dc692c3e43f3a02b80c116a1298e593f5f887b97e6c983a", size = 399300 }, + { url = "https://files.pythonhosted.org/packages/dc/b9/aa6365859c96881d02f6ad6ab8aa38eaced5887f3cc431165ed9ee7b35ea/grimp-3.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:aba7ce7839b72efb4c6d06404d2b2a3026e28dd89816f4e546b3cd6626cbeeb1", size = 405288 }, + { url = "https://files.pythonhosted.org/packages/7d/f8/32446714ed0f23a1b8067f9f627c55f7c186d3477baa5c6e6312c95060ab/grimp-3.5-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eaedfba320a70d87b14acb25a685c8629586b943129c71ffd02b47d9531c11ce", size = 416565 }, + { url = "https://files.pythonhosted.org/packages/93/3c/185a0292111350823fdd7d61481157295d90260e248888ab274a00f84456/grimp-3.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:60a9afd3dd00ad1d301a07f97e26bc9ecdc3d2db39ab6ac46c315a7dea0a96cb", size = 437249 }, + { url = "https://files.pythonhosted.org/packages/1a/21/55ede2511dc111e9ca90d23e479e263e0845dc7e389515587f9575c5b139/grimp-3.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:11c66039c0475e5c9fc6a086264f11870181ae79f603caa5dffa1411ddad636b", size = 443560 }, + { url = "https://files.pythonhosted.org/packages/12/e1/79f6a4212d46ae6db6281c5b179990d2dd53c72eed7587592c0968f0d3f0/grimp-3.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bff39a0061790f074f86a7841cd8e6064aa7b2208cb1ee5c3f2e685dead2b66e", size = 393185 }, + { url = "https://files.pythonhosted.org/packages/46/62/1a7f8b54d73f3ef6595dd8bfa10686321c04b562fbb997e9d46f95fe7b96/grimp-3.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cf7f5367c4a87b8e9f08c09e7401d2d73f21cb65d6142445819f9df0d6ab3f6b", size = 573292 }, + { url = "https://files.pythonhosted.org/packages/24/1d/b90853a0b7eb7aa08073dafd896551d26a416e922a68eb1cab35319593c7/grimp-3.5-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:edee4b087f007dab8b65461caf6a1b67b2f9480cceb5f6aceea87008d8f283c4", size = 664226 }, + { url = "https://files.pythonhosted.org/packages/6b/7b/ce6cf6ba2da6305725692ea2a7a4080da60ac075d23353fc07cae7adb5b7/grimp-3.5-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:6af125013ad2a56c18f2f53a3fcabbfbe96c70374abecd6f14b82dc444726ebe", size = 589077 }, + { url = "https://files.pythonhosted.org/packages/ad/29/03156310a1bf278e8dd46e2e02311bad1d1551b1555d245e28008c318086/grimp-3.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:24aabae0183ca5fd5a710257ff37120b55d8e6d6d4cbb2c08481552832e5c901", size = 560463 }, + { url = "https://files.pythonhosted.org/packages/1e/29/300daf9b6a7f3f979a379cda69e04374912031fd85f3b6f0e03d9b637263/grimp-3.5-cp313-none-win32.whl", hash = "sha256:506091bfd600dd7ad427586998ef5e54a2098485148a1499bd9af5943d2fb0b7", size = 222673 }, + { url = "https://files.pythonhosted.org/packages/e5/8c/15834d919407c1ab11590367e03ac9d1c162d518a0f54f2970e315e0d623/grimp-3.5-cp313-none-win_amd64.whl", hash = "sha256:099388df82d922ddc589f362f1a523ab053c8dee5d29a6b622b2cddf481c6a2f", size = 236037 }, +] + [[package]] name = "identify" version = "2.6.5" @@ -89,6 +114,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/fa/dce098f4cdf7621aa8f7b4f919ce545891f489482f0bfa5102f3eca8608b/identify-2.6.5-py2.py3-none-any.whl", hash = "sha256:14181a47091eb75b337af4c23078c9d09225cd4c48929f521f3bf16b09d02566", size = 99078 }, ] +[[package]] +name = "import-linter" +version = "2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "grimp" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5e/d3/546d27e9390061cbc692541612bbaa3f69a3ad23f592cfbfefe0517ff5b0/import_linter-2.1.tar.gz", hash = "sha256:393fadb2e91304c22c5ceab575213ed2b6a794abc7bd05ab345f2430930b2eae", size = 29160 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d8/38/137d5310c52c1b79d1588afa2ce5038cb3a10dbfd1a642962f3bc3ccbe61/import_linter-2.1-py3-none-any.whl", hash = "sha256:324d65035f0252a8e432f60256b0d0d32d8d5d6e4f8fd29716688b09d7a2217a", size = 41402 }, +] + [[package]] name = "iniconfig" version = "2.0.0" @@ -153,36 +192,40 @@ version = "0.1" source = { editable = "." } dependencies = [ { name = "click" }, - { name = "faker" }, { name = "pydantic" }, { name = "pyinstaller" }, ] [package.dev-dependencies] dev = [ + { name = "import-linter" }, { name = "mypy" }, { name = "pre-commit" }, { name = "ruff" }, ] test = [ + { name = "faker" }, { name = "pytest" }, ] [package.metadata] requires-dist = [ { name = "click", specifier = ">=8.1.8,<9" }, - { name = "faker", specifier = ">=35.0.0" }, { name = "pydantic", specifier = ">=2.10.6" }, { name = "pyinstaller", specifier = ">=6.11.1" }, ] [package.metadata.requires-dev] dev = [ + { name = "import-linter", specifier = ">=2.1" }, { name = "mypy", specifier = ">=1.14.1,<2" }, { name = "pre-commit", specifier = ">=4.1.0,<5" }, { name = "ruff", specifier = ">=0.9.3" }, ] -test = [{ name = "pytest", specifier = ">=8.3.4,<9" }] +test = [ + { name = "faker", specifier = ">=35.0.0" }, + { name = "pytest", specifier = ">=8.3.4,<9" }, +] [[package]] name = "packaging" From b58ba849c9d92eb43b34d958a33f3fa6ab511a80 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 12:41:00 +0100 Subject: [PATCH 20/58] refactor: dedicated package for input formats This allows to write a rule for import linter which automatically extends on new packages --- .importlinter | 3 +- src/opossum_lib/core/input_reader.py | 6 ++- src/opossum_lib/core/opossum_generator.py | 2 +- src/opossum_lib/core/opossum_model.py | 6 +-- .../{opossum => input_formats}/__init__.py | 0 .../opossum}/__init__.py | 0 .../{ => input_formats}/opossum/constants.py | 0 .../opossum/opossum_file.py | 0 .../opossum/opossum_file_content.py | 9 +++-- .../opossum_file_to_opossum_converter.py | 40 +++++++++---------- .../opossum/opossum_file_writer.py | 4 +- .../opossum/opossum_format_reader.py | 4 +- .../opossum/output_model.py | 0 .../input_formats/scancode/__init__.py | 0 .../{ => input_formats}/scancode/constants.py | 0 .../{ => input_formats}/scancode/model.py | 0 .../scancode_data_to_opossum_converter.py | 9 ++++- .../scancode/scancode_format_reader.py | 4 +- tests/core/test_opossum_model.py | 2 +- .../opossum_provider.py | 2 +- tests/test_cli.py | 9 +++-- .../generators/generate_file_information.py | 2 +- .../generate_opossum_file_content.py | 6 +-- .../generate_outfile_information.py | 2 +- .../test_opossum/test_conversion_roundtrip.py | 4 +- tests/test_opossum/test_file_generation.py | 9 +++-- .../test_opossum_file_to_opossum_converter.py | 7 +++- tests/test_opossum/test_read_opossum_file.py | 2 +- .../generators/generate_scancode_file.py | 2 +- tests/test_scancode/model_helpers.py | 2 +- .../test_convert_scancode_to_opossum.py | 2 +- .../test_get_attribution_info.py | 6 +-- 32 files changed, 80 insertions(+), 64 deletions(-) rename src/opossum_lib/{opossum => input_formats}/__init__.py (100%) rename src/opossum_lib/{scancode => input_formats/opossum}/__init__.py (100%) rename src/opossum_lib/{ => input_formats}/opossum/constants.py (100%) rename src/opossum_lib/{ => input_formats}/opossum/opossum_file.py (100%) rename src/opossum_lib/{ => input_formats}/opossum/opossum_file_content.py (90%) rename src/opossum_lib/{ => input_formats}/opossum/opossum_file_to_opossum_converter.py (88%) rename src/opossum_lib/{ => input_formats}/opossum/opossum_file_writer.py (91%) rename src/opossum_lib/{ => input_formats}/opossum/opossum_format_reader.py (80%) rename src/opossum_lib/{ => input_formats}/opossum/output_model.py (100%) create mode 100644 src/opossum_lib/input_formats/scancode/__init__.py rename src/opossum_lib/{ => input_formats}/scancode/constants.py (100%) rename src/opossum_lib/{ => input_formats}/scancode/model.py (100%) rename src/opossum_lib/{ => input_formats}/scancode/scancode_data_to_opossum_converter.py (94%) rename src/opossum_lib/{ => input_formats}/scancode/scancode_format_reader.py (90%) diff --git a/.importlinter b/.importlinter index 1c2be2da..42d49924 100644 --- a/.importlinter +++ b/.importlinter @@ -12,7 +12,6 @@ exclude_type_checking_imports = True name = The different file formats should be independent type = independence modules = - opossum_lib.opossum - opossum_lib.scancode + opossum_lib.input_formats.* ignore_imports = ** -> opossum_lib.core.opossum_model diff --git a/src/opossum_lib/core/input_reader.py b/src/opossum_lib/core/input_reader.py index d036575d..9ed5dcd5 100644 --- a/src/opossum_lib/core/input_reader.py +++ b/src/opossum_lib/core/input_reader.py @@ -5,8 +5,10 @@ from opossum_lib.core.input_file import InputFile from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum -from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader -from opossum_lib.scancode.scancode_format_reader import ScancodeFormatReader +from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader +from opossum_lib.input_formats.scancode.scancode_format_reader import ( + ScancodeFormatReader, +) class InputReader: diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 451ed8f1..8a913229 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -5,7 +5,7 @@ from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments -from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter class OpossumGenerator: diff --git a/src/opossum_lib/core/opossum_model.py b/src/opossum_lib/core/opossum_model.py index 43ece1fa..5e3cdf31 100644 --- a/src/opossum_lib/core/opossum_model.py +++ b/src/opossum_lib/core/opossum_model.py @@ -15,9 +15,9 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.opossum.opossum_file as opossum_file -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.output_model import OpossumOutputFile +import opossum_lib.input_formats.opossum.opossum_file as opossum_file +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile type OpossumPackageIdentifier = str type ResourcePath = str diff --git a/src/opossum_lib/opossum/__init__.py b/src/opossum_lib/input_formats/__init__.py similarity index 100% rename from src/opossum_lib/opossum/__init__.py rename to src/opossum_lib/input_formats/__init__.py diff --git a/src/opossum_lib/scancode/__init__.py b/src/opossum_lib/input_formats/opossum/__init__.py similarity index 100% rename from src/opossum_lib/scancode/__init__.py rename to src/opossum_lib/input_formats/opossum/__init__.py diff --git a/src/opossum_lib/opossum/constants.py b/src/opossum_lib/input_formats/opossum/constants.py similarity index 100% rename from src/opossum_lib/opossum/constants.py rename to src/opossum_lib/input_formats/opossum/constants.py diff --git a/src/opossum_lib/opossum/opossum_file.py b/src/opossum_lib/input_formats/opossum/opossum_file.py similarity index 100% rename from src/opossum_lib/opossum/opossum_file.py rename to src/opossum_lib/input_formats/opossum/opossum_file.py diff --git a/src/opossum_lib/opossum/opossum_file_content.py b/src/opossum_lib/input_formats/opossum/opossum_file_content.py similarity index 90% rename from src/opossum_lib/opossum/opossum_file_content.py rename to src/opossum_lib/input_formats/opossum/opossum_file_content.py index b951c7d8..93e56a75 100644 --- a/src/opossum_lib/opossum/opossum_file_content.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_content.py @@ -11,9 +11,12 @@ from pydantic import BaseModel, TypeAdapter -from opossum_lib.opossum.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME -from opossum_lib.opossum.opossum_file import OpossumInformation -from opossum_lib.opossum.output_model import OpossumOutputFile +from opossum_lib.input_formats.opossum.constants import ( + INPUT_JSON_NAME, + OUTPUT_JSON_NAME, +) +from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation +from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile class OpossumFileContent(BaseModel): diff --git a/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py similarity index 88% rename from src/opossum_lib/opossum/opossum_file_to_opossum_converter.py rename to src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py index 97a82b35..6fe06042 100644 --- a/src/opossum_lib/opossum/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py @@ -5,7 +5,7 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.opossum.opossum_file +import opossum_lib.input_formats.opossum.opossum_file as opossum_file_package from opossum_lib.core.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, @@ -20,11 +20,11 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.opossum.opossum_file import ( +from opossum_lib.input_formats.opossum.opossum_file import ( ExternalAttributionSource as FileExternalAttributionSource, ) -from opossum_lib.opossum.opossum_file import OpossumInformation -from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent class OpossumFileToOpossumConverter: @@ -101,8 +101,8 @@ def _convert_to_opossum_scan_results( def _get_unassigned_attributions( used_attribution_ids: set[OpossumPackageIdentifier], external_attributions: dict[ - opossum_lib.opossum.opossum_file.OpossumPackageIdentifier, - opossum_lib.opossum.opossum_file.OpossumPackage, + opossum_file_package.OpossumPackageIdentifier, + opossum_file_package.OpossumPackage, ], ) -> list[OpossumPackage] | None: available_attribution_ids = external_attributions.keys() @@ -125,9 +125,7 @@ def _convert_external_attribution_source( @staticmethod def _convert_frequent_licenses_to_model_frequent_licenses( - frequent_licenses_infile: list[ - opossum_lib.opossum.opossum_file.FrequentLicense - ], + frequent_licenses_infile: list[opossum_file_package.FrequentLicense], ) -> list[FrequentLicense]: frequent_licenses: list[FrequentLicense] = [ OpossumFileToOpossumConverter._convert_frequent_license(license) @@ -137,27 +135,27 @@ def _convert_frequent_licenses_to_model_frequent_licenses( @staticmethod def _convert_to_opossum_model_metadata( - infile_metadata: opossum_lib.opossum.opossum_file.Metadata, + infile_metadata: opossum_file_package.Metadata, ) -> Metadata: return Metadata(**infile_metadata.model_dump()) @staticmethod def _convert_to_opossum_model_resource_tree( - resources: opossum_lib.opossum.opossum_file.ResourceInFile, + resources: opossum_file_package.ResourceInFile, external_attributions: dict[ - opossum_lib.opossum.opossum_file.OpossumPackageIdentifier, - opossum_lib.opossum.opossum_file.OpossumPackage, + opossum_file_package.OpossumPackageIdentifier, + opossum_file_package.OpossumPackage, ], resources_to_attributions: dict[ - opossum_lib.opossum.opossum_file.ResourcePath, - list[opossum_lib.opossum.opossum_file.OpossumPackageIdentifier], + opossum_file_package.ResourcePath, + list[opossum_file_package.OpossumPackageIdentifier], ], ) -> tuple[list[Resource], set[OpossumPackageIdentifier]]: used_attribution_ids = set() def generate_child_resource( current_path: PurePath, - to_insert: opossum_lib.opossum.opossum_file.ResourceInFile, + to_insert: opossum_file_package.ResourceInFile, ) -> Resource: path = current_path current_path_as_string = _convert_path_to_str(current_path) @@ -216,8 +214,8 @@ def _get_applicable_attributions( @staticmethod def _convert_to_attribution_with_id( external_attributions: dict[ - opossum_lib.opossum.opossum_file.OpossumPackageIdentifier, - opossum_lib.opossum.opossum_file.OpossumPackage, + opossum_file_package.OpossumPackageIdentifier, + opossum_file_package.OpossumPackage, ], ) -> dict[OpossumPackage, str]: result = {} @@ -234,7 +232,7 @@ def _convert_to_attribution_with_id( @staticmethod def _convert_frequent_license( - infile_frequent_license: opossum_lib.opossum.opossum_file.FrequentLicense, + infile_frequent_license: opossum_file_package.FrequentLicense, ) -> FrequentLicense: return FrequentLicense( full_name=infile_frequent_license.full_name, @@ -244,7 +242,7 @@ def _convert_frequent_license( @staticmethod def _convert_package( - infile_package: opossum_lib.opossum.opossum_file.OpossumPackage, + infile_package: opossum_file_package.OpossumPackage, ) -> OpossumPackage: pass return OpossumPackage( @@ -272,7 +270,7 @@ def _convert_package( @staticmethod def _convert_source( - infile_source_info: opossum_lib.opossum.opossum_file.SourceInfo, + infile_source_info: opossum_file_package.SourceInfo, ) -> SourceInfo: return SourceInfo( name=infile_source_info.name, diff --git a/src/opossum_lib/opossum/opossum_file_writer.py b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py similarity index 91% rename from src/opossum_lib/opossum/opossum_file_writer.py rename to src/opossum_lib/input_formats/opossum/opossum_file_writer.py index b832e81f..c3d462f3 100644 --- a/src/opossum_lib/opossum/opossum_file_writer.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py @@ -4,12 +4,12 @@ from pathlib import Path from zipfile import ZIP_DEFLATED, ZipFile -from opossum_lib.opossum.constants import ( +from opossum_lib.input_formats.opossum.constants import ( COMPRESSION_LEVEL, INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent class OpossumFileWriter: diff --git a/src/opossum_lib/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py similarity index 80% rename from src/opossum_lib/opossum/opossum_format_reader.py rename to src/opossum_lib/input_formats/opossum/opossum_format_reader.py index a1e0245f..79ba6530 100644 --- a/src/opossum_lib/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py @@ -6,8 +6,8 @@ from opossum_lib.core.input_file import FileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) diff --git a/src/opossum_lib/opossum/output_model.py b/src/opossum_lib/input_formats/opossum/output_model.py similarity index 100% rename from src/opossum_lib/opossum/output_model.py rename to src/opossum_lib/input_formats/opossum/output_model.py diff --git a/src/opossum_lib/input_formats/scancode/__init__.py b/src/opossum_lib/input_formats/scancode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/scancode/constants.py b/src/opossum_lib/input_formats/scancode/constants.py similarity index 100% rename from src/opossum_lib/scancode/constants.py rename to src/opossum_lib/input_formats/scancode/constants.py diff --git a/src/opossum_lib/scancode/model.py b/src/opossum_lib/input_formats/scancode/model.py similarity index 100% rename from src/opossum_lib/scancode/model.py rename to src/opossum_lib/input_formats/scancode/model.py diff --git a/src/opossum_lib/scancode/scancode_data_to_opossum_converter.py b/src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py similarity index 94% rename from src/opossum_lib/scancode/scancode_data_to_opossum_converter.py rename to src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py index 97f64528..aa023008 100644 --- a/src/opossum_lib/scancode/scancode_data_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py @@ -16,8 +16,13 @@ ScanResults, SourceInfo, ) -from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.scancode.model import File, FileType, Header, ScanCodeData +from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME +from opossum_lib.input_formats.scancode.model import ( + File, + FileType, + Header, + ScanCodeData, +) class ScancodeDataToOpossumConverter: diff --git a/src/opossum_lib/scancode/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py similarity index 90% rename from src/opossum_lib/scancode/scancode_format_reader.py rename to src/opossum_lib/input_formats/scancode/scancode_format_reader.py index c2ef19c5..1b6077a0 100644 --- a/src/opossum_lib/scancode/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py @@ -12,8 +12,8 @@ from opossum_lib.core.opossum_model import ( Opossum, ) -from opossum_lib.scancode.model import ScanCodeData -from opossum_lib.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.input_formats.scancode.model import ScanCodeData +from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( ScancodeDataToOpossumConverter, ) diff --git a/tests/core/test_opossum_model.py b/tests/core/test_opossum_model.py index f6928fb9..4d23cd9b 100644 --- a/tests/core/test_opossum_model.py +++ b/tests/core/test_opossum_model.py @@ -4,7 +4,7 @@ import json from copy import deepcopy -from opossum_lib.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) from tests.test_setup.opossum_faker_setup import OpossumFaker diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index 77e2d461..2009a627 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -6,7 +6,7 @@ from faker.providers import BaseProvider from opossum_lib.core.opossum_model import Opossum, ScanResults -from opossum_lib.opossum.output_model import OpossumOutputFile +from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, diff --git a/tests/test_cli.py b/tests/test_cli.py index 554e7ac1..6f83fcac 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,9 +11,12 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate -from opossum_lib.opossum.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME -from opossum_lib.opossum.opossum_file import OpossumPackage -from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.input_formats.opossum.constants import ( + INPUT_JSON_NAME, + OUTPUT_JSON_NAME, +) +from opossum_lib.input_formats.opossum.opossum_file import OpossumPackage +from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index 1664ea95..e02fe76a 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,7 +13,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.opossum.opossum_file import ( +from opossum_lib.input_formats.opossum.opossum_file import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index 9ec17e68..5460728f 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,9 +5,9 @@ from faker.providers import BaseProvider -from opossum_lib.opossum.opossum_file import OpossumInformation -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.output_model import OpossumOutputFile +from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, ) diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/test_opossum/generators/generate_outfile_information.py index dc2a5781..b3f7137a 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/test_opossum/generators/generate_outfile_information.py @@ -9,7 +9,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.opossum.output_model import ( +from opossum_lib.input_formats.opossum.output_model import ( ManualAttributions, Metadata, OpossumOutputFile, diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index a96ac68d..76ab748e 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index 6f614e4d..10950606 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -5,9 +5,12 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.opossum.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME -from opossum_lib.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.input_formats.opossum.constants import ( + INPUT_JSON_NAME, + OUTPUT_JSON_NAME, +) +from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index f62de679..cf6fe8b8 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -3,8 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.opossum.opossum_file import OpossumPackage, OpossumPackageIdentifier -from opossum_lib.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.opossum_file import ( + OpossumPackage, + OpossumPackageIdentifier, +) +from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/test_opossum/test_read_opossum_file.py index c5911a76..b6cd32ae 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/test_opossum/test_read_opossum_file.py @@ -6,7 +6,7 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.opossum.opossum_format_reader import OpossumFormatReader +from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader TEST_DATA_DIRECTORY = Path(__file__).resolve().parent.parent / "data" diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/test_scancode/generators/generate_scancode_file.py index 9cc092d7..85b4999b 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/test_scancode/generators/generate_scancode_file.py @@ -20,7 +20,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.scancode.model import ( +from opossum_lib.input_formats.scancode.model import ( Copyright, Email, ExtraData, diff --git a/tests/test_scancode/model_helpers.py b/tests/test_scancode/model_helpers.py index 40a3a1b2..e3a718cf 100644 --- a/tests/test_scancode/model_helpers.py +++ b/tests/test_scancode/model_helpers.py @@ -5,7 +5,7 @@ from pathlib import PurePath -from opossum_lib.scancode.model import ( +from opossum_lib.input_formats.scancode.model import ( Copyright, File, FileBasedLicenseDetection, diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/test_scancode/test_convert_scancode_to_opossum.py index 44290a66..cd434b44 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/test_scancode/test_convert_scancode_to_opossum.py @@ -7,7 +7,7 @@ from _pytest.logging import LogCaptureFixture from opossum_lib.core.opossum_model import Resource -from opossum_lib.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( ScancodeDataToOpossumConverter, ) from tests.test_setup.scancode_faker_setup import ScanCodeFaker diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index d5101f55..8c77352d 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -3,9 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.opossum.opossum_file import OpossumPackage, SourceInfo -from opossum_lib.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.opossum_file import OpossumPackage, SourceInfo +from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME +from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( ScancodeDataToOpossumConverter, ) from tests.test_setup.scancode_faker_setup import ScanCodeFaker From 18bb9d49977aec32fe7ae287b0348e582a4cdb34 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 16:53:14 +0100 Subject: [PATCH 21/58] refactor: review-comment: Rename input file type --- src/opossum_lib/core/input_file.py | 4 ++-- src/opossum_lib/core/input_format_reader.py | 4 ++-- src/opossum_lib/core/opossum_generation_arguments.py | 7 ++++--- .../input_formats/opossum/opossum_format_reader.py | 6 +++--- .../input_formats/scancode/scancode_format_reader.py | 6 +++--- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/opossum_lib/core/input_file.py b/src/opossum_lib/core/input_file.py index f0275552..6def8f69 100644 --- a/src/opossum_lib/core/input_file.py +++ b/src/opossum_lib/core/input_file.py @@ -7,11 +7,11 @@ from pydantic import BaseModel -class FileType(Enum): +class InputFileType(Enum): OPOSSUM = (auto(),) SCAN_CODE = (auto(),) class InputFile(BaseModel): path: Path - type: FileType + type: InputFileType diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/input_format_reader.py index 50872a86..e2493f47 100644 --- a/src/opossum_lib/core/input_format_reader.py +++ b/src/opossum_lib/core/input_format_reader.py @@ -6,13 +6,13 @@ from pathlib import Path from typing import Protocol -from opossum_lib.core.input_file import FileType +from opossum_lib.core.input_file import InputFileType from opossum_lib.core.opossum_model import Opossum class InputFormatReader(Protocol): @abstractmethod - def can_handle(self, file_type: FileType) -> bool: ... + def can_handle(self, file_type: InputFileType) -> bool: ... @abstractmethod def read(self, path: Path) -> Opossum: ... diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 895d582c..2947a33e 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -8,7 +8,7 @@ from pydantic import BaseModel -from opossum_lib.core.input_file import FileType, InputFile +from opossum_lib.core.input_file import InputFile, InputFileType class OpossumGenerationArguments(BaseModel): @@ -35,10 +35,11 @@ def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: def input_files(self) -> list[InputFile]: result = [] result += [ - InputFile(path=path, type=FileType.SCAN_CODE) + InputFile(path=path, type=InputFileType.SCAN_CODE) for path in self.scancode_json_files ] result += [ - InputFile(path=path, type=FileType.OPOSSUM) for path in self.opossum_files + InputFile(path=path, type=InputFileType.OPOSSUM) + for path in self.opossum_files ] return result diff --git a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py index 79ba6530..9fbe4d73 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from opossum_lib.core.input_file import FileType +from opossum_lib.core.input_file import InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent @@ -13,8 +13,8 @@ class OpossumFormatReader(InputFormatReader): - def can_handle(self, file_type: FileType) -> bool: - return file_type == FileType.OPOSSUM + def can_handle(self, file_type: InputFileType) -> bool: + return file_type == InputFileType.OPOSSUM def read(self, path: Path) -> Opossum: opossum_input_file = OpossumFileContent.from_file(path=path) diff --git a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py index 1b6077a0..79cd8c45 100644 --- a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py @@ -7,7 +7,7 @@ import sys from pathlib import Path -from opossum_lib.core.input_file import FileType +from opossum_lib.core.input_file import InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import ( Opossum, @@ -19,8 +19,8 @@ class ScancodeFormatReader(InputFormatReader): - def can_handle(self, file_type: FileType) -> bool: - return file_type == FileType.SCAN_CODE + def can_handle(self, file_type: InputFileType) -> bool: + return file_type == InputFileType.SCAN_CODE def read(self, path: Path) -> Opossum: logging.info(f"Converting scancode to opossum {path}") From 308d17510f1912df966963b3b098ea3dff44801f Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 16:57:53 +0100 Subject: [PATCH 22/58] refactor: review-comment: Using property annotation --- src/opossum_lib/core/opossum_generation_arguments.py | 1 + src/opossum_lib/core/opossum_generator.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 2947a33e..6b56ef37 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -32,6 +32,7 @@ def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: if Path.is_file(Path(self.outfile)): logging.warning(f"{self.outfile} already exists and will be overwritten.") + @property def input_files(self) -> list[InputFile]: result = [] result += [ diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index 8a913229..c6736305 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -15,7 +15,7 @@ def generate( self, opossum_generation_arguments: OpossumGenerationArguments ) -> None: opossum_generation_arguments.validate_and_exit_on_error() - input_files = opossum_generation_arguments.input_files() + input_files = opossum_generation_arguments.input_files opossum = self.input_reader.read(input_files[0]) From c46f1a21b5aa16de64acd77b903cdb2513b97e3d Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 17:30:57 +0100 Subject: [PATCH 23/58] feat: Remove misleading log statement * Remove the warning log if the output file is overwritten * It is not useful for the user as it she/he cannot react on the message * Instead, put a clearer warning in the docs + help message --- README.md | 5 +++-- src/opossum_lib/cli.py | 3 ++- .../core/opossum_generation_arguments.py | 7 ------- src/opossum_lib/core/opossum_generator.py | 1 - .../opossum/opossum_file_writer.py | 5 +++++ tests/test_cli.py | 21 ------------------- 6 files changed, 10 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 67050fda..b2dc5515 100644 --- a/README.md +++ b/README.md @@ -64,8 +64,9 @@ Options: that you would like to include in the final output. Option can be repeated. -o, --outfile TEXT The file path to write the generated opossum document - to. If appropriate, the extension ".opossum" will be - appended. [default: output.opossum] + to. If appropriate, the extension ".opossum" is + appended.If the output file already exists, it is + overwritten. [default: output.opossum] --help Show this message and exit. diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 3cb13916..ec8540ce 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -40,7 +40,8 @@ def opossum_file() -> None: default="output.opossum", show_default=True, help="The file path to write the generated opossum document to. " - 'If appropriate, the extension ".opossum" will be appended.', + 'If appropriate, the extension ".opossum" is appended.' + "If the output file already exists, it is overwritten.", ) def generate( scancode_json_files: list[Path], diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index 6b56ef37..d4abaa0b 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -25,13 +25,6 @@ def validate_and_exit_on_error(self) -> None: logging.error("Merging of multiple files not yet supported!") sys.exit(1) - def add_outfile_ending_and_warn_on_existing_outfile(self) -> None: - if self.outfile.suffix != ".opossum": - self.outfile = self.outfile.with_suffix(".opossum") - - if Path.is_file(Path(self.outfile)): - logging.warning(f"{self.outfile} already exists and will be overwritten.") - @property def input_files(self) -> list[InputFile]: result = [] diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index c6736305..aabaacf1 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -20,7 +20,6 @@ def generate( opossum = self.input_reader.read(input_files[0]) opossum_file_content = opossum.to_opossum_file_format() - opossum_generation_arguments.add_outfile_ending_and_warn_on_existing_outfile() OpossumFileWriter.write( opossum_file_content, opossum_generation_arguments.outfile ) diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py index c3d462f3..c625f5bd 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py @@ -15,6 +15,7 @@ class OpossumFileWriter: @staticmethod def write(opossum_file_content: OpossumFileContent, file_path: Path) -> None: + file_path = OpossumFileWriter._ensure_outfile_suffix(file_path) with ZipFile( file_path, "w", compression=ZIP_DEFLATED, compresslevel=COMPRESSION_LEVEL ) as zip_file: @@ -47,3 +48,7 @@ def _write_input_json( by_alias=True, ), ) + + @staticmethod + def _ensure_outfile_suffix(outfile_path: Path) -> Path: + return outfile_path.with_suffix(".opossum") diff --git a/tests/test_cli.py b/tests/test_cli.py index 6f83fcac..d351b043 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -190,27 +190,6 @@ def test_cli_no_output_file_provided(opossum_file_faker: OpossumFileFaker) -> No assert Path.is_file(Path("output.opossum")) -def test_cli_warning_if_outfile_already_exists( - caplog: LogCaptureFixture, opossum_file_faker: OpossumFileFaker -) -> None: - runner = CliRunner() - - with runner.isolated_filesystem(): - file_path = "input.opossum" - opossum_file = opossum_file_faker.opossum_file_content() - OpossumFileWriter.write(opossum_file, Path(file_path)) - with open("output.opossum", "w") as f: - f.write("") - result = runner.invoke( - generate, - "--opossum " + file_path + " -o output.opossum", - ) - - assert result.exit_code == 0 - - assert caplog.messages == ["output.opossum already exists and will be overwritten."] - - @pytest.mark.parametrize( "options", [ From 51a4603d02eeee509e62b1690fcb0899d865a99b Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 17:44:54 +0100 Subject: [PATCH 24/58] refactor: review comment: further improve the readability of the generator funktion --- .../core/opossum_generation_arguments.py | 21 ++++++++++--------- src/opossum_lib/core/opossum_generator.py | 3 +-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/opossum_generation_arguments.py index d4abaa0b..02aecd11 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/opossum_generation_arguments.py @@ -16,17 +16,9 @@ class OpossumGenerationArguments(BaseModel): opossum_files: list[Path] outfile: Path - def validate_and_exit_on_error(self) -> None: - total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) - @property - def input_files(self) -> list[InputFile]: + def valid_input_files(self) -> list[InputFile]: + self._validate_and_exit_on_error() result = [] result += [ InputFile(path=path, type=InputFileType.SCAN_CODE) @@ -37,3 +29,12 @@ def input_files(self) -> list[InputFile]: for path in self.opossum_files ] return result + + def _validate_and_exit_on_error(self) -> None: + total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index aabaacf1..bc86f239 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -14,8 +14,7 @@ class OpossumGenerator: def generate( self, opossum_generation_arguments: OpossumGenerationArguments ) -> None: - opossum_generation_arguments.validate_and_exit_on_error() - input_files = opossum_generation_arguments.input_files + input_files = opossum_generation_arguments.valid_input_files opossum = self.input_reader.read(input_files[0]) From 93ecfe140c02c6b8fe0a55eb76707df34e583a2b Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Tue, 28 Jan 2025 18:14:22 +0100 Subject: [PATCH 25/58] refactor: review comment: remove for loop to select reader * Note: Needed to switch the InputFileType enum to an int enum to be able to use it as a key in a dictionary --- src/opossum_lib/core/input_file.py | 4 ++-- src/opossum_lib/core/input_format_reader.py | 4 ---- src/opossum_lib/core/input_reader.py | 15 +++++++-------- .../opossum/opossum_format_reader.py | 4 ---- .../scancode/scancode_format_reader.py | 4 ---- 5 files changed, 9 insertions(+), 22 deletions(-) diff --git a/src/opossum_lib/core/input_file.py b/src/opossum_lib/core/input_file.py index 6def8f69..89ff4d60 100644 --- a/src/opossum_lib/core/input_file.py +++ b/src/opossum_lib/core/input_file.py @@ -1,13 +1,13 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 -from enum import Enum, auto +from enum import IntEnum, auto from pathlib import Path from pydantic import BaseModel -class InputFileType(Enum): +class InputFileType(IntEnum): OPOSSUM = (auto(),) SCAN_CODE = (auto(),) diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/input_format_reader.py index e2493f47..9d0af5c7 100644 --- a/src/opossum_lib/core/input_format_reader.py +++ b/src/opossum_lib/core/input_format_reader.py @@ -6,13 +6,9 @@ from pathlib import Path from typing import Protocol -from opossum_lib.core.input_file import InputFileType from opossum_lib.core.opossum_model import Opossum class InputFormatReader(Protocol): - @abstractmethod - def can_handle(self, file_type: InputFileType) -> bool: ... - @abstractmethod def read(self, path: Path) -> Opossum: ... diff --git a/src/opossum_lib/core/input_reader.py b/src/opossum_lib/core/input_reader.py index 9ed5dcd5..38a827ce 100644 --- a/src/opossum_lib/core/input_reader.py +++ b/src/opossum_lib/core/input_reader.py @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.core.input_file import InputFile +from opossum_lib.core.input_file import InputFile, InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader @@ -12,13 +12,12 @@ class InputReader: - input_format_readers: list[InputFormatReader] = [ - OpossumFormatReader(), - ScancodeFormatReader(), - ] + input_format_readers: dict[InputFileType, InputFormatReader] = { + InputFileType.OPOSSUM: OpossumFormatReader(), + InputFileType.SCAN_CODE: ScancodeFormatReader(), + } def read(self, input_file: InputFile) -> Opossum: - for input_format_reader in self.input_format_readers: - if input_format_reader.can_handle(input_file.type): - return input_format_reader.read(input_file.path) + if input_file.type in self.input_format_readers: + return self.input_format_readers[input_file.type].read(input_file.path) raise NotImplementedError(f"Unsupported file type: {input_file.type}") diff --git a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py index 9fbe4d73..6d285801 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py @@ -3,7 +3,6 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from opossum_lib.core.input_file import InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent @@ -13,9 +12,6 @@ class OpossumFormatReader(InputFormatReader): - def can_handle(self, file_type: InputFileType) -> bool: - return file_type == InputFileType.OPOSSUM - def read(self, path: Path) -> Opossum: opossum_input_file = OpossumFileContent.from_file(path=path) return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py index 79cd8c45..7ff1afd2 100644 --- a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/scancode_format_reader.py @@ -7,7 +7,6 @@ import sys from pathlib import Path -from opossum_lib.core.input_file import InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import ( Opossum, @@ -19,9 +18,6 @@ class ScancodeFormatReader(InputFormatReader): - def can_handle(self, file_type: InputFileType) -> bool: - return file_type == InputFileType.SCAN_CODE - def read(self, path: Path) -> Opossum: logging.info(f"Converting scancode to opossum {path}") From f58490a230b6041018d715dde5fd1f5cdb4e1e1f Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 07:26:05 +0100 Subject: [PATCH 26/58] refactor: move opossum file model to dedicated package * to remove dependency between core model and input models --- src/opossum_lib/core/opossum_model.py | 6 +++--- .../opossum/opossum_file_to_opossum_converter.py | 8 ++++---- .../input_formats/opossum/opossum_file_writer.py | 2 +- .../input_formats/opossum/opossum_format_reader.py | 2 +- src/opossum_lib/opossum_file_model/__init__.py | 0 .../opossum => opossum_file_model}/opossum_file.py | 0 .../opossum_file_content.py | 4 ++-- .../opossum => opossum_file_model}/output_model.py | 0 tests/opossum_model_generators/opossum_provider.py | 2 +- tests/test_cli.py | 2 +- .../test_opossum/generators/generate_file_information.py | 2 +- .../generators/generate_opossum_file_content.py | 6 +++--- .../generators/generate_outfile_information.py | 2 +- tests/test_opossum/test_conversion_roundtrip.py | 2 +- tests/test_opossum/test_file_generation.py | 2 +- .../test_opossum_file_to_opossum_converter.py | 8 ++++---- tests/test_scancode/test_get_attribution_info.py | 2 +- 17 files changed, 25 insertions(+), 25 deletions(-) create mode 100644 src/opossum_lib/opossum_file_model/__init__.py rename src/opossum_lib/{input_formats/opossum => opossum_file_model}/opossum_file.py (100%) rename src/opossum_lib/{input_formats/opossum => opossum_file_model}/opossum_file_content.py (94%) rename src/opossum_lib/{input_formats/opossum => opossum_file_model}/output_model.py (100%) diff --git a/src/opossum_lib/core/opossum_model.py b/src/opossum_lib/core/opossum_model.py index 5e3cdf31..858568d3 100644 --- a/src/opossum_lib/core/opossum_model.py +++ b/src/opossum_lib/core/opossum_model.py @@ -15,9 +15,9 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.input_formats.opossum.opossum_file as opossum_file -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile +import opossum_lib.opossum_file_model.opossum_file as opossum_file +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.output_model import OpossumOutputFile type OpossumPackageIdentifier = str type ResourcePath = str diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py index 6fe06042..166e9fba 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py @@ -5,7 +5,7 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.input_formats.opossum.opossum_file as opossum_file_package +import opossum_lib.opossum_file_model.opossum_file as opossum_file_package from opossum_lib.core.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, @@ -20,11 +20,11 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.input_formats.opossum.opossum_file import ( +from opossum_lib.opossum_file_model.opossum_file import ( ExternalAttributionSource as FileExternalAttributionSource, ) -from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.opossum_file import OpossumInformation +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent class OpossumFileToOpossumConverter: diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py index c625f5bd..132932e6 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_writer.py @@ -9,7 +9,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent class OpossumFileWriter: diff --git a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py index 6d285801..429df3af 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py @@ -5,10 +5,10 @@ from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent class OpossumFormatReader(InputFormatReader): diff --git a/src/opossum_lib/opossum_file_model/__init__.py b/src/opossum_lib/opossum_file_model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/input_formats/opossum/opossum_file.py b/src/opossum_lib/opossum_file_model/opossum_file.py similarity index 100% rename from src/opossum_lib/input_formats/opossum/opossum_file.py rename to src/opossum_lib/opossum_file_model/opossum_file.py diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_content.py b/src/opossum_lib/opossum_file_model/opossum_file_content.py similarity index 94% rename from src/opossum_lib/input_formats/opossum/opossum_file_content.py rename to src/opossum_lib/opossum_file_model/opossum_file_content.py index 93e56a75..aa80b70f 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_content.py +++ b/src/opossum_lib/opossum_file_model/opossum_file_content.py @@ -15,8 +15,8 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation -from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile +from opossum_lib.opossum_file_model.opossum_file import OpossumInformation +from opossum_lib.opossum_file_model.output_model import OpossumOutputFile class OpossumFileContent(BaseModel): diff --git a/src/opossum_lib/input_formats/opossum/output_model.py b/src/opossum_lib/opossum_file_model/output_model.py similarity index 100% rename from src/opossum_lib/input_formats/opossum/output_model.py rename to src/opossum_lib/opossum_file_model/output_model.py diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index 2009a627..bf7d6c6a 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -6,7 +6,7 @@ from faker.providers import BaseProvider from opossum_lib.core.opossum_model import Opossum, ScanResults -from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile +from opossum_lib.opossum_file_model.output_model import OpossumOutputFile from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, diff --git a/tests/test_cli.py b/tests/test_cli.py index d351b043..7e1b986f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,8 +15,8 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file import OpossumPackage from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.opossum_file_model.opossum_file import OpossumPackage from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index e02fe76a..4c9dc6fb 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,7 +13,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.input_formats.opossum.opossum_file import ( +from opossum_lib.opossum_file_model.opossum_file import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index 5460728f..d39d7720 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,9 +5,9 @@ from faker.providers import BaseProvider -from opossum_lib.input_formats.opossum.opossum_file import OpossumInformation -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent -from opossum_lib.input_formats.opossum.output_model import OpossumOutputFile +from opossum_lib.opossum_file_model.opossum_file import OpossumInformation +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.output_model import OpossumOutputFile from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, ) diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/test_opossum/generators/generate_outfile_information.py index b3f7137a..1008b3e0 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/test_opossum/generators/generate_outfile_information.py @@ -9,7 +9,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.input_formats.opossum.output_model import ( +from opossum_lib.opossum_file_model.output_model import ( ManualAttributions, Metadata, OpossumOutputFile, diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index 76ab748e..8193f121 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -3,10 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index 10950606..a377c50c 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -9,8 +9,8 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file_content import OpossumFileContent from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index cf6fe8b8..6b0cdd95 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -3,13 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.input_formats.opossum.opossum_file import ( - OpossumPackage, - OpossumPackageIdentifier, -) from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) +from opossum_lib.opossum_file_model.opossum_file import ( + OpossumPackage, + OpossumPackageIdentifier, +) from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index 8c77352d..bc038c2b 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.input_formats.opossum.opossum_file import OpossumPackage, SourceInfo from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( ScancodeDataToOpossumConverter, ) +from opossum_lib.opossum_file_model.opossum_file import OpossumPackage, SourceInfo from tests.test_setup.scancode_faker_setup import ScanCodeFaker From d24c10596bccbcd429e887b8eaa8739a175987fa Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 07:36:58 +0100 Subject: [PATCH 27/58] refactor: move also file writing into the package --- src/opossum_lib/core/opossum_generator.py | 2 +- .../opossum/opossum_file_reader.py | 65 +++++++++++++++++++ .../opossum/opossum_format_reader.py | 4 +- .../constants.py | 0 .../opossum_file_content.py | 64 +----------------- .../opossum_file_writer.py | 2 +- tests/test_cli.py | 4 +- tests/test_opossum/test_file_generation.py | 4 +- 8 files changed, 74 insertions(+), 71 deletions(-) create mode 100644 src/opossum_lib/input_formats/opossum/opossum_file_reader.py rename src/opossum_lib/{input_formats/opossum => opossum_file_model}/constants.py (100%) rename src/opossum_lib/{input_formats/opossum => opossum_file_model}/opossum_file_writer.py (96%) diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index bc86f239..ae679b37 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -5,7 +5,7 @@ from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments -from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter +from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter class OpossumGenerator: diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/opossum_file_reader.py new file mode 100644 index 00000000..a5ce8dda --- /dev/null +++ b/src/opossum_lib/input_formats/opossum/opossum_file_reader.py @@ -0,0 +1,65 @@ +import json +import logging +import sys +from pathlib import Path +from zipfile import ZipFile + +from opossum_lib.opossum_file_model.constants import ( + INPUT_JSON_NAME, + OUTPUT_JSON_NAME, +) +from opossum_lib.opossum_file_model.opossum_file import OpossumInformation +from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.output_model import OpossumOutputFile + + +class OpossumFileReader: + @staticmethod + def from_file(path: Path) -> OpossumFileContent: + logging.info(f"Converting opossum to opossum {path}") + + try: + with ( + ZipFile(path, "r") as input_zip_file, + ): + OpossumFileReader._validate_zip_file_contents(input_zip_file) + input_file = OpossumFileReader._read_input_json_from_zip_file( + input_zip_file + ) + return OpossumFileContent( + input_file=input_file, + output_file=OpossumFileReader._read_output_json_if_exists( + input_zip_file + ), + ) + except Exception as e: + print(f"Error reading file {path}: {e}") + sys.exit(1) + + @staticmethod + def _read_input_json_from_zip_file(input_zip_file: ZipFile) -> OpossumInformation: + with input_zip_file.open(INPUT_JSON_NAME) as input_json_file: + input_json = json.load(input_json_file) + input_file = OpossumInformation.model_validate(input_json) + return input_file + + @staticmethod + def _read_output_json_if_exists( + input_zip_file: ZipFile, + ) -> OpossumOutputFile | None: + if OUTPUT_JSON_NAME in input_zip_file.namelist(): + with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: + output_json = json.load(output_json_file) + output_file = OpossumOutputFile.model_validate(output_json) + else: + output_file = None + return output_file + + @staticmethod + def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: + if INPUT_JSON_NAME not in input_zip_file.namelist(): + logging.error( + f"Opossum file {input_zip_file.filename} is corrupt" + f" and does not contain '{INPUT_JSON_NAME}'" + ) + sys.exit(1) diff --git a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py index 429df3af..26001e9d 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_format_reader.py @@ -5,13 +5,13 @@ from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum +from opossum_lib.input_formats.opossum.opossum_file_reader import OpossumFileReader from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( OpossumFileToOpossumConverter, ) -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent class OpossumFormatReader(InputFormatReader): def read(self, path: Path) -> Opossum: - opossum_input_file = OpossumFileContent.from_file(path=path) + opossum_input_file = OpossumFileReader.from_file(path=path) return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/input_formats/opossum/constants.py b/src/opossum_lib/opossum_file_model/constants.py similarity index 100% rename from src/opossum_lib/input_formats/opossum/constants.py rename to src/opossum_lib/opossum_file_model/constants.py diff --git a/src/opossum_lib/opossum_file_model/opossum_file_content.py b/src/opossum_lib/opossum_file_model/opossum_file_content.py index aa80b70f..01c3e235 100644 --- a/src/opossum_lib/opossum_file_model/opossum_file_content.py +++ b/src/opossum_lib/opossum_file_model/opossum_file_content.py @@ -3,18 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import json -import logging -import sys -from pathlib import Path -from zipfile import ZipFile +from pydantic import BaseModel -from pydantic import BaseModel, TypeAdapter - -from opossum_lib.input_formats.opossum.constants import ( - INPUT_JSON_NAME, - OUTPUT_JSON_NAME, -) from opossum_lib.opossum_file_model.opossum_file import OpossumInformation from opossum_lib.opossum_file_model.output_model import OpossumOutputFile @@ -22,55 +12,3 @@ class OpossumFileContent(BaseModel): input_file: OpossumInformation output_file: OpossumOutputFile | None = None - - @staticmethod - def from_file(path: Path) -> OpossumFileContent: - logging.info(f"Converting opossum to opossum {path}") - - try: - with ( - ZipFile(path, "r") as input_zip_file, - ): - OpossumFileContent._validate_zip_file_contents(input_zip_file) - input_file = OpossumFileContent._read_input_json_from_zip_file( - input_zip_file - ) - return OpossumFileContent( - input_file=input_file, - output_file=OpossumFileContent._read_output_json_if_exists( - input_zip_file - ), - ) - except Exception as e: - print(f"Error reading file {path}: {e}") - sys.exit(1) - - @staticmethod - def _read_input_json_from_zip_file(input_zip_file: ZipFile) -> OpossumInformation: - with input_zip_file.open(INPUT_JSON_NAME) as input_json_file: - input_json = json.load(input_json_file) - input_file = OpossumInformation.model_validate(input_json) - return input_file - - @staticmethod - def _read_output_json_if_exists( - input_zip_file: ZipFile, - ) -> OpossumOutputFile | None: - if OUTPUT_JSON_NAME in input_zip_file.namelist(): - with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: - output_json = json.load(output_json_file) - output_file = TypeAdapter(OpossumOutputFile).validate_python( - output_json - ) - else: - output_file = None - return output_file - - @staticmethod - def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: - if INPUT_JSON_NAME not in input_zip_file.namelist(): - logging.error( - f"Opossum file {input_zip_file.filename} is corrupt" - f" and does not contain '{INPUT_JSON_NAME}'" - ) - sys.exit(1) diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py b/src/opossum_lib/opossum_file_model/opossum_file_writer.py similarity index 96% rename from src/opossum_lib/input_formats/opossum/opossum_file_writer.py rename to src/opossum_lib/opossum_file_model/opossum_file_writer.py index 132932e6..3a641527 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_writer.py +++ b/src/opossum_lib/opossum_file_model/opossum_file_writer.py @@ -4,7 +4,7 @@ from pathlib import Path from zipfile import ZIP_DEFLATED, ZipFile -from opossum_lib.input_formats.opossum.constants import ( +from opossum_lib.opossum_file_model.constants import ( COMPRESSION_LEVEL, INPUT_JSON_NAME, OUTPUT_JSON_NAME, diff --git a/tests/test_cli.py b/tests/test_cli.py index 7e1b986f..582b795a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,12 +11,12 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate -from opossum_lib.input_formats.opossum.constants import ( +from opossum_lib.opossum_file_model.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter from opossum_lib.opossum_file_model.opossum_file import OpossumPackage +from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index a377c50c..af1df5f6 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -5,12 +5,12 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.input_formats.opossum.constants import ( +from opossum_lib.opossum_file_model.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.input_formats.opossum.opossum_file_writer import OpossumFileWriter from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker From 5dd9461b28aa428087a87a074b03b5a95da351b0 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 07:45:45 +0100 Subject: [PATCH 28/58] refactor: DIY dependency injection: Define input readers at top level * This also allows to activate the next importlinter rule --- .importlinter | 9 +++++++++ src/opossum_lib/cli.py | 22 +++++++++++++++++----- src/opossum_lib/core/input_reader.py | 12 ++++-------- src/opossum_lib/core/opossum_generator.py | 9 ++++++--- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/.importlinter b/.importlinter index 42d49924..69a36107 100644 --- a/.importlinter +++ b/.importlinter @@ -15,3 +15,12 @@ modules = opossum_lib.input_formats.* ignore_imports = ** -> opossum_lib.core.opossum_model + + +[importlinter:contract:core-should-not-depend -on-input-files] +name = Core should not depend on input files +type = forbidden +source_modules = + opossum_lib.core +forbidden_modules = + opossum_lib.input_formats diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index ec8540ce..21002958 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -6,10 +6,15 @@ # SPDX-License-Identifier: Apache-2.0 import click +from opossum_lib.core.input_file import InputFileType from opossum_lib.core.opossum_generator import ( OpossumGenerationArguments, OpossumGenerator, ) +from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader +from opossum_lib.input_formats.scancode.scancode_format_reader import ( + ScancodeFormatReader, +) @click.group() @@ -56,11 +61,18 @@ def generate( - ScanCode - Opossum """ - OpossumGenerator().generate( - opossum_generation_arguments=OpossumGenerationArguments( - opossum_files=opossum_files, - scancode_json_files=scancode_json_files, - outfile=outfile, + ( + OpossumGenerator( + input_format_readers={ + InputFileType.OPOSSUM: OpossumFormatReader(), + InputFileType.SCAN_CODE: ScancodeFormatReader(), + } + ).generate( + opossum_generation_arguments=OpossumGenerationArguments( + opossum_files=opossum_files, + scancode_json_files=scancode_json_files, + outfile=outfile, + ) ) ) diff --git a/src/opossum_lib/core/input_reader.py b/src/opossum_lib/core/input_reader.py index 38a827ce..066a200f 100644 --- a/src/opossum_lib/core/input_reader.py +++ b/src/opossum_lib/core/input_reader.py @@ -5,17 +5,13 @@ from opossum_lib.core.input_file import InputFile, InputFileType from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.opossum_model import Opossum -from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader -from opossum_lib.input_formats.scancode.scancode_format_reader import ( - ScancodeFormatReader, -) class InputReader: - input_format_readers: dict[InputFileType, InputFormatReader] = { - InputFileType.OPOSSUM: OpossumFormatReader(), - InputFileType.SCAN_CODE: ScancodeFormatReader(), - } + input_format_readers: dict[InputFileType, InputFormatReader] + + def __init__(self, input_format_readers: dict[InputFileType, InputFormatReader]): + self.input_format_readers = input_format_readers def read(self, input_file: InputFile) -> Opossum: if input_file.type in self.input_format_readers: diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/opossum_generator.py index ae679b37..ebd5dc13 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/opossum_generator.py @@ -1,15 +1,18 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 - - +from opossum_lib.core.input_file import InputFileType +from opossum_lib.core.input_format_reader import InputFormatReader from opossum_lib.core.input_reader import InputReader from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter class OpossumGenerator: - input_reader: InputReader = InputReader() + input_reader: InputReader + + def __init__(self, input_format_readers: dict[InputFileType, InputFormatReader]): + self.input_reader = InputReader(input_format_readers) def generate( self, opossum_generation_arguments: OpossumGenerationArguments From 780a80b8edb00ba6ee6adf3c759152373db7e76d Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 07:51:03 +0100 Subject: [PATCH 29/58] feat: Add importlinter rule to keep file format independent --- .importlinter | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.importlinter b/.importlinter index 69a36107..a81be45d 100644 --- a/.importlinter +++ b/.importlinter @@ -24,3 +24,11 @@ source_modules = opossum_lib.core forbidden_modules = opossum_lib.input_formats + +[importlinter:contract:opossum_file_format_should_not_depend_on_input_formats] +name = Opossum file format should not depend on input formats +type = forbidden +source_modules = + opossum_lib.opossum_file_model +forbidden_modules = + opossum_lib.input_formats From bf6ab6e603bf9bb5037dbc68a4b18015ea7631d1 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 09:11:22 +0100 Subject: [PATCH 30/58] fix: Add missing license header --- src/opossum_lib/input_formats/opossum/opossum_file_reader.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/opossum_file_reader.py index a5ce8dda..6c68bcc4 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_reader.py @@ -1,3 +1,8 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + + import json import logging import sys From 137a3d0d7c82182f384154197b7ffc5747ece504 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 12:57:45 +0100 Subject: [PATCH 31/58] fix: Remove surplus pass --- .../input_formats/opossum/opossum_file_to_opossum_converter.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py index 166e9fba..9a6d6db2 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py @@ -244,7 +244,6 @@ def _convert_frequent_license( def _convert_package( infile_package: opossum_file_package.OpossumPackage, ) -> OpossumPackage: - pass return OpossumPackage( source=OpossumFileToOpossumConverter._convert_source(infile_package.source), attribution_confidence=infile_package.attribution_confidence, From 9d5f9c9ea7f26fb88aff916612e9282728cc3382 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:25:53 +0100 Subject: [PATCH 32/58] feat: review-comment: New package structure * For each package add a substructure * entities * services * Move the global model to a shared package --- .importlinter | 8 +++----- src/opossum_lib/cli.py | 10 ++++++---- .../entities}/__init__.py | 0 src/opossum_lib/core/{ => entities}/input_file.py | 0 .../{ => entities}/opossum_generation_arguments.py | 2 +- src/opossum_lib/core/{ => entities}/opossum_model.py | 6 +++--- src/opossum_lib/core/services/__init__.py | 0 .../core/{ => services}/input_format_reader.py | 2 +- src/opossum_lib/core/{ => services}/input_reader.py | 6 +++--- .../core/{ => services}/opossum_generator.py | 12 +++++++----- .../input_formats/opossum/entities/__init__.py | 0 .../input_formats/opossum/services/__init__.py | 0 .../opossum/{ => services}/opossum_file_reader.py | 8 ++++---- .../opossum_file_to_opossum_converter.py | 10 +++++----- .../opossum/{ => services}/opossum_format_reader.py | 10 ++++++---- .../input_formats/scancode/entities/__init__.py | 0 .../input_formats/scancode/{ => entities}/model.py | 0 .../input_formats/scancode/services/__init__.py | 0 .../scancode_data_to_opossum_converter.py | 4 ++-- .../{ => services}/scancode_format_reader.py | 8 ++++---- src/opossum_lib/shared/__init__.py | 0 .../{opossum_file_model => shared}/constants.py | 0 src/opossum_lib/shared/entities/__init__.py | 0 .../entities}/opossum_file.py | 0 .../entities}/opossum_file_content.py | 4 ++-- .../entities}/output_model.py | 0 src/opossum_lib/shared/services/__init__.py | 0 .../services}/opossum_file_writer.py | 4 ++-- tests/core/test_opossum_model.py | 2 +- .../external_attribution_source_provider.py | 2 +- tests/opossum_model_generators/metadata_provider.py | 2 +- tests/opossum_model_generators/opossum_provider.py | 4 ++-- tests/opossum_model_generators/package_provider.py | 2 +- tests/opossum_model_generators/resource_provider.py | 6 +++++- .../scan_results_provider.py | 2 +- .../opossum_model_generators/source_info_provider.py | 2 +- tests/test_cli.py | 6 +++--- .../generators/generate_file_information.py | 2 +- .../generators/generate_opossum_file_content.py | 6 +++--- .../generators/generate_outfile_information.py | 2 +- tests/test_opossum/test_conversion_roundtrip.py | 4 ++-- tests/test_opossum/test_file_generation.py | 6 +++--- .../test_opossum_file_to_opossum_converter.py | 4 ++-- tests/test_opossum/test_read_opossum_file.py | 4 +++- .../generators/generate_scancode_file.py | 2 +- tests/test_scancode/model_helpers.py | 2 +- .../test_convert_scancode_to_opossum.py | 4 ++-- tests/test_scancode/test_get_attribution_info.py | 4 ++-- 48 files changed, 86 insertions(+), 76 deletions(-) rename src/opossum_lib/{opossum_file_model => core/entities}/__init__.py (100%) rename src/opossum_lib/core/{ => entities}/input_file.py (100%) rename src/opossum_lib/core/{ => entities}/opossum_generation_arguments.py (94%) rename src/opossum_lib/core/{ => entities}/opossum_model.py (98%) create mode 100644 src/opossum_lib/core/services/__init__.py rename src/opossum_lib/core/{ => services}/input_format_reader.py (83%) rename src/opossum_lib/core/{ => services}/input_reader.py (74%) rename src/opossum_lib/core/{ => services}/opossum_generator.py (65%) create mode 100644 src/opossum_lib/input_formats/opossum/entities/__init__.py create mode 100644 src/opossum_lib/input_formats/opossum/services/__init__.py rename src/opossum_lib/input_formats/opossum/{ => services}/opossum_file_reader.py (88%) rename src/opossum_lib/input_formats/opossum/{ => services}/opossum_file_to_opossum_converter.py (96%) rename src/opossum_lib/input_formats/opossum/{ => services}/opossum_format_reader.py (55%) create mode 100644 src/opossum_lib/input_formats/scancode/entities/__init__.py rename src/opossum_lib/input_formats/scancode/{ => entities}/model.py (100%) create mode 100644 src/opossum_lib/input_formats/scancode/services/__init__.py rename src/opossum_lib/input_formats/scancode/{ => services}/scancode_data_to_opossum_converter.py (96%) rename src/opossum_lib/input_formats/scancode/{ => services}/scancode_format_reader.py (77%) create mode 100644 src/opossum_lib/shared/__init__.py rename src/opossum_lib/{opossum_file_model => shared}/constants.py (100%) create mode 100644 src/opossum_lib/shared/entities/__init__.py rename src/opossum_lib/{opossum_file_model => shared/entities}/opossum_file.py (100%) rename src/opossum_lib/{opossum_file_model => shared/entities}/opossum_file_content.py (67%) rename src/opossum_lib/{opossum_file_model => shared/entities}/output_model.py (100%) create mode 100644 src/opossum_lib/shared/services/__init__.py rename src/opossum_lib/{opossum_file_model => shared/services}/opossum_file_writer.py (92%) diff --git a/.importlinter b/.importlinter index a81be45d..77a0db24 100644 --- a/.importlinter +++ b/.importlinter @@ -13,8 +13,6 @@ name = The different file formats should be independent type = independence modules = opossum_lib.input_formats.* -ignore_imports = - ** -> opossum_lib.core.opossum_model [importlinter:contract:core-should-not-depend -on-input-files] @@ -25,10 +23,10 @@ source_modules = forbidden_modules = opossum_lib.input_formats -[importlinter:contract:opossum_file_format_should_not_depend_on_input_formats] -name = Opossum file format should not depend on input formats +[importlinter:contract:shared_objects_not_depend_on_input_formats] +name = Shared package not depend on input formats type = forbidden source_modules = - opossum_lib.opossum_file_model + opossum_lib.shared forbidden_modules = opossum_lib.input_formats diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 21002958..b2cf151d 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -6,13 +6,15 @@ # SPDX-License-Identifier: Apache-2.0 import click -from opossum_lib.core.input_file import InputFileType -from opossum_lib.core.opossum_generator import ( +from opossum_lib.core.entities.input_file import InputFileType +from opossum_lib.core.services.opossum_generator import ( OpossumGenerationArguments, OpossumGenerator, ) -from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader -from opossum_lib.input_formats.scancode.scancode_format_reader import ( +from opossum_lib.input_formats.opossum.services.opossum_format_reader import ( + OpossumFormatReader, +) +from opossum_lib.input_formats.scancode.services.scancode_format_reader import ( ScancodeFormatReader, ) diff --git a/src/opossum_lib/opossum_file_model/__init__.py b/src/opossum_lib/core/entities/__init__.py similarity index 100% rename from src/opossum_lib/opossum_file_model/__init__.py rename to src/opossum_lib/core/entities/__init__.py diff --git a/src/opossum_lib/core/input_file.py b/src/opossum_lib/core/entities/input_file.py similarity index 100% rename from src/opossum_lib/core/input_file.py rename to src/opossum_lib/core/entities/input_file.py diff --git a/src/opossum_lib/core/opossum_generation_arguments.py b/src/opossum_lib/core/entities/opossum_generation_arguments.py similarity index 94% rename from src/opossum_lib/core/opossum_generation_arguments.py rename to src/opossum_lib/core/entities/opossum_generation_arguments.py index 02aecd11..f15f49ff 100644 --- a/src/opossum_lib/core/opossum_generation_arguments.py +++ b/src/opossum_lib/core/entities/opossum_generation_arguments.py @@ -8,7 +8,7 @@ from pydantic import BaseModel -from opossum_lib.core.input_file import InputFile, InputFileType +from opossum_lib.core.entities.input_file import InputFile, InputFileType class OpossumGenerationArguments(BaseModel): diff --git a/src/opossum_lib/core/opossum_model.py b/src/opossum_lib/core/entities/opossum_model.py similarity index 98% rename from src/opossum_lib/core/opossum_model.py rename to src/opossum_lib/core/entities/opossum_model.py index 858568d3..5329cd4b 100644 --- a/src/opossum_lib/core/opossum_model.py +++ b/src/opossum_lib/core/entities/opossum_model.py @@ -15,9 +15,9 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.opossum_file_model.opossum_file as opossum_file -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent -from opossum_lib.opossum_file_model.output_model import OpossumOutputFile +import opossum_lib.shared.entities.opossum_file as opossum_file +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.output_model import OpossumOutputFile type OpossumPackageIdentifier = str type ResourcePath = str diff --git a/src/opossum_lib/core/services/__init__.py b/src/opossum_lib/core/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/core/input_format_reader.py b/src/opossum_lib/core/services/input_format_reader.py similarity index 83% rename from src/opossum_lib/core/input_format_reader.py rename to src/opossum_lib/core/services/input_format_reader.py index 9d0af5c7..7b5a6a6b 100644 --- a/src/opossum_lib/core/input_format_reader.py +++ b/src/opossum_lib/core/services/input_format_reader.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Protocol -from opossum_lib.core.opossum_model import Opossum +from opossum_lib.core.entities.opossum_model import Opossum class InputFormatReader(Protocol): diff --git a/src/opossum_lib/core/input_reader.py b/src/opossum_lib/core/services/input_reader.py similarity index 74% rename from src/opossum_lib/core/input_reader.py rename to src/opossum_lib/core/services/input_reader.py index 066a200f..91c6c693 100644 --- a/src/opossum_lib/core/input_reader.py +++ b/src/opossum_lib/core/services/input_reader.py @@ -2,9 +2,9 @@ # # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.core.input_file import InputFile, InputFileType -from opossum_lib.core.input_format_reader import InputFormatReader -from opossum_lib.core.opossum_model import Opossum +from opossum_lib.core.entities.input_file import InputFile, InputFileType +from opossum_lib.core.entities.opossum_model import Opossum +from opossum_lib.core.services.input_format_reader import InputFormatReader class InputReader: diff --git a/src/opossum_lib/core/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py similarity index 65% rename from src/opossum_lib/core/opossum_generator.py rename to src/opossum_lib/core/services/opossum_generator.py index ebd5dc13..83902a2e 100644 --- a/src/opossum_lib/core/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -1,11 +1,13 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.core.input_file import InputFileType -from opossum_lib.core.input_format_reader import InputFormatReader -from opossum_lib.core.input_reader import InputReader -from opossum_lib.core.opossum_generation_arguments import OpossumGenerationArguments -from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter +from opossum_lib.core.entities.input_file import InputFileType +from opossum_lib.core.entities.opossum_generation_arguments import ( + OpossumGenerationArguments, +) +from opossum_lib.core.services.input_format_reader import InputFormatReader +from opossum_lib.core.services.input_reader import InputReader +from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter class OpossumGenerator: diff --git a/src/opossum_lib/input_formats/opossum/entities/__init__.py b/src/opossum_lib/input_formats/opossum/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/input_formats/opossum/services/__init__.py b/src/opossum_lib/input_formats/opossum/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py similarity index 88% rename from src/opossum_lib/input_formats/opossum/opossum_file_reader.py rename to src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 6c68bcc4..7999cd16 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -9,13 +9,13 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.opossum_file_model.constants import ( +from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.opossum_file_model.opossum_file import OpossumInformation -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent -from opossum_lib.opossum_file_model.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_file import OpossumInformation +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.output_model import OpossumOutputFile class OpossumFileReader: diff --git a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py similarity index 96% rename from src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py rename to src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py index 9a6d6db2..cb54e57b 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py @@ -5,8 +5,8 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.opossum_file_model.opossum_file as opossum_file_package -from opossum_lib.core.opossum_model import ( +import opossum_lib.shared.entities.opossum_file as opossum_file_package +from opossum_lib.core.entities.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, @@ -20,11 +20,11 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.opossum_file_model.opossum_file import ( +from opossum_lib.shared.entities.opossum_file import ( ExternalAttributionSource as FileExternalAttributionSource, ) -from opossum_lib.opossum_file_model.opossum_file import OpossumInformation -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file import OpossumInformation +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent class OpossumFileToOpossumConverter: diff --git a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py similarity index 55% rename from src/opossum_lib/input_formats/opossum/opossum_format_reader.py rename to src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py index 26001e9d..e3dc7aae 100644 --- a/src/opossum_lib/input_formats/opossum/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py @@ -3,10 +3,12 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from opossum_lib.core.input_format_reader import InputFormatReader -from opossum_lib.core.opossum_model import Opossum -from opossum_lib.input_formats.opossum.opossum_file_reader import OpossumFileReader -from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.core.entities.opossum_model import Opossum +from opossum_lib.core.services.input_format_reader import InputFormatReader +from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( + OpossumFileReader, +) +from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) diff --git a/src/opossum_lib/input_formats/scancode/entities/__init__.py b/src/opossum_lib/input_formats/scancode/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/input_formats/scancode/model.py b/src/opossum_lib/input_formats/scancode/entities/model.py similarity index 100% rename from src/opossum_lib/input_formats/scancode/model.py rename to src/opossum_lib/input_formats/scancode/entities/model.py diff --git a/src/opossum_lib/input_formats/scancode/services/__init__.py b/src/opossum_lib/input_formats/scancode/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py similarity index 96% rename from src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py rename to src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py index aa023008..25ef66dd 100644 --- a/src/opossum_lib/input_formats/scancode/scancode_data_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py @@ -7,7 +7,7 @@ import uuid from pathlib import PurePath -from opossum_lib.core.opossum_model import ( +from opossum_lib.core.entities.opossum_model import ( Metadata, Opossum, OpossumPackage, @@ -17,7 +17,7 @@ SourceInfo, ) from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.model import ( +from opossum_lib.input_formats.scancode.entities.model import ( File, FileType, Header, diff --git a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py similarity index 77% rename from src/opossum_lib/input_formats/scancode/scancode_format_reader.py rename to src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index 7ff1afd2..e235785e 100644 --- a/src/opossum_lib/input_formats/scancode/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -7,12 +7,12 @@ import sys from pathlib import Path -from opossum_lib.core.input_format_reader import InputFormatReader -from opossum_lib.core.opossum_model import ( +from opossum_lib.core.entities.opossum_model import ( Opossum, ) -from opossum_lib.input_formats.scancode.model import ScanCodeData -from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.core.services.input_format_reader import InputFormatReader +from opossum_lib.input_formats.scancode.entities.model import ScanCodeData +from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) diff --git a/src/opossum_lib/shared/__init__.py b/src/opossum_lib/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/opossum_file_model/constants.py b/src/opossum_lib/shared/constants.py similarity index 100% rename from src/opossum_lib/opossum_file_model/constants.py rename to src/opossum_lib/shared/constants.py diff --git a/src/opossum_lib/shared/entities/__init__.py b/src/opossum_lib/shared/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/opossum_file_model/opossum_file.py b/src/opossum_lib/shared/entities/opossum_file.py similarity index 100% rename from src/opossum_lib/opossum_file_model/opossum_file.py rename to src/opossum_lib/shared/entities/opossum_file.py diff --git a/src/opossum_lib/opossum_file_model/opossum_file_content.py b/src/opossum_lib/shared/entities/opossum_file_content.py similarity index 67% rename from src/opossum_lib/opossum_file_model/opossum_file_content.py rename to src/opossum_lib/shared/entities/opossum_file_content.py index 01c3e235..f8909b37 100644 --- a/src/opossum_lib/opossum_file_model/opossum_file_content.py +++ b/src/opossum_lib/shared/entities/opossum_file_content.py @@ -5,8 +5,8 @@ from pydantic import BaseModel -from opossum_lib.opossum_file_model.opossum_file import OpossumInformation -from opossum_lib.opossum_file_model.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_file import OpossumInformation +from opossum_lib.shared.entities.output_model import OpossumOutputFile class OpossumFileContent(BaseModel): diff --git a/src/opossum_lib/opossum_file_model/output_model.py b/src/opossum_lib/shared/entities/output_model.py similarity index 100% rename from src/opossum_lib/opossum_file_model/output_model.py rename to src/opossum_lib/shared/entities/output_model.py diff --git a/src/opossum_lib/shared/services/__init__.py b/src/opossum_lib/shared/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/opossum_lib/opossum_file_model/opossum_file_writer.py b/src/opossum_lib/shared/services/opossum_file_writer.py similarity index 92% rename from src/opossum_lib/opossum_file_model/opossum_file_writer.py rename to src/opossum_lib/shared/services/opossum_file_writer.py index 3a641527..760d811c 100644 --- a/src/opossum_lib/opossum_file_model/opossum_file_writer.py +++ b/src/opossum_lib/shared/services/opossum_file_writer.py @@ -4,12 +4,12 @@ from pathlib import Path from zipfile import ZIP_DEFLATED, ZipFile -from opossum_lib.opossum_file_model.constants import ( +from opossum_lib.shared.constants import ( COMPRESSION_LEVEL, INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent class OpossumFileWriter: diff --git a/tests/core/test_opossum_model.py b/tests/core/test_opossum_model.py index 4d23cd9b..02285658 100644 --- a/tests/core/test_opossum_model.py +++ b/tests/core/test_opossum_model.py @@ -4,7 +4,7 @@ import json from copy import deepcopy -from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) from tests.test_setup.opossum_faker_setup import OpossumFaker diff --git a/tests/opossum_model_generators/external_attribution_source_provider.py b/tests/opossum_model_generators/external_attribution_source_provider.py index 0ff8f184..34073205 100644 --- a/tests/opossum_model_generators/external_attribution_source_provider.py +++ b/tests/opossum_model_generators/external_attribution_source_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.core.opossum_model import ExternalAttributionSource +from opossum_lib.core.entities.opossum_model import ExternalAttributionSource from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/metadata_provider.py b/tests/opossum_model_generators/metadata_provider.py index 51b97714..35e5c0ab 100644 --- a/tests/opossum_model_generators/metadata_provider.py +++ b/tests/opossum_model_generators/metadata_provider.py @@ -8,7 +8,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.opossum_model import Metadata +from opossum_lib.core.entities.opossum_model import Metadata from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index bf7d6c6a..2c0ebb54 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -5,8 +5,8 @@ from faker.providers import BaseProvider -from opossum_lib.core.opossum_model import Opossum, ScanResults -from opossum_lib.opossum_file_model.output_model import OpossumOutputFile +from opossum_lib.core.entities.opossum_model import Opossum, ScanResults +from opossum_lib.shared.entities.output_model import OpossumOutputFile from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, diff --git a/tests/opossum_model_generators/package_provider.py b/tests/opossum_model_generators/package_provider.py index 0566bff5..0cb553f8 100644 --- a/tests/opossum_model_generators/package_provider.py +++ b/tests/opossum_model_generators/package_provider.py @@ -10,7 +10,7 @@ from faker.providers.misc.en_US import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.opossum_model import OpossumPackage, SourceInfo +from opossum_lib.core.entities.opossum_model import OpossumPackage, SourceInfo from tests.opossum_model_generators.source_info_provider import SourceInfoProvider from tests.util.generator_helpers import entry_or_none, random_list diff --git a/tests/opossum_model_generators/resource_provider.py b/tests/opossum_model_generators/resource_provider.py index 067de70f..04068a07 100644 --- a/tests/opossum_model_generators/resource_provider.py +++ b/tests/opossum_model_generators/resource_provider.py @@ -8,7 +8,11 @@ from faker.providers.file import Provider as FileProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.opossum_model import OpossumPackage, Resource, ResourceType +from opossum_lib.core.entities.opossum_model import ( + OpossumPackage, + Resource, + ResourceType, +) from tests.opossum_model_generators.package_provider import PackageProvider from tests.util.generator_helpers import random_list diff --git a/tests/opossum_model_generators/scan_results_provider.py b/tests/opossum_model_generators/scan_results_provider.py index ba08d319..162e50cc 100644 --- a/tests/opossum_model_generators/scan_results_provider.py +++ b/tests/opossum_model_generators/scan_results_provider.py @@ -8,7 +8,7 @@ from faker.providers.file.en_US import Provider as FileProvider from faker.providers.misc.en_US import Provider as MiscProvider -from opossum_lib.core.opossum_model import ( +from opossum_lib.core.entities.opossum_model import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/opossum_model_generators/source_info_provider.py b/tests/opossum_model_generators/source_info_provider.py index a67f96ad..cf3a5822 100644 --- a/tests/opossum_model_generators/source_info_provider.py +++ b/tests/opossum_model_generators/source_info_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.opossum_model import SourceInfo +from opossum_lib.core.entities.opossum_model import SourceInfo from tests.util.generator_helpers import entry_or_none diff --git a/tests/test_cli.py b/tests/test_cli.py index 582b795a..31de5ed7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,12 +11,12 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate -from opossum_lib.opossum_file_model.constants import ( +from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.opossum_file_model.opossum_file import OpossumPackage -from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter +from opossum_lib.shared.entities.opossum_file import OpossumPackage +from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index 4c9dc6fb..d869d3d1 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,7 +13,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.opossum_file_model.opossum_file import ( +from opossum_lib.shared.entities.opossum_file import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index d39d7720..e6b4a00e 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,9 +5,9 @@ from faker.providers import BaseProvider -from opossum_lib.opossum_file_model.opossum_file import OpossumInformation -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent -from opossum_lib.opossum_file_model.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_file import OpossumInformation +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.output_model import OpossumOutputFile from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, ) diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/test_opossum/generators/generate_outfile_information.py index 1008b3e0..801331cf 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/test_opossum/generators/generate_outfile_information.py @@ -9,7 +9,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.opossum_file_model.output_model import ( +from opossum_lib.shared.entities.output_model import ( ManualAttributions, Metadata, OpossumOutputFile, diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index 8193f121..95496f32 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -3,10 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index af1df5f6..f3e49017 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -5,12 +5,12 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.opossum_file_model.constants import ( +from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.opossum_file_model.opossum_file_content import OpossumFileContent -from opossum_lib.opossum_file_model.opossum_file_writer import OpossumFileWriter +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index 6b0cdd95..c3284225 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -3,10 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.input_formats.opossum.opossum_file_to_opossum_converter import ( +from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.opossum_file_model.opossum_file import ( +from opossum_lib.shared.entities.opossum_file import ( OpossumPackage, OpossumPackageIdentifier, ) diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/test_opossum/test_read_opossum_file.py index b6cd32ae..5632502b 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/test_opossum/test_read_opossum_file.py @@ -6,7 +6,9 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.input_formats.opossum.opossum_format_reader import OpossumFormatReader +from opossum_lib.input_formats.opossum.services.opossum_format_reader import ( + OpossumFormatReader, +) TEST_DATA_DIRECTORY = Path(__file__).resolve().parent.parent / "data" diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/test_scancode/generators/generate_scancode_file.py index 85b4999b..cd77952f 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/test_scancode/generators/generate_scancode_file.py @@ -20,7 +20,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.input_formats.scancode.model import ( +from opossum_lib.input_formats.scancode.entities.model import ( Copyright, Email, ExtraData, diff --git a/tests/test_scancode/model_helpers.py b/tests/test_scancode/model_helpers.py index e3a718cf..7ab434c8 100644 --- a/tests/test_scancode/model_helpers.py +++ b/tests/test_scancode/model_helpers.py @@ -5,7 +5,7 @@ from pathlib import PurePath -from opossum_lib.input_formats.scancode.model import ( +from opossum_lib.input_formats.scancode.entities.model import ( Copyright, File, FileBasedLicenseDetection, diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/test_scancode/test_convert_scancode_to_opossum.py index cd434b44..f56d81c1 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/test_scancode/test_convert_scancode_to_opossum.py @@ -6,8 +6,8 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.core.opossum_model import Resource -from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.core.entities.opossum_model import Resource +from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) from tests.test_setup.scancode_faker_setup import ScanCodeFaker diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index bc038c2b..31ce8799 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -4,10 +4,10 @@ from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.scancode_data_to_opossum_converter import ( +from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) -from opossum_lib.opossum_file_model.opossum_file import OpossumPackage, SourceInfo +from opossum_lib.shared.entities.opossum_file import OpossumPackage, SourceInfo from tests.test_setup.scancode_faker_setup import ScanCodeFaker From 7707e8f52ae4c9bb0cf5a1bb5104d197048f5e99 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:31:46 +0100 Subject: [PATCH 33/58] refactor: review-comment: Filename to match main class name --- .../core/entities/{opossum_model.py => opossum.py} | 4 ++-- src/opossum_lib/core/services/input_format_reader.py | 2 +- src/opossum_lib/core/services/input_reader.py | 2 +- .../opossum/services/opossum_file_reader.py | 4 ++-- .../services/opossum_file_to_opossum_converter.py | 10 +++++----- .../opossum/services/opossum_format_reader.py | 2 +- .../entities/{model.py => scan_code_data_raw.py} | 2 +- .../services/scancode_data_to_opossum_converter.py | 12 ++++++------ .../scancode/services/scancode_format_reader.py | 10 ++++++---- .../shared/entities/opossum_file_content.py | 4 ++-- .../{opossum_file.py => opossum_information.py} | 0 .../{output_model.py => opossum_output_file.py} | 0 .../external_attribution_source_provider.py | 2 +- tests/opossum_model_generators/metadata_provider.py | 2 +- tests/opossum_model_generators/opossum_provider.py | 4 ++-- tests/opossum_model_generators/package_provider.py | 2 +- tests/opossum_model_generators/resource_provider.py | 2 +- .../scan_results_provider.py | 2 +- .../opossum_model_generators/source_info_provider.py | 2 +- tests/test_cli.py | 2 +- .../generators/generate_file_information.py | 2 +- .../generators/generate_opossum_file_content.py | 4 ++-- .../generators/generate_outfile_information.py | 2 +- .../test_opossum_file_to_opossum_converter.py | 2 +- .../generators/generate_scancode_file.py | 8 ++++---- tests/test_scancode/model_helpers.py | 2 +- .../test_convert_scancode_to_opossum.py | 2 +- tests/test_scancode/test_get_attribution_info.py | 2 +- 28 files changed, 48 insertions(+), 46 deletions(-) rename src/opossum_lib/core/entities/{opossum_model.py => opossum.py} (98%) rename src/opossum_lib/input_formats/scancode/entities/{model.py => scan_code_data_raw.py} (98%) rename src/opossum_lib/shared/entities/{opossum_file.py => opossum_information.py} (100%) rename src/opossum_lib/shared/entities/{output_model.py => opossum_output_file.py} (100%) diff --git a/src/opossum_lib/core/entities/opossum_model.py b/src/opossum_lib/core/entities/opossum.py similarity index 98% rename from src/opossum_lib/core/entities/opossum_model.py rename to src/opossum_lib/core/entities/opossum.py index 5329cd4b..3ccafc2c 100644 --- a/src/opossum_lib/core/entities/opossum_model.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -15,9 +15,9 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.shared.entities.opossum_file as opossum_file +import opossum_lib.shared.entities.opossum_information as opossum_file from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile type OpossumPackageIdentifier = str type ResourcePath = str diff --git a/src/opossum_lib/core/services/input_format_reader.py b/src/opossum_lib/core/services/input_format_reader.py index 7b5a6a6b..70b5f798 100644 --- a/src/opossum_lib/core/services/input_format_reader.py +++ b/src/opossum_lib/core/services/input_format_reader.py @@ -6,7 +6,7 @@ from pathlib import Path from typing import Protocol -from opossum_lib.core.entities.opossum_model import Opossum +from opossum_lib.core.entities.opossum import Opossum class InputFormatReader(Protocol): diff --git a/src/opossum_lib/core/services/input_reader.py b/src/opossum_lib/core/services/input_reader.py index 91c6c693..bca9530d 100644 --- a/src/opossum_lib/core/services/input_reader.py +++ b/src/opossum_lib/core/services/input_reader.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from opossum_lib.core.entities.input_file import InputFile, InputFileType -from opossum_lib.core.entities.opossum_model import Opossum +from opossum_lib.core.entities.opossum import Opossum from opossum_lib.core.services.input_format_reader import InputFormatReader diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 7999cd16..06a366e6 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -13,9 +13,9 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file import OpossumInformation from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile class OpossumFileReader: diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py index cb54e57b..24dba189 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py @@ -5,8 +5,8 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.shared.entities.opossum_file as opossum_file_package -from opossum_lib.core.entities.opossum_model import ( +import opossum_lib.shared.entities.opossum_information as opossum_file_package +from opossum_lib.core.entities.opossum import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, @@ -20,11 +20,11 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.shared.entities.opossum_file import ( +from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_information import ( ExternalAttributionSource as FileExternalAttributionSource, ) -from opossum_lib.shared.entities.opossum_file import OpossumInformation -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_information import OpossumInformation class OpossumFileToOpossumConverter: diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py index e3dc7aae..ec921a34 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from pathlib import Path -from opossum_lib.core.entities.opossum_model import Opossum +from opossum_lib.core.entities.opossum import Opossum from opossum_lib.core.services.input_format_reader import InputFormatReader from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( OpossumFileReader, diff --git a/src/opossum_lib/input_formats/scancode/entities/model.py b/src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py similarity index 98% rename from src/opossum_lib/input_formats/scancode/entities/model.py rename to src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py index 45b6031b..29251574 100644 --- a/src/opossum_lib/input_formats/scancode/entities/model.py +++ b/src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py @@ -155,7 +155,7 @@ class File(BaseModel): urls: list[Url] -class ScanCodeData(BaseModel): +class ScanCodeDataRaw(BaseModel): dependencies: list files: list[File] license_detections: list[GlobalLicenseDetection] diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py index 25ef66dd..a05a44df 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py @@ -7,7 +7,7 @@ import uuid from pathlib import PurePath -from opossum_lib.core.entities.opossum_model import ( +from opossum_lib.core.entities.opossum import ( Metadata, Opossum, OpossumPackage, @@ -17,17 +17,17 @@ SourceInfo, ) from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.entities.model import ( +from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( File, FileType, Header, - ScanCodeData, + ScanCodeDataRaw, ) class ScancodeDataToOpossumConverter: @staticmethod - def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> Opossum: + def convert_scancode_to_opossum(scancode_data: ScanCodeDataRaw) -> Opossum: resources = ScancodeDataToOpossumConverter.extract_opossum_resources( scancode_data ) @@ -49,7 +49,7 @@ def convert_scancode_to_opossum(scancode_data: ScanCodeData) -> Opossum: ) @staticmethod - def extract_scancode_header(scancode_data: ScanCodeData) -> Header: + def extract_scancode_header(scancode_data: ScanCodeDataRaw) -> Header: if len(scancode_data.headers) != 1: logging.error("Headers of ScanCode file are invalid.") sys.exit(1) @@ -57,7 +57,7 @@ def extract_scancode_header(scancode_data: ScanCodeData) -> Header: @staticmethod def extract_opossum_resources( - scancode_data: ScanCodeData, + scancode_data: ScanCodeDataRaw, ) -> list[Resource]: temp_root = Resource(path=PurePath("")) for file in scancode_data.files: diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index e235785e..2e24ea05 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -7,11 +7,13 @@ import sys from pathlib import Path -from opossum_lib.core.entities.opossum_model import ( +from opossum_lib.core.entities.opossum import ( Opossum, ) from opossum_lib.core.services.input_format_reader import InputFormatReader -from opossum_lib.input_formats.scancode.entities.model import ScanCodeData +from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( + ScanCodeDataRaw, +) from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) @@ -26,7 +28,7 @@ def read(self, path: Path) -> Opossum: return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) @staticmethod - def load_scancode_json(path: Path) -> ScanCodeData: + def load_scancode_json(path: Path) -> ScanCodeDataRaw: try: with open(path) as inp: json_data = json.load(inp) @@ -37,6 +39,6 @@ def load_scancode_json(path: Path) -> ScanCodeData: logging.error(f"Error decoding json for file {path}.") sys.exit(1) - scancode_data = ScanCodeData.model_validate(json_data) + scancode_data = ScanCodeDataRaw.model_validate(json_data) return scancode_data diff --git a/src/opossum_lib/shared/entities/opossum_file_content.py b/src/opossum_lib/shared/entities/opossum_file_content.py index f8909b37..8af47b7b 100644 --- a/src/opossum_lib/shared/entities/opossum_file_content.py +++ b/src/opossum_lib/shared/entities/opossum_file_content.py @@ -5,8 +5,8 @@ from pydantic import BaseModel -from opossum_lib.shared.entities.opossum_file import OpossumInformation -from opossum_lib.shared.entities.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile class OpossumFileContent(BaseModel): diff --git a/src/opossum_lib/shared/entities/opossum_file.py b/src/opossum_lib/shared/entities/opossum_information.py similarity index 100% rename from src/opossum_lib/shared/entities/opossum_file.py rename to src/opossum_lib/shared/entities/opossum_information.py diff --git a/src/opossum_lib/shared/entities/output_model.py b/src/opossum_lib/shared/entities/opossum_output_file.py similarity index 100% rename from src/opossum_lib/shared/entities/output_model.py rename to src/opossum_lib/shared/entities/opossum_output_file.py diff --git a/tests/opossum_model_generators/external_attribution_source_provider.py b/tests/opossum_model_generators/external_attribution_source_provider.py index 34073205..3d24ac47 100644 --- a/tests/opossum_model_generators/external_attribution_source_provider.py +++ b/tests/opossum_model_generators/external_attribution_source_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.core.entities.opossum_model import ExternalAttributionSource +from opossum_lib.core.entities.opossum import ExternalAttributionSource from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/metadata_provider.py b/tests/opossum_model_generators/metadata_provider.py index 35e5c0ab..44d30625 100644 --- a/tests/opossum_model_generators/metadata_provider.py +++ b/tests/opossum_model_generators/metadata_provider.py @@ -8,7 +8,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.entities.opossum_model import Metadata +from opossum_lib.core.entities.opossum import Metadata from tests.util.generator_helpers import entry_or_none diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index 2c0ebb54..55ffb41e 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -5,8 +5,8 @@ from faker.providers import BaseProvider -from opossum_lib.core.entities.opossum_model import Opossum, ScanResults -from opossum_lib.shared.entities.output_model import OpossumOutputFile +from opossum_lib.core.entities.opossum import Opossum, ScanResults +from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, diff --git a/tests/opossum_model_generators/package_provider.py b/tests/opossum_model_generators/package_provider.py index 0cb553f8..b644d7ff 100644 --- a/tests/opossum_model_generators/package_provider.py +++ b/tests/opossum_model_generators/package_provider.py @@ -10,7 +10,7 @@ from faker.providers.misc.en_US import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.entities.opossum_model import OpossumPackage, SourceInfo +from opossum_lib.core.entities.opossum import OpossumPackage, SourceInfo from tests.opossum_model_generators.source_info_provider import SourceInfoProvider from tests.util.generator_helpers import entry_or_none, random_list diff --git a/tests/opossum_model_generators/resource_provider.py b/tests/opossum_model_generators/resource_provider.py index 04068a07..22dd31cb 100644 --- a/tests/opossum_model_generators/resource_provider.py +++ b/tests/opossum_model_generators/resource_provider.py @@ -8,7 +8,7 @@ from faker.providers.file import Provider as FileProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.entities.opossum_model import ( +from opossum_lib.core.entities.opossum import ( OpossumPackage, Resource, ResourceType, diff --git a/tests/opossum_model_generators/scan_results_provider.py b/tests/opossum_model_generators/scan_results_provider.py index 162e50cc..c8473f19 100644 --- a/tests/opossum_model_generators/scan_results_provider.py +++ b/tests/opossum_model_generators/scan_results_provider.py @@ -8,7 +8,7 @@ from faker.providers.file.en_US import Provider as FileProvider from faker.providers.misc.en_US import Provider as MiscProvider -from opossum_lib.core.entities.opossum_model import ( +from opossum_lib.core.entities.opossum import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/opossum_model_generators/source_info_provider.py b/tests/opossum_model_generators/source_info_provider.py index cf3a5822..90c07801 100644 --- a/tests/opossum_model_generators/source_info_provider.py +++ b/tests/opossum_model_generators/source_info_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.entities.opossum_model import SourceInfo +from opossum_lib.core.entities.opossum import SourceInfo from tests.util.generator_helpers import entry_or_none diff --git a/tests/test_cli.py b/tests/test_cli.py index 31de5ed7..ac4c218c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,7 +15,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file import OpossumPackage +from opossum_lib.shared.entities.opossum_information import OpossumPackage from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index d869d3d1..96100a74 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,7 +13,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.shared.entities.opossum_file import ( +from opossum_lib.shared.entities.opossum_information import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index e6b4a00e..1218047a 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,9 +5,9 @@ from faker.providers import BaseProvider -from opossum_lib.shared.entities.opossum_file import OpossumInformation from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.output_model import OpossumOutputFile +from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, ) diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/test_opossum/generators/generate_outfile_information.py index 801331cf..4a73eac2 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/test_opossum/generators/generate_outfile_information.py @@ -9,7 +9,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.shared.entities.output_model import ( +from opossum_lib.shared.entities.opossum_output_file import ( ManualAttributions, Metadata, OpossumOutputFile, diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index c3284225..37f3601e 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -6,7 +6,7 @@ from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_file import ( +from opossum_lib.shared.entities.opossum_information import ( OpossumPackage, OpossumPackageIdentifier, ) diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/test_scancode/generators/generate_scancode_file.py index cd77952f..563aec62 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/test_scancode/generators/generate_scancode_file.py @@ -20,7 +20,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.input_formats.scancode.entities.model import ( +from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( Copyright, Email, ExtraData, @@ -33,7 +33,7 @@ Match, Options, ReferenceMatch, - ScanCodeData, + ScanCodeDataRaw, SystemEnvironment, Url, ) @@ -68,7 +68,7 @@ def scancode_data( headers: list[Header] | None = None, packages: list | None = None, options: Options | None = None, - ) -> ScanCodeData: + ) -> ScanCodeDataRaw: # TODO: #184 depending on which options are passed in additional_options # we need to generate different fields, e.g. --licenses # out of scope for now @@ -76,7 +76,7 @@ def scancode_data( license_detections = license_detections or self.global_license_detections(files) if headers is None: headers = [self.header(options=options)] - return ScanCodeData( + return ScanCodeDataRaw( dependencies=dependencies or [], files=files, license_detections=license_detections, diff --git a/tests/test_scancode/model_helpers.py b/tests/test_scancode/model_helpers.py index 7ab434c8..f942a12f 100644 --- a/tests/test_scancode/model_helpers.py +++ b/tests/test_scancode/model_helpers.py @@ -5,7 +5,7 @@ from pathlib import PurePath -from opossum_lib.input_formats.scancode.entities.model import ( +from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( Copyright, File, FileBasedLicenseDetection, diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/test_scancode/test_convert_scancode_to_opossum.py index f56d81c1..1606fa4b 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/test_scancode/test_convert_scancode_to_opossum.py @@ -6,7 +6,7 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.core.entities.opossum_model import Resource +from opossum_lib.core.entities.opossum import Resource from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index 31ce8799..3c4fa67e 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -7,7 +7,7 @@ from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_file import OpossumPackage, SourceInfo +from opossum_lib.shared.entities.opossum_information import OpossumPackage, SourceInfo from tests.test_setup.scancode_faker_setup import ScanCodeFaker From 2348a1a8603259df6431364c6a59e680cbe210f5 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:33:49 +0100 Subject: [PATCH 34/58] refactor: review-comment: Move file writer to core --- .../{shared => core}/services/opossum_file_writer.py | 0 src/opossum_lib/core/services/opossum_generator.py | 2 +- tests/test_cli.py | 2 +- tests/test_opossum/test_file_generation.py | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename src/opossum_lib/{shared => core}/services/opossum_file_writer.py (100%) diff --git a/src/opossum_lib/shared/services/opossum_file_writer.py b/src/opossum_lib/core/services/opossum_file_writer.py similarity index 100% rename from src/opossum_lib/shared/services/opossum_file_writer.py rename to src/opossum_lib/core/services/opossum_file_writer.py diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index 83902a2e..047228ef 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -7,7 +7,7 @@ ) from opossum_lib.core.services.input_format_reader import InputFormatReader from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter +from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter class OpossumGenerator: diff --git a/tests/test_cli.py b/tests/test_cli.py index ac4c218c..d24b6945 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,12 +11,12 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate +from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) from opossum_lib.shared.entities.opossum_information import OpossumPackage -from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index f3e49017..d0f2f939 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -5,12 +5,12 @@ from pathlib import Path from zipfile import ZipFile +from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.services.opossum_file_writer import OpossumFileWriter from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker From 72dd03e71f349fde36b00a23fb9e28844b253348 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:38:54 +0100 Subject: [PATCH 35/58] refactor: review-comment: Fuse closely connected classes --- .../entities/opossum_generation_arguments.py | 40 ----------------- .../core/services/input_format_reader.py | 14 ------ src/opossum_lib/core/services/input_reader.py | 9 +++- .../core/services/opossum_generator.py | 43 ++++++++++++++++--- .../opossum/services/opossum_format_reader.py | 2 +- .../services/scancode_format_reader.py | 2 +- 6 files changed, 47 insertions(+), 63 deletions(-) delete mode 100644 src/opossum_lib/core/entities/opossum_generation_arguments.py delete mode 100644 src/opossum_lib/core/services/input_format_reader.py diff --git a/src/opossum_lib/core/entities/opossum_generation_arguments.py b/src/opossum_lib/core/entities/opossum_generation_arguments.py deleted file mode 100644 index f15f49ff..00000000 --- a/src/opossum_lib/core/entities/opossum_generation_arguments.py +++ /dev/null @@ -1,40 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging -import sys -from pathlib import Path - -from pydantic import BaseModel - -from opossum_lib.core.entities.input_file import InputFile, InputFileType - - -class OpossumGenerationArguments(BaseModel): - scancode_json_files: list[Path] - opossum_files: list[Path] - outfile: Path - - @property - def valid_input_files(self) -> list[InputFile]: - self._validate_and_exit_on_error() - result = [] - result += [ - InputFile(path=path, type=InputFileType.SCAN_CODE) - for path in self.scancode_json_files - ] - result += [ - InputFile(path=path, type=InputFileType.OPOSSUM) - for path in self.opossum_files - ] - return result - - def _validate_and_exit_on_error(self) -> None: - total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) diff --git a/src/opossum_lib/core/services/input_format_reader.py b/src/opossum_lib/core/services/input_format_reader.py deleted file mode 100644 index 70b5f798..00000000 --- a/src/opossum_lib/core/services/input_format_reader.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from abc import abstractmethod -from pathlib import Path -from typing import Protocol - -from opossum_lib.core.entities.opossum import Opossum - - -class InputFormatReader(Protocol): - @abstractmethod - def read(self, path: Path) -> Opossum: ... diff --git a/src/opossum_lib/core/services/input_reader.py b/src/opossum_lib/core/services/input_reader.py index bca9530d..e431a6d4 100644 --- a/src/opossum_lib/core/services/input_reader.py +++ b/src/opossum_lib/core/services/input_reader.py @@ -1,10 +1,17 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 +from abc import abstractmethod +from asyncio import Protocol +from pathlib import Path from opossum_lib.core.entities.input_file import InputFile, InputFileType from opossum_lib.core.entities.opossum import Opossum -from opossum_lib.core.services.input_format_reader import InputFormatReader + + +class InputFormatReader(Protocol): + @abstractmethod + def read(self, path: Path) -> Opossum: ... class InputReader: diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index 047228ef..f1f14f33 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -1,15 +1,46 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 -from opossum_lib.core.entities.input_file import InputFileType -from opossum_lib.core.entities.opossum_generation_arguments import ( - OpossumGenerationArguments, -) -from opossum_lib.core.services.input_format_reader import InputFormatReader -from opossum_lib.core.services.input_reader import InputReader +import logging +import sys +from pathlib import Path + +from pydantic import BaseModel + +from opossum_lib.core.entities.input_file import InputFile, InputFileType +from opossum_lib.core.services.input_reader import InputFormatReader, InputReader from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter +class OpossumGenerationArguments(BaseModel): + scancode_json_files: list[Path] + opossum_files: list[Path] + outfile: Path + + @property + def valid_input_files(self) -> list[InputFile]: + self._validate_and_exit_on_error() + result = [] + result += [ + InputFile(path=path, type=InputFileType.SCAN_CODE) + for path in self.scancode_json_files + ] + result += [ + InputFile(path=path, type=InputFileType.OPOSSUM) + for path in self.opossum_files + ] + return result + + def _validate_and_exit_on_error(self) -> None: + total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) + + class OpossumGenerator: input_reader: InputReader diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py index ec921a34..98a7c162 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py @@ -4,7 +4,7 @@ from pathlib import Path from opossum_lib.core.entities.opossum import Opossum -from opossum_lib.core.services.input_format_reader import InputFormatReader +from opossum_lib.core.services.input_reader import InputFormatReader from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( OpossumFileReader, ) diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index 2e24ea05..db49229f 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -10,7 +10,7 @@ from opossum_lib.core.entities.opossum import ( Opossum, ) -from opossum_lib.core.services.input_format_reader import InputFormatReader +from opossum_lib.core.services.input_reader import InputFormatReader from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( ScanCodeDataRaw, ) From b8848d530131889b019c2af2f052f39598d093c0 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:39:49 +0100 Subject: [PATCH 36/58] refactor: review-comment: Fix typo --- README.md | 2 +- src/opossum_lib/cli.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b2dc5515..e992241d 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ Options: Option can be repeated. -o, --outfile TEXT The file path to write the generated opossum document to. If appropriate, the extension ".opossum" is - appended.If the output file already exists, it is + appended. If the output file already exists, it is overwritten. [default: output.opossum] --help Show this message and exit. diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index b2cf151d..50c5c35e 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -47,7 +47,7 @@ def opossum_file() -> None: default="output.opossum", show_default=True, help="The file path to write the generated opossum document to. " - 'If appropriate, the extension ".opossum" is appended.' + 'If appropriate, the extension ".opossum" is appended. ' "If the output file already exists, it is overwritten.", ) def generate( From 7c952359f122ad4598b36e44fdd53b3f55c13cf4 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 13:42:51 +0100 Subject: [PATCH 37/58] refactor: review-comment: Smaller fixes * renaming * fix surplus plus * added comment --- src/opossum_lib/core/services/opossum_generator.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index f1f14f33..412c12b2 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -18,7 +18,7 @@ class OpossumGenerationArguments(BaseModel): outfile: Path @property - def valid_input_files(self) -> list[InputFile]: + def input_files(self) -> list[InputFile]: self._validate_and_exit_on_error() result = [] result += [ @@ -32,7 +32,7 @@ def valid_input_files(self) -> list[InputFile]: return result def _validate_and_exit_on_error(self) -> None: - total_number_of_files = +len(self.scancode_json_files) + len(self.opossum_files) + total_number_of_files = len(self.scancode_json_files) + len(self.opossum_files) if total_number_of_files == 0: logging.warning("No input provided. Exiting.") sys.exit(1) @@ -50,8 +50,10 @@ def __init__(self, input_format_readers: dict[InputFileType, InputFormatReader]) def generate( self, opossum_generation_arguments: OpossumGenerationArguments ) -> None: - input_files = opossum_generation_arguments.valid_input_files + input_files = opossum_generation_arguments.input_files + # currently this converts only one file (validated in the arguments) + # for the future a merge step is planned after reading the files opossum = self.input_reader.read(input_files[0]) opossum_file_content = opossum.to_opossum_file_format() From 7b1547a33c5d6ee51793c9f07cbe8a3d6d2e4a83 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 15:11:39 +0100 Subject: [PATCH 38/58] refactor: review-comment: rename opossum model files To better match name and content --- src/opossum_lib/core/entities/opossum.py | 12 ++++++------ .../core/services/opossum_file_writer.py | 14 +++++++------- .../opossum/services/opossum_file_reader.py | 12 ++++++------ .../services/opossum_file_to_opossum_converter.py | 12 ++++++------ .../{opossum_file_content.py => opossum_file.py} | 6 +++--- ...possum_information.py => opossum_input_file.py} | 2 +- tests/test_cli.py | 2 +- .../generators/generate_file_information.py | 8 ++++---- .../generators/generate_opossum_file_content.py | 10 +++++----- tests/test_opossum/test_conversion_roundtrip.py | 6 +++--- tests/test_opossum/test_file_generation.py | 4 ++-- .../test_opossum_file_to_opossum_converter.py | 2 +- tests/test_scancode/test_get_attribution_info.py | 2 +- 13 files changed, 46 insertions(+), 46 deletions(-) rename src/opossum_lib/shared/entities/{opossum_file_content.py => opossum_file.py} (67%) rename src/opossum_lib/shared/entities/{opossum_information.py => opossum_input_file.py} (99%) diff --git a/src/opossum_lib/core/entities/opossum.py b/src/opossum_lib/core/entities/opossum.py index 3ccafc2c..ea63376d 100644 --- a/src/opossum_lib/core/entities/opossum.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -15,8 +15,8 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.shared.entities.opossum_information as opossum_file -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +import opossum_lib.shared.entities.opossum_input_file as opossum_file +from opossum_lib.shared.entities.opossum_file import OpossumFileModel from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile type OpossumPackageIdentifier = str @@ -36,8 +36,8 @@ class Opossum(BaseModel): scan_results: ScanResults review_results: OpossumOutputFile | None = None - def to_opossum_file_format(self) -> OpossumFileContent: - return OpossumFileContent( + def to_opossum_file_format(self) -> OpossumFileModel: + return OpossumFileModel( input_file=self.scan_results.to_opossum_file_format(), output_file=self.review_results, ) @@ -57,7 +57,7 @@ class ScanResults(BaseModel): ) unassigned_attributions: list[OpossumPackage] = [] - def to_opossum_file_format(self) -> opossum_file.OpossumInformation: + def to_opossum_file_format(self) -> opossum_file.OpossumInputFile: external_attributions, resources_to_attributions = ( self.create_attribution_mapping(self.resources) ) @@ -78,7 +78,7 @@ def to_opossum_file_format(self) -> opossum_file.OpossumInformation: for (key, val) in self.external_attribution_sources.items() } - return opossum_file.OpossumInformation( + return opossum_file.OpossumInputFile( metadata=self.metadata.to_opossum_file_format(), resources={ str(resource.path): resource.to_opossum_file_format() diff --git a/src/opossum_lib/core/services/opossum_file_writer.py b/src/opossum_lib/core/services/opossum_file_writer.py index 760d811c..db5c18b6 100644 --- a/src/opossum_lib/core/services/opossum_file_writer.py +++ b/src/opossum_lib/core/services/opossum_file_writer.py @@ -9,24 +9,24 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file import OpossumFileModel class OpossumFileWriter: @staticmethod - def write(opossum_file_content: OpossumFileContent, file_path: Path) -> None: + def write(opossum_file_model: OpossumFileModel, file_path: Path) -> None: file_path = OpossumFileWriter._ensure_outfile_suffix(file_path) with ZipFile( file_path, "w", compression=ZIP_DEFLATED, compresslevel=COMPRESSION_LEVEL ) as zip_file: - OpossumFileWriter._write_input_json(opossum_file_content, zip_file) + OpossumFileWriter._write_input_json(opossum_file_model, zip_file) OpossumFileWriter._write_output_json_if_existing( - opossum_file_content, zip_file + opossum_file_model, zip_file ) @staticmethod def _write_output_json_if_existing( - opossum_file_content: OpossumFileContent, zip_file: ZipFile + opossum_file_content: OpossumFileModel, zip_file: ZipFile ) -> None: if opossum_file_content.output_file: zip_file.writestr( @@ -38,11 +38,11 @@ def _write_output_json_if_existing( @staticmethod def _write_input_json( - opossum_file_content: OpossumFileContent, zip_file: ZipFile + opossum_file_model: OpossumFileModel, zip_file: ZipFile ) -> None: zip_file.writestr( INPUT_JSON_NAME, - opossum_file_content.input_file.model_dump_json( + opossum_file_model.input_file.model_dump_json( indent=4, exclude_none=True, by_alias=True, diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 06a366e6..4b6acd3c 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -13,14 +13,14 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile class OpossumFileReader: @staticmethod - def from_file(path: Path) -> OpossumFileContent: + def from_file(path: Path) -> OpossumFileModel: logging.info(f"Converting opossum to opossum {path}") try: @@ -31,7 +31,7 @@ def from_file(path: Path) -> OpossumFileContent: input_file = OpossumFileReader._read_input_json_from_zip_file( input_zip_file ) - return OpossumFileContent( + return OpossumFileModel( input_file=input_file, output_file=OpossumFileReader._read_output_json_if_exists( input_zip_file @@ -42,10 +42,10 @@ def from_file(path: Path) -> OpossumFileContent: sys.exit(1) @staticmethod - def _read_input_json_from_zip_file(input_zip_file: ZipFile) -> OpossumInformation: + def _read_input_json_from_zip_file(input_zip_file: ZipFile) -> OpossumInputFile: with input_zip_file.open(INPUT_JSON_NAME) as input_json_file: input_json = json.load(input_json_file) - input_file = OpossumInformation.model_validate(input_json) + input_file = OpossumInputFile.model_validate(input_json) return input_file @staticmethod diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py index 24dba189..6ffd4889 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py @@ -5,7 +5,7 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.shared.entities.opossum_information as opossum_file_package +import opossum_lib.shared.entities.opossum_input_file as opossum_file_package from opossum_lib.core.entities.opossum import ( BaseUrlsForSources, ExternalAttributionSource, @@ -20,16 +20,16 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.opossum_information import ( +from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file import ( ExternalAttributionSource as FileExternalAttributionSource, ) -from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile class OpossumFileToOpossumConverter: @staticmethod - def convert_to_opossum(opossum_file: OpossumFileContent) -> Opossum: + def convert_to_opossum(opossum_file: OpossumFileModel) -> Opossum: opossum = Opossum( scan_results=OpossumFileToOpossumConverter._convert_to_opossum_scan_results( opossum_file.input_file @@ -40,7 +40,7 @@ def convert_to_opossum(opossum_file: OpossumFileContent) -> Opossum: @staticmethod def _convert_to_opossum_scan_results( - opossum_information: OpossumInformation, + opossum_information: OpossumInputFile, ) -> ScanResults: resources, used_attribution_ids = ( OpossumFileToOpossumConverter._convert_to_opossum_model_resource_tree( diff --git a/src/opossum_lib/shared/entities/opossum_file_content.py b/src/opossum_lib/shared/entities/opossum_file.py similarity index 67% rename from src/opossum_lib/shared/entities/opossum_file_content.py rename to src/opossum_lib/shared/entities/opossum_file.py index 8af47b7b..5a7e4c19 100644 --- a/src/opossum_lib/shared/entities/opossum_file_content.py +++ b/src/opossum_lib/shared/entities/opossum_file.py @@ -5,10 +5,10 @@ from pydantic import BaseModel -from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile -class OpossumFileContent(BaseModel): - input_file: OpossumInformation +class OpossumFileModel(BaseModel): + input_file: OpossumInputFile output_file: OpossumOutputFile | None = None diff --git a/src/opossum_lib/shared/entities/opossum_information.py b/src/opossum_lib/shared/entities/opossum_input_file.py similarity index 99% rename from src/opossum_lib/shared/entities/opossum_information.py rename to src/opossum_lib/shared/entities/opossum_input_file.py index 513bbe9f..1b07ce0e 100644 --- a/src/opossum_lib/shared/entities/opossum_information.py +++ b/src/opossum_lib/shared/entities/opossum_input_file.py @@ -22,7 +22,7 @@ class CamelBaseModel(BaseModel): ) -class OpossumInformation(CamelBaseModel): +class OpossumInputFile(CamelBaseModel): metadata: Metadata resources: ResourceInFile external_attributions: dict[OpossumPackageIdentifier, OpossumPackage] diff --git a/tests/test_cli.py b/tests/test_cli.py index d24b6945..b8f62674 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_information import OpossumPackage +from opossum_lib.shared.entities.opossum_input_file import OpossumPackage from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index 96100a74..6cdb76e0 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,12 +13,12 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.shared.entities.opossum_information import ( +from opossum_lib.shared.entities.opossum_input_file import ( BaseUrlsForSources, ExternalAttributionSource, FrequentLicense, Metadata, - OpossumInformation, + OpossumInputFile, OpossumPackage, OpossumPackageIdentifier, ResourceInFile, @@ -99,12 +99,12 @@ def opossum_file_information( frequent_licenses: list[FrequentLicense] | None = None, files_with_children: list[str] | None = None, base_urls_for_sources: BaseUrlsForSources | None = None, - ) -> OpossumInformation: + ) -> OpossumInputFile: generated_resources = resources or self.resource_in_file() attributions = external_attributions or self.external_attributions( min_number_of_attributions=25 ) - return OpossumInformation( + return OpossumInputFile( metadata=metadata or self.metadata_provider.opossum_input_metadata(), resources=generated_resources, external_attributions=attributions, diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index 1218047a..b44c0b85 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,8 +5,8 @@ from faker.providers import BaseProvider -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent -from opossum_lib.shared.entities.opossum_information import OpossumInformation +from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, @@ -27,10 +27,10 @@ def __init__(self, generator: Any): def opossum_file_content( self, - in_file: OpossumInformation | None = None, + in_file: OpossumInputFile | None = None, out_file: OpossumOutputFile | None = None, - ) -> OpossumFileContent: - return OpossumFileContent( + ) -> OpossumFileModel: + return OpossumFileModel( input_file=in_file or self.infile_provider.opossum_file_information(), output_file=out_file or self.outfile_provider.output_file(), ) diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index 95496f32..bbf90ade 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -6,7 +6,7 @@ from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file import OpossumFileModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker @@ -16,7 +16,7 @@ def test_roundtrip(self, opossum_file_faker: OpossumFileFaker) -> None: TestConversionRoundtrip._check_round_trip(start_file_content) def test_input_file_only(self, opossum_file_faker: OpossumFileFaker) -> None: - start_file_content = OpossumFileContent( + start_file_content = OpossumFileModel( input_file=opossum_file_faker.opossum_file_information() ) TestConversionRoundtrip._check_round_trip(start_file_content) @@ -29,7 +29,7 @@ def test_surplus_attributions(self, opossum_file_faker: OpossumFileFaker) -> Non TestConversionRoundtrip._check_round_trip(start_file_content) @staticmethod - def _check_round_trip(start_file_content: OpossumFileContent) -> None: + def _check_round_trip(start_file_content: OpossumFileModel) -> None: expected_file_content = deepcopy(start_file_content) result = OpossumFileToOpossumConverter.convert_to_opossum( start_file_content diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index d0f2f939..e5273c2d 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -10,14 +10,14 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file_content import OpossumFileContent +from opossum_lib.shared.entities.opossum_file import OpossumFileModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker def test_only_input_information_available_writes_only_input_information( tmp_path: Path, opossum_file_faker: OpossumFileFaker ) -> None: - opossum_file_content = OpossumFileContent( + opossum_file_content = OpossumFileModel( input_file=opossum_file_faker.opossum_file_information() ) output_path = tmp_path / "output.opossum" diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index 37f3601e..df491dd6 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -6,7 +6,7 @@ from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_information import ( +from opossum_lib.shared.entities.opossum_input_file import ( OpossumPackage, OpossumPackageIdentifier, ) diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index 3c4fa67e..31a50e4e 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -7,7 +7,7 @@ from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_information import OpossumPackage, SourceInfo +from opossum_lib.shared.entities.opossum_input_file import OpossumPackage, SourceInfo from tests.test_setup.scancode_faker_setup import ScanCodeFaker From bc8a7d6cf49a229055950fe30fe71dbefbe7a4d6 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 15:21:58 +0100 Subject: [PATCH 39/58] refactor: review-comment: rename all file specific models to end with Model --- src/opossum_lib/core/entities/opossum.py | 42 ++--- .../core/services/opossum_file_writer.py | 2 +- .../opossum/services/opossum_file_reader.py | 16 +- .../opossum_file_to_opossum_converter.py | 28 ++-- ...de_data_raw.py => scan_code_data_model.py} | 58 +++---- .../scancode_data_to_opossum_converter.py | 24 +-- .../services/scancode_format_reader.py | 8 +- .../shared/entities/opossum_file.py | 14 -- .../shared/entities/opossum_file_model.py | 14 ++ ...ut_file.py => opossum_input_file_model.py} | 52 +++---- ...t_file.py => opossum_output_file_model.py} | 2 +- .../opossum_provider.py | 4 +- tests/test_cli.py | 6 +- .../generators/generate_file_information.py | 54 +++---- .../generate_opossum_file_content.py | 10 +- .../generate_outfile_information.py | 8 +- .../test_opossum/test_conversion_roundtrip.py | 2 +- tests/test_opossum/test_file_generation.py | 2 +- .../test_opossum_file_to_opossum_converter.py | 6 +- .../generators/generate_scancode_file.py | 146 +++++++++--------- tests/test_scancode/model_helpers.py | 40 ++--- .../test_get_attribution_info.py | 13 +- 22 files changed, 277 insertions(+), 274 deletions(-) rename src/opossum_lib/input_formats/scancode/entities/{scan_code_data_raw.py => scan_code_data_model.py} (69%) delete mode 100644 src/opossum_lib/shared/entities/opossum_file.py create mode 100644 src/opossum_lib/shared/entities/opossum_file_model.py rename src/opossum_lib/shared/entities/{opossum_input_file.py => opossum_input_file_model.py} (82%) rename src/opossum_lib/shared/entities/{opossum_output_file.py => opossum_output_file_model.py} (99%) diff --git a/src/opossum_lib/core/entities/opossum.py b/src/opossum_lib/core/entities/opossum.py index ea63376d..77598d8d 100644 --- a/src/opossum_lib/core/entities/opossum.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -15,9 +15,9 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.shared.entities.opossum_input_file as opossum_file -from opossum_lib.shared.entities.opossum_file import OpossumFileModel -from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile +import opossum_lib.shared.entities.opossum_input_file_model as opossum_file +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel type OpossumPackageIdentifier = str type ResourcePath = str @@ -34,7 +34,7 @@ def default_attribution_id_mapper() -> dict[OpossumPackage, str]: class Opossum(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") scan_results: ScanResults - review_results: OpossumOutputFile | None = None + review_results: OpossumOutputFileModel | None = None def to_opossum_file_format(self) -> OpossumFileModel: return OpossumFileModel( @@ -57,7 +57,7 @@ class ScanResults(BaseModel): ) unassigned_attributions: list[OpossumPackage] = [] - def to_opossum_file_format(self) -> opossum_file.OpossumInputFile: + def to_opossum_file_format(self) -> opossum_file.OpossumInputFileModel: external_attributions, resources_to_attributions = ( self.create_attribution_mapping(self.resources) ) @@ -78,7 +78,7 @@ def to_opossum_file_format(self) -> opossum_file.OpossumInputFile: for (key, val) in self.external_attribution_sources.items() } - return opossum_file.OpossumInputFile( + return opossum_file.OpossumInputFileModel( metadata=self.metadata.to_opossum_file_format(), resources={ str(resource.path): resource.to_opossum_file_format() @@ -95,7 +95,7 @@ def to_opossum_file_format(self) -> opossum_file.OpossumInputFile: def _get_unassigned_attributions( self, - ) -> dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackage]: + ) -> dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel]: if self.unassigned_attributions: result = {} for unassigned_attribution in self.unassigned_attributions: @@ -118,11 +118,11 @@ def create_attribution_mapping( self, root_nodes: list[Resource], ) -> tuple[ - dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackage], + dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel], dict[opossum_file.ResourcePath, list[opossum_file.OpossumPackageIdentifier]], ]: external_attributions: dict[ - opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackage + opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel ] = {} resources_to_attributions: dict[ opossum_file.ResourcePath, list[opossum_file.OpossumPackageIdentifier] @@ -227,8 +227,8 @@ def _update(self, other: Resource) -> None: class BaseUrlsForSources(BaseModel): model_config = ConfigDict(frozen=True, extra="allow") - def to_opossum_file_format(self) -> opossum_file.BaseUrlsForSources: - return opossum_file.BaseUrlsForSources(**self.model_dump()) + def to_opossum_file_format(self) -> opossum_file.BaseUrlsForSourcesModel: + return opossum_file.BaseUrlsForSourcesModel(**self.model_dump()) class FrequentLicense(BaseModel): @@ -237,8 +237,8 @@ class FrequentLicense(BaseModel): short_name: str default_text: str - def to_opossum_file_format(self) -> opossum_file.FrequentLicense: - return opossum_file.FrequentLicense( + def to_opossum_file_format(self) -> opossum_file.FrequentLicenseModel: + return opossum_file.FrequentLicenseModel( full_name=self.full_name, short_name=self.short_name, default_text=self.default_text, @@ -251,8 +251,8 @@ class SourceInfo(BaseModel): document_confidence: int | float | None = 0 additional_name: str | None = None - def to_opossum_file_format(self) -> opossum_file.SourceInfo: - return opossum_file.SourceInfo( + def to_opossum_file_format(self) -> opossum_file.SourceInfoModel: + return opossum_file.SourceInfoModel( name=self.name, document_confidence=self.document_confidence, additional_name=self.additional_name, @@ -282,8 +282,8 @@ class OpossumPackage(BaseModel): criticality: Literal["high"] | Literal["medium"] | None = None was_preferred: bool | None = None - def to_opossum_file_format(self) -> opossum_file.OpossumPackage: - return opossum_file.OpossumPackage( + def to_opossum_file_format(self) -> opossum_file.OpossumPackageModel: + return opossum_file.OpossumPackageModel( source=self.source.to_opossum_file_format(), attribution_confidence=self.attribution_confidence, comment=self.comment, @@ -316,8 +316,8 @@ class Metadata(BaseModel): expected_release_date: str | None = None build_date: str | None = None - def to_opossum_file_format(self) -> opossum_file.Metadata: - return opossum_file.Metadata(**self.model_dump()) + def to_opossum_file_format(self) -> opossum_file.MetadataModel: + return opossum_file.MetadataModel(**self.model_dump()) class ExternalAttributionSource(BaseModel): @@ -326,8 +326,8 @@ class ExternalAttributionSource(BaseModel): priority: int is_relevant_for_preferred: bool | None = None - def to_opossum_file_format(self) -> opossum_file.ExternalAttributionSource: - return opossum_file.ExternalAttributionSource( + def to_opossum_file_format(self) -> opossum_file.ExternalAttributionSourceModel: + return opossum_file.ExternalAttributionSourceModel( name=self.name, priority=self.priority, is_relevant_for_preferred=self.is_relevant_for_preferred, diff --git a/src/opossum_lib/core/services/opossum_file_writer.py b/src/opossum_lib/core/services/opossum_file_writer.py index db5c18b6..95651fae 100644 --- a/src/opossum_lib/core/services/opossum_file_writer.py +++ b/src/opossum_lib/core/services/opossum_file_writer.py @@ -9,7 +9,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel class OpossumFileWriter: diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 4b6acd3c..feeee775 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -13,9 +13,9 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile -from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel class OpossumFileReader: @@ -42,20 +42,20 @@ def from_file(path: Path) -> OpossumFileModel: sys.exit(1) @staticmethod - def _read_input_json_from_zip_file(input_zip_file: ZipFile) -> OpossumInputFile: - with input_zip_file.open(INPUT_JSON_NAME) as input_json_file: + def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: + with zip_file.open(INPUT_JSON_NAME) as input_json_file: input_json = json.load(input_json_file) - input_file = OpossumInputFile.model_validate(input_json) + input_file = OpossumInputFileModel.model_validate(input_json) return input_file @staticmethod def _read_output_json_if_exists( input_zip_file: ZipFile, - ) -> OpossumOutputFile | None: + ) -> OpossumOutputFileModel | None: if OUTPUT_JSON_NAME in input_zip_file.namelist(): with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: output_json = json.load(output_json_file) - output_file = OpossumOutputFile.model_validate(output_json) + output_file = OpossumOutputFileModel.model_validate(output_json) else: output_file = None return output_file diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py index 6ffd4889..80be66a1 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py @@ -5,7 +5,7 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.shared.entities.opossum_input_file as opossum_file_package +import opossum_lib.shared.entities.opossum_input_file_model as opossum_file_package from opossum_lib.core.entities.opossum import ( BaseUrlsForSources, ExternalAttributionSource, @@ -20,11 +20,11 @@ SourceInfo, _convert_path_to_str, ) -from opossum_lib.shared.entities.opossum_file import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file import ( - ExternalAttributionSource as FileExternalAttributionSource, +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import ( + ExternalAttributionSourceModel as FileExternalAttributionSource, ) -from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel class OpossumFileToOpossumConverter: @@ -40,7 +40,7 @@ def convert_to_opossum(opossum_file: OpossumFileModel) -> Opossum: @staticmethod def _convert_to_opossum_scan_results( - opossum_information: OpossumInputFile, + opossum_information: OpossumInputFileModel, ) -> ScanResults: resources, used_attribution_ids = ( OpossumFileToOpossumConverter._convert_to_opossum_model_resource_tree( @@ -102,7 +102,7 @@ def _get_unassigned_attributions( used_attribution_ids: set[OpossumPackageIdentifier], external_attributions: dict[ opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackage, + opossum_file_package.OpossumPackageModel, ], ) -> list[OpossumPackage] | None: available_attribution_ids = external_attributions.keys() @@ -125,7 +125,7 @@ def _convert_external_attribution_source( @staticmethod def _convert_frequent_licenses_to_model_frequent_licenses( - frequent_licenses_infile: list[opossum_file_package.FrequentLicense], + frequent_licenses_infile: list[opossum_file_package.FrequentLicenseModel], ) -> list[FrequentLicense]: frequent_licenses: list[FrequentLicense] = [ OpossumFileToOpossumConverter._convert_frequent_license(license) @@ -135,7 +135,7 @@ def _convert_frequent_licenses_to_model_frequent_licenses( @staticmethod def _convert_to_opossum_model_metadata( - infile_metadata: opossum_file_package.Metadata, + infile_metadata: opossum_file_package.MetadataModel, ) -> Metadata: return Metadata(**infile_metadata.model_dump()) @@ -144,7 +144,7 @@ def _convert_to_opossum_model_resource_tree( resources: opossum_file_package.ResourceInFile, external_attributions: dict[ opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackage, + opossum_file_package.OpossumPackageModel, ], resources_to_attributions: dict[ opossum_file_package.ResourcePath, @@ -215,7 +215,7 @@ def _get_applicable_attributions( def _convert_to_attribution_with_id( external_attributions: dict[ opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackage, + opossum_file_package.OpossumPackageModel, ], ) -> dict[OpossumPackage, str]: result = {} @@ -232,7 +232,7 @@ def _convert_to_attribution_with_id( @staticmethod def _convert_frequent_license( - infile_frequent_license: opossum_file_package.FrequentLicense, + infile_frequent_license: opossum_file_package.FrequentLicenseModel, ) -> FrequentLicense: return FrequentLicense( full_name=infile_frequent_license.full_name, @@ -242,7 +242,7 @@ def _convert_frequent_license( @staticmethod def _convert_package( - infile_package: opossum_file_package.OpossumPackage, + infile_package: opossum_file_package.OpossumPackageModel, ) -> OpossumPackage: return OpossumPackage( source=OpossumFileToOpossumConverter._convert_source(infile_package.source), @@ -269,7 +269,7 @@ def _convert_package( @staticmethod def _convert_source( - infile_source_info: opossum_file_package.SourceInfo, + infile_source_info: opossum_file_package.SourceInfoModel, ) -> SourceInfo: return SourceInfo( name=infile_source_info.name, diff --git a/src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py b/src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py similarity index 69% rename from src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py rename to src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py index 29251574..a0ea9002 100644 --- a/src/opossum_lib/input_formats/scancode/entities/scan_code_data_raw.py +++ b/src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py @@ -10,11 +10,11 @@ from pydantic import BaseModel -class Options(BaseModel, extra="ignore"): +class OptionsModel(BaseModel, extra="ignore"): input: list[str] -class SystemEnvironment(BaseModel): +class SystemEnvironmentModel(BaseModel): cpu_architecture: str operating_system: str platform: str @@ -22,20 +22,20 @@ class SystemEnvironment(BaseModel): python_version: str -class ExtraData(BaseModel): +class ExtraDataModel(BaseModel): files_count: int spdx_license_list_version: str - system_environment: SystemEnvironment + system_environment: SystemEnvironmentModel -class Header(BaseModel): +class HeaderModel(BaseModel): duration: float end_timestamp: str errors: list - extra_data: ExtraData + extra_data: ExtraDataModel message: Any notice: str - options: Options + options: OptionsModel output_format_version: str start_timestamp: str tool_name: str @@ -43,7 +43,7 @@ class Header(BaseModel): warnings: list -class ReferenceMatch(BaseModel): +class ReferenceMatchModel(BaseModel): end_line: int from_file: str license_expression: str @@ -58,7 +58,7 @@ class ReferenceMatch(BaseModel): start_line: int -class Match(BaseModel): +class MatchModel(BaseModel): end_line: int from_file: str license_expression: str @@ -73,64 +73,64 @@ class Match(BaseModel): start_line: int -class GlobalLicenseDetection(BaseModel): +class GlobalLicenseDetectionModel(BaseModel): detection_count: int identifier: str license_expression: str license_expression_spdx: str - reference_matches: list[ReferenceMatch] + reference_matches: list[ReferenceMatchModel] -class FileBasedLicenseDetection(BaseModel): +class FileBasedLicenseDetectionModel(BaseModel): license_expression: str license_expression_spdx: str - matches: list[Match] + matches: list[MatchModel] identifier: str -class Copyright(BaseModel): +class CopyrightModel(BaseModel): copyright: str end_line: int start_line: int -class Holder(BaseModel): +class HolderModel(BaseModel): end_line: int holder: str start_line: int -class Url(BaseModel): +class UrlModel(BaseModel): end_line: int start_line: int url: str -class Email(BaseModel): +class EmailModel(BaseModel): email: str end_line: int start_line: int -class FileType(Enum): +class FileTypeModel(Enum): FILE = "file" DIRECTORY = "directory" -class File(BaseModel): +class FileModel(BaseModel): authors: list base_name: str - copyrights: list[Copyright] + copyrights: list[CopyrightModel] date: str | None detected_license_expression: str | None detected_license_expression_spdx: str | None dirs_count: int - emails: list[Email] + emails: list[EmailModel] extension: str files_count: int file_type: str | None for_packages: list - holders: list[Holder] + holders: list[HolderModel] is_archive: bool is_binary: bool is_media: bool @@ -138,7 +138,7 @@ class File(BaseModel): is_source: bool is_text: bool license_clues: list - license_detections: list[FileBasedLicenseDetection] + license_detections: list[FileBasedLicenseDetectionModel] md5: str | None mime_type: str | None name: str @@ -151,13 +151,13 @@ class File(BaseModel): sha256: str | None size: int size_count: int - type: FileType - urls: list[Url] + type: FileTypeModel + urls: list[UrlModel] -class ScanCodeDataRaw(BaseModel): +class ScanCodeDataModel(BaseModel): dependencies: list - files: list[File] - license_detections: list[GlobalLicenseDetection] - headers: list[Header] + files: list[FileModel] + license_detections: list[GlobalLicenseDetectionModel] + headers: list[HeaderModel] packages: list diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py index a05a44df..aa5bd4a7 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py @@ -17,17 +17,17 @@ SourceInfo, ) from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( - File, - FileType, - Header, - ScanCodeDataRaw, +from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( + FileModel, + FileTypeModel, + HeaderModel, + ScanCodeDataModel, ) class ScancodeDataToOpossumConverter: @staticmethod - def convert_scancode_to_opossum(scancode_data: ScanCodeDataRaw) -> Opossum: + def convert_scancode_to_opossum(scancode_data: ScanCodeDataModel) -> Opossum: resources = ScancodeDataToOpossumConverter.extract_opossum_resources( scancode_data ) @@ -49,7 +49,7 @@ def convert_scancode_to_opossum(scancode_data: ScanCodeDataRaw) -> Opossum: ) @staticmethod - def extract_scancode_header(scancode_data: ScanCodeDataRaw) -> Header: + def extract_scancode_header(scancode_data: ScanCodeDataModel) -> HeaderModel: if len(scancode_data.headers) != 1: logging.error("Headers of ScanCode file are invalid.") sys.exit(1) @@ -57,7 +57,7 @@ def extract_scancode_header(scancode_data: ScanCodeDataRaw) -> Header: @staticmethod def extract_opossum_resources( - scancode_data: ScanCodeDataRaw, + scancode_data: ScanCodeDataModel, ) -> list[Resource]: temp_root = Resource(path=PurePath("")) for file in scancode_data.files: @@ -71,15 +71,15 @@ def extract_opossum_resources( return list(temp_root.children.values()) @staticmethod - def convert_resource_type(file_type: FileType) -> ResourceType: - if file_type == FileType.FILE: + def convert_resource_type(file_type: FileTypeModel) -> ResourceType: + if file_type == FileTypeModel.FILE: return ResourceType.FILE else: return ResourceType.FOLDER @staticmethod - def get_attribution_info(file: File) -> list[OpossumPackage]: - if file.type == FileType.DIRECTORY: + def get_attribution_info(file: FileModel) -> list[OpossumPackage]: + if file.type == FileTypeModel.DIRECTORY: return [] copyright = "\n".join(c.copyright for c in file.copyrights) source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index db49229f..3a55db50 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -11,8 +11,8 @@ Opossum, ) from opossum_lib.core.services.input_reader import InputFormatReader -from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( - ScanCodeDataRaw, +from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( + ScanCodeDataModel, ) from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, @@ -28,7 +28,7 @@ def read(self, path: Path) -> Opossum: return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) @staticmethod - def load_scancode_json(path: Path) -> ScanCodeDataRaw: + def load_scancode_json(path: Path) -> ScanCodeDataModel: try: with open(path) as inp: json_data = json.load(inp) @@ -39,6 +39,6 @@ def load_scancode_json(path: Path) -> ScanCodeDataRaw: logging.error(f"Error decoding json for file {path}.") sys.exit(1) - scancode_data = ScanCodeDataRaw.model_validate(json_data) + scancode_data = ScanCodeDataModel.model_validate(json_data) return scancode_data diff --git a/src/opossum_lib/shared/entities/opossum_file.py b/src/opossum_lib/shared/entities/opossum_file.py deleted file mode 100644 index 5a7e4c19..00000000 --- a/src/opossum_lib/shared/entities/opossum_file.py +++ /dev/null @@ -1,14 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - -from pydantic import BaseModel - -from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile -from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile - - -class OpossumFileModel(BaseModel): - input_file: OpossumInputFile - output_file: OpossumOutputFile | None = None diff --git a/src/opossum_lib/shared/entities/opossum_file_model.py b/src/opossum_lib/shared/entities/opossum_file_model.py new file mode 100644 index 00000000..ea114f33 --- /dev/null +++ b/src/opossum_lib/shared/entities/opossum_file_model.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from pydantic import BaseModel + +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel + + +class OpossumFileModel(BaseModel): + input_file: OpossumInputFileModel + output_file: OpossumOutputFileModel | None = None diff --git a/src/opossum_lib/shared/entities/opossum_input_file.py b/src/opossum_lib/shared/entities/opossum_input_file_model.py similarity index 82% rename from src/opossum_lib/shared/entities/opossum_input_file.py rename to src/opossum_lib/shared/entities/opossum_input_file_model.py index 1b07ce0e..91091b3b 100644 --- a/src/opossum_lib/shared/entities/opossum_input_file.py +++ b/src/opossum_lib/shared/entities/opossum_input_file_model.py @@ -22,21 +22,21 @@ class CamelBaseModel(BaseModel): ) -class OpossumInputFile(CamelBaseModel): - metadata: Metadata +class OpossumInputFileModel(CamelBaseModel): + metadata: MetadataModel resources: ResourceInFile - external_attributions: dict[OpossumPackageIdentifier, OpossumPackage] + external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel] resources_to_attributions: dict[ResourcePath, list[OpossumPackageIdentifier]] attribution_breakpoints: list[str] = field(default_factory=list) - external_attribution_sources: dict[str, ExternalAttributionSource] = field( + external_attribution_sources: dict[str, ExternalAttributionSourceModel] = field( default_factory=dict ) - frequent_licenses: list[FrequentLicense] | None = None + frequent_licenses: list[FrequentLicenseModel] | None = None files_with_children: list[str] | None = None - base_urls_for_sources: BaseUrlsForSources | None = None + base_urls_for_sources: BaseUrlsForSourcesModel | None = None -class BaseUrlsForSources(CamelBaseModel): +class BaseUrlsForSourcesModel(CamelBaseModel): @model_serializer def serialize(self) -> dict: # hack to override not serializing keys with corresponding value none: @@ -46,20 +46,20 @@ def serialize(self) -> dict: model_config = ConfigDict(extra="allow") -class FrequentLicense(CamelBaseModel): +class FrequentLicenseModel(CamelBaseModel): full_name: str short_name: str default_text: str -class SourceInfo(CamelBaseModel): +class SourceInfoModel(CamelBaseModel): name: str document_confidence: int | float | None = 0 additional_name: str | None = None -class OpossumPackage(CamelBaseModel): - source: SourceInfo +class OpossumPackageModel(CamelBaseModel): + source: SourceInfoModel attribution_confidence: int | None = None comment: str | None = None package_name: str | None = None @@ -81,7 +81,7 @@ class OpossumPackage(CamelBaseModel): was_preferred: bool | None = None -class Metadata(CamelBaseModel): +class MetadataModel(CamelBaseModel): model_config = ConfigDict(extra="allow") project_id: str file_creation_date: str @@ -91,20 +91,20 @@ class Metadata(CamelBaseModel): build_date: str | None = None -class ResourceType(Enum): +class ResourceTypeModel(Enum): FILE = auto() FOLDER = auto() TOP_LEVEL = auto() OTHER = auto() -class Resource(CamelBaseModel): - type: ResourceType - children: dict[str, Resource] = field(default_factory=dict) +class ResourceModel(CamelBaseModel): + type: ResourceTypeModel + children: dict[str, ResourceModel] = field(default_factory=dict) def add_path( - self, path_with_resource_types: list[tuple[str, ResourceType]] - ) -> Resource: + self, path_with_resource_types: list[tuple[str, ResourceTypeModel]] + ) -> ResourceModel: resource = deepcopy(self) if len(path_with_resource_types) == 0: return resource @@ -118,21 +118,21 @@ def add_path( " the same path differ." ) if first not in self.children: - resource.children[first] = Resource(type=resource_type) + resource.children[first] = ResourceModel(type=resource_type) resource.children[first] = resource.children[first].add_path(rest) return resource def element_exists_but_resource_type_differs( - self, element: str, resource_type: ResourceType + self, element: str, resource_type: ResourceTypeModel ) -> bool: if element in self.children: return self.children[element].type != resource_type return False def drop_element( - self, path_to_element_to_drop: list[tuple[str, ResourceType]] - ) -> Resource: + self, path_to_element_to_drop: list[tuple[str, ResourceTypeModel]] + ) -> ResourceModel: paths_in_resource = self.get_paths_of_all_leaf_nodes_with_types() if path_to_element_to_drop not in paths_in_resource: raise ValueError( @@ -140,7 +140,7 @@ def drop_element( ) else: - resource = Resource(type=ResourceType.TOP_LEVEL) + resource = ResourceModel(type=ResourceTypeModel.TOP_LEVEL) paths_in_resource.remove(path_to_element_to_drop) paths_in_resource.append(path_to_element_to_drop[:-1]) @@ -151,7 +151,7 @@ def drop_element( def to_dict(self) -> ResourceInFile: if not self.has_children(): - if self.type == ResourceType.FOLDER: + if self.type == ResourceTypeModel.FOLDER: return {} else: return 1 @@ -162,7 +162,7 @@ def to_dict(self) -> ResourceInFile: def get_paths_of_all_leaf_nodes_with_types( self, - ) -> list[list[tuple[str, ResourceType]]]: + ) -> list[list[tuple[str, ResourceTypeModel]]]: paths = [] for name, resource in self.children.items(): path = [(name, resource.type)] @@ -184,7 +184,7 @@ def convert_to_file_resource(self) -> ResourceInFile: return self.to_dict() -class ExternalAttributionSource(CamelBaseModel): +class ExternalAttributionSourceModel(CamelBaseModel): model_config = ConfigDict(frozen=True) name: str priority: int diff --git a/src/opossum_lib/shared/entities/opossum_output_file.py b/src/opossum_lib/shared/entities/opossum_output_file_model.py similarity index 99% rename from src/opossum_lib/shared/entities/opossum_output_file.py rename to src/opossum_lib/shared/entities/opossum_output_file_model.py index a00121f6..de9da8b1 100644 --- a/src/opossum_lib/shared/entities/opossum_output_file.py +++ b/src/opossum_lib/shared/entities/opossum_output_file_model.py @@ -129,7 +129,7 @@ class ManualAttributions(CamelBaseModel): ) -class OpossumOutputFile(CamelBaseModel): +class OpossumOutputFileModel(CamelBaseModel): model_config = ConfigDict(extra="allow") metadata: Metadata diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/opossum_model_generators/opossum_provider.py index 55ffb41e..a37075ca 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/opossum_model_generators/opossum_provider.py @@ -6,7 +6,7 @@ from faker.providers import BaseProvider from opossum_lib.core.entities.opossum import Opossum, ScanResults -from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider from tests.test_opossum.generators.generate_outfile_information import ( OpossumOutputFileProvider, @@ -25,7 +25,7 @@ def __init__(self, generator: Any): def opossum( self, scan_results: ScanResults | None = None, - review_results: OpossumOutputFile | None = None, + review_results: OpossumOutputFileModel | None = None, ) -> Opossum: return Opossum( scan_results=scan_results or self.scan_results_provider.scan_results(), diff --git a/tests/test_cli.py b/tests/test_cli.py index b8f62674..ebc9b9bb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -16,7 +16,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_input_file import OpossumPackage +from opossum_lib.shared.entities.opossum_input_file_model import OpossumPackageModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" @@ -129,12 +129,12 @@ def assert_output_json_matches_expectations(output_file: str) -> None: def inline_attributions_into_resources( *, resources_with_ids: dict[str, list[str]], all_attributions: dict[str, Any] -) -> dict[str, set[OpossumPackage]]: +) -> dict[str, set[OpossumPackageModel]]: resource_with_inlined_attributions = {} for path, ids in resources_with_ids.items(): attributions = [] for id in ids: - attribution = OpossumPackage(**all_attributions[id]) + attribution = OpossumPackageModel(**all_attributions[id]) attributions.append(attribution) resource_with_inlined_attributions[path] = set(attributions) return resource_with_inlined_attributions diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index 6cdb76e0..525108ce 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -13,17 +13,17 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.shared.entities.opossum_input_file import ( - BaseUrlsForSources, - ExternalAttributionSource, - FrequentLicense, - Metadata, - OpossumInputFile, - OpossumPackage, +from opossum_lib.shared.entities.opossum_input_file_model import ( + BaseUrlsForSourcesModel, + ExternalAttributionSourceModel, + FrequentLicenseModel, + MetadataModel, + OpossumInputFileModel, OpossumPackageIdentifier, + OpossumPackageModel, ResourceInFile, ResourcePath, - SourceInfo, + SourceInfoModel, ) from tests.util.generator_helpers import entry_or_none, random_list @@ -48,8 +48,8 @@ def opossum_input_metadata( project_version: str | None = None, expected_release_date: str | None = None, build_date: str | None = None, - ) -> Metadata: - return Metadata( + ) -> MetadataModel: + return MetadataModel( project_id=project_id or "project-id-" + self.lorem_provider.word(), file_creation_date=file_creation_date or self.date_time_provider.date_time().isoformat(), @@ -87,24 +87,24 @@ def __init__(self, generator: Any): def opossum_file_information( self, *, - metadata: Metadata | None = None, + metadata: MetadataModel | None = None, resources: ResourceInFile | None = None, - external_attributions: dict[OpossumPackageIdentifier, OpossumPackage] + external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel] | None = None, resources_to_attributions: dict[ResourcePath, list[OpossumPackageIdentifier]] | None = None, attribution_breakpoints: list[str] | None = None, - external_attribution_sources: dict[str, ExternalAttributionSource] + external_attribution_sources: dict[str, ExternalAttributionSourceModel] | None = None, - frequent_licenses: list[FrequentLicense] | None = None, + frequent_licenses: list[FrequentLicenseModel] | None = None, files_with_children: list[str] | None = None, - base_urls_for_sources: BaseUrlsForSources | None = None, - ) -> OpossumInputFile: + base_urls_for_sources: BaseUrlsForSourcesModel | None = None, + ) -> OpossumInputFileModel: generated_resources = resources or self.resource_in_file() attributions = external_attributions or self.external_attributions( min_number_of_attributions=25 ) - return OpossumInputFile( + return OpossumInputFileModel( metadata=metadata or self.metadata_provider.opossum_input_metadata(), resources=generated_resources, external_attributions=attributions, @@ -153,8 +153,8 @@ def source_info( name: str | None = None, document_confidence: int | float | None = None, additional_name: str | None = None, - ) -> SourceInfo: - return SourceInfo( + ) -> SourceInfoModel: + return SourceInfoModel( name=name or self.person_provider.name(), document_confidence=document_confidence or entry_or_none(self.misc_provider, self.random_int(0, 100)), @@ -164,7 +164,7 @@ def source_info( def opossum_package( self, - source: SourceInfo | None = None, + source: SourceInfoModel | None = None, attribution_confidence: int | None = None, comment: str | None = None, package_name: str | None = None, @@ -184,8 +184,8 @@ def opossum_package( origin_ids: list[str] | None = None, criticality: Literal["high"] | Literal["medium"] | None = None, was_preferred: bool | None = None, - ) -> OpossumPackage: - return OpossumPackage( + ) -> OpossumPackageModel: + return OpossumPackageModel( source=source or self.source_info(), attribution_confidence=attribution_confidence or entry_or_none(self.misc_provider, self.random_int()), @@ -242,7 +242,7 @@ def opossum_package( def external_attributions( self, max_number_of_attributions: int = 50, min_number_of_attributions: int = 5 - ) -> dict[OpossumPackageIdentifier, OpossumPackage]: + ) -> dict[OpossumPackageIdentifier, OpossumPackageModel]: number_of_attributions = self.random_int( min_number_of_attributions, max_number_of_attributions ) @@ -254,7 +254,7 @@ def external_attributions( def resources_to_attributions( self, resources: ResourceInFile, - external_attributions: dict[OpossumPackageIdentifier, OpossumPackage], + external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel], ) -> dict[ResourcePath, list[OpossumPackageIdentifier]]: def get_file_paths(resource: ResourceInFile, current_path: str) -> list[str]: if isinstance(resource, int): @@ -294,8 +294,8 @@ def external_attribution_source( name: str | None = None, priority: int | None = None, is_relevant_for_preferred: bool | None = None, - ) -> ExternalAttributionSource: - return ExternalAttributionSource( + ) -> ExternalAttributionSourceModel: + return ExternalAttributionSourceModel( name=name or self.person_provider.name(), priority=priority or self.random_int(1, 100), is_relevant_for_preferred=is_relevant_for_preferred @@ -304,7 +304,7 @@ def external_attribution_source( def external_attribution_sources( self, max_nb_of_external_attributions: int = 5 - ) -> dict[str, ExternalAttributionSource]: + ) -> dict[str, ExternalAttributionSourceModel]: nb_of_external_attributions = self.random_int( 1, max_nb_of_external_attributions ) diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/test_opossum/generators/generate_opossum_file_content.py index b44c0b85..ee2eeec6 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/test_opossum/generators/generate_opossum_file_content.py @@ -5,9 +5,9 @@ from faker.providers import BaseProvider -from opossum_lib.shared.entities.opossum_file import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file import OpossumInputFile -from opossum_lib.shared.entities.opossum_output_file import OpossumOutputFile +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel from tests.test_opossum.generators.generate_file_information import ( FileInformationProvider, ) @@ -27,8 +27,8 @@ def __init__(self, generator: Any): def opossum_file_content( self, - in_file: OpossumInputFile | None = None, - out_file: OpossumOutputFile | None = None, + in_file: OpossumInputFileModel | None = None, + out_file: OpossumOutputFileModel | None = None, ) -> OpossumFileModel: return OpossumFileModel( input_file=in_file or self.infile_provider.opossum_file_information(), diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/test_opossum/generators/generate_outfile_information.py index 4a73eac2..f9a3260a 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/test_opossum/generators/generate_outfile_information.py @@ -9,10 +9,10 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.shared.entities.opossum_output_file import ( +from opossum_lib.shared.entities.opossum_output_file_model import ( ManualAttributions, Metadata, - OpossumOutputFile, + OpossumOutputFileModel, ) from tests.util.generator_helpers import entry_or_none @@ -34,8 +34,8 @@ def output_file( manual_attributions: dict[str, ManualAttributions] | None = None, resources_to_attributions: dict[str, list[str]] | None = None, resolved_external_attributions: list[str] | None = None, - ) -> OpossumOutputFile: - return OpossumOutputFile( + ) -> OpossumOutputFileModel: + return OpossumOutputFileModel( metadata=metadata or self.outfile_metadata(), manual_attributions=manual_attributions or {}, resources_to_attributions=resources_to_attributions or {}, diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index bbf90ade..41ba4f73 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -6,7 +6,7 @@ from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_file_generation.py b/tests/test_opossum/test_file_generation.py index e5273c2d..2483752f 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/test_opossum/test_file_generation.py @@ -10,7 +10,7 @@ INPUT_JSON_NAME, OUTPUT_JSON_NAME, ) -from opossum_lib.shared.entities.opossum_file import OpossumFileModel +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index df491dd6..4cd1da22 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -6,9 +6,9 @@ from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 OpossumFileToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_input_file import ( - OpossumPackage, +from opossum_lib.shared.entities.opossum_input_file_model import ( OpossumPackageIdentifier, + OpossumPackageModel, ) from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker @@ -39,7 +39,7 @@ def test_throws_on_duplicate_attributions( @staticmethod def _fake_duplicate_external_attributions( opossum_file_faker: OpossumFileFaker, - ) -> dict[OpossumPackageIdentifier, OpossumPackage]: + ) -> dict[OpossumPackageIdentifier, OpossumPackageModel]: external_attributions = opossum_file_faker.external_attributions( min_number_of_attributions=2 ) diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/test_scancode/generators/generate_scancode_file.py index 563aec62..22b064f0 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/test_scancode/generators/generate_scancode_file.py @@ -20,22 +20,22 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( - Copyright, - Email, - ExtraData, - File, - FileBasedLicenseDetection, - FileType, - GlobalLicenseDetection, - Header, - Holder, - Match, - Options, - ReferenceMatch, - ScanCodeDataRaw, - SystemEnvironment, - Url, +from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( + CopyrightModel, + EmailModel, + ExtraDataModel, + FileBasedLicenseDetectionModel, + FileModel, + FileTypeModel, + GlobalLicenseDetectionModel, + HeaderModel, + HolderModel, + MatchModel, + OptionsModel, + ReferenceMatchModel, + ScanCodeDataModel, + SystemEnvironmentModel, + UrlModel, ) from tests.util.generator_helpers import entry_or_none, random_list @@ -63,12 +63,12 @@ def scancode_data( self, *, dependencies: list | None = None, - files: list[File] | None = None, - license_detections: list[GlobalLicenseDetection] | None = None, - headers: list[Header] | None = None, + files: list[FileModel] | None = None, + license_detections: list[GlobalLicenseDetectionModel] | None = None, + headers: list[HeaderModel] | None = None, packages: list | None = None, - options: Options | None = None, - ) -> ScanCodeDataRaw: + options: OptionsModel | None = None, + ) -> ScanCodeDataModel: # TODO: #184 depending on which options are passed in additional_options # we need to generate different fields, e.g. --licenses # out of scope for now @@ -76,7 +76,7 @@ def scancode_data( license_detections = license_detections or self.global_license_detections(files) if headers is None: headers = [self.header(options=options)] - return ScanCodeDataRaw( + return ScanCodeDataModel( dependencies=dependencies or [], files=files, license_detections=license_detections, @@ -90,17 +90,17 @@ def header( duration: float | None = None, end_timestamp: str | None = None, errors: list | None = None, - extra_data: ExtraData | None = None, + extra_data: ExtraDataModel | None = None, message: Any | None = None, notice: str | None = None, - options: Options | None = None, + options: OptionsModel | None = None, output_format_version: str | None = None, start_timestamp: str | None = None, tool_name: str | None = None, tool_version: str | None = None, warnings: list | None = None, - ) -> Header: - return Header( + ) -> HeaderModel: + return HeaderModel( duration=duration or self.random_int(max=9999999) / 1e3, end_timestamp=end_timestamp or self.date_provider.iso8601(), errors=errors or [], @@ -117,8 +117,8 @@ def header( def options( self, *, input: list[str] | None = None, **additional_options: Any - ) -> Options: - return Options( + ) -> OptionsModel: + return OptionsModel( input=input or [ self.file_provider.file_path( @@ -133,9 +133,9 @@ def extra_data( *, files_count: int | None = None, spdx_license_list_version: str | None = None, - system_environment: SystemEnvironment | None = None, - ) -> ExtraData: - return ExtraData( + system_environment: SystemEnvironmentModel | None = None, + ) -> ExtraDataModel: + return ExtraDataModel( files_count=files_count or self.random_int(), spdx_license_list_version=spdx_license_list_version or self.numerify("#.##"), @@ -150,11 +150,11 @@ def system_environment( platform: str | None = None, platform_version: str | None = None, python_version: str | None = None, - ) -> SystemEnvironment: + ) -> SystemEnvironmentModel: operating_system = operating_system or self.random_element( ["linux", "windows", "macos"] ) - return SystemEnvironment( + return SystemEnvironmentModel( cpu_architecture=cpu_architecture or self.random_element(["32", "64"]), operating_system=operating_system, platform=platform @@ -164,10 +164,10 @@ def system_environment( ) def global_license_detections( - self, files: list[File] - ) -> list[GlobalLicenseDetection]: + self, files: list[FileModel] + ) -> list[GlobalLicenseDetectionModel]: license_counter: dict[str, int] = defaultdict(int) - id_to_license_detection: dict[str, FileBasedLicenseDetection] = {} + id_to_license_detection: dict[str, FileBasedLicenseDetectionModel] = {} for file in files: for ld in file.license_detections: license_counter[ld.identifier] += 1 @@ -176,13 +176,13 @@ def global_license_detections( global_license_detections = [] for id, count in license_counter.items(): ld = id_to_license_detection[id] - gld = GlobalLicenseDetection( + gld = GlobalLicenseDetectionModel( detection_count=count, license_expression=ld.license_expression, license_expression_spdx=ld.license_expression_spdx, identifier=ld.identifier, reference_matches=[ - ReferenceMatch( + ReferenceMatchModel( end_line=match.end_line, from_file=match.from_file, license_expression=match.license_expression, @@ -228,11 +228,11 @@ def generate_path_structure( folders[folder_name] = children return {**files, **folders} - def files(self, path_tree: TempPathTree | None = None) -> list[File]: + def files(self, path_tree: TempPathTree | None = None) -> list[FileModel]: path_tree = path_tree or self.generate_path_structure() - def process_path(current_path: str, path_tree: TempPathTree) -> list[File]: - files: list[File] = [] + def process_path(current_path: str, path_tree: TempPathTree) -> list[FileModel]: + files: list[FileModel] = [] for name, data in path_tree.items(): path = current_path + name if data: @@ -240,8 +240,8 @@ def process_path(current_path: str, path_tree: TempPathTree) -> list[File]: child_types = [c.type for c in child_files] folder = self.single_folder( path=path, - dirs_count=child_types.count(FileType.DIRECTORY), - files_count=child_types.count(FileType.FILE), + dirs_count=child_types.count(FileTypeModel.DIRECTORY), + files_count=child_types.count(FileTypeModel.FILE), size_count=sum(c.size for c in child_files), ) files.append(folder) @@ -261,17 +261,17 @@ def single_folder( path: str, authors: list | None = None, base_name: str | None = None, - copyrights: list[Copyright] | None = None, + copyrights: list[CopyrightModel] | None = None, date: str | None = None, detected_license_expression: str | None = None, detected_license_expression_spdx: str | None = None, dirs_count: int = 0, - emails: list[Email] | None = None, + emails: list[EmailModel] | None = None, extension: str = "", files_count: int = 0, file_type: str | None = None, for_packages: list | None = None, - holders: list[Holder] | None = None, + holders: list[HolderModel] | None = None, is_archive: bool = False, is_binary: bool = False, is_media: bool = False, @@ -279,7 +279,7 @@ def single_folder( is_source: bool = False, is_text: bool = False, license_clues: list | None = None, - license_detections: list[FileBasedLicenseDetection] | None = None, + license_detections: list[FileBasedLicenseDetectionModel] | None = None, md5: str | None = None, mime_type: str | None = None, name: str | None = None, @@ -291,9 +291,9 @@ def single_folder( sha256: str | None = None, size: int = 0, size_count: int = 0, - urls: list[Url] | None = None, - ) -> File: - return File( + urls: list[UrlModel] | None = None, + ) -> FileModel: + return FileModel( authors=authors or [], base_name=base_name or PurePath(PurePath(path).name).stem, copyrights=copyrights or [], @@ -327,7 +327,7 @@ def single_folder( sha256=sha256, size=size, size_count=size_count, - type=FileType.DIRECTORY, + type=FileTypeModel.DIRECTORY, urls=urls or [], ) @@ -337,17 +337,17 @@ def single_file( path: str, authors: list | None = None, base_name: str | None = None, - copyrights: list[Copyright] | None = None, + copyrights: list[CopyrightModel] | None = None, date: str | None = None, detected_license_expression: str | None = None, detected_license_expression_spdx: str | None = None, dirs_count: int = 0, - emails: list[Email] | None = None, + emails: list[EmailModel] | None = None, extension: str | None = None, files_count: int = 0, file_type: str | None = None, for_packages: list | None = None, - holders: list[Holder] | None = None, + holders: list[HolderModel] | None = None, is_archive: bool | None = None, is_binary: bool | None = None, is_media: bool | None = None, @@ -355,7 +355,7 @@ def single_file( is_source: bool | None = None, is_text: bool | None = None, license_clues: list | None = None, - license_detections: list[FileBasedLicenseDetection] | None = None, + license_detections: list[FileBasedLicenseDetectionModel] | None = None, md5: str | None = None, mime_type: str | None = None, name: str | None = None, @@ -367,14 +367,14 @@ def single_file( sha256: str | None = None, size: int | None = None, size_count: int = 0, - urls: list[Url] | None = None, - ) -> File: + urls: list[UrlModel] | None = None, + ) -> FileModel: if copyrights is None and holders is None: holders = [] for _ in range(self.random_int(max=3)): start_line = self.random_int() end_line = start_line + self.random_int(max=2) - holder = Holder( + holder = HolderModel( holder=self.company_provider.company(), start_line=start_line, end_line=end_line, @@ -383,7 +383,7 @@ def single_file( if copyrights is None: assert holders is not None # can never trigger but makes mypy happy copyrights = [ - Copyright( + CopyrightModel( copyright="Copyright " + h.holder, start_line=h.start_line, end_line=h.end_line, @@ -392,7 +392,7 @@ def single_file( ] if holders is None: holders = [ - Holder( + HolderModel( holder=cr.copyright, start_line=cr.start_line, end_line=cr.end_line, @@ -435,7 +435,7 @@ def single_file( self.misc_provider, self.random_element(["Java", "Typescript", "HTML", "Python"]), ) - return File( + return FileModel( authors=authors or [], base_name=base_name or PurePath(PurePath(path).name).stem, copyrights=copyrights, @@ -469,7 +469,7 @@ def single_file( sha256=sha256 if sha256 is not None else self.misc_provider.sha256(), size=size if size is not None else self.random_int(max=10**9), size_count=size_count, - type=FileType.FILE, + type=FileTypeModel.FILE, urls=urls if urls is not None else random_list(self, self.url), ) @@ -478,10 +478,10 @@ def copyright( copyright: str | None = None, end_line: int | None = None, start_line: int | None = None, - ) -> Copyright: + ) -> CopyrightModel: start_line = start_line or self.random_int() end_line = start_line + self.random_int(max=50) - return Copyright( + return CopyrightModel( copyright=copyright or "Copyright " + self.company_provider.company(), end_line=end_line, start_line=start_line, @@ -492,10 +492,10 @@ def email( email: str | None = None, end_line: int | None = None, start_line: int | None = None, - ) -> Email: + ) -> EmailModel: start_line = start_line or self.random_int() end_line = start_line + self.random_int(max=2) - return Email( + return EmailModel( email=email or self.internet_provider.email(), end_line=end_line, start_line=start_line, @@ -506,10 +506,10 @@ def url( url: str | None = None, end_line: int | None = None, start_line: int | None = None, - ) -> Url: + ) -> UrlModel: start_line = start_line or self.random_int() end_line = start_line + self.random_int(max=2) - return Url( + return UrlModel( url=url or self.internet_provider.url(), end_line=end_line, start_line=start_line, @@ -519,10 +519,10 @@ def license_detection( self, license_expression: str | None = None, license_expression_spdx: str | None = None, - matches: list[Match] | None = None, + matches: list[MatchModel] | None = None, identifier: str | None = None, path: str | None = None, - ) -> FileBasedLicenseDetection: + ) -> FileBasedLicenseDetectionModel: if path is None and matches is None: raise RuntimeError( "Neither path nor matches given which is likely a user error. " @@ -546,7 +546,7 @@ def license_detection( ), min_number_of_entries=1, ) - return FileBasedLicenseDetection( + return FileBasedLicenseDetectionModel( license_expression=license_expression, license_expression_spdx=license_expression_spdx, matches=matches, @@ -568,12 +568,12 @@ def match( rule_url: Any | None = None, score: float | None = None, start_line: int | None = None, - ) -> Match: + ) -> MatchModel: start_line = start_line or self.random_int() end_line = start_line + self.random_int() if license_expression_spdx is None: license_expression_spdx = self.lexify("???? License") - return Match( + return MatchModel( end_line=end_line, from_file=from_file, license_expression=license_expression or "", diff --git a/tests/test_scancode/model_helpers.py b/tests/test_scancode/model_helpers.py index f942a12f..347b3fca 100644 --- a/tests/test_scancode/model_helpers.py +++ b/tests/test_scancode/model_helpers.py @@ -5,29 +5,29 @@ from pathlib import PurePath -from opossum_lib.input_formats.scancode.entities.scan_code_data_raw import ( - Copyright, - File, - FileBasedLicenseDetection, - FileType, - Holder, - Url, +from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( + CopyrightModel, + FileBasedLicenseDetectionModel, + FileModel, + FileTypeModel, + HolderModel, + UrlModel, ) -def _create_reference_scancode_files() -> list[File]: +def _create_reference_scancode_files() -> list[FileModel]: return [ - _create_file("A", FileType.DIRECTORY), - _create_file("A/B", FileType.DIRECTORY), - _create_file("A/file1", FileType.FILE), - _create_file("A/file2.txt", FileType.FILE), - _create_file("A/B/file3", FileType.FILE), + _create_file("A", FileTypeModel.DIRECTORY), + _create_file("A/B", FileTypeModel.DIRECTORY), + _create_file("A/file1", FileTypeModel.FILE), + _create_file("A/file2.txt", FileTypeModel.FILE), + _create_file("A/B/file3", FileTypeModel.FILE), ] def _create_file( path: str, - type: FileType, + type: FileTypeModel, *, name: str | None = None, base_name: str | None = None, @@ -50,19 +50,19 @@ def _create_file( for_packages: list | None = None, detected_license_expression: str | None = None, detected_license_expression_spdx: str | None = None, - license_detections: list[FileBasedLicenseDetection] | None = None, + license_detections: list[FileBasedLicenseDetectionModel] | None = None, license_clues: list | None = None, percentage_of_license_text: float = 0.0, - copyrights: list[Copyright] | None = None, - holders: list[Holder] | None = None, + copyrights: list[CopyrightModel] | None = None, + holders: list[HolderModel] | None = None, authors: list | None = None, emails: list | None = None, - urls: list[Url] | None = None, + urls: list[UrlModel] | None = None, files_count: int = 0, dirs_count: int = 0, size_count: int = 0, scan_errors: list | None = None, -) -> File: +) -> FileModel: if package_data is None: package_data = [] if for_packages is None: @@ -89,7 +89,7 @@ def _create_file( base_name = PurePath(PurePath(path).name).stem if extension is None: extension = PurePath(path).suffix - return File( + return FileModel( authors=authors, base_name=base_name, copyrights=copyrights, diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index 31a50e4e..d1dd4413 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -7,7 +7,10 @@ from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) -from opossum_lib.shared.entities.opossum_input_file import OpossumPackage, SourceInfo +from opossum_lib.shared.entities.opossum_input_file_model import ( + OpossumPackageModel, + SourceInfoModel, +) from tests.test_setup.scancode_faker_setup import ScanCodeFaker @@ -70,14 +73,14 @@ def test_get_attribution_info_file_multiple(scancode_faker: ScanCodeFaker) -> No opossum.to_opossum_file_format().input_file.external_attributions.values() ) - expected1 = OpossumPackage( - source=SourceInfo(name=SCANCODE_SOURCE_NAME), + expected1 = OpossumPackageModel( + source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), license_name="Apache-2.0", copyright="Me\nMyself\nI", attribution_confidence=95, ) - expected2 = OpossumPackage( - source=SourceInfo(name=SCANCODE_SOURCE_NAME), + expected2 = OpossumPackageModel( + source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), license_name="MIT", copyright="Me\nMyself\nI", attribution_confidence=50, From 378c276a6fabf346169b83ab5e5318d21403ce33 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 16:12:28 +0100 Subject: [PATCH 40/58] refactor: review-comment: use the renamings in converters --- src/opossum_lib/core/entities/opossum.py | 78 ++++++----- .../core/services/opossum_generator.py | 2 +- ...possum_file_model_to_opossum_converter.py} | 126 ++++++++++-------- .../opossum/services/opossum_format_reader.py | 8 +- .../entities/opossum_input_file_model.py | 18 +-- tests/core/test_opossum_model.py | 14 +- .../generators/generate_file_information.py | 28 ++-- .../test_opossum/test_conversion_roundtrip.py | 8 +- .../test_opossum_file_to_opossum_converter.py | 12 +- .../test_get_attribution_info.py | 4 +- 10 files changed, 163 insertions(+), 135 deletions(-) rename src/opossum_lib/input_formats/opossum/services/{opossum_file_to_opossum_converter.py => opossum_file_model_to_opossum_converter.py} (67%) diff --git a/src/opossum_lib/core/entities/opossum.py b/src/opossum_lib/core/entities/opossum.py index 77598d8d..5f99e0f2 100644 --- a/src/opossum_lib/core/entities/opossum.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -15,8 +15,19 @@ from pydantic import BaseModel, ConfigDict -import opossum_lib.shared.entities.opossum_input_file_model as opossum_file from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import ( + BaseUrlsForSourcesModel, + ExternalAttributionSourceModel, + FrequentLicenseModel, + MetadataModel, + OpossumInputFileModel, + OpossumPackageIdentifierModel, + OpossumPackageModel, + ResourceInFileModel, + ResourcePathModel, + SourceInfoModel, +) from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel type OpossumPackageIdentifier = str @@ -36,9 +47,9 @@ class Opossum(BaseModel): scan_results: ScanResults review_results: OpossumOutputFileModel | None = None - def to_opossum_file_format(self) -> OpossumFileModel: + def to_opossum_model(self) -> OpossumFileModel: return OpossumFileModel( - input_file=self.scan_results.to_opossum_file_format(), + input_file=self.scan_results.to_opossum_model(), output_file=self.review_results, ) @@ -57,7 +68,7 @@ class ScanResults(BaseModel): ) unassigned_attributions: list[OpossumPackage] = [] - def to_opossum_file_format(self) -> opossum_file.OpossumInputFileModel: + def to_opossum_model(self) -> OpossumInputFileModel: external_attributions, resources_to_attributions = ( self.create_attribution_mapping(self.resources) ) @@ -66,22 +77,21 @@ def to_opossum_file_format(self) -> opossum_file.OpossumInputFileModel: frequent_licenses = None if self.frequent_licenses: frequent_licenses = [ - license.to_opossum_file_format() for license in self.frequent_licenses + license.to_opossum_model() for license in self.frequent_licenses ] base_urls_for_sources = ( - self.base_urls_for_sources - and self.base_urls_for_sources.to_opossum_file_format() + self.base_urls_for_sources and self.base_urls_for_sources.to_opossum_model() ) external_attribution_sources = { - key: val.to_opossum_file_format() + key: val.to_opossum_model() for (key, val) in self.external_attribution_sources.items() } - return opossum_file.OpossumInputFileModel( - metadata=self.metadata.to_opossum_file_format(), + return OpossumInputFileModel( + metadata=self.metadata.to_opossum_model(), resources={ - str(resource.path): resource.to_opossum_file_format() + str(resource.path): resource.to_opossum_model() for resource in self.resources }, external_attributions=external_attributions, @@ -95,20 +105,20 @@ def to_opossum_file_format(self) -> opossum_file.OpossumInputFileModel: def _get_unassigned_attributions( self, - ) -> dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel]: + ) -> dict[OpossumPackageIdentifierModel, OpossumPackageModel]: if self.unassigned_attributions: result = {} for unassigned_attribution in self.unassigned_attributions: if unassigned_attribution in self.attribution_to_id: package_identifier = self.attribution_to_id[unassigned_attribution] result[package_identifier] = ( - unassigned_attribution.to_opossum_file_format() + unassigned_attribution.to_opossum_model() ) else: package_identifier = str(uuid.uuid4()) self.attribution_to_id[unassigned_attribution] = package_identifier result[package_identifier] = ( - unassigned_attribution.to_opossum_file_format() + unassigned_attribution.to_opossum_model() ) return result else: @@ -118,14 +128,14 @@ def create_attribution_mapping( self, root_nodes: list[Resource], ) -> tuple[ - dict[opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel], - dict[opossum_file.ResourcePath, list[opossum_file.OpossumPackageIdentifier]], + dict[OpossumPackageIdentifierModel, OpossumPackageModel], + dict[ResourcePathModel, list[OpossumPackageIdentifierModel]], ]: external_attributions: dict[ - opossum_file.OpossumPackageIdentifier, opossum_file.OpossumPackageModel + OpossumPackageIdentifierModel, OpossumPackageModel ] = {} resources_to_attributions: dict[ - opossum_file.ResourcePath, list[opossum_file.OpossumPackageIdentifier] + ResourcePathModel, list[OpossumPackageIdentifierModel] ] = {} def process_node(node: Resource) -> None: @@ -135,7 +145,7 @@ def process_node(node: Resource) -> None: path = "/" + path node_attributions_by_id = { - self.get_attribution_key(a): a.to_opossum_file_format() + self.get_attribution_key(a): a.to_opossum_model() for a in node.attributions } external_attributions.update(node_attributions_by_id) @@ -171,12 +181,12 @@ class Resource(BaseModel): attributions: list[OpossumPackage] = [] children: dict[str, Resource] = {} - def to_opossum_file_format(self) -> opossum_file.ResourceInFile: + def to_opossum_model(self) -> ResourceInFileModel: if self.children or self.type == ResourceType.FOLDER: return { _convert_path_to_str( child.path.relative_to(self.path) - ): child.to_opossum_file_format() + ): child.to_opossum_model() for child in self.children.values() } else: @@ -227,8 +237,8 @@ def _update(self, other: Resource) -> None: class BaseUrlsForSources(BaseModel): model_config = ConfigDict(frozen=True, extra="allow") - def to_opossum_file_format(self) -> opossum_file.BaseUrlsForSourcesModel: - return opossum_file.BaseUrlsForSourcesModel(**self.model_dump()) + def to_opossum_model(self) -> BaseUrlsForSourcesModel: + return BaseUrlsForSourcesModel(**self.model_dump()) class FrequentLicense(BaseModel): @@ -237,8 +247,8 @@ class FrequentLicense(BaseModel): short_name: str default_text: str - def to_opossum_file_format(self) -> opossum_file.FrequentLicenseModel: - return opossum_file.FrequentLicenseModel( + def to_opossum_model(self) -> FrequentLicenseModel: + return FrequentLicenseModel( full_name=self.full_name, short_name=self.short_name, default_text=self.default_text, @@ -251,8 +261,8 @@ class SourceInfo(BaseModel): document_confidence: int | float | None = 0 additional_name: str | None = None - def to_opossum_file_format(self) -> opossum_file.SourceInfoModel: - return opossum_file.SourceInfoModel( + def to_opossum_model(self) -> SourceInfoModel: + return SourceInfoModel( name=self.name, document_confidence=self.document_confidence, additional_name=self.additional_name, @@ -282,9 +292,9 @@ class OpossumPackage(BaseModel): criticality: Literal["high"] | Literal["medium"] | None = None was_preferred: bool | None = None - def to_opossum_file_format(self) -> opossum_file.OpossumPackageModel: - return opossum_file.OpossumPackageModel( - source=self.source.to_opossum_file_format(), + def to_opossum_model(self) -> OpossumPackageModel: + return OpossumPackageModel( + source=self.source.to_opossum_model(), attribution_confidence=self.attribution_confidence, comment=self.comment, package_name=self.package_name, @@ -316,8 +326,8 @@ class Metadata(BaseModel): expected_release_date: str | None = None build_date: str | None = None - def to_opossum_file_format(self) -> opossum_file.MetadataModel: - return opossum_file.MetadataModel(**self.model_dump()) + def to_opossum_model(self) -> MetadataModel: + return MetadataModel(**self.model_dump()) class ExternalAttributionSource(BaseModel): @@ -326,8 +336,8 @@ class ExternalAttributionSource(BaseModel): priority: int is_relevant_for_preferred: bool | None = None - def to_opossum_file_format(self) -> opossum_file.ExternalAttributionSourceModel: - return opossum_file.ExternalAttributionSourceModel( + def to_opossum_model(self) -> ExternalAttributionSourceModel: + return ExternalAttributionSourceModel( name=self.name, priority=self.priority, is_relevant_for_preferred=self.is_relevant_for_preferred, diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index 412c12b2..0c7f27ee 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -56,7 +56,7 @@ def generate( # for the future a merge step is planned after reading the files opossum = self.input_reader.read(input_files[0]) - opossum_file_content = opossum.to_opossum_file_format() + opossum_file_content = opossum.to_opossum_model() OpossumFileWriter.write( opossum_file_content, opossum_generation_arguments.outfile ) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py similarity index 67% rename from src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py rename to src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py index 80be66a1..910a91fd 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_to_opossum_converter.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py @@ -5,7 +5,6 @@ from copy import deepcopy from pathlib import PurePath -import opossum_lib.shared.entities.opossum_input_file_model as opossum_file_package from opossum_lib.core.entities.opossum import ( BaseUrlsForSources, ExternalAttributionSource, @@ -13,7 +12,7 @@ Metadata, Opossum, OpossumPackage, - OpossumPackageIdentifier, + OpossumPackageIdentifierModel, Resource, ResourceType, ScanResults, @@ -24,91 +23,102 @@ from opossum_lib.shared.entities.opossum_input_file_model import ( ExternalAttributionSourceModel as FileExternalAttributionSource, ) -from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_input_file_model import ( + FrequentLicenseModel, + MetadataModel, + OpossumInputFileModel, + OpossumPackageModel, + ResourceInFileModel, + ResourcePathModel, + SourceInfoModel, +) -class OpossumFileToOpossumConverter: +class OpossumFileModelToOpossumConverter: @staticmethod - def convert_to_opossum(opossum_file: OpossumFileModel) -> Opossum: + def convert_to_opossum(opossum_file_model: OpossumFileModel) -> Opossum: opossum = Opossum( - scan_results=OpossumFileToOpossumConverter._convert_to_opossum_scan_results( - opossum_file.input_file + scan_results=OpossumFileModelToOpossumConverter._convert_to_scan_results( + opossum_file_model.input_file ), - review_results=opossum_file.output_file, + review_results=opossum_file_model.output_file, ) return opossum @staticmethod - def _convert_to_opossum_scan_results( - opossum_information: OpossumInputFileModel, + def _convert_to_scan_results( + opossum_input_file_model: OpossumInputFileModel, ) -> ScanResults: resources, used_attribution_ids = ( - OpossumFileToOpossumConverter._convert_to_opossum_model_resource_tree( - resources=opossum_information.resources, - external_attributions=opossum_information.external_attributions, - resources_to_attributions=opossum_information.resources_to_attributions, + OpossumFileModelToOpossumConverter._convert_to_resource_tree( + resources=opossum_input_file_model.resources, + external_attributions=opossum_input_file_model.external_attributions, + resources_to_attributions=opossum_input_file_model.resources_to_attributions, ) ) # fmt: off frequent_licenses = ( - opossum_information.frequent_licenses - and OpossumFileToOpossumConverter - ._convert_frequent_licenses_to_model_frequent_licenses( - opossum_information.frequent_licenses + opossum_input_file_model.frequent_licenses + and OpossumFileModelToOpossumConverter + ._convert_frequent_licenses( + opossum_input_file_model.frequent_licenses ) ) # fmt: on base_urls_for_sources = ( - opossum_information.base_urls_for_sources + opossum_input_file_model.base_urls_for_sources and BaseUrlsForSources( - **(opossum_information.base_urls_for_sources.model_dump()) + **(opossum_input_file_model.base_urls_for_sources.model_dump()) ) ) - file_attribution_sources = opossum_information.external_attribution_sources + file_attribution_sources = opossum_input_file_model.external_attribution_sources external_attribution_sources = { - name: OpossumFileToOpossumConverter._convert_external_attribution_source( + # noqa required due to clash between linter and formatter + name: OpossumFileModelToOpossumConverter._convert_external_attribution_source( # noqa: E501 attribution_source ) for name, attribution_source in file_attribution_sources.items() } attribution_with_id = ( - OpossumFileToOpossumConverter._convert_to_attribution_with_id( - opossum_information.external_attributions + OpossumFileModelToOpossumConverter._convert_to_attribution_with_id( + opossum_input_file_model.external_attributions ) ) return ScanResults( - metadata=OpossumFileToOpossumConverter._convert_to_opossum_model_metadata( - opossum_information.metadata + metadata=OpossumFileModelToOpossumConverter._convert_to_metadata( + opossum_input_file_model.metadata ), resources=resources, attribution_breakpoints=deepcopy( - opossum_information.attribution_breakpoints + opossum_input_file_model.attribution_breakpoints ), external_attribution_sources=external_attribution_sources, frequent_licenses=frequent_licenses, - files_with_children=deepcopy(opossum_information.files_with_children), + files_with_children=deepcopy(opossum_input_file_model.files_with_children), base_urls_for_sources=base_urls_for_sources, attribution_to_id=attribution_with_id, - unassigned_attributions=OpossumFileToOpossumConverter._get_unassigned_attributions( - used_attribution_ids, opossum_information.external_attributions + unassigned_attributions=OpossumFileModelToOpossumConverter._get_unassigned_attributions( + used_attribution_ids, opossum_input_file_model.external_attributions ), ) @staticmethod def _get_unassigned_attributions( - used_attribution_ids: set[OpossumPackageIdentifier], + used_attribution_ids: set[OpossumPackageIdentifierModel], external_attributions: dict[ - opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackageModel, + OpossumPackageIdentifierModel, + OpossumPackageModel, ], ) -> list[OpossumPackage] | None: available_attribution_ids = external_attributions.keys() unused_attributions_ids = set(available_attribution_ids) - used_attribution_ids unused_attributions = [ - OpossumFileToOpossumConverter._convert_package(external_attributions[id]) + OpossumFileModelToOpossumConverter._convert_package( + external_attributions[id] + ) for id in unused_attributions_ids ] return unused_attributions @@ -124,38 +134,38 @@ def _convert_external_attribution_source( ) @staticmethod - def _convert_frequent_licenses_to_model_frequent_licenses( - frequent_licenses_infile: list[opossum_file_package.FrequentLicenseModel], + def _convert_frequent_licenses( + frequent_licenses_infile: list[FrequentLicenseModel], ) -> list[FrequentLicense]: frequent_licenses: list[FrequentLicense] = [ - OpossumFileToOpossumConverter._convert_frequent_license(license) + OpossumFileModelToOpossumConverter._convert_frequent_license(license) for license in frequent_licenses_infile ] return frequent_licenses @staticmethod - def _convert_to_opossum_model_metadata( - infile_metadata: opossum_file_package.MetadataModel, + def _convert_to_metadata( + infile_metadata: MetadataModel, ) -> Metadata: return Metadata(**infile_metadata.model_dump()) @staticmethod - def _convert_to_opossum_model_resource_tree( - resources: opossum_file_package.ResourceInFile, + def _convert_to_resource_tree( + resources: ResourceInFileModel, external_attributions: dict[ - opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackageModel, + OpossumPackageIdentifierModel, + OpossumPackageModel, ], resources_to_attributions: dict[ - opossum_file_package.ResourcePath, - list[opossum_file_package.OpossumPackageIdentifier], + ResourcePathModel, + list[OpossumPackageIdentifierModel], ], - ) -> tuple[list[Resource], set[OpossumPackageIdentifier]]: + ) -> tuple[list[Resource], set[OpossumPackageIdentifierModel]]: used_attribution_ids = set() def generate_child_resource( current_path: PurePath, - to_insert: opossum_file_package.ResourceInFile, + to_insert: ResourceInFileModel, ) -> Resource: path = current_path current_path_as_string = _convert_path_to_str(current_path) @@ -188,13 +198,13 @@ def generate_child_resource( def _get_applicable_attributions( current_path_as_string: str, - ) -> tuple[list[OpossumPackage], set[OpossumPackageIdentifier]]: + ) -> tuple[list[OpossumPackage], set[OpossumPackageIdentifierModel]]: attributions = [] attribution_ids: list[str] = [] if current_path_as_string in resources_to_attributions: attribution_ids = resources_to_attributions[current_path_as_string] attributions = [ - OpossumFileToOpossumConverter._convert_package( + OpossumFileModelToOpossumConverter._convert_package( external_attributions[id] ) for id in attribution_ids @@ -214,13 +224,15 @@ def _get_applicable_attributions( @staticmethod def _convert_to_attribution_with_id( external_attributions: dict[ - opossum_file_package.OpossumPackageIdentifier, - opossum_file_package.OpossumPackageModel, + OpossumPackageIdentifierModel, + OpossumPackageModel, ], ) -> dict[OpossumPackage, str]: result = {} for package_identifier, package in external_attributions.items(): - converted_package = OpossumFileToOpossumConverter._convert_package(package) + converted_package = OpossumFileModelToOpossumConverter._convert_package( + package + ) if converted_package not in result: result[converted_package] = package_identifier else: @@ -232,7 +244,7 @@ def _convert_to_attribution_with_id( @staticmethod def _convert_frequent_license( - infile_frequent_license: opossum_file_package.FrequentLicenseModel, + infile_frequent_license: FrequentLicenseModel, ) -> FrequentLicense: return FrequentLicense( full_name=infile_frequent_license.full_name, @@ -242,10 +254,12 @@ def _convert_frequent_license( @staticmethod def _convert_package( - infile_package: opossum_file_package.OpossumPackageModel, + infile_package: OpossumPackageModel, ) -> OpossumPackage: return OpossumPackage( - source=OpossumFileToOpossumConverter._convert_source(infile_package.source), + source=OpossumFileModelToOpossumConverter._convert_source( + infile_package.source + ), attribution_confidence=infile_package.attribution_confidence, comment=infile_package.comment, package_name=infile_package.package_name, @@ -269,7 +283,7 @@ def _convert_package( @staticmethod def _convert_source( - infile_source_info: opossum_file_package.SourceInfoModel, + infile_source_info: SourceInfoModel, ) -> SourceInfo: return SourceInfo( name=infile_source_info.name, diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py index 98a7c162..cd066720 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py @@ -5,15 +5,15 @@ from opossum_lib.core.entities.opossum import Opossum from opossum_lib.core.services.input_reader import InputFormatReader +from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 + OpossumFileModelToOpossumConverter, +) from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( OpossumFileReader, ) -from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 - OpossumFileToOpossumConverter, -) class OpossumFormatReader(InputFormatReader): def read(self, path: Path) -> Opossum: opossum_input_file = OpossumFileReader.from_file(path=path) - return OpossumFileToOpossumConverter.convert_to_opossum(opossum_input_file) + return OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/shared/entities/opossum_input_file_model.py b/src/opossum_lib/shared/entities/opossum_input_file_model.py index 91091b3b..39398dc1 100644 --- a/src/opossum_lib/shared/entities/opossum_input_file_model.py +++ b/src/opossum_lib/shared/entities/opossum_input_file_model.py @@ -11,9 +11,9 @@ from pydantic import BaseModel, ConfigDict, model_serializer from pydantic.alias_generators import to_camel -type OpossumPackageIdentifier = str -type ResourcePath = str -type ResourceInFile = dict[str, ResourceInFile] | int +type OpossumPackageIdentifierModel = str +type ResourcePathModel = str +type ResourceInFileModel = dict[str, ResourceInFileModel] | int class CamelBaseModel(BaseModel): @@ -24,9 +24,11 @@ class CamelBaseModel(BaseModel): class OpossumInputFileModel(CamelBaseModel): metadata: MetadataModel - resources: ResourceInFile - external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel] - resources_to_attributions: dict[ResourcePath, list[OpossumPackageIdentifier]] + resources: ResourceInFileModel + external_attributions: dict[OpossumPackageIdentifierModel, OpossumPackageModel] + resources_to_attributions: dict[ + ResourcePathModel, list[OpossumPackageIdentifierModel] + ] attribution_breakpoints: list[str] = field(default_factory=list) external_attribution_sources: dict[str, ExternalAttributionSourceModel] = field( default_factory=dict @@ -149,7 +151,7 @@ def drop_element( return resource - def to_dict(self) -> ResourceInFile: + def to_dict(self) -> ResourceInFileModel: if not self.has_children(): if self.type == ResourceTypeModel.FOLDER: return {} @@ -180,7 +182,7 @@ def get_paths_of_all_leaf_nodes_with_types( def has_children(self) -> bool: return len(self.children) > 0 - def convert_to_file_resource(self) -> ResourceInFile: + def convert_to_file_resource(self) -> ResourceInFileModel: return self.to_dict() diff --git a/tests/core/test_opossum_model.py b/tests/core/test_opossum_model.py index 02285658..da5c3f06 100644 --- a/tests/core/test_opossum_model.py +++ b/tests/core/test_opossum_model.py @@ -4,8 +4,8 @@ import json from copy import deepcopy -from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 - OpossumFileToOpossumConverter, +from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 + OpossumFileModelToOpossumConverter, ) from tests.test_setup.opossum_faker_setup import OpossumFaker @@ -14,7 +14,7 @@ class TestOpossumModelToOpossumFileConversion: def test_moves_outfile(self, opossum_faker: OpossumFaker) -> None: opossum = opossum_faker.opossum() - result = opossum.to_opossum_file_format() + result = opossum.to_opossum_model() assert result.output_file == opossum.review_results @@ -22,9 +22,9 @@ def test_roundtrip(self, opossum_faker: OpossumFaker) -> None: opossum = opossum_faker.opossum() expected_result = deepcopy(opossum) - opossum_file = opossum.to_opossum_file_format() + opossum_file = opossum.to_opossum_model() - result = OpossumFileToOpossumConverter.convert_to_opossum(opossum_file) + result = OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_file) ## this can change due to the generation of new ids result_json = result.model_dump_json() @@ -51,8 +51,8 @@ def test_roundtrip_with_resource_ids(self, opossum_faker: OpossumFaker) -> None: ) expected_result = deepcopy(opossum) - opossum_file = opossum.to_opossum_file_format() + opossum_file = opossum.to_opossum_model() - result = OpossumFileToOpossumConverter.convert_to_opossum(opossum_file) + result = OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_file) assert result == expected_result diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/test_opossum/generators/generate_file_information.py index 525108ce..bf63a490 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/test_opossum/generators/generate_file_information.py @@ -19,10 +19,10 @@ FrequentLicenseModel, MetadataModel, OpossumInputFileModel, - OpossumPackageIdentifier, + OpossumPackageIdentifierModel, OpossumPackageModel, - ResourceInFile, - ResourcePath, + ResourceInFileModel, + ResourcePathModel, SourceInfoModel, ) from tests.util.generator_helpers import entry_or_none, random_list @@ -88,10 +88,12 @@ def opossum_file_information( self, *, metadata: MetadataModel | None = None, - resources: ResourceInFile | None = None, - external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel] + resources: ResourceInFileModel | None = None, + external_attributions: dict[OpossumPackageIdentifierModel, OpossumPackageModel] | None = None, - resources_to_attributions: dict[ResourcePath, list[OpossumPackageIdentifier]] + resources_to_attributions: dict[ + ResourcePathModel, list[OpossumPackageIdentifierModel] + ] | None = None, attribution_breakpoints: list[str] | None = None, external_attribution_sources: dict[str, ExternalAttributionSourceModel] @@ -126,7 +128,7 @@ def resource_in_file( depth: int = 3, max_folders_per_level: int = 3, max_files_per_level: int = 3, - ) -> ResourceInFile: + ) -> ResourceInFileModel: if depth == 0: files = self.random_int(0, max_files_per_level) return { @@ -242,7 +244,7 @@ def opossum_package( def external_attributions( self, max_number_of_attributions: int = 50, min_number_of_attributions: int = 5 - ) -> dict[OpossumPackageIdentifier, OpossumPackageModel]: + ) -> dict[OpossumPackageIdentifierModel, OpossumPackageModel]: number_of_attributions = self.random_int( min_number_of_attributions, max_number_of_attributions ) @@ -253,10 +255,12 @@ def external_attributions( def resources_to_attributions( self, - resources: ResourceInFile, - external_attributions: dict[OpossumPackageIdentifier, OpossumPackageModel], - ) -> dict[ResourcePath, list[OpossumPackageIdentifier]]: - def get_file_paths(resource: ResourceInFile, current_path: str) -> list[str]: + resources: ResourceInFileModel, + external_attributions: dict[OpossumPackageIdentifierModel, OpossumPackageModel], + ) -> dict[ResourcePathModel, list[OpossumPackageIdentifierModel]]: + def get_file_paths( + resource: ResourceInFileModel, current_path: str + ) -> list[str]: if isinstance(resource, int): return [] resulting_file_paths = [] diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/test_opossum/test_conversion_roundtrip.py index 41ba4f73..2d0dd8fa 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/test_opossum/test_conversion_roundtrip.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 - OpossumFileToOpossumConverter, +from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 + OpossumFileModelToOpossumConverter, ) from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker @@ -31,7 +31,7 @@ def test_surplus_attributions(self, opossum_file_faker: OpossumFileFaker) -> Non @staticmethod def _check_round_trip(start_file_content: OpossumFileModel) -> None: expected_file_content = deepcopy(start_file_content) - result = OpossumFileToOpossumConverter.convert_to_opossum( + result = OpossumFileModelToOpossumConverter.convert_to_opossum( start_file_content - ).to_opossum_file_format() + ).to_opossum_model() assert result == expected_file_content diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/test_opossum/test_opossum_file_to_opossum_converter.py index 4cd1da22..40d8c62e 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/test_opossum/test_opossum_file_to_opossum_converter.py @@ -3,11 +3,11 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.input_formats.opossum.services.opossum_file_to_opossum_converter import ( # noqa: E501 - OpossumFileToOpossumConverter, +from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 + OpossumFileModelToOpossumConverter, ) from opossum_lib.shared.entities.opossum_input_file_model import ( - OpossumPackageIdentifier, + OpossumPackageIdentifierModel, OpossumPackageModel, ) from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker @@ -18,7 +18,7 @@ def test_output_file_moved(self, opossum_file_faker: OpossumFileFaker) -> None: output_file = opossum_file_faker.output_file() input_file = opossum_file_faker.opossum_file_content(out_file=output_file) - result = OpossumFileToOpossumConverter.convert_to_opossum(input_file) + result = OpossumFileModelToOpossumConverter.convert_to_opossum(input_file) assert result.review_results == output_file @@ -34,12 +34,12 @@ def test_throws_on_duplicate_attributions( input_file = opossum_file_faker.opossum_file_content(in_file=file_information) with pytest.raises(RuntimeError, match=r".*attribution was duplicated.*"): - OpossumFileToOpossumConverter.convert_to_opossum(input_file) + OpossumFileModelToOpossumConverter.convert_to_opossum(input_file) @staticmethod def _fake_duplicate_external_attributions( opossum_file_faker: OpossumFileFaker, - ) -> dict[OpossumPackageIdentifier, OpossumPackageModel]: + ) -> dict[OpossumPackageIdentifierModel, OpossumPackageModel]: external_attributions = opossum_file_faker.external_attributions( min_number_of_attributions=2 ) diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/test_scancode/test_get_attribution_info.py index d1dd4413..1836a027 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/test_scancode/test_get_attribution_info.py @@ -69,9 +69,7 @@ def test_get_attribution_info_file_multiple(scancode_faker: ScanCodeFaker) -> No ) scancode_data = scancode_faker.scancode_data(files=[file]) opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) - attributions = ( - opossum.to_opossum_file_format().input_file.external_attributions.values() - ) + attributions = opossum.to_opossum_model().input_file.external_attributions.values() expected1 = OpossumPackageModel( source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), From c4fd6994ccc860d5ff13b34e3bb786eb2200f656 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Wed, 29 Jan 2025 16:47:48 +0100 Subject: [PATCH 41/58] refactor: review-comment: Make tests adhere to the new structure --- tests/conftest.py | 6 +- .../entities}/__init__.py | 0 .../entities/generators}/__init__.py | 0 .../external_attribution_source_provider.py | 2 +- .../entities/generators}/metadata_provider.py | 2 +- .../entities/generators}/opossum_provider.py | 4 +- .../entities/generators}/package_provider.py | 4 +- .../entities/generators}/resource_provider.py | 4 +- .../generators}/scan_results_provider.py | 10 +- .../generators}/source_info_provider.py | 2 +- .../test_opossum.py} | 4 +- .../generators => core/services}/__init__.py | 0 .../services/test_opossum_file_writer.py} | 2 +- .../__init__.py | 0 .../opossum}/__init__.py | 0 .../opossum/entities}/__init__.py | 0 .../opossum/entities/generators}/__init__.py | 0 .../generators/generate_file_information.py | 2 +- .../generate_opossum_file_content.py | 4 +- .../generate_outfile_information.py | 2 +- .../opossum/entities/services/__init__.py | 0 .../services}/test_conversion_roundtrip.py | 2 +- ...possum_file_model_to_opossum_converter.py} | 4 +- .../services/test_opossum_format_reader.py} | 6 +- tests/input_formats/scancode/__init__.py | 0 .../scancode/entities/__init__.py | 0 .../scancode/entities/generators/__init__.py | 0 .../generators/generate_scancode_file.py | 2 +- .../scancode/entities/services/__init__.py | 0 .../services}/test_get_attribution_info.py | 2 +- ...est_scancode_data_to_opossum_converter.py} | 2 +- tests/setup/__init__.py | 0 .../opossum_faker_setup.py | 4 +- .../opossum_file_faker_setup.py | 6 +- .../scancode_faker_setup.py | 4 +- tests/shared/__init__.py | 0 tests/{util => shared}/generator_helpers.py | 0 tests/test_cli.py | 2 +- tests/test_scancode/model_helpers.py | 128 ------------------ 39 files changed, 42 insertions(+), 168 deletions(-) rename tests/{opossum_model_generators => core/entities}/__init__.py (100%) rename tests/{test_opossum => core/entities/generators}/__init__.py (100%) rename tests/{opossum_model_generators => core/entities/generators}/external_attribution_source_provider.py (96%) rename tests/{opossum_model_generators => core/entities/generators}/metadata_provider.py (97%) rename tests/{opossum_model_generators => core/entities/generators}/opossum_provider.py (86%) rename tests/{opossum_model_generators => core/entities/generators}/package_provider.py (97%) rename tests/{opossum_model_generators => core/entities/generators}/resource_provider.py (96%) rename tests/{opossum_model_generators => core/entities/generators}/scan_results_provider.py (94%) rename tests/{opossum_model_generators => core/entities/generators}/source_info_provider.py (95%) rename tests/core/{test_opossum_model.py => entities/test_opossum.py} (95%) rename tests/{test_opossum/generators => core/services}/__init__.py (100%) rename tests/{test_opossum/test_file_generation.py => core/services/test_opossum_file_writer.py} (95%) rename tests/{test_scancode => input_formats}/__init__.py (100%) rename tests/{test_scancode/generators => input_formats/opossum}/__init__.py (100%) rename tests/{test_setup => input_formats/opossum/entities}/__init__.py (100%) rename tests/{util => input_formats/opossum/entities/generators}/__init__.py (100%) rename tests/{test_opossum => input_formats/opossum/entities}/generators/generate_file_information.py (99%) rename tests/{test_opossum => input_formats/opossum/entities}/generators/generate_opossum_file_content.py (86%) rename tests/{test_opossum => input_formats/opossum/entities}/generators/generate_outfile_information.py (97%) create mode 100644 tests/input_formats/opossum/entities/services/__init__.py rename tests/{test_opossum => input_formats/opossum/entities/services}/test_conversion_roundtrip.py (95%) rename tests/{test_opossum/test_opossum_file_to_opossum_converter.py => input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py} (94%) rename tests/{test_opossum/test_read_opossum_file.py => input_formats/opossum/entities/services/test_opossum_format_reader.py} (81%) create mode 100644 tests/input_formats/scancode/__init__.py create mode 100644 tests/input_formats/scancode/entities/__init__.py create mode 100644 tests/input_formats/scancode/entities/generators/__init__.py rename tests/{test_scancode => input_formats/scancode/entities}/generators/generate_scancode_file.py (99%) create mode 100644 tests/input_formats/scancode/entities/services/__init__.py rename tests/{test_scancode => input_formats/scancode/entities/services}/test_get_attribution_info.py (98%) rename tests/{test_scancode/test_convert_scancode_to_opossum.py => input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py} (97%) create mode 100644 tests/setup/__init__.py rename tests/{test_setup => setup}/opossum_faker_setup.py (91%) rename tests/{test_setup => setup}/opossum_file_faker_setup.py (90%) rename tests/{test_setup => setup}/scancode_faker_setup.py (94%) create mode 100644 tests/shared/__init__.py rename tests/{util => shared}/generator_helpers.py (100%) delete mode 100644 tests/test_scancode/model_helpers.py diff --git a/tests/conftest.py b/tests/conftest.py index 83850a84..2e2ad4c3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,12 +5,12 @@ import pytest from faker.proxy import Faker -from tests.test_setup.opossum_faker_setup import OpossumFaker, setup_opossum_faker -from tests.test_setup.opossum_file_faker_setup import ( +from tests.setup.opossum_faker_setup import OpossumFaker, setup_opossum_faker +from tests.setup.opossum_file_faker_setup import ( OpossumFileFaker, setup_opossum_file_faker, ) -from tests.test_setup.scancode_faker_setup import ScanCodeFaker, setup_scancode_faker +from tests.setup.scancode_faker_setup import ScanCodeFaker, setup_scancode_faker @pytest.fixture diff --git a/tests/opossum_model_generators/__init__.py b/tests/core/entities/__init__.py similarity index 100% rename from tests/opossum_model_generators/__init__.py rename to tests/core/entities/__init__.py diff --git a/tests/test_opossum/__init__.py b/tests/core/entities/generators/__init__.py similarity index 100% rename from tests/test_opossum/__init__.py rename to tests/core/entities/generators/__init__.py diff --git a/tests/opossum_model_generators/external_attribution_source_provider.py b/tests/core/entities/generators/external_attribution_source_provider.py similarity index 96% rename from tests/opossum_model_generators/external_attribution_source_provider.py rename to tests/core/entities/generators/external_attribution_source_provider.py index 3d24ac47..b52e3984 100644 --- a/tests/opossum_model_generators/external_attribution_source_provider.py +++ b/tests/core/entities/generators/external_attribution_source_provider.py @@ -8,7 +8,7 @@ from faker.providers.person import Provider as PersonProvider from opossum_lib.core.entities.opossum import ExternalAttributionSource -from tests.util.generator_helpers import entry_or_none +from tests.shared.generator_helpers import entry_or_none class ExternalAttributionSourceProvider(BaseProvider): diff --git a/tests/opossum_model_generators/metadata_provider.py b/tests/core/entities/generators/metadata_provider.py similarity index 97% rename from tests/opossum_model_generators/metadata_provider.py rename to tests/core/entities/generators/metadata_provider.py index 44d30625..77b54dbb 100644 --- a/tests/opossum_model_generators/metadata_provider.py +++ b/tests/core/entities/generators/metadata_provider.py @@ -9,7 +9,7 @@ from faker.providers.misc import Provider as MiscProvider from opossum_lib.core.entities.opossum import Metadata -from tests.util.generator_helpers import entry_or_none +from tests.shared.generator_helpers import entry_or_none class MetadataProvider(BaseProvider): diff --git a/tests/opossum_model_generators/opossum_provider.py b/tests/core/entities/generators/opossum_provider.py similarity index 86% rename from tests/opossum_model_generators/opossum_provider.py rename to tests/core/entities/generators/opossum_provider.py index a37075ca..e6d527aa 100644 --- a/tests/opossum_model_generators/opossum_provider.py +++ b/tests/core/entities/generators/opossum_provider.py @@ -7,8 +7,8 @@ from opossum_lib.core.entities.opossum import Opossum, ScanResults from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel -from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider -from tests.test_opossum.generators.generate_outfile_information import ( +from tests.core.entities.generators.scan_results_provider import ScanResultsProvider +from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa: E501 OpossumOutputFileProvider, ) diff --git a/tests/opossum_model_generators/package_provider.py b/tests/core/entities/generators/package_provider.py similarity index 97% rename from tests/opossum_model_generators/package_provider.py rename to tests/core/entities/generators/package_provider.py index b644d7ff..3ca9b9b7 100644 --- a/tests/opossum_model_generators/package_provider.py +++ b/tests/core/entities/generators/package_provider.py @@ -11,8 +11,8 @@ from faker.providers.person.en_US import Provider as PersonProvider from opossum_lib.core.entities.opossum import OpossumPackage, SourceInfo -from tests.opossum_model_generators.source_info_provider import SourceInfoProvider -from tests.util.generator_helpers import entry_or_none, random_list +from tests.core.entities.generators.source_info_provider import SourceInfoProvider +from tests.shared.generator_helpers import entry_or_none, random_list class PackageProvider(BaseProvider): diff --git a/tests/opossum_model_generators/resource_provider.py b/tests/core/entities/generators/resource_provider.py similarity index 96% rename from tests/opossum_model_generators/resource_provider.py rename to tests/core/entities/generators/resource_provider.py index 22dd31cb..b9d6069b 100644 --- a/tests/opossum_model_generators/resource_provider.py +++ b/tests/core/entities/generators/resource_provider.py @@ -13,8 +13,8 @@ Resource, ResourceType, ) -from tests.opossum_model_generators.package_provider import PackageProvider -from tests.util.generator_helpers import random_list +from tests.core.entities.generators.package_provider import PackageProvider +from tests.shared.generator_helpers import random_list class ResourceProvider(BaseProvider): diff --git a/tests/opossum_model_generators/scan_results_provider.py b/tests/core/entities/generators/scan_results_provider.py similarity index 94% rename from tests/opossum_model_generators/scan_results_provider.py rename to tests/core/entities/generators/scan_results_provider.py index c8473f19..96f26a49 100644 --- a/tests/opossum_model_generators/scan_results_provider.py +++ b/tests/core/entities/generators/scan_results_provider.py @@ -18,13 +18,13 @@ ResourceType, ScanResults, ) -from tests.opossum_model_generators.external_attribution_source_provider import ( +from tests.core.entities.generators.external_attribution_source_provider import ( ExternalAttributionSourceProvider, ) -from tests.opossum_model_generators.metadata_provider import MetadataProvider -from tests.opossum_model_generators.package_provider import PackageProvider -from tests.opossum_model_generators.resource_provider import ResourceProvider -from tests.util.generator_helpers import entry_or_none, random_list +from tests.core.entities.generators.metadata_provider import MetadataProvider +from tests.core.entities.generators.package_provider import PackageProvider +from tests.core.entities.generators.resource_provider import ResourceProvider +from tests.shared.generator_helpers import entry_or_none, random_list class ScanResultsProvider(BaseProvider): diff --git a/tests/opossum_model_generators/source_info_provider.py b/tests/core/entities/generators/source_info_provider.py similarity index 95% rename from tests/opossum_model_generators/source_info_provider.py rename to tests/core/entities/generators/source_info_provider.py index 90c07801..226107c3 100644 --- a/tests/opossum_model_generators/source_info_provider.py +++ b/tests/core/entities/generators/source_info_provider.py @@ -8,7 +8,7 @@ from faker.providers.person.en_US import Provider as PersonProvider from opossum_lib.core.entities.opossum import SourceInfo -from tests.util.generator_helpers import entry_or_none +from tests.shared.generator_helpers import entry_or_none class SourceInfoProvider(BaseProvider): diff --git a/tests/core/test_opossum_model.py b/tests/core/entities/test_opossum.py similarity index 95% rename from tests/core/test_opossum_model.py rename to tests/core/entities/test_opossum.py index da5c3f06..6fd8b30d 100644 --- a/tests/core/test_opossum_model.py +++ b/tests/core/entities/test_opossum.py @@ -7,10 +7,10 @@ from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 OpossumFileModelToOpossumConverter, ) -from tests.test_setup.opossum_faker_setup import OpossumFaker +from tests.setup.opossum_faker_setup import OpossumFaker -class TestOpossumModelToOpossumFileConversion: +class TestOpossumToOpossumModelConversion: def test_moves_outfile(self, opossum_faker: OpossumFaker) -> None: opossum = opossum_faker.opossum() diff --git a/tests/test_opossum/generators/__init__.py b/tests/core/services/__init__.py similarity index 100% rename from tests/test_opossum/generators/__init__.py rename to tests/core/services/__init__.py diff --git a/tests/test_opossum/test_file_generation.py b/tests/core/services/test_opossum_file_writer.py similarity index 95% rename from tests/test_opossum/test_file_generation.py rename to tests/core/services/test_opossum_file_writer.py index 2483752f..44ba2125 100644 --- a/tests/test_opossum/test_file_generation.py +++ b/tests/core/services/test_opossum_file_writer.py @@ -11,7 +11,7 @@ OUTPUT_JSON_NAME, ) from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker +from tests.setup.opossum_file_faker_setup import OpossumFileFaker def test_only_input_information_available_writes_only_input_information( diff --git a/tests/test_scancode/__init__.py b/tests/input_formats/__init__.py similarity index 100% rename from tests/test_scancode/__init__.py rename to tests/input_formats/__init__.py diff --git a/tests/test_scancode/generators/__init__.py b/tests/input_formats/opossum/__init__.py similarity index 100% rename from tests/test_scancode/generators/__init__.py rename to tests/input_formats/opossum/__init__.py diff --git a/tests/test_setup/__init__.py b/tests/input_formats/opossum/entities/__init__.py similarity index 100% rename from tests/test_setup/__init__.py rename to tests/input_formats/opossum/entities/__init__.py diff --git a/tests/util/__init__.py b/tests/input_formats/opossum/entities/generators/__init__.py similarity index 100% rename from tests/util/__init__.py rename to tests/input_formats/opossum/entities/generators/__init__.py diff --git a/tests/test_opossum/generators/generate_file_information.py b/tests/input_formats/opossum/entities/generators/generate_file_information.py similarity index 99% rename from tests/test_opossum/generators/generate_file_information.py rename to tests/input_formats/opossum/entities/generators/generate_file_information.py index bf63a490..05418225 100644 --- a/tests/test_opossum/generators/generate_file_information.py +++ b/tests/input_formats/opossum/entities/generators/generate_file_information.py @@ -25,7 +25,7 @@ ResourcePathModel, SourceInfoModel, ) -from tests.util.generator_helpers import entry_or_none, random_list +from tests.shared.generator_helpers import entry_or_none, random_list class MetadataProvider(BaseProvider): diff --git a/tests/test_opossum/generators/generate_opossum_file_content.py b/tests/input_formats/opossum/entities/generators/generate_opossum_file_content.py similarity index 86% rename from tests/test_opossum/generators/generate_opossum_file_content.py rename to tests/input_formats/opossum/entities/generators/generate_opossum_file_content.py index ee2eeec6..3d6a77e1 100644 --- a/tests/test_opossum/generators/generate_opossum_file_content.py +++ b/tests/input_formats/opossum/entities/generators/generate_opossum_file_content.py @@ -8,10 +8,10 @@ from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel -from tests.test_opossum.generators.generate_file_information import ( +from tests.input_formats.opossum.entities.generators.generate_file_information import ( FileInformationProvider, ) -from tests.test_opossum.generators.generate_outfile_information import ( +from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa: E501 OpossumOutputFileProvider, ) diff --git a/tests/test_opossum/generators/generate_outfile_information.py b/tests/input_formats/opossum/entities/generators/generate_outfile_information.py similarity index 97% rename from tests/test_opossum/generators/generate_outfile_information.py rename to tests/input_formats/opossum/entities/generators/generate_outfile_information.py index f9a3260a..3a56e008 100644 --- a/tests/test_opossum/generators/generate_outfile_information.py +++ b/tests/input_formats/opossum/entities/generators/generate_outfile_information.py @@ -14,7 +14,7 @@ Metadata, OpossumOutputFileModel, ) -from tests.util.generator_helpers import entry_or_none +from tests.shared.generator_helpers import entry_or_none class OpossumOutputFileProvider(BaseProvider): diff --git a/tests/input_formats/opossum/entities/services/__init__.py b/tests/input_formats/opossum/entities/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_opossum/test_conversion_roundtrip.py b/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py similarity index 95% rename from tests/test_opossum/test_conversion_roundtrip.py rename to tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py index 2d0dd8fa..e85950cf 100644 --- a/tests/test_opossum/test_conversion_roundtrip.py +++ b/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py @@ -7,7 +7,7 @@ OpossumFileModelToOpossumConverter, ) from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker +from tests.setup.opossum_file_faker_setup import OpossumFileFaker class TestConversionRoundtrip: diff --git a/tests/test_opossum/test_opossum_file_to_opossum_converter.py b/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py similarity index 94% rename from tests/test_opossum/test_opossum_file_to_opossum_converter.py rename to tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py index 40d8c62e..9fba7481 100644 --- a/tests/test_opossum/test_opossum_file_to_opossum_converter.py +++ b/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py @@ -10,10 +10,10 @@ OpossumPackageIdentifierModel, OpossumPackageModel, ) -from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker +from tests.setup.opossum_file_faker_setup import OpossumFileFaker -class TestOpossumFileToOpossumConverter: +class TestOpossumFileModelToOpossumConverter: def test_output_file_moved(self, opossum_file_faker: OpossumFileFaker) -> None: output_file = opossum_file_faker.output_file() input_file = opossum_file_faker.opossum_file_content(out_file=output_file) diff --git a/tests/test_opossum/test_read_opossum_file.py b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py similarity index 81% rename from tests/test_opossum/test_read_opossum_file.py rename to tests/input_formats/opossum/entities/services/test_opossum_format_reader.py index 5632502b..4aba605d 100644 --- a/tests/test_opossum/test_read_opossum_file.py +++ b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py @@ -10,11 +10,11 @@ OpossumFormatReader, ) -TEST_DATA_DIRECTORY = Path(__file__).resolve().parent.parent / "data" +TEST_DATA_DIR = Path(__file__).resolve().parent.parent.parent.parent.parent / "data" def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: - input_path = TEST_DATA_DIRECTORY / "opossum_input_corrupt.opossum" + input_path = TEST_DATA_DIR / "opossum_input_corrupt.opossum" opossum_format_reader = OpossumFormatReader() with pytest.raises(SystemExit) as system_exit: @@ -24,7 +24,7 @@ def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> def test_read_opossum_file_containing_output_json() -> None: - input_path = TEST_DATA_DIRECTORY / "opossum_input_with_result.opossum" + input_path = TEST_DATA_DIR / "opossum_input_with_result.opossum" opossum_format_reader = OpossumFormatReader() result = opossum_format_reader.read(input_path) diff --git a/tests/input_formats/scancode/__init__.py b/tests/input_formats/scancode/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/input_formats/scancode/entities/__init__.py b/tests/input_formats/scancode/entities/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/input_formats/scancode/entities/generators/__init__.py b/tests/input_formats/scancode/entities/generators/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scancode/generators/generate_scancode_file.py b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py similarity index 99% rename from tests/test_scancode/generators/generate_scancode_file.py rename to tests/input_formats/scancode/entities/generators/generate_scancode_file.py index 22b064f0..5e283b70 100644 --- a/tests/test_scancode/generators/generate_scancode_file.py +++ b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py @@ -37,7 +37,7 @@ SystemEnvironmentModel, UrlModel, ) -from tests.util.generator_helpers import entry_or_none, random_list +from tests.shared.generator_helpers import entry_or_none, random_list type TempPathTree = dict[str, TempPathTree | None] diff --git a/tests/input_formats/scancode/entities/services/__init__.py b/tests/input_formats/scancode/entities/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_scancode/test_get_attribution_info.py b/tests/input_formats/scancode/entities/services/test_get_attribution_info.py similarity index 98% rename from tests/test_scancode/test_get_attribution_info.py rename to tests/input_formats/scancode/entities/services/test_get_attribution_info.py index 1836a027..bf6d0c41 100644 --- a/tests/test_scancode/test_get_attribution_info.py +++ b/tests/input_formats/scancode/entities/services/test_get_attribution_info.py @@ -11,7 +11,7 @@ OpossumPackageModel, SourceInfoModel, ) -from tests.test_setup.scancode_faker_setup import ScanCodeFaker +from tests.setup.scancode_faker_setup import ScanCodeFaker def test_get_attribution_info_directory(scancode_faker: ScanCodeFaker) -> None: diff --git a/tests/test_scancode/test_convert_scancode_to_opossum.py b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py similarity index 97% rename from tests/test_scancode/test_convert_scancode_to_opossum.py rename to tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py index 1606fa4b..787836e8 100644 --- a/tests/test_scancode/test_convert_scancode_to_opossum.py +++ b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py @@ -10,7 +10,7 @@ from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 ScancodeDataToOpossumConverter, ) -from tests.test_setup.scancode_faker_setup import ScanCodeFaker +from tests.setup.scancode_faker_setup import ScanCodeFaker def test_extract_scancode_header_produces_expected_result( diff --git a/tests/setup/__init__.py b/tests/setup/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_setup/opossum_faker_setup.py b/tests/setup/opossum_faker_setup.py similarity index 91% rename from tests/test_setup/opossum_faker_setup.py rename to tests/setup/opossum_faker_setup.py index 72ae37c7..3dbabf04 100644 --- a/tests/test_setup/opossum_faker_setup.py +++ b/tests/setup/opossum_faker_setup.py @@ -7,8 +7,8 @@ from faker import Faker, Generator -from tests.opossum_model_generators.opossum_provider import OpossumProvider -from tests.opossum_model_generators.scan_results_provider import ScanResultsProvider +from tests.core.entities.generators.opossum_provider import OpossumProvider +from tests.core.entities.generators.scan_results_provider import ScanResultsProvider class OpossumFaker(Faker): diff --git a/tests/test_setup/opossum_file_faker_setup.py b/tests/setup/opossum_file_faker_setup.py similarity index 90% rename from tests/test_setup/opossum_file_faker_setup.py rename to tests/setup/opossum_file_faker_setup.py index f1a1ea6d..b471c2d9 100644 --- a/tests/test_setup/opossum_file_faker_setup.py +++ b/tests/setup/opossum_file_faker_setup.py @@ -8,14 +8,14 @@ from faker import Faker, Generator -from tests.test_opossum.generators.generate_file_information import ( +from tests.input_formats.opossum.entities.generators.generate_file_information import ( FileInformationProvider, MetadataProvider, ) -from tests.test_opossum.generators.generate_opossum_file_content import ( +from tests.input_formats.opossum.entities.generators.generate_opossum_file_content import ( # noqa: E501 OpossumFileContentProvider, ) -from tests.test_opossum.generators.generate_outfile_information import ( +from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa: E501 OpossumOutputFileProvider, ) diff --git a/tests/test_setup/scancode_faker_setup.py b/tests/setup/scancode_faker_setup.py similarity index 94% rename from tests/test_setup/scancode_faker_setup.py rename to tests/setup/scancode_faker_setup.py index 80fc39d6..07da5f92 100644 --- a/tests/test_setup/scancode_faker_setup.py +++ b/tests/setup/scancode_faker_setup.py @@ -8,7 +8,9 @@ from faker import Faker, Generator -from tests.test_scancode.generators.generate_scancode_file import ScanCodeDataProvider +from tests.input_formats.scancode.entities.generators.generate_scancode_file import ( + ScanCodeDataProvider, +) class ScanCodeFaker(Faker): diff --git a/tests/shared/__init__.py b/tests/shared/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/util/generator_helpers.py b/tests/shared/generator_helpers.py similarity index 100% rename from tests/util/generator_helpers.py rename to tests/shared/generator_helpers.py diff --git a/tests/test_cli.py b/tests/test_cli.py index ebc9b9bb..199e61de 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -17,7 +17,7 @@ OUTPUT_JSON_NAME, ) from opossum_lib.shared.entities.opossum_input_file_model import OpossumPackageModel -from tests.test_setup.opossum_file_faker_setup import OpossumFileFaker +from tests.setup.opossum_file_faker_setup import OpossumFileFaker test_data_path = Path(__file__).resolve().parent / "data" diff --git a/tests/test_scancode/model_helpers.py b/tests/test_scancode/model_helpers.py deleted file mode 100644 index 347b3fca..00000000 --- a/tests/test_scancode/model_helpers.py +++ /dev/null @@ -1,128 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -from pathlib import PurePath - -from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( - CopyrightModel, - FileBasedLicenseDetectionModel, - FileModel, - FileTypeModel, - HolderModel, - UrlModel, -) - - -def _create_reference_scancode_files() -> list[FileModel]: - return [ - _create_file("A", FileTypeModel.DIRECTORY), - _create_file("A/B", FileTypeModel.DIRECTORY), - _create_file("A/file1", FileTypeModel.FILE), - _create_file("A/file2.txt", FileTypeModel.FILE), - _create_file("A/B/file3", FileTypeModel.FILE), - ] - - -def _create_file( - path: str, - type: FileTypeModel, - *, - name: str | None = None, - base_name: str | None = None, - extension: str | None = None, - size: int = 0, - date: str | None = None, - sha1: str | None = None, - md5: str | None = None, - sha256: str | None = None, - mime_type: str | None = None, - file_type: str | None = None, - programming_language: str | None = None, - is_binary: bool = False, - is_text: bool = False, - is_archive: bool = False, - is_media: bool = False, - is_source: bool = False, - is_script: bool = False, - package_data: list | None = None, - for_packages: list | None = None, - detected_license_expression: str | None = None, - detected_license_expression_spdx: str | None = None, - license_detections: list[FileBasedLicenseDetectionModel] | None = None, - license_clues: list | None = None, - percentage_of_license_text: float = 0.0, - copyrights: list[CopyrightModel] | None = None, - holders: list[HolderModel] | None = None, - authors: list | None = None, - emails: list | None = None, - urls: list[UrlModel] | None = None, - files_count: int = 0, - dirs_count: int = 0, - size_count: int = 0, - scan_errors: list | None = None, -) -> FileModel: - if package_data is None: - package_data = [] - if for_packages is None: - for_packages = [] - if license_clues is None: - license_clues = [] - if license_detections is None: - license_detections = [] - if scan_errors is None: - scan_errors = [] - if urls is None: - urls = [] - if emails is None: - emails = [] - if authors is None: - authors = [] - if holders is None: - holders = [] - if copyrights is None: - copyrights = [] - if name is None: - name = PurePath(path).name - if base_name is None: - base_name = PurePath(PurePath(path).name).stem - if extension is None: - extension = PurePath(path).suffix - return FileModel( - authors=authors, - base_name=base_name, - copyrights=copyrights, - date=date, - detected_license_expression=detected_license_expression, - detected_license_expression_spdx=detected_license_expression_spdx, - dirs_count=dirs_count, - emails=emails, - extension=extension, - files_count=files_count, - file_type=file_type, - for_packages=for_packages, - holders=holders, - is_archive=is_archive, - is_binary=is_binary, - is_media=is_media, - is_script=is_script, - is_source=is_source, - is_text=is_text, - license_clues=license_clues, - license_detections=license_detections, - md5=md5, - mime_type=mime_type, - name=name, - package_data=package_data, - path=path, - percentage_of_license_text=percentage_of_license_text, - programming_language=programming_language, - scan_errors=scan_errors, - sha1=sha1, - sha256=sha256, - size=size, - size_count=size_count, - type=type, - urls=urls, - ) From ea536f113e523b929f4b5b259ba63208545c34e2 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 13:13:45 +0100 Subject: [PATCH 42/58] refactor: review -- inline validation logic --- src/opossum_lib/cli.py | 34 +++++------ src/opossum_lib/core/entities/input_file.py | 17 ------ src/opossum_lib/core/services/input_reader.py | 18 +----- .../core/services/opossum_generator.py | 60 +++---------------- .../opossum/services/opossum_format_reader.py | 13 ++-- .../services/scancode_format_reader.py | 15 +++-- .../services/test_opossum_format_reader.py | 8 +-- 7 files changed, 49 insertions(+), 116 deletions(-) delete mode 100644 src/opossum_lib/core/entities/input_file.py diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 50c5c35e..37a88220 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import logging +import sys from pathlib import Path # SPDX-FileCopyrightText: TNG Technology Consulting GmbH @@ -6,10 +8,9 @@ # SPDX-License-Identifier: Apache-2.0 import click -from opossum_lib.core.entities.input_file import InputFileType +from opossum_lib.core.services.input_reader import InputReader from opossum_lib.core.services.opossum_generator import ( - OpossumGenerationArguments, - OpossumGenerator, + generate_impl, ) from opossum_lib.input_formats.opossum.services.opossum_format_reader import ( OpossumFormatReader, @@ -63,20 +64,19 @@ def generate( - ScanCode - Opossum """ - ( - OpossumGenerator( - input_format_readers={ - InputFileType.OPOSSUM: OpossumFormatReader(), - InputFileType.SCAN_CODE: ScancodeFormatReader(), - } - ).generate( - opossum_generation_arguments=OpossumGenerationArguments( - opossum_files=opossum_files, - scancode_json_files=scancode_json_files, - outfile=outfile, - ) - ) - ) + + total_number_of_files = len(scancode_json_files) + len(opossum_files) + if total_number_of_files == 0: + logging.warning("No input provided. Exiting.") + sys.exit(1) + if total_number_of_files > 1: + logging.error("Merging of multiple files not yet supported!") + sys.exit(1) + input_readers: list[InputReader] = [] + input_readers += [ScancodeFormatReader(path=path) for path in scancode_json_files] + input_readers += [OpossumFormatReader(path=path) for path in opossum_files] + + generate_impl(input_readers=input_readers, output_file=Path(outfile)) if __name__ == "__main__": diff --git a/src/opossum_lib/core/entities/input_file.py b/src/opossum_lib/core/entities/input_file.py deleted file mode 100644 index 89ff4d60..00000000 --- a/src/opossum_lib/core/entities/input_file.py +++ /dev/null @@ -1,17 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from enum import IntEnum, auto -from pathlib import Path - -from pydantic import BaseModel - - -class InputFileType(IntEnum): - OPOSSUM = (auto(),) - SCAN_CODE = (auto(),) - - -class InputFile(BaseModel): - path: Path - type: InputFileType diff --git a/src/opossum_lib/core/services/input_reader.py b/src/opossum_lib/core/services/input_reader.py index e431a6d4..6ec0f465 100644 --- a/src/opossum_lib/core/services/input_reader.py +++ b/src/opossum_lib/core/services/input_reader.py @@ -3,24 +3,10 @@ # SPDX-License-Identifier: Apache-2.0 from abc import abstractmethod from asyncio import Protocol -from pathlib import Path -from opossum_lib.core.entities.input_file import InputFile, InputFileType from opossum_lib.core.entities.opossum import Opossum -class InputFormatReader(Protocol): +class InputReader(Protocol): @abstractmethod - def read(self, path: Path) -> Opossum: ... - - -class InputReader: - input_format_readers: dict[InputFileType, InputFormatReader] - - def __init__(self, input_format_readers: dict[InputFileType, InputFormatReader]): - self.input_format_readers = input_format_readers - - def read(self, input_file: InputFile) -> Opossum: - if input_file.type in self.input_format_readers: - return self.input_format_readers[input_file.type].read(input_file.path) - raise NotImplementedError(f"Unsupported file type: {input_file.type}") + def read(self) -> Opossum: ... diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index 0c7f27ee..01504baf 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -1,62 +1,16 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 -import logging -import sys from pathlib import Path -from pydantic import BaseModel - -from opossum_lib.core.entities.input_file import InputFile, InputFileType -from opossum_lib.core.services.input_reader import InputFormatReader, InputReader +from opossum_lib.core.services.input_reader import InputReader from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter -class OpossumGenerationArguments(BaseModel): - scancode_json_files: list[Path] - opossum_files: list[Path] - outfile: Path - - @property - def input_files(self) -> list[InputFile]: - self._validate_and_exit_on_error() - result = [] - result += [ - InputFile(path=path, type=InputFileType.SCAN_CODE) - for path in self.scancode_json_files - ] - result += [ - InputFile(path=path, type=InputFileType.OPOSSUM) - for path in self.opossum_files - ] - return result - - def _validate_and_exit_on_error(self) -> None: - total_number_of_files = len(self.scancode_json_files) + len(self.opossum_files) - if total_number_of_files == 0: - logging.warning("No input provided. Exiting.") - sys.exit(1) - if total_number_of_files > 1: - logging.error("Merging of multiple files not yet supported!") - sys.exit(1) - - -class OpossumGenerator: - input_reader: InputReader - - def __init__(self, input_format_readers: dict[InputFileType, InputFormatReader]): - self.input_reader = InputReader(input_format_readers) - - def generate( - self, opossum_generation_arguments: OpossumGenerationArguments - ) -> None: - input_files = opossum_generation_arguments.input_files - - # currently this converts only one file (validated in the arguments) - # for the future a merge step is planned after reading the files - opossum = self.input_reader.read(input_files[0]) +def generate_impl(input_readers: list[InputReader], output_file: Path) -> None: + # currently this converts only one file (validated in the arguments) + # for the future a merge step is planned after reading the files + opossum = input_readers[0].read() - opossum_file_content = opossum.to_opossum_model() - OpossumFileWriter.write( - opossum_file_content, opossum_generation_arguments.outfile - ) + opossum_file_content = opossum.to_opossum_model() + OpossumFileWriter.write(opossum_file_content, output_file) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py index cd066720..3755eca6 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py @@ -4,7 +4,7 @@ from pathlib import Path from opossum_lib.core.entities.opossum import Opossum -from opossum_lib.core.services.input_reader import InputFormatReader +from opossum_lib.core.services.input_reader import InputReader from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 OpossumFileModelToOpossumConverter, ) @@ -13,7 +13,12 @@ ) -class OpossumFormatReader(InputFormatReader): - def read(self, path: Path) -> Opossum: - opossum_input_file = OpossumFileReader.from_file(path=path) +class OpossumFormatReader(InputReader): + path: Path + + def __init__(self, path: Path): + self.path = path + + def read(self) -> Opossum: + opossum_input_file = OpossumFileReader.from_file(path=self.path) return OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index 3a55db50..555ad947 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -10,7 +10,7 @@ from opossum_lib.core.entities.opossum import ( Opossum, ) -from opossum_lib.core.services.input_reader import InputFormatReader +from opossum_lib.core.services.input_reader import InputReader from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( ScanCodeDataModel, ) @@ -19,11 +19,16 @@ ) -class ScancodeFormatReader(InputFormatReader): - def read(self, path: Path) -> Opossum: - logging.info(f"Converting scancode to opossum {path}") +class ScancodeFormatReader(InputReader): + path: Path - scancode_data = ScancodeFormatReader.load_scancode_json(path) + def __init__(self, path: Path): + self.path = path + + def read(self) -> Opossum: + logging.info(f"Converting scancode to opossum {self.path}") + + scancode_data = ScancodeFormatReader.load_scancode_json(self.path) return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) diff --git a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py index 4aba605d..05c2a4a8 100644 --- a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py +++ b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py @@ -15,19 +15,19 @@ def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: input_path = TEST_DATA_DIR / "opossum_input_corrupt.opossum" - opossum_format_reader = OpossumFormatReader() + opossum_format_reader = OpossumFormatReader(input_path) with pytest.raises(SystemExit) as system_exit: - opossum_format_reader.read(input_path) + opossum_format_reader.read() assert system_exit.value.code == 1 assert "is corrupt and does not contain 'input.json'" in caplog.messages[0] def test_read_opossum_file_containing_output_json() -> None: input_path = TEST_DATA_DIR / "opossum_input_with_result.opossum" - opossum_format_reader = OpossumFormatReader() + opossum_format_reader = OpossumFormatReader(input_path) - result = opossum_format_reader.read(input_path) + result = opossum_format_reader.read() assert result is not None assert result.scan_results is not None From 30bd42a9dcbcf8d8b3e22692e43f417c297df37a Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 13:45:10 +0100 Subject: [PATCH 43/58] refactor: review -- static only classes to methods --- src/opossum_lib/cli.py | 13 +- .../core/services/opossum_file_writer.py | 66 ++-- .../core/services/opossum_generator.py | 4 +- .../opossum/services/convert_to_opossum.py | 267 ++++++++++++++++ ...opossum_file_model_to_opossum_converter.py | 292 ------------------ .../opossum/services/opossum_file_reader.py | 74 +---- .../opossum/services/opossum_format_reader.py | 24 -- .../opossum/services/read_opossum_file.py | 64 ++++ .../scancode/services/convert_to_opossum.py | 95 ++++++ .../scancode_data_to_opossum_converter.py | 101 ------ .../services/scancode_format_reader.py | 6 +- tests/core/entities/test_opossum.py | 8 +- .../core/services/test_opossum_file_writer.py | 6 +- .../services/test_conversion_roundtrip.py | 8 +- ...opossum_file_model_to_opossum_converter.py | 8 +- .../services/test_opossum_format_reader.py | 8 +- .../services/test_get_attribution_info.py | 10 +- ...test_scancode_data_to_opossum_converter.py | 14 +- tests/test_cli.py | 4 +- 19 files changed, 513 insertions(+), 559 deletions(-) create mode 100644 src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py delete mode 100644 src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py delete mode 100644 src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py create mode 100644 src/opossum_lib/input_formats/opossum/services/read_opossum_file.py create mode 100644 src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py delete mode 100644 src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 37a88220..9b731178 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -1,19 +1,20 @@ #!/usr/bin/env python3 -import logging -import sys -from pathlib import Path # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 +import logging +import sys +from pathlib import Path + import click from opossum_lib.core.services.input_reader import InputReader from opossum_lib.core.services.opossum_generator import ( generate_impl, ) -from opossum_lib.input_formats.opossum.services.opossum_format_reader import ( - OpossumFormatReader, +from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( + OpossumFileReader, ) from opossum_lib.input_formats.scancode.services.scancode_format_reader import ( ScancodeFormatReader, @@ -74,7 +75,7 @@ def generate( sys.exit(1) input_readers: list[InputReader] = [] input_readers += [ScancodeFormatReader(path=path) for path in scancode_json_files] - input_readers += [OpossumFormatReader(path=path) for path in opossum_files] + input_readers += [OpossumFileReader(path=path) for path in opossum_files] generate_impl(input_readers=input_readers, output_file=Path(outfile)) diff --git a/src/opossum_lib/core/services/opossum_file_writer.py b/src/opossum_lib/core/services/opossum_file_writer.py index 95651fae..9f011819 100644 --- a/src/opossum_lib/core/services/opossum_file_writer.py +++ b/src/opossum_lib/core/services/opossum_file_writer.py @@ -12,43 +12,37 @@ from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -class OpossumFileWriter: - @staticmethod - def write(opossum_file_model: OpossumFileModel, file_path: Path) -> None: - file_path = OpossumFileWriter._ensure_outfile_suffix(file_path) - with ZipFile( - file_path, "w", compression=ZIP_DEFLATED, compresslevel=COMPRESSION_LEVEL - ) as zip_file: - OpossumFileWriter._write_input_json(opossum_file_model, zip_file) - OpossumFileWriter._write_output_json_if_existing( - opossum_file_model, zip_file - ) - - @staticmethod - def _write_output_json_if_existing( - opossum_file_content: OpossumFileModel, zip_file: ZipFile - ) -> None: - if opossum_file_content.output_file: - zip_file.writestr( - OUTPUT_JSON_NAME, - opossum_file_content.output_file.model_dump_json( - exclude_none=True, indent=4, by_alias=True - ), - ) - - @staticmethod - def _write_input_json( - opossum_file_model: OpossumFileModel, zip_file: ZipFile - ) -> None: +def write_opossum_file(opossum_file_model: OpossumFileModel, file_path: Path) -> None: + file_path = _ensure_outfile_suffix(file_path) + with ZipFile( + file_path, "w", compression=ZIP_DEFLATED, compresslevel=COMPRESSION_LEVEL + ) as zip_file: + _write_input_json(opossum_file_model, zip_file) + _write_output_json_if_existing(opossum_file_model, zip_file) + + +def _write_output_json_if_existing( + opossum_file_model: OpossumFileModel, zip_file: ZipFile +) -> None: + if opossum_file_model.output_file: zip_file.writestr( - INPUT_JSON_NAME, - opossum_file_model.input_file.model_dump_json( - indent=4, - exclude_none=True, - by_alias=True, + OUTPUT_JSON_NAME, + opossum_file_model.output_file.model_dump_json( + exclude_none=True, indent=4, by_alias=True ), ) - @staticmethod - def _ensure_outfile_suffix(outfile_path: Path) -> Path: - return outfile_path.with_suffix(".opossum") + +def _write_input_json(opossum_file_model: OpossumFileModel, zip_file: ZipFile) -> None: + zip_file.writestr( + INPUT_JSON_NAME, + opossum_file_model.input_file.model_dump_json( + indent=4, + exclude_none=True, + by_alias=True, + ), + ) + + +def _ensure_outfile_suffix(outfile_path: Path) -> Path: + return outfile_path.with_suffix(".opossum") diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/opossum_generator.py index 01504baf..176c7b7e 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/opossum_generator.py @@ -4,7 +4,7 @@ from pathlib import Path from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter +from opossum_lib.core.services.opossum_file_writer import write_opossum_file def generate_impl(input_readers: list[InputReader], output_file: Path) -> None: @@ -13,4 +13,4 @@ def generate_impl(input_readers: list[InputReader], output_file: Path) -> None: opossum = input_readers[0].read() opossum_file_content = opossum.to_opossum_model() - OpossumFileWriter.write(opossum_file_content, output_file) + write_opossum_file(opossum_file_content, output_file) diff --git a/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py new file mode 100644 index 00000000..ef3387ea --- /dev/null +++ b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py @@ -0,0 +1,267 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy +from pathlib import PurePath + +from opossum_lib.core.entities.opossum import ( + BaseUrlsForSources, + ExternalAttributionSource, + FrequentLicense, + Metadata, + Opossum, + OpossumPackage, + OpossumPackageIdentifierModel, + Resource, + ResourceType, + ScanResults, + SourceInfo, + _convert_path_to_str, +) +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import ( + ExternalAttributionSourceModel as FileExternalAttributionSource, +) +from opossum_lib.shared.entities.opossum_input_file_model import ( + FrequentLicenseModel, + MetadataModel, + OpossumInputFileModel, + OpossumPackageModel, + ResourceInFileModel, + ResourcePathModel, + SourceInfoModel, +) + + +def convert_to_opossum(opossum_file_model: OpossumFileModel) -> Opossum: + opossum = Opossum( + scan_results=_convert_to_scan_results(opossum_file_model.input_file), + review_results=opossum_file_model.output_file, + ) + return opossum + + +def _convert_to_scan_results( + opossum_input_file_model: OpossumInputFileModel, +) -> ScanResults: + resources, used_attribution_ids = _convert_to_resource_tree( + resources=opossum_input_file_model.resources, + external_attributions=opossum_input_file_model.external_attributions, + resources_to_attributions=opossum_input_file_model.resources_to_attributions, + ) + # fmt: off + frequent_licenses = ( + opossum_input_file_model.frequent_licenses + and _convert_frequent_licenses(opossum_input_file_model.frequent_licenses) + ) + # fmt: on + + base_urls_for_sources = ( + opossum_input_file_model.base_urls_for_sources + and BaseUrlsForSources( + **(opossum_input_file_model.base_urls_for_sources.model_dump()) + ) + ) + + file_attribution_sources = opossum_input_file_model.external_attribution_sources + external_attribution_sources = { + # noqa required due to clash between linter and formatter + name: _convert_external_attribution_source( # noqa: E501 + attribution_source + ) + for name, attribution_source in file_attribution_sources.items() + } + + attribution_with_id = _convert_to_attribution_with_id( + opossum_input_file_model.external_attributions + ) + return ScanResults( + metadata=_convert_to_metadata(opossum_input_file_model.metadata), + resources=resources, + attribution_breakpoints=deepcopy( + opossum_input_file_model.attribution_breakpoints + ), + external_attribution_sources=external_attribution_sources, + frequent_licenses=frequent_licenses, + files_with_children=deepcopy(opossum_input_file_model.files_with_children), + base_urls_for_sources=base_urls_for_sources, + attribution_to_id=attribution_with_id, + unassigned_attributions=_get_unassigned_attributions( + used_attribution_ids, opossum_input_file_model.external_attributions + ), + ) + + +def _get_unassigned_attributions( + used_attribution_ids: set[OpossumPackageIdentifierModel], + external_attributions: dict[ + OpossumPackageIdentifierModel, + OpossumPackageModel, + ], +) -> list[OpossumPackage] | None: + available_attribution_ids = external_attributions.keys() + unused_attributions_ids = set(available_attribution_ids) - used_attribution_ids + unused_attributions = [ + _convert_package(external_attributions[id]) for id in unused_attributions_ids + ] + return unused_attributions + + +def _convert_external_attribution_source( + external_attribution_source: FileExternalAttributionSource, +) -> ExternalAttributionSource: + return ExternalAttributionSource( + name=external_attribution_source.name, + priority=external_attribution_source.priority, + is_relevant_for_preferred=external_attribution_source.is_relevant_for_preferred, + ) + + +def _convert_frequent_licenses( + frequent_licenses_infile: list[FrequentLicenseModel], +) -> list[FrequentLicense]: + frequent_licenses: list[FrequentLicense] = [ + _convert_frequent_license(license) for license in frequent_licenses_infile + ] + return frequent_licenses + + +def _convert_to_metadata( + infile_metadata: MetadataModel, +) -> Metadata: + return Metadata(**infile_metadata.model_dump()) + + +def _convert_to_resource_tree( + resources: ResourceInFileModel, + external_attributions: dict[ + OpossumPackageIdentifierModel, + OpossumPackageModel, + ], + resources_to_attributions: dict[ + ResourcePathModel, + list[OpossumPackageIdentifierModel], + ], +) -> tuple[list[Resource], set[OpossumPackageIdentifierModel]]: + used_attribution_ids = set() + + def generate_child_resource( + current_path: PurePath, + to_insert: ResourceInFileModel, + ) -> Resource: + path = current_path + current_path_as_string = _convert_path_to_str(current_path) + if not current_path_as_string.startswith("/"): + current_path_as_string = "/" + current_path_as_string + attributions, attribution_ids = _get_applicable_attributions( + current_path_as_string + ) + used_attribution_ids.update(attribution_ids) + if isinstance(to_insert, int): + resource_type = ResourceType.FILE + return Resource( + type=resource_type, + path=path, + attributions=attributions, + ) + else: + resource_type = ResourceType.FOLDER + return Resource( + type=resource_type, + path=path, + attributions=attributions, + children={ + relative_path: generate_child_resource( + current_path / relative_path, child + ) + for relative_path, child in to_insert.items() + }, + ) + + def _get_applicable_attributions( + current_path_as_string: str, + ) -> tuple[list[OpossumPackage], set[OpossumPackageIdentifierModel]]: + attributions = [] + attribution_ids: list[str] = [] + if current_path_as_string in resources_to_attributions: + attribution_ids = resources_to_attributions[current_path_as_string] + attributions = [ + _convert_package(external_attributions[id]) for id in attribution_ids + ] + return attributions, set(attribution_ids) + + root_path = PurePath("") + + if isinstance(resources, dict): + return [ + generate_child_resource(root_path / relative_path, child) + for relative_path, child in resources.items() + ], used_attribution_ids + else: + raise RuntimeError("Root node must not be of file type") + + +def _convert_to_attribution_with_id( + external_attributions: dict[ + OpossumPackageIdentifierModel, + OpossumPackageModel, + ], +) -> dict[OpossumPackage, str]: + result = {} + for package_identifier, package in external_attributions.items(): + converted_package = _convert_package(package) + if converted_package not in result: + result[converted_package] = package_identifier + else: + raise RuntimeError( + "An attribution was duplicated in the scan breaking internal assertions" + ) + return result + + +def _convert_frequent_license( + infile_frequent_license: FrequentLicenseModel, +) -> FrequentLicense: + return FrequentLicense( + full_name=infile_frequent_license.full_name, + short_name=infile_frequent_license.short_name, + default_text=infile_frequent_license.default_text, + ) + + +def _convert_package( + infile_package: OpossumPackageModel, +) -> OpossumPackage: + return OpossumPackage( + source=_convert_source(infile_package.source), + attribution_confidence=infile_package.attribution_confidence, + comment=infile_package.comment, + package_name=infile_package.package_name, + package_version=infile_package.package_version, + package_namespace=infile_package.package_namespace, + package_type=infile_package.package_type, + package_purl_appendix=infile_package.package_p_u_r_l_appendix, + copyright=infile_package.copyright, + license_name=infile_package.license_name, + license_text=infile_package.license_text, + url=infile_package.url, + first_party=infile_package.first_party, + exclude_from_notice=infile_package.exclude_from_notice, + pre_selected=infile_package.pre_selected, + follow_up=infile_package.follow_up, + origin_id=infile_package.origin_id, + origin_ids=infile_package.origin_ids, + criticality=infile_package.criticality, + was_preferred=infile_package.was_preferred, + ) + + +def _convert_source( + infile_source_info: SourceInfoModel, +) -> SourceInfo: + return SourceInfo( + name=infile_source_info.name, + document_confidence=infile_source_info.document_confidence, + additional_name=infile_source_info.additional_name, + ) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py deleted file mode 100644 index 910a91fd..00000000 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_model_to_opossum_converter.py +++ /dev/null @@ -1,292 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -from copy import deepcopy -from pathlib import PurePath - -from opossum_lib.core.entities.opossum import ( - BaseUrlsForSources, - ExternalAttributionSource, - FrequentLicense, - Metadata, - Opossum, - OpossumPackage, - OpossumPackageIdentifierModel, - Resource, - ResourceType, - ScanResults, - SourceInfo, - _convert_path_to_str, -) -from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file_model import ( - ExternalAttributionSourceModel as FileExternalAttributionSource, -) -from opossum_lib.shared.entities.opossum_input_file_model import ( - FrequentLicenseModel, - MetadataModel, - OpossumInputFileModel, - OpossumPackageModel, - ResourceInFileModel, - ResourcePathModel, - SourceInfoModel, -) - - -class OpossumFileModelToOpossumConverter: - @staticmethod - def convert_to_opossum(opossum_file_model: OpossumFileModel) -> Opossum: - opossum = Opossum( - scan_results=OpossumFileModelToOpossumConverter._convert_to_scan_results( - opossum_file_model.input_file - ), - review_results=opossum_file_model.output_file, - ) - return opossum - - @staticmethod - def _convert_to_scan_results( - opossum_input_file_model: OpossumInputFileModel, - ) -> ScanResults: - resources, used_attribution_ids = ( - OpossumFileModelToOpossumConverter._convert_to_resource_tree( - resources=opossum_input_file_model.resources, - external_attributions=opossum_input_file_model.external_attributions, - resources_to_attributions=opossum_input_file_model.resources_to_attributions, - ) - ) - # fmt: off - frequent_licenses = ( - opossum_input_file_model.frequent_licenses - and OpossumFileModelToOpossumConverter - ._convert_frequent_licenses( - opossum_input_file_model.frequent_licenses - ) - ) - # fmt: on - - base_urls_for_sources = ( - opossum_input_file_model.base_urls_for_sources - and BaseUrlsForSources( - **(opossum_input_file_model.base_urls_for_sources.model_dump()) - ) - ) - - file_attribution_sources = opossum_input_file_model.external_attribution_sources - external_attribution_sources = { - # noqa required due to clash between linter and formatter - name: OpossumFileModelToOpossumConverter._convert_external_attribution_source( # noqa: E501 - attribution_source - ) - for name, attribution_source in file_attribution_sources.items() - } - - attribution_with_id = ( - OpossumFileModelToOpossumConverter._convert_to_attribution_with_id( - opossum_input_file_model.external_attributions - ) - ) - return ScanResults( - metadata=OpossumFileModelToOpossumConverter._convert_to_metadata( - opossum_input_file_model.metadata - ), - resources=resources, - attribution_breakpoints=deepcopy( - opossum_input_file_model.attribution_breakpoints - ), - external_attribution_sources=external_attribution_sources, - frequent_licenses=frequent_licenses, - files_with_children=deepcopy(opossum_input_file_model.files_with_children), - base_urls_for_sources=base_urls_for_sources, - attribution_to_id=attribution_with_id, - unassigned_attributions=OpossumFileModelToOpossumConverter._get_unassigned_attributions( - used_attribution_ids, opossum_input_file_model.external_attributions - ), - ) - - @staticmethod - def _get_unassigned_attributions( - used_attribution_ids: set[OpossumPackageIdentifierModel], - external_attributions: dict[ - OpossumPackageIdentifierModel, - OpossumPackageModel, - ], - ) -> list[OpossumPackage] | None: - available_attribution_ids = external_attributions.keys() - unused_attributions_ids = set(available_attribution_ids) - used_attribution_ids - unused_attributions = [ - OpossumFileModelToOpossumConverter._convert_package( - external_attributions[id] - ) - for id in unused_attributions_ids - ] - return unused_attributions - - @staticmethod - def _convert_external_attribution_source( - external_attribution_source: FileExternalAttributionSource, - ) -> ExternalAttributionSource: - return ExternalAttributionSource( - name=external_attribution_source.name, - priority=external_attribution_source.priority, - is_relevant_for_preferred=external_attribution_source.is_relevant_for_preferred, - ) - - @staticmethod - def _convert_frequent_licenses( - frequent_licenses_infile: list[FrequentLicenseModel], - ) -> list[FrequentLicense]: - frequent_licenses: list[FrequentLicense] = [ - OpossumFileModelToOpossumConverter._convert_frequent_license(license) - for license in frequent_licenses_infile - ] - return frequent_licenses - - @staticmethod - def _convert_to_metadata( - infile_metadata: MetadataModel, - ) -> Metadata: - return Metadata(**infile_metadata.model_dump()) - - @staticmethod - def _convert_to_resource_tree( - resources: ResourceInFileModel, - external_attributions: dict[ - OpossumPackageIdentifierModel, - OpossumPackageModel, - ], - resources_to_attributions: dict[ - ResourcePathModel, - list[OpossumPackageIdentifierModel], - ], - ) -> tuple[list[Resource], set[OpossumPackageIdentifierModel]]: - used_attribution_ids = set() - - def generate_child_resource( - current_path: PurePath, - to_insert: ResourceInFileModel, - ) -> Resource: - path = current_path - current_path_as_string = _convert_path_to_str(current_path) - if not current_path_as_string.startswith("/"): - current_path_as_string = "/" + current_path_as_string - attributions, attribution_ids = _get_applicable_attributions( - current_path_as_string - ) - used_attribution_ids.update(attribution_ids) - if isinstance(to_insert, int): - resource_type = ResourceType.FILE - return Resource( - type=resource_type, - path=path, - attributions=attributions, - ) - else: - resource_type = ResourceType.FOLDER - return Resource( - type=resource_type, - path=path, - attributions=attributions, - children={ - relative_path: generate_child_resource( - current_path / relative_path, child - ) - for relative_path, child in to_insert.items() - }, - ) - - def _get_applicable_attributions( - current_path_as_string: str, - ) -> tuple[list[OpossumPackage], set[OpossumPackageIdentifierModel]]: - attributions = [] - attribution_ids: list[str] = [] - if current_path_as_string in resources_to_attributions: - attribution_ids = resources_to_attributions[current_path_as_string] - attributions = [ - OpossumFileModelToOpossumConverter._convert_package( - external_attributions[id] - ) - for id in attribution_ids - ] - return attributions, set(attribution_ids) - - root_path = PurePath("") - - if isinstance(resources, dict): - return [ - generate_child_resource(root_path / relative_path, child) - for relative_path, child in resources.items() - ], used_attribution_ids - else: - raise RuntimeError("Root node must not be of file type") - - @staticmethod - def _convert_to_attribution_with_id( - external_attributions: dict[ - OpossumPackageIdentifierModel, - OpossumPackageModel, - ], - ) -> dict[OpossumPackage, str]: - result = {} - for package_identifier, package in external_attributions.items(): - converted_package = OpossumFileModelToOpossumConverter._convert_package( - package - ) - if converted_package not in result: - result[converted_package] = package_identifier - else: - raise RuntimeError( - "An attribution was duplicated in the " - "scan breaking internal assertions" - ) - return result - - @staticmethod - def _convert_frequent_license( - infile_frequent_license: FrequentLicenseModel, - ) -> FrequentLicense: - return FrequentLicense( - full_name=infile_frequent_license.full_name, - short_name=infile_frequent_license.short_name, - default_text=infile_frequent_license.default_text, - ) - - @staticmethod - def _convert_package( - infile_package: OpossumPackageModel, - ) -> OpossumPackage: - return OpossumPackage( - source=OpossumFileModelToOpossumConverter._convert_source( - infile_package.source - ), - attribution_confidence=infile_package.attribution_confidence, - comment=infile_package.comment, - package_name=infile_package.package_name, - package_version=infile_package.package_version, - package_namespace=infile_package.package_namespace, - package_type=infile_package.package_type, - package_purl_appendix=infile_package.package_p_u_r_l_appendix, - copyright=infile_package.copyright, - license_name=infile_package.license_name, - license_text=infile_package.license_text, - url=infile_package.url, - first_party=infile_package.first_party, - exclude_from_notice=infile_package.exclude_from_notice, - pre_selected=infile_package.pre_selected, - follow_up=infile_package.follow_up, - origin_id=infile_package.origin_id, - origin_ids=infile_package.origin_ids, - criticality=infile_package.criticality, - was_preferred=infile_package.was_preferred, - ) - - @staticmethod - def _convert_source( - infile_source_info: SourceInfoModel, - ) -> SourceInfo: - return SourceInfo( - name=infile_source_info.name, - document_confidence=infile_source_info.document_confidence, - additional_name=infile_source_info.additional_name, - ) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index feeee775..ba81d7b5 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -1,70 +1,24 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 - - -import json -import logging -import sys from pathlib import Path -from zipfile import ZipFile -from opossum_lib.shared.constants import ( - INPUT_JSON_NAME, - OUTPUT_JSON_NAME, +from opossum_lib.core.entities.opossum import Opossum +from opossum_lib.core.services.input_reader import InputReader +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, +) +from opossum_lib.input_formats.opossum.services.read_opossum_file import ( + read_opossum_file, ) -from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel -from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel - - -class OpossumFileReader: - @staticmethod - def from_file(path: Path) -> OpossumFileModel: - logging.info(f"Converting opossum to opossum {path}") - try: - with ( - ZipFile(path, "r") as input_zip_file, - ): - OpossumFileReader._validate_zip_file_contents(input_zip_file) - input_file = OpossumFileReader._read_input_json_from_zip_file( - input_zip_file - ) - return OpossumFileModel( - input_file=input_file, - output_file=OpossumFileReader._read_output_json_if_exists( - input_zip_file - ), - ) - except Exception as e: - print(f"Error reading file {path}: {e}") - sys.exit(1) - @staticmethod - def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: - with zip_file.open(INPUT_JSON_NAME) as input_json_file: - input_json = json.load(input_json_file) - input_file = OpossumInputFileModel.model_validate(input_json) - return input_file +class OpossumFileReader(InputReader): + path: Path - @staticmethod - def _read_output_json_if_exists( - input_zip_file: ZipFile, - ) -> OpossumOutputFileModel | None: - if OUTPUT_JSON_NAME in input_zip_file.namelist(): - with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: - output_json = json.load(output_json_file) - output_file = OpossumOutputFileModel.model_validate(output_json) - else: - output_file = None - return output_file + def __init__(self, path: Path): + self.path = path - @staticmethod - def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: - if INPUT_JSON_NAME not in input_zip_file.namelist(): - logging.error( - f"Opossum file {input_zip_file.filename} is corrupt" - f" and does not contain '{INPUT_JSON_NAME}'" - ) - sys.exit(1) + def read(self) -> Opossum: + opossum_input_file = read_opossum_file(path=self.path) + return convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py deleted file mode 100644 index 3755eca6..00000000 --- a/src/opossum_lib/input_formats/opossum/services/opossum_format_reader.py +++ /dev/null @@ -1,24 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 -from pathlib import Path - -from opossum_lib.core.entities.opossum import Opossum -from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 - OpossumFileModelToOpossumConverter, -) -from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( - OpossumFileReader, -) - - -class OpossumFormatReader(InputReader): - path: Path - - def __init__(self, path: Path): - self.path = path - - def read(self) -> Opossum: - opossum_input_file = OpossumFileReader.from_file(path=self.path) - return OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_input_file) diff --git a/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py b/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py new file mode 100644 index 00000000..535acadc --- /dev/null +++ b/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py @@ -0,0 +1,64 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + + +import json +import logging +import sys +from pathlib import Path +from zipfile import ZipFile + +from opossum_lib.shared.constants import ( + INPUT_JSON_NAME, + OUTPUT_JSON_NAME, +) +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel + + +def read_opossum_file(path: Path) -> OpossumFileModel: + logging.info(f"Converting opossum to opossum {path}") + + try: + with ( + ZipFile(path, "r") as input_zip_file, + ): + _validate_zip_file_contents(input_zip_file) + input_file = _read_input_json_from_zip_file(input_zip_file) + return OpossumFileModel( + input_file=input_file, + output_file=_read_output_json_if_exists(input_zip_file), + ) + except Exception as e: + print(f"Error reading file {path}: {e}") + sys.exit(1) + + +def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: + with zip_file.open(INPUT_JSON_NAME) as input_json_file: + input_json = json.load(input_json_file) + input_file = OpossumInputFileModel.model_validate(input_json) + return input_file + + +def _read_output_json_if_exists( + input_zip_file: ZipFile, +) -> OpossumOutputFileModel | None: + if OUTPUT_JSON_NAME in input_zip_file.namelist(): + with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: + output_json = json.load(output_json_file) + output_file = OpossumOutputFileModel.model_validate(output_json) + else: + output_file = None + return output_file + + +def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: + if INPUT_JSON_NAME not in input_zip_file.namelist(): + logging.error( + f"Opossum file {input_zip_file.filename} is corrupt" + f" and does not contain '{INPUT_JSON_NAME}'" + ) + sys.exit(1) diff --git a/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py new file mode 100644 index 00000000..5ca1677c --- /dev/null +++ b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py @@ -0,0 +1,95 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import sys +import uuid +from pathlib import PurePath + +from opossum_lib.core.entities.opossum import ( + Metadata, + Opossum, + OpossumPackage, + Resource, + ResourceType, + ScanResults, + SourceInfo, +) +from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME +from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( + FileModel, + FileTypeModel, + HeaderModel, + ScanCodeDataModel, +) + + +def convert_to_opossum(scancode_data: ScanCodeDataModel) -> Opossum: + resources = _extract_opossum_resources(scancode_data) + + scancode_header = _extract_scancode_header(scancode_data) + metadata = Metadata( + project_id=str(uuid.uuid4()), + file_creation_date=scancode_header.end_timestamp, + project_title="ScanCode file", + ) + + return Opossum( + scan_results=ScanResults( + metadata=metadata, + resources=resources, + ) + ) + + +def _extract_scancode_header(scancode_data: ScanCodeDataModel) -> HeaderModel: + if len(scancode_data.headers) != 1: + logging.error("Headers of ScanCode file are invalid.") + sys.exit(1) + return scancode_data.headers[0] + + +def _extract_opossum_resources( + scancode_data: ScanCodeDataModel, +) -> list[Resource]: + temp_root = Resource(path=PurePath("")) + for file in scancode_data.files: + resource = Resource( + path=PurePath(file.path), + attributions=_get_attribution_info(file), + type=_convert_resource_type(file.type), + ) + temp_root.add_resource(resource) + + return list(temp_root.children.values()) + + +def _convert_resource_type(file_type: FileTypeModel) -> ResourceType: + if file_type == FileTypeModel.FILE: + return ResourceType.FILE + else: + return ResourceType.FOLDER + + +def _get_attribution_info(file: FileModel) -> list[OpossumPackage]: + if file.type == FileTypeModel.DIRECTORY: + return [] + copyright = "\n".join(c.copyright for c in file.copyrights) + source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) + + attribution_infos = [] + for license_detection in file.license_detections: + license_name = license_detection.license_expression_spdx + max_score = max(m.score for m in license_detection.matches) + attribution_confidence = int(max_score) + + package = OpossumPackage( + source=source_info, + license_name=license_name, + attribution_confidence=attribution_confidence, + copyright=copyright, + ) + attribution_infos.append(package) + + return attribution_infos diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py b/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py deleted file mode 100644 index aa5bd4a7..00000000 --- a/src/opossum_lib/input_formats/scancode/services/scancode_data_to_opossum_converter.py +++ /dev/null @@ -1,101 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - -import logging -import sys -import uuid -from pathlib import PurePath - -from opossum_lib.core.entities.opossum import ( - Metadata, - Opossum, - OpossumPackage, - Resource, - ResourceType, - ScanResults, - SourceInfo, -) -from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( - FileModel, - FileTypeModel, - HeaderModel, - ScanCodeDataModel, -) - - -class ScancodeDataToOpossumConverter: - @staticmethod - def convert_scancode_to_opossum(scancode_data: ScanCodeDataModel) -> Opossum: - resources = ScancodeDataToOpossumConverter.extract_opossum_resources( - scancode_data - ) - - scancode_header = ScancodeDataToOpossumConverter.extract_scancode_header( - scancode_data - ) - metadata = Metadata( - project_id=str(uuid.uuid4()), - file_creation_date=scancode_header.end_timestamp, - project_title="ScanCode file", - ) - - return Opossum( - scan_results=ScanResults( - metadata=metadata, - resources=resources, - ) - ) - - @staticmethod - def extract_scancode_header(scancode_data: ScanCodeDataModel) -> HeaderModel: - if len(scancode_data.headers) != 1: - logging.error("Headers of ScanCode file are invalid.") - sys.exit(1) - return scancode_data.headers[0] - - @staticmethod - def extract_opossum_resources( - scancode_data: ScanCodeDataModel, - ) -> list[Resource]: - temp_root = Resource(path=PurePath("")) - for file in scancode_data.files: - resource = Resource( - path=PurePath(file.path), - attributions=ScancodeDataToOpossumConverter.get_attribution_info(file), - type=ScancodeDataToOpossumConverter.convert_resource_type(file.type), - ) - temp_root.add_resource(resource) - - return list(temp_root.children.values()) - - @staticmethod - def convert_resource_type(file_type: FileTypeModel) -> ResourceType: - if file_type == FileTypeModel.FILE: - return ResourceType.FILE - else: - return ResourceType.FOLDER - - @staticmethod - def get_attribution_info(file: FileModel) -> list[OpossumPackage]: - if file.type == FileTypeModel.DIRECTORY: - return [] - copyright = "\n".join(c.copyright for c in file.copyrights) - source_info = SourceInfo(name=SCANCODE_SOURCE_NAME) - - attribution_infos = [] - for license_detection in file.license_detections: - license_name = license_detection.license_expression_spdx - max_score = max(m.score for m in license_detection.matches) - attribution_confidence = int(max_score) - - package = OpossumPackage( - source=source_info, - license_name=license_name, - attribution_confidence=attribution_confidence, - copyright=copyright, - ) - attribution_infos.append(package) - - return attribution_infos diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index 555ad947..d3cb7147 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -14,8 +14,8 @@ from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( ScanCodeDataModel, ) -from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 - ScancodeDataToOpossumConverter, +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) @@ -30,7 +30,7 @@ def read(self) -> Opossum: scancode_data = ScancodeFormatReader.load_scancode_json(self.path) - return ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + return convert_to_opossum(scancode_data) @staticmethod def load_scancode_json(path: Path) -> ScanCodeDataModel: diff --git a/tests/core/entities/test_opossum.py b/tests/core/entities/test_opossum.py index 6fd8b30d..0a264592 100644 --- a/tests/core/entities/test_opossum.py +++ b/tests/core/entities/test_opossum.py @@ -4,8 +4,8 @@ import json from copy import deepcopy -from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 - OpossumFileModelToOpossumConverter, +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) from tests.setup.opossum_faker_setup import OpossumFaker @@ -24,7 +24,7 @@ def test_roundtrip(self, opossum_faker: OpossumFaker) -> None: opossum_file = opossum.to_opossum_model() - result = OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_file) + result = convert_to_opossum(opossum_file) ## this can change due to the generation of new ids result_json = result.model_dump_json() @@ -53,6 +53,6 @@ def test_roundtrip_with_resource_ids(self, opossum_faker: OpossumFaker) -> None: opossum_file = opossum.to_opossum_model() - result = OpossumFileModelToOpossumConverter.convert_to_opossum(opossum_file) + result = convert_to_opossum(opossum_file) assert result == expected_result diff --git a/tests/core/services/test_opossum_file_writer.py b/tests/core/services/test_opossum_file_writer.py index 44ba2125..0793ecb4 100644 --- a/tests/core/services/test_opossum_file_writer.py +++ b/tests/core/services/test_opossum_file_writer.py @@ -5,7 +5,7 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter +from opossum_lib.core.services.opossum_file_writer import write_opossum_file from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, @@ -22,7 +22,7 @@ def test_only_input_information_available_writes_only_input_information( ) output_path = tmp_path / "output.opossum" - OpossumFileWriter.write(opossum_file_content, output_path) + write_opossum_file(opossum_file_content, output_path) with ZipFile(output_path, "r") as zip_file: assert zip_file.namelist() == [INPUT_JSON_NAME] @@ -34,7 +34,7 @@ def test_input_and_output_information_available_writes_both( opossum_file_content = opossum_file_faker.opossum_file_content() output_path = tmp_path / "output.opossum" - OpossumFileWriter.write(opossum_file_content, output_path) + write_opossum_file(opossum_file_content, output_path) with ZipFile(output_path, "r") as zip_file: assert INPUT_JSON_NAME in zip_file.namelist() diff --git a/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py b/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py index e85950cf..d0ac02f7 100644 --- a/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py +++ b/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 - OpossumFileModelToOpossumConverter, +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from tests.setup.opossum_file_faker_setup import OpossumFileFaker @@ -31,7 +31,5 @@ def test_surplus_attributions(self, opossum_file_faker: OpossumFileFaker) -> Non @staticmethod def _check_round_trip(start_file_content: OpossumFileModel) -> None: expected_file_content = deepcopy(start_file_content) - result = OpossumFileModelToOpossumConverter.convert_to_opossum( - start_file_content - ).to_opossum_model() + result = convert_to_opossum(start_file_content).to_opossum_model() assert result == expected_file_content diff --git a/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py b/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py index 9fba7481..e2f32fa2 100644 --- a/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py +++ b/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.input_formats.opossum.services.opossum_file_model_to_opossum_converter import ( # noqa: E501 - OpossumFileModelToOpossumConverter, +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) from opossum_lib.shared.entities.opossum_input_file_model import ( OpossumPackageIdentifierModel, @@ -18,7 +18,7 @@ def test_output_file_moved(self, opossum_file_faker: OpossumFileFaker) -> None: output_file = opossum_file_faker.output_file() input_file = opossum_file_faker.opossum_file_content(out_file=output_file) - result = OpossumFileModelToOpossumConverter.convert_to_opossum(input_file) + result = convert_to_opossum(input_file) assert result.review_results == output_file @@ -34,7 +34,7 @@ def test_throws_on_duplicate_attributions( input_file = opossum_file_faker.opossum_file_content(in_file=file_information) with pytest.raises(RuntimeError, match=r".*attribution was duplicated.*"): - OpossumFileModelToOpossumConverter.convert_to_opossum(input_file) + convert_to_opossum(input_file) @staticmethod def _fake_duplicate_external_attributions( diff --git a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py index 05c2a4a8..9c3bc7c6 100644 --- a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py +++ b/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py @@ -6,8 +6,8 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.input_formats.opossum.services.opossum_format_reader import ( - OpossumFormatReader, +from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( + OpossumFileReader, ) TEST_DATA_DIR = Path(__file__).resolve().parent.parent.parent.parent.parent / "data" @@ -15,7 +15,7 @@ def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: input_path = TEST_DATA_DIR / "opossum_input_corrupt.opossum" - opossum_format_reader = OpossumFormatReader(input_path) + opossum_format_reader = OpossumFileReader(input_path) with pytest.raises(SystemExit) as system_exit: opossum_format_reader.read() @@ -25,7 +25,7 @@ def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> def test_read_opossum_file_containing_output_json() -> None: input_path = TEST_DATA_DIR / "opossum_input_with_result.opossum" - opossum_format_reader = OpossumFormatReader(input_path) + opossum_format_reader = OpossumFileReader(input_path) result = opossum_format_reader.read() diff --git a/tests/input_formats/scancode/entities/services/test_get_attribution_info.py b/tests/input_formats/scancode/entities/services/test_get_attribution_info.py index bf6d0c41..869fbbc8 100644 --- a/tests/input_formats/scancode/entities/services/test_get_attribution_info.py +++ b/tests/input_formats/scancode/entities/services/test_get_attribution_info.py @@ -4,8 +4,8 @@ from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 - ScancodeDataToOpossumConverter, +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) from opossum_lib.shared.entities.opossum_input_file_model import ( OpossumPackageModel, @@ -17,7 +17,7 @@ def test_get_attribution_info_directory(scancode_faker: ScanCodeFaker) -> None: folder = scancode_faker.single_folder(path="some/single/folder") scancode_data = scancode_faker.scancode_data(files=[folder]) - opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + opossum = convert_to_opossum(scancode_data) assert len(opossum.scan_results.resources) == 1 assert opossum.scan_results.resources[0].attributions == [] @@ -27,7 +27,7 @@ def test_get_attribution_info_from_file_without_detections( ) -> None: file = scancode_faker.single_file(path="some/single/file", license_detections=[]) scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + opossum = convert_to_opossum(scancode_data) assert len(opossum.scan_results.resources) == 1 assert opossum.scan_results.resources[0].attributions == [] @@ -68,7 +68,7 @@ def test_get_attribution_info_file_multiple(scancode_faker: ScanCodeFaker) -> No copyrights=[copyright1, copyright2, copyright3], ) scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + opossum = convert_to_opossum(scancode_data) attributions = opossum.to_opossum_model().input_file.external_attributions.values() expected1 = OpossumPackageModel( diff --git a/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py index 787836e8..683d57ec 100644 --- a/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py +++ b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py @@ -7,8 +7,8 @@ from _pytest.logging import LogCaptureFixture from opossum_lib.core.entities.opossum import Resource -from opossum_lib.input_formats.scancode.services.scancode_data_to_opossum_converter import ( # noqa: E501 - ScancodeDataToOpossumConverter, +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 + convert_to_opossum, ) from tests.setup.scancode_faker_setup import ScanCodeFaker @@ -17,7 +17,7 @@ def test_extract_scancode_header_produces_expected_result( scancode_faker: ScanCodeFaker, ) -> None: scancode_data = scancode_faker.scancode_data() - opossum = ScancodeDataToOpossumConverter.convert_scancode_to_opossum( + opossum = convert_to_opossum( scancode_data, ) metadata = opossum.scan_results.metadata @@ -32,7 +32,7 @@ def test_extract_scancode_header_errors_with_missing_header( scancode_data = scancode_faker.scancode_data(headers=[]) with pytest.raises(SystemExit): - ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + convert_to_opossum(scancode_data) assert "header" in caplog.messages[0].lower() @@ -45,7 +45,7 @@ def test_extract_scancode_header_error_with_multiple_headers( scancode_data = scancode_faker.scancode_data(headers=[header1, header2]) with pytest.raises(SystemExit): - ScancodeDataToOpossumConverter.convert_scancode_to_opossum(scancode_data) + convert_to_opossum(scancode_data) assert "header" in caplog.messages[0].lower() @@ -64,9 +64,7 @@ def test_convert_scancode_produces_expected_result( scancode_faker: ScanCodeFaker, ) -> None: scancode_data = scancode_faker.scancode_data() - opossum_data = ScancodeDataToOpossumConverter.convert_scancode_to_opossum( - scancode_data - ) + opossum_data = convert_to_opossum(scancode_data) assert opossum_data.review_results is None scan_results = opossum_data.scan_results diff --git a/tests/test_cli.py b/tests/test_cli.py index 199e61de..2b635a47 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,7 +11,7 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate -from opossum_lib.core.services.opossum_file_writer import OpossumFileWriter +from opossum_lib.core.services.opossum_file_writer import write_opossum_file from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, @@ -180,7 +180,7 @@ def test_cli_no_output_file_provided(opossum_file_faker: OpossumFileFaker) -> No with runner.isolated_filesystem(): file_path = "input.opossum" opossum_file = opossum_file_faker.opossum_file_content() - OpossumFileWriter.write(opossum_file, Path(file_path)) + write_opossum_file(opossum_file, Path(file_path)) result = runner.invoke( generate, "--opossum " + file_path, From df51d5ab4a47f90ec871b939c350e1ce2e403a4a Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 14:27:02 +0100 Subject: [PATCH 44/58] refactor: review -- expand opossum model to multiple files --- .../core/entities/base_url_for_sources.py | 16 + .../entities/external_attribution_source.py | 25 ++ .../core/entities/frequent_license.py | 23 ++ src/opossum_lib/core/entities/metadata.py | 22 ++ src/opossum_lib/core/entities/opossum.py | 320 +----------------- .../core/entities/opossum_package.py | 60 ++++ src/opossum_lib/core/entities/resource.py | 84 +++++ src/opossum_lib/core/entities/scan_results.py | 146 ++++++++ src/opossum_lib/core/entities/source_info.py | 23 ++ .../opossum/services/convert_to_opossum.py | 20 +- .../scancode/services/convert_to_opossum.py | 11 +- .../external_attribution_source_provider.py | 4 +- .../entities/generators/metadata_provider.py | 2 +- .../entities/generators/opossum_provider.py | 3 +- .../entities/generators/package_provider.py | 3 +- .../entities/generators/resource_provider.py | 7 +- .../generators/scan_results_provider.py | 15 +- .../generators/source_info_provider.py | 2 +- ...test_scancode_data_to_opossum_converter.py | 2 +- 19 files changed, 436 insertions(+), 352 deletions(-) create mode 100644 src/opossum_lib/core/entities/base_url_for_sources.py create mode 100644 src/opossum_lib/core/entities/external_attribution_source.py create mode 100644 src/opossum_lib/core/entities/frequent_license.py create mode 100644 src/opossum_lib/core/entities/metadata.py create mode 100644 src/opossum_lib/core/entities/opossum_package.py create mode 100644 src/opossum_lib/core/entities/resource.py create mode 100644 src/opossum_lib/core/entities/scan_results.py create mode 100644 src/opossum_lib/core/entities/source_info.py diff --git a/src/opossum_lib/core/entities/base_url_for_sources.py b/src/opossum_lib/core/entities/base_url_for_sources.py new file mode 100644 index 00000000..0e952bb6 --- /dev/null +++ b/src/opossum_lib/core/entities/base_url_for_sources.py @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.shared.entities.opossum_input_file_model import BaseUrlsForSourcesModel + + +class BaseUrlsForSources(BaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + + def to_opossum_model(self) -> BaseUrlsForSourcesModel: + return BaseUrlsForSourcesModel(**self.model_dump()) diff --git a/src/opossum_lib/core/entities/external_attribution_source.py b/src/opossum_lib/core/entities/external_attribution_source.py new file mode 100644 index 00000000..3fec8151 --- /dev/null +++ b/src/opossum_lib/core/entities/external_attribution_source.py @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.shared.entities.opossum_input_file_model import ( + ExternalAttributionSourceModel, +) + + +class ExternalAttributionSource(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + name: str + priority: int + is_relevant_for_preferred: bool | None = None + + def to_opossum_model(self) -> ExternalAttributionSourceModel: + return ExternalAttributionSourceModel( + name=self.name, + priority=self.priority, + is_relevant_for_preferred=self.is_relevant_for_preferred, + ) diff --git a/src/opossum_lib/core/entities/frequent_license.py b/src/opossum_lib/core/entities/frequent_license.py new file mode 100644 index 00000000..af3feea6 --- /dev/null +++ b/src/opossum_lib/core/entities/frequent_license.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.shared.entities.opossum_input_file_model import FrequentLicenseModel + + +class FrequentLicense(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + full_name: str + short_name: str + default_text: str + + def to_opossum_model(self) -> FrequentLicenseModel: + return FrequentLicenseModel( + full_name=self.full_name, + short_name=self.short_name, + default_text=self.default_text, + ) diff --git a/src/opossum_lib/core/entities/metadata.py b/src/opossum_lib/core/entities/metadata.py new file mode 100644 index 00000000..463b59a9 --- /dev/null +++ b/src/opossum_lib/core/entities/metadata.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.shared.entities.opossum_input_file_model import MetadataModel + + +class Metadata(BaseModel): + model_config = ConfigDict(frozen=True, extra="allow") + project_id: str + file_creation_date: str + project_title: str + project_version: str | None = None + expected_release_date: str | None = None + build_date: str | None = None + + def to_opossum_model(self) -> MetadataModel: + return MetadataModel(**self.model_dump()) diff --git a/src/opossum_lib/core/entities/opossum.py b/src/opossum_lib/core/entities/opossum.py index 5f99e0f2..e5a06db9 100644 --- a/src/opossum_lib/core/entities/opossum.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -4,44 +4,16 @@ from __future__ import annotations -import uuid -from collections import defaultdict -from collections.abc import Iterable -from copy import deepcopy -from dataclasses import field -from enum import Enum, auto -from pathlib import PurePath -from typing import Literal - from pydantic import BaseModel, ConfigDict +from opossum_lib.core.entities.scan_results import ScanResults from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file_model import ( - BaseUrlsForSourcesModel, - ExternalAttributionSourceModel, - FrequentLicenseModel, - MetadataModel, - OpossumInputFileModel, - OpossumPackageIdentifierModel, - OpossumPackageModel, - ResourceInFileModel, - ResourcePathModel, - SourceInfoModel, -) from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel type OpossumPackageIdentifier = str type ResourcePath = str -def _convert_path_to_str(path: PurePath) -> str: - return str(path).replace("\\", "/") - - -def default_attribution_id_mapper() -> dict[OpossumPackage, str]: - return defaultdict(lambda: str(uuid.uuid4())) - - class Opossum(BaseModel): model_config = ConfigDict(frozen=True, extra="forbid") scan_results: ScanResults @@ -52,293 +24,3 @@ def to_opossum_model(self) -> OpossumFileModel: input_file=self.scan_results.to_opossum_model(), output_file=self.review_results, ) - - -class ScanResults(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - metadata: Metadata - resources: list[Resource] - attribution_breakpoints: list[str] = [] - external_attribution_sources: dict[str, ExternalAttributionSource] = {} - frequent_licenses: list[FrequentLicense] | None = None - files_with_children: list[str] | None = None - base_urls_for_sources: BaseUrlsForSources | None = None - attribution_to_id: dict[OpossumPackage, str] = field( - default_factory=default_attribution_id_mapper - ) - unassigned_attributions: list[OpossumPackage] = [] - - def to_opossum_model(self) -> OpossumInputFileModel: - external_attributions, resources_to_attributions = ( - self.create_attribution_mapping(self.resources) - ) - external_attributions.update(self._get_unassigned_attributions()) - - frequent_licenses = None - if self.frequent_licenses: - frequent_licenses = [ - license.to_opossum_model() for license in self.frequent_licenses - ] - base_urls_for_sources = ( - self.base_urls_for_sources and self.base_urls_for_sources.to_opossum_model() - ) - - external_attribution_sources = { - key: val.to_opossum_model() - for (key, val) in self.external_attribution_sources.items() - } - - return OpossumInputFileModel( - metadata=self.metadata.to_opossum_model(), - resources={ - str(resource.path): resource.to_opossum_model() - for resource in self.resources - }, - external_attributions=external_attributions, - resources_to_attributions=resources_to_attributions, - attribution_breakpoints=deepcopy(self.attribution_breakpoints), - external_attribution_sources=external_attribution_sources, - frequent_licenses=frequent_licenses, - files_with_children=deepcopy(self.files_with_children), - base_urls_for_sources=base_urls_for_sources, - ) - - def _get_unassigned_attributions( - self, - ) -> dict[OpossumPackageIdentifierModel, OpossumPackageModel]: - if self.unassigned_attributions: - result = {} - for unassigned_attribution in self.unassigned_attributions: - if unassigned_attribution in self.attribution_to_id: - package_identifier = self.attribution_to_id[unassigned_attribution] - result[package_identifier] = ( - unassigned_attribution.to_opossum_model() - ) - else: - package_identifier = str(uuid.uuid4()) - self.attribution_to_id[unassigned_attribution] = package_identifier - result[package_identifier] = ( - unassigned_attribution.to_opossum_model() - ) - return result - else: - return {} - - def create_attribution_mapping( - self, - root_nodes: list[Resource], - ) -> tuple[ - dict[OpossumPackageIdentifierModel, OpossumPackageModel], - dict[ResourcePathModel, list[OpossumPackageIdentifierModel]], - ]: - external_attributions: dict[ - OpossumPackageIdentifierModel, OpossumPackageModel - ] = {} - resources_to_attributions: dict[ - ResourcePathModel, list[OpossumPackageIdentifierModel] - ] = {} - - def process_node(node: Resource) -> None: - path = _convert_path_to_str(node.path) - if not path.startswith("/"): - # the / is required by OpossumUI - path = "/" + path - - node_attributions_by_id = { - self.get_attribution_key(a): a.to_opossum_model() - for a in node.attributions - } - external_attributions.update(node_attributions_by_id) - - if len(node_attributions_by_id) > 0: - resources_to_attributions[path] = list(node_attributions_by_id.keys()) - - for child in node.children.values(): - process_node(child) - - for root in root_nodes: - process_node(root) - - return external_attributions, resources_to_attributions - - def get_attribution_key( - self, attribution: OpossumPackage - ) -> OpossumPackageIdentifier: - id = self.attribution_to_id[attribution] - self.attribution_to_id[attribution] = id - return id - - -class ResourceType(Enum): - FILE = auto() - FOLDER = auto() - - -class Resource(BaseModel): - model_config = ConfigDict(frozen=False, extra="forbid") - path: PurePath - type: ResourceType | None = None - attributions: list[OpossumPackage] = [] - children: dict[str, Resource] = {} - - def to_opossum_model(self) -> ResourceInFileModel: - if self.children or self.type == ResourceType.FOLDER: - return { - _convert_path_to_str( - child.path.relative_to(self.path) - ): child.to_opossum_model() - for child in self.children.values() - } - else: - return 1 - - def add_resource(self, resource: Resource) -> None: - if not resource.path.is_relative_to(self.path): - raise RuntimeError( - f"The path {resource.path} is not a child of this node at {self.path}." - ) - remaining_path_parts = resource.path.relative_to(self.path).parts - if remaining_path_parts: - self._add_resource(resource, remaining_path_parts) - else: - self._update(resource) - - def _add_resource( - self, resource: Resource, remaining_path_parts: Iterable[str] - ) -> None: - if not remaining_path_parts: - self._update(resource) - return - next, *rest_parts = remaining_path_parts - if next not in self.children: - self.children[next] = Resource(path=self.path / next) - self.children[next]._add_resource(resource, rest_parts) - - def _update(self, other: Resource) -> None: - if self.path != other.path: - raise RuntimeError( - "Trying to merge nodes with different paths: " - + f"{self.path} vs. {other.path}" - ) - if self.type and other.type and self.type != other.type: - raise RuntimeError( - "Trying to merge incompatible node types. " - + f"Current node is {self.type}. Other is {other.type}" - ) - self.type = self.type or other.type - self.attributions.extend(other.attributions) - for key, child in other.children.items(): - if key in self.children: - self.children[key]._update(child) - else: - self.children[key] = child - - -class BaseUrlsForSources(BaseModel): - model_config = ConfigDict(frozen=True, extra="allow") - - def to_opossum_model(self) -> BaseUrlsForSourcesModel: - return BaseUrlsForSourcesModel(**self.model_dump()) - - -class FrequentLicense(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - full_name: str - short_name: str - default_text: str - - def to_opossum_model(self) -> FrequentLicenseModel: - return FrequentLicenseModel( - full_name=self.full_name, - short_name=self.short_name, - default_text=self.default_text, - ) - - -class SourceInfo(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - name: str - document_confidence: int | float | None = 0 - additional_name: str | None = None - - def to_opossum_model(self) -> SourceInfoModel: - return SourceInfoModel( - name=self.name, - document_confidence=self.document_confidence, - additional_name=self.additional_name, - ) - - -class OpossumPackage(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - source: SourceInfo - attribution_confidence: int | None = None - comment: str | None = None - package_name: str | None = None - package_version: str | None = None - package_namespace: str | None = None - package_type: str | None = None - package_purl_appendix: str | None = None - copyright: str | None = None - license_name: str | None = None - license_text: str | None = None - url: str | None = None - first_party: bool | None = None - exclude_from_notice: bool | None = None - pre_selected: bool | None = None - follow_up: Literal["FOLLOW_UP"] | None = None - origin_id: str | None = None - origin_ids: tuple[str, ...] | None = None - criticality: Literal["high"] | Literal["medium"] | None = None - was_preferred: bool | None = None - - def to_opossum_model(self) -> OpossumPackageModel: - return OpossumPackageModel( - source=self.source.to_opossum_model(), - attribution_confidence=self.attribution_confidence, - comment=self.comment, - package_name=self.package_name, - package_version=self.package_version, - package_namespace=self.package_namespace, - package_type=self.package_type, - package_p_u_r_l_appendix=self.package_purl_appendix, - copyright=self.copyright, - license_name=self.license_name, - license_text=self.license_text, - url=self.url, - first_party=self.first_party, - exclude_from_notice=self.exclude_from_notice, - pre_selected=self.pre_selected, - follow_up=self.follow_up, - origin_id=self.origin_id, - origin_ids=self.origin_ids, - criticality=self.criticality, - was_preferred=self.was_preferred, - ) - - -class Metadata(BaseModel): - model_config = ConfigDict(frozen=True, extra="allow") - project_id: str - file_creation_date: str - project_title: str - project_version: str | None = None - expected_release_date: str | None = None - build_date: str | None = None - - def to_opossum_model(self) -> MetadataModel: - return MetadataModel(**self.model_dump()) - - -class ExternalAttributionSource(BaseModel): - model_config = ConfigDict(frozen=True, extra="forbid") - name: str - priority: int - is_relevant_for_preferred: bool | None = None - - def to_opossum_model(self) -> ExternalAttributionSourceModel: - return ExternalAttributionSourceModel( - name=self.name, - priority=self.priority, - is_relevant_for_preferred=self.is_relevant_for_preferred, - ) diff --git a/src/opossum_lib/core/entities/opossum_package.py b/src/opossum_lib/core/entities/opossum_package.py new file mode 100644 index 00000000..cffe9f6d --- /dev/null +++ b/src/opossum_lib/core/entities/opossum_package.py @@ -0,0 +1,60 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Literal + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.core.entities.source_info import SourceInfo +from opossum_lib.shared.entities.opossum_input_file_model import OpossumPackageModel + + +class OpossumPackage(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + source: SourceInfo + attribution_confidence: int | None = None + comment: str | None = None + package_name: str | None = None + package_version: str | None = None + package_namespace: str | None = None + package_type: str | None = None + package_purl_appendix: str | None = None + copyright: str | None = None + license_name: str | None = None + license_text: str | None = None + url: str | None = None + first_party: bool | None = None + exclude_from_notice: bool | None = None + pre_selected: bool | None = None + follow_up: Literal["FOLLOW_UP"] | None = None + origin_id: str | None = None + origin_ids: tuple[str, ...] | None = None + criticality: Literal["high"] | Literal["medium"] | None = None + was_preferred: bool | None = None + + def to_opossum_model(self) -> OpossumPackageModel: + return OpossumPackageModel( + source=self.source.to_opossum_model(), + attribution_confidence=self.attribution_confidence, + comment=self.comment, + package_name=self.package_name, + package_version=self.package_version, + package_namespace=self.package_namespace, + package_type=self.package_type, + package_p_u_r_l_appendix=self.package_purl_appendix, + copyright=self.copyright, + license_name=self.license_name, + license_text=self.license_text, + url=self.url, + first_party=self.first_party, + exclude_from_notice=self.exclude_from_notice, + pre_selected=self.pre_selected, + follow_up=self.follow_up, + origin_id=self.origin_id, + origin_ids=self.origin_ids, + criticality=self.criticality, + was_preferred=self.was_preferred, + ) diff --git a/src/opossum_lib/core/entities/resource.py b/src/opossum_lib/core/entities/resource.py new file mode 100644 index 00000000..ef533c19 --- /dev/null +++ b/src/opossum_lib/core/entities/resource.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + + +from __future__ import annotations + +from collections.abc import Iterable +from enum import Enum, auto +from pathlib import PurePath + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.shared.entities.opossum_input_file_model import ResourceInFileModel + + +def _convert_path_to_str(path: PurePath) -> str: + return str(path).replace("\\", "/") + + +class ResourceType(Enum): + FILE = auto() + FOLDER = auto() + + +class Resource(BaseModel): + model_config = ConfigDict(frozen=False, extra="forbid") + path: PurePath + type: ResourceType | None = None + attributions: list[OpossumPackage] = [] + children: dict[str, Resource] = {} + + def to_opossum_model(self) -> ResourceInFileModel: + if self.children or self.type == ResourceType.FOLDER: + return { + _convert_path_to_str( + child.path.relative_to(self.path) + ): child.to_opossum_model() + for child in self.children.values() + } + else: + return 1 + + def add_resource(self, resource: Resource) -> None: + if not resource.path.is_relative_to(self.path): + raise RuntimeError( + f"The path {resource.path} is not a child of this node at {self.path}." + ) + remaining_path_parts = resource.path.relative_to(self.path).parts + if remaining_path_parts: + self._add_resource(resource, remaining_path_parts) + else: + self._update(resource) + + def _add_resource( + self, resource: Resource, remaining_path_parts: Iterable[str] + ) -> None: + if not remaining_path_parts: + self._update(resource) + return + next, *rest_parts = remaining_path_parts + if next not in self.children: + self.children[next] = Resource(path=self.path / next) + self.children[next]._add_resource(resource, rest_parts) + + def _update(self, other: Resource) -> None: + if self.path != other.path: + raise RuntimeError( + "Trying to merge nodes with different paths: " + + f"{self.path} vs. {other.path}" + ) + if self.type and other.type and self.type != other.type: + raise RuntimeError( + "Trying to merge incompatible node types. " + + f"Current node is {self.type}. Other is {other.type}" + ) + self.type = self.type or other.type + self.attributions.extend(other.attributions) + for key, child in other.children.items(): + if key in self.children: + self.children[key]._update(child) + else: + self.children[key] = child diff --git a/src/opossum_lib/core/entities/scan_results.py b/src/opossum_lib/core/entities/scan_results.py new file mode 100644 index 00000000..6c7bfffe --- /dev/null +++ b/src/opossum_lib/core/entities/scan_results.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import uuid +from collections import defaultdict +from copy import deepcopy +from dataclasses import field + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.core.entities.base_url_for_sources import BaseUrlsForSources +from opossum_lib.core.entities.external_attribution_source import ( + ExternalAttributionSource, +) +from opossum_lib.core.entities.frequent_license import FrequentLicense +from opossum_lib.core.entities.metadata import Metadata +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.resource import Resource, _convert_path_to_str +from opossum_lib.shared.entities.opossum_input_file_model import ( + OpossumInputFileModel, + OpossumPackageIdentifierModel, + OpossumPackageModel, + ResourcePathModel, +) + + +def _default_attribution_id_mapper() -> dict[OpossumPackage, str]: + return defaultdict(lambda: str(uuid.uuid4())) + + +class ScanResults(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + metadata: Metadata + resources: list[Resource] + attribution_breakpoints: list[str] = [] + external_attribution_sources: dict[str, ExternalAttributionSource] = {} + frequent_licenses: list[FrequentLicense] | None = None + files_with_children: list[str] | None = None + base_urls_for_sources: BaseUrlsForSources | None = None + attribution_to_id: dict[OpossumPackage, str] = field( + default_factory=_default_attribution_id_mapper + ) + unassigned_attributions: list[OpossumPackage] = [] + + def to_opossum_model(self) -> OpossumInputFileModel: + external_attributions, resources_to_attributions = ( + self.create_attribution_mapping(self.resources) + ) + external_attributions.update(self._get_unassigned_attributions()) + + frequent_licenses = None + if self.frequent_licenses: + frequent_licenses = [ + license.to_opossum_model() for license in self.frequent_licenses + ] + base_urls_for_sources = ( + self.base_urls_for_sources and self.base_urls_for_sources.to_opossum_model() + ) + + external_attribution_sources = { + key: val.to_opossum_model() + for (key, val) in self.external_attribution_sources.items() + } + + return OpossumInputFileModel( + metadata=self.metadata.to_opossum_model(), + resources={ + str(resource.path): resource.to_opossum_model() + for resource in self.resources + }, + external_attributions=external_attributions, + resources_to_attributions=resources_to_attributions, + attribution_breakpoints=deepcopy(self.attribution_breakpoints), + external_attribution_sources=external_attribution_sources, + frequent_licenses=frequent_licenses, + files_with_children=deepcopy(self.files_with_children), + base_urls_for_sources=base_urls_for_sources, + ) + + def _get_unassigned_attributions( + self, + ) -> dict[OpossumPackageIdentifierModel, OpossumPackageModel]: + if self.unassigned_attributions: + result = {} + for unassigned_attribution in self.unassigned_attributions: + if unassigned_attribution in self.attribution_to_id: + package_identifier = self.attribution_to_id[unassigned_attribution] + result[package_identifier] = ( + unassigned_attribution.to_opossum_model() + ) + else: + package_identifier = str(uuid.uuid4()) + self.attribution_to_id[unassigned_attribution] = package_identifier + result[package_identifier] = ( + unassigned_attribution.to_opossum_model() + ) + return result + else: + return {} + + def create_attribution_mapping( + self, + root_nodes: list[Resource], + ) -> tuple[ + dict[OpossumPackageIdentifierModel, OpossumPackageModel], + dict[ResourcePathModel, list[OpossumPackageIdentifierModel]], + ]: + external_attributions: dict[ + OpossumPackageIdentifierModel, OpossumPackageModel + ] = {} + resources_to_attributions: dict[ + ResourcePathModel, list[OpossumPackageIdentifierModel] + ] = {} + + def process_node(node: Resource) -> None: + path = _convert_path_to_str(node.path) + if not path.startswith("/"): + # the / is required by OpossumUI + path = "/" + path + + node_attributions_by_id = { + self.get_attribution_key(a): a.to_opossum_model() + for a in node.attributions + } + external_attributions.update(node_attributions_by_id) + + if len(node_attributions_by_id) > 0: + resources_to_attributions[path] = list(node_attributions_by_id.keys()) + + for child in node.children.values(): + process_node(child) + + for root in root_nodes: + process_node(root) + + return external_attributions, resources_to_attributions + + def get_attribution_key( + self, attribution: OpossumPackage + ) -> OpossumPackageIdentifierModel: + id = self.attribution_to_id[attribution] + self.attribution_to_id[attribution] = id + return id diff --git a/src/opossum_lib/core/entities/source_info.py b/src/opossum_lib/core/entities/source_info.py new file mode 100644 index 00000000..f83aa6ae --- /dev/null +++ b/src/opossum_lib/core/entities/source_info.py @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from opossum_lib.shared.entities.opossum_input_file_model import SourceInfoModel + + +class SourceInfo(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + name: str + document_confidence: int | float | None = 0 + additional_name: str | None = None + + def to_opossum_model(self) -> SourceInfoModel: + return SourceInfoModel( + name=self.name, + document_confidence=self.document_confidence, + additional_name=self.additional_name, + ) diff --git a/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py index ef3387ea..1712030d 100644 --- a/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py +++ b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py @@ -5,20 +5,23 @@ from copy import deepcopy from pathlib import PurePath -from opossum_lib.core.entities.opossum import ( - BaseUrlsForSources, +from opossum_lib.core.entities.base_url_for_sources import BaseUrlsForSources +from opossum_lib.core.entities.external_attribution_source import ( ExternalAttributionSource, - FrequentLicense, - Metadata, +) +from opossum_lib.core.entities.frequent_license import FrequentLicense +from opossum_lib.core.entities.metadata import Metadata +from opossum_lib.core.entities.opossum import ( Opossum, - OpossumPackage, - OpossumPackageIdentifierModel, +) +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.resource import ( Resource, ResourceType, - ScanResults, - SourceInfo, _convert_path_to_str, ) +from opossum_lib.core.entities.scan_results import ScanResults +from opossum_lib.core.entities.source_info import SourceInfo from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel from opossum_lib.shared.entities.opossum_input_file_model import ( ExternalAttributionSourceModel as FileExternalAttributionSource, @@ -27,6 +30,7 @@ FrequentLicenseModel, MetadataModel, OpossumInputFileModel, + OpossumPackageIdentifierModel, OpossumPackageModel, ResourceInFileModel, ResourcePathModel, diff --git a/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py index 5ca1677c..c7663e91 100644 --- a/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py +++ b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py @@ -7,15 +7,14 @@ import uuid from pathlib import PurePath +from opossum_lib.core.entities.metadata import Metadata from opossum_lib.core.entities.opossum import ( - Metadata, Opossum, - OpossumPackage, - Resource, - ResourceType, - ScanResults, - SourceInfo, ) +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.resource import Resource, ResourceType +from opossum_lib.core.entities.scan_results import ScanResults +from opossum_lib.core.entities.source_info import SourceInfo from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( FileModel, diff --git a/tests/core/entities/generators/external_attribution_source_provider.py b/tests/core/entities/generators/external_attribution_source_provider.py index b52e3984..10699dfc 100644 --- a/tests/core/entities/generators/external_attribution_source_provider.py +++ b/tests/core/entities/generators/external_attribution_source_provider.py @@ -7,7 +7,9 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person import Provider as PersonProvider -from opossum_lib.core.entities.opossum import ExternalAttributionSource +from opossum_lib.core.entities.external_attribution_source import ( + ExternalAttributionSource, +) from tests.shared.generator_helpers import entry_or_none diff --git a/tests/core/entities/generators/metadata_provider.py b/tests/core/entities/generators/metadata_provider.py index 77b54dbb..2be31f7c 100644 --- a/tests/core/entities/generators/metadata_provider.py +++ b/tests/core/entities/generators/metadata_provider.py @@ -8,7 +8,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.entities.opossum import Metadata +from opossum_lib.core.entities.metadata import Metadata from tests.shared.generator_helpers import entry_or_none diff --git a/tests/core/entities/generators/opossum_provider.py b/tests/core/entities/generators/opossum_provider.py index e6d527aa..8f0ab056 100644 --- a/tests/core/entities/generators/opossum_provider.py +++ b/tests/core/entities/generators/opossum_provider.py @@ -5,7 +5,8 @@ from faker.providers import BaseProvider -from opossum_lib.core.entities.opossum import Opossum, ScanResults +from opossum_lib.core.entities.opossum import Opossum +from opossum_lib.core.entities.scan_results import ScanResults from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel from tests.core.entities.generators.scan_results_provider import ScanResultsProvider from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa: E501 diff --git a/tests/core/entities/generators/package_provider.py b/tests/core/entities/generators/package_provider.py index 3ca9b9b7..1878085f 100644 --- a/tests/core/entities/generators/package_provider.py +++ b/tests/core/entities/generators/package_provider.py @@ -10,7 +10,8 @@ from faker.providers.misc.en_US import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.entities.opossum import OpossumPackage, SourceInfo +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.source_info import SourceInfo from tests.core.entities.generators.source_info_provider import SourceInfoProvider from tests.shared.generator_helpers import entry_or_none, random_list diff --git a/tests/core/entities/generators/resource_provider.py b/tests/core/entities/generators/resource_provider.py index b9d6069b..79bf4f97 100644 --- a/tests/core/entities/generators/resource_provider.py +++ b/tests/core/entities/generators/resource_provider.py @@ -8,11 +8,8 @@ from faker.providers.file import Provider as FileProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.core.entities.opossum import ( - OpossumPackage, - Resource, - ResourceType, -) +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.resource import Resource, ResourceType from tests.core.entities.generators.package_provider import PackageProvider from tests.shared.generator_helpers import random_list diff --git a/tests/core/entities/generators/scan_results_provider.py b/tests/core/entities/generators/scan_results_provider.py index 96f26a49..00d2a435 100644 --- a/tests/core/entities/generators/scan_results_provider.py +++ b/tests/core/entities/generators/scan_results_provider.py @@ -8,16 +8,15 @@ from faker.providers.file.en_US import Provider as FileProvider from faker.providers.misc.en_US import Provider as MiscProvider -from opossum_lib.core.entities.opossum import ( - BaseUrlsForSources, +from opossum_lib.core.entities.base_url_for_sources import BaseUrlsForSources +from opossum_lib.core.entities.external_attribution_source import ( ExternalAttributionSource, - FrequentLicense, - Metadata, - OpossumPackage, - Resource, - ResourceType, - ScanResults, ) +from opossum_lib.core.entities.frequent_license import FrequentLicense +from opossum_lib.core.entities.metadata import Metadata +from opossum_lib.core.entities.opossum_package import OpossumPackage +from opossum_lib.core.entities.resource import Resource, ResourceType +from opossum_lib.core.entities.scan_results import ScanResults from tests.core.entities.generators.external_attribution_source_provider import ( ExternalAttributionSourceProvider, ) diff --git a/tests/core/entities/generators/source_info_provider.py b/tests/core/entities/generators/source_info_provider.py index 226107c3..a61a0cad 100644 --- a/tests/core/entities/generators/source_info_provider.py +++ b/tests/core/entities/generators/source_info_provider.py @@ -7,7 +7,7 @@ from faker.providers.misc import Provider as MiscProvider from faker.providers.person.en_US import Provider as PersonProvider -from opossum_lib.core.entities.opossum import SourceInfo +from opossum_lib.core.entities.source_info import SourceInfo from tests.shared.generator_helpers import entry_or_none diff --git a/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py index 683d57ec..00188081 100644 --- a/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py +++ b/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py @@ -6,7 +6,7 @@ import pytest from _pytest.logging import LogCaptureFixture -from opossum_lib.core.entities.opossum import Resource +from opossum_lib.core.entities.resource import Resource from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 convert_to_opossum, ) From c06bb6d2415ed1df2f31b927f752439b913c13a8 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 14:37:20 +0100 Subject: [PATCH 45/58] refactor: review -- improve writing to opossum files --- src/opossum_lib/cli.py | 4 ++-- .../{opossum_generator.py => generate_impl.py} | 2 +- ...sum_file_writer.py => write_opossum_file.py} | 17 +++++++++-------- tests/core/services/test_opossum_file_writer.py | 2 +- tests/test_cli.py | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) rename src/opossum_lib/core/services/{opossum_generator.py => generate_impl.py} (88%) rename src/opossum_lib/core/services/{opossum_file_writer.py => write_opossum_file.py} (78%) diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 9b731178..443d4361 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -9,10 +9,10 @@ import click -from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.core.services.opossum_generator import ( +from opossum_lib.core.services.generate_impl import ( generate_impl, ) +from opossum_lib.core.services.input_reader import InputReader from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( OpossumFileReader, ) diff --git a/src/opossum_lib/core/services/opossum_generator.py b/src/opossum_lib/core/services/generate_impl.py similarity index 88% rename from src/opossum_lib/core/services/opossum_generator.py rename to src/opossum_lib/core/services/generate_impl.py index 176c7b7e..977927ea 100644 --- a/src/opossum_lib/core/services/opossum_generator.py +++ b/src/opossum_lib/core/services/generate_impl.py @@ -4,7 +4,7 @@ from pathlib import Path from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.core.services.opossum_file_writer import write_opossum_file +from opossum_lib.core.services.write_opossum_file import write_opossum_file def generate_impl(input_readers: list[InputReader], output_file: Path) -> None: diff --git a/src/opossum_lib/core/services/opossum_file_writer.py b/src/opossum_lib/core/services/write_opossum_file.py similarity index 78% rename from src/opossum_lib/core/services/opossum_file_writer.py rename to src/opossum_lib/core/services/write_opossum_file.py index 9f011819..534a9526 100644 --- a/src/opossum_lib/core/services/opossum_file_writer.py +++ b/src/opossum_lib/core/services/write_opossum_file.py @@ -4,6 +4,8 @@ from pathlib import Path from zipfile import ZIP_DEFLATED, ZipFile +from pydantic import BaseModel + from opossum_lib.shared.constants import ( COMPRESSION_LEVEL, INPUT_JSON_NAME, @@ -25,18 +27,17 @@ def _write_output_json_if_existing( opossum_file_model: OpossumFileModel, zip_file: ZipFile ) -> None: if opossum_file_model.output_file: - zip_file.writestr( - OUTPUT_JSON_NAME, - opossum_file_model.output_file.model_dump_json( - exclude_none=True, indent=4, by_alias=True - ), - ) + _write_json_to_zip(zip_file, OUTPUT_JSON_NAME, opossum_file_model.output_file) def _write_input_json(opossum_file_model: OpossumFileModel, zip_file: ZipFile) -> None: + _write_json_to_zip(zip_file, INPUT_JSON_NAME, opossum_file_model.input_file) + + +def _write_json_to_zip(zip_file: ZipFile, sub_file_name: str, model: BaseModel) -> None: zip_file.writestr( - INPUT_JSON_NAME, - opossum_file_model.input_file.model_dump_json( + sub_file_name, + model.model_dump_json( indent=4, exclude_none=True, by_alias=True, diff --git a/tests/core/services/test_opossum_file_writer.py b/tests/core/services/test_opossum_file_writer.py index 0793ecb4..ce37df31 100644 --- a/tests/core/services/test_opossum_file_writer.py +++ b/tests/core/services/test_opossum_file_writer.py @@ -5,7 +5,7 @@ from pathlib import Path from zipfile import ZipFile -from opossum_lib.core.services.opossum_file_writer import write_opossum_file +from opossum_lib.core.services.write_opossum_file import write_opossum_file from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, diff --git a/tests/test_cli.py b/tests/test_cli.py index 2b635a47..f80b161a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,7 +11,7 @@ from click.testing import CliRunner, Result from opossum_lib.cli import generate -from opossum_lib.core.services.opossum_file_writer import write_opossum_file +from opossum_lib.core.services.write_opossum_file import write_opossum_file from opossum_lib.shared.constants import ( INPUT_JSON_NAME, OUTPUT_JSON_NAME, From 927b3eb32a1e9b71859fd4941127ac1b45dba3a7 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 14:40:09 +0100 Subject: [PATCH 46/58] refactor: review -- rename scancode data model * remove filler word data * scancode is treated as one word --- .../{scan_code_data_model.py => scancode_model.py} | 2 +- .../scancode/services/convert_to_opossum.py | 10 +++++----- .../scancode/services/scancode_format_reader.py | 8 ++++---- .../entities/generators/generate_scancode_file.py | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) rename src/opossum_lib/input_formats/scancode/entities/{scan_code_data_model.py => scancode_model.py} (98%) diff --git a/src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py b/src/opossum_lib/input_formats/scancode/entities/scancode_model.py similarity index 98% rename from src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py rename to src/opossum_lib/input_formats/scancode/entities/scancode_model.py index a0ea9002..1e825971 100644 --- a/src/opossum_lib/input_formats/scancode/entities/scan_code_data_model.py +++ b/src/opossum_lib/input_formats/scancode/entities/scancode_model.py @@ -155,7 +155,7 @@ class FileModel(BaseModel): urls: list[UrlModel] -class ScanCodeDataModel(BaseModel): +class ScancodeModel(BaseModel): dependencies: list files: list[FileModel] license_detections: list[GlobalLicenseDetectionModel] diff --git a/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py index c7663e91..19cb0149 100644 --- a/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py +++ b/src/opossum_lib/input_formats/scancode/services/convert_to_opossum.py @@ -16,15 +16,15 @@ from opossum_lib.core.entities.scan_results import ScanResults from opossum_lib.core.entities.source_info import SourceInfo from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( +from opossum_lib.input_formats.scancode.entities.scancode_model import ( FileModel, FileTypeModel, HeaderModel, - ScanCodeDataModel, + ScancodeModel, ) -def convert_to_opossum(scancode_data: ScanCodeDataModel) -> Opossum: +def convert_to_opossum(scancode_data: ScancodeModel) -> Opossum: resources = _extract_opossum_resources(scancode_data) scancode_header = _extract_scancode_header(scancode_data) @@ -42,7 +42,7 @@ def convert_to_opossum(scancode_data: ScanCodeDataModel) -> Opossum: ) -def _extract_scancode_header(scancode_data: ScanCodeDataModel) -> HeaderModel: +def _extract_scancode_header(scancode_data: ScancodeModel) -> HeaderModel: if len(scancode_data.headers) != 1: logging.error("Headers of ScanCode file are invalid.") sys.exit(1) @@ -50,7 +50,7 @@ def _extract_scancode_header(scancode_data: ScanCodeDataModel) -> HeaderModel: def _extract_opossum_resources( - scancode_data: ScanCodeDataModel, + scancode_data: ScancodeModel, ) -> list[Resource]: temp_root = Resource(path=PurePath("")) for file in scancode_data.files: diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py index d3cb7147..3738582a 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py @@ -11,8 +11,8 @@ Opossum, ) from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( - ScanCodeDataModel, +from opossum_lib.input_formats.scancode.entities.scancode_model import ( + ScancodeModel, ) from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 convert_to_opossum, @@ -33,7 +33,7 @@ def read(self) -> Opossum: return convert_to_opossum(scancode_data) @staticmethod - def load_scancode_json(path: Path) -> ScanCodeDataModel: + def load_scancode_json(path: Path) -> ScancodeModel: try: with open(path) as inp: json_data = json.load(inp) @@ -44,6 +44,6 @@ def load_scancode_json(path: Path) -> ScanCodeDataModel: logging.error(f"Error decoding json for file {path}.") sys.exit(1) - scancode_data = ScanCodeDataModel.model_validate(json_data) + scancode_data = ScancodeModel.model_validate(json_data) return scancode_data diff --git a/tests/input_formats/scancode/entities/generators/generate_scancode_file.py b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py index 5e283b70..9327293a 100644 --- a/tests/input_formats/scancode/entities/generators/generate_scancode_file.py +++ b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py @@ -20,7 +20,7 @@ from faker.providers.lorem.en_US import Provider as LoremProvider from faker.providers.misc import Provider as MiscProvider -from opossum_lib.input_formats.scancode.entities.scan_code_data_model import ( +from opossum_lib.input_formats.scancode.entities.scancode_model import ( CopyrightModel, EmailModel, ExtraDataModel, @@ -33,7 +33,7 @@ MatchModel, OptionsModel, ReferenceMatchModel, - ScanCodeDataModel, + ScancodeModel, SystemEnvironmentModel, UrlModel, ) @@ -68,7 +68,7 @@ def scancode_data( headers: list[HeaderModel] | None = None, packages: list | None = None, options: OptionsModel | None = None, - ) -> ScanCodeDataModel: + ) -> ScancodeModel: # TODO: #184 depending on which options are passed in additional_options # we need to generate different fields, e.g. --licenses # out of scope for now @@ -76,7 +76,7 @@ def scancode_data( license_detections = license_detections or self.global_license_detections(files) if headers is None: headers = [self.header(options=options)] - return ScanCodeDataModel( + return ScancodeModel( dependencies=dependencies or [], files=files, license_detections=license_detections, From d774d133c352eb89dbe543cfa6a694ccbe235d9d Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 14:50:25 +0100 Subject: [PATCH 47/58] refactor: review -- improve scancode file reader * rename to file reader * make the member function private --- src/opossum_lib/cli.py | 6 +++--- ...e_format_reader.py => scancode_file_reader.py} | 15 +++++++-------- .../opossum/entities/services/__init__.py | 0 tests/input_formats/opossum/services/__init__.py | 4 ++++ .../services/test_conversion_roundtrip.py | 0 ...est_opossum_file_model_to_opossum_converter.py | 0 .../services/test_opossum_format_reader.py | 6 ++++-- 7 files changed, 18 insertions(+), 13 deletions(-) rename src/opossum_lib/input_formats/scancode/services/{scancode_format_reader.py => scancode_file_reader.py} (70%) delete mode 100644 tests/input_formats/opossum/entities/services/__init__.py create mode 100644 tests/input_formats/opossum/services/__init__.py rename tests/input_formats/opossum/{entities => }/services/test_conversion_roundtrip.py (100%) rename tests/input_formats/opossum/{entities => }/services/test_opossum_file_model_to_opossum_converter.py (100%) rename tests/input_formats/opossum/{entities => }/services/test_opossum_format_reader.py (88%) diff --git a/src/opossum_lib/cli.py b/src/opossum_lib/cli.py index 443d4361..7cf60bd9 100644 --- a/src/opossum_lib/cli.py +++ b/src/opossum_lib/cli.py @@ -16,8 +16,8 @@ from opossum_lib.input_formats.opossum.services.opossum_file_reader import ( OpossumFileReader, ) -from opossum_lib.input_formats.scancode.services.scancode_format_reader import ( - ScancodeFormatReader, +from opossum_lib.input_formats.scancode.services.scancode_file_reader import ( + ScancodeFileReader, ) @@ -74,7 +74,7 @@ def generate( logging.error("Merging of multiple files not yet supported!") sys.exit(1) input_readers: list[InputReader] = [] - input_readers += [ScancodeFormatReader(path=path) for path in scancode_json_files] + input_readers += [ScancodeFileReader(path=path) for path in scancode_json_files] input_readers += [OpossumFileReader(path=path) for path in opossum_files] generate_impl(input_readers=input_readers, output_file=Path(outfile)) diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py similarity index 70% rename from src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py rename to src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py index 3738582a..6309277a 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_format_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py @@ -19,7 +19,7 @@ ) -class ScancodeFormatReader(InputReader): +class ScancodeFileReader(InputReader): path: Path def __init__(self, path: Path): @@ -28,20 +28,19 @@ def __init__(self, path: Path): def read(self) -> Opossum: logging.info(f"Converting scancode to opossum {self.path}") - scancode_data = ScancodeFormatReader.load_scancode_json(self.path) + scancode_data = self._load_scancode_json() return convert_to_opossum(scancode_data) - @staticmethod - def load_scancode_json(path: Path) -> ScancodeModel: + def _load_scancode_json(self) -> ScancodeModel: try: - with open(path) as inp: - json_data = json.load(inp) + with open(self.path) as input_file: + json_data = json.load(input_file) except json.JSONDecodeError as e: - logging.error(f"Error decoding json for file {path}. Message: {e.msg}") + logging.error(f"Error decoding json for file {self.path}. Message: {e.msg}") sys.exit(1) except UnicodeDecodeError: - logging.error(f"Error decoding json for file {path}.") + logging.error(f"Error decoding json for file {self.path}.") sys.exit(1) scancode_data = ScancodeModel.model_validate(json_data) diff --git a/tests/input_formats/opossum/entities/services/__init__.py b/tests/input_formats/opossum/entities/services/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/input_formats/opossum/services/__init__.py b/tests/input_formats/opossum/services/__init__.py new file mode 100644 index 00000000..698616e0 --- /dev/null +++ b/tests/input_formats/opossum/services/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + diff --git a/tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py b/tests/input_formats/opossum/services/test_conversion_roundtrip.py similarity index 100% rename from tests/input_formats/opossum/entities/services/test_conversion_roundtrip.py rename to tests/input_formats/opossum/services/test_conversion_roundtrip.py diff --git a/tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py b/tests/input_formats/opossum/services/test_opossum_file_model_to_opossum_converter.py similarity index 100% rename from tests/input_formats/opossum/entities/services/test_opossum_file_model_to_opossum_converter.py rename to tests/input_formats/opossum/services/test_opossum_file_model_to_opossum_converter.py diff --git a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py b/tests/input_formats/opossum/services/test_opossum_format_reader.py similarity index 88% rename from tests/input_formats/opossum/entities/services/test_opossum_format_reader.py rename to tests/input_formats/opossum/services/test_opossum_format_reader.py index 9c3bc7c6..cf0d5d64 100644 --- a/tests/input_formats/opossum/entities/services/test_opossum_format_reader.py +++ b/tests/input_formats/opossum/services/test_opossum_format_reader.py @@ -1,4 +1,6 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 # # SPDX-License-Identifier: Apache-2.0 from pathlib import Path @@ -10,7 +12,7 @@ OpossumFileReader, ) -TEST_DATA_DIR = Path(__file__).resolve().parent.parent.parent.parent.parent / "data" +TEST_DATA_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data" def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: From a2afebb4db3556ee550e34cc6df6a74db31836ab Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 14:55:45 +0100 Subject: [PATCH 48/58] refactor: review -- extract shared CamelBaseModel --- .../shared/entities/camel_base_model.py | 14 ++++++++++++++ .../shared/entities/opossum_input_file_model.py | 11 +++-------- .../shared/entities/opossum_output_file_model.py | 9 ++------- 3 files changed, 19 insertions(+), 15 deletions(-) create mode 100644 src/opossum_lib/shared/entities/camel_base_model.py diff --git a/src/opossum_lib/shared/entities/camel_base_model.py b/src/opossum_lib/shared/entities/camel_base_model.py new file mode 100644 index 00000000..4ef53eee --- /dev/null +++ b/src/opossum_lib/shared/entities/camel_base_model.py @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict +from pydantic.alias_generators import to_camel + + +class CamelBaseModel(BaseModel): + model_config = ConfigDict( + alias_generator=to_camel, populate_by_name=True, extra="forbid", frozen=True + ) diff --git a/src/opossum_lib/shared/entities/opossum_input_file_model.py b/src/opossum_lib/shared/entities/opossum_input_file_model.py index 39398dc1..b664e368 100644 --- a/src/opossum_lib/shared/entities/opossum_input_file_model.py +++ b/src/opossum_lib/shared/entities/opossum_input_file_model.py @@ -8,20 +8,15 @@ from enum import Enum, auto from typing import Literal -from pydantic import BaseModel, ConfigDict, model_serializer -from pydantic.alias_generators import to_camel +from pydantic import ConfigDict, model_serializer + +from opossum_lib.shared.entities.camel_base_model import CamelBaseModel type OpossumPackageIdentifierModel = str type ResourcePathModel = str type ResourceInFileModel = dict[str, ResourceInFileModel] | int -class CamelBaseModel(BaseModel): - model_config = ConfigDict( - alias_generator=to_camel, populate_by_name=True, extra="forbid", frozen=True - ) - - class OpossumInputFileModel(CamelBaseModel): metadata: MetadataModel resources: ResourceInFileModel diff --git a/src/opossum_lib/shared/entities/opossum_output_file_model.py b/src/opossum_lib/shared/entities/opossum_output_file_model.py index de9da8b1..e9f9337e 100644 --- a/src/opossum_lib/shared/entities/opossum_output_file_model.py +++ b/src/opossum_lib/shared/entities/opossum_output_file_model.py @@ -9,14 +9,9 @@ from enum import Enum from typing import Annotated -from pydantic import BaseModel, ConfigDict, Field -from pydantic.alias_generators import to_camel +from pydantic import ConfigDict, Field - -class CamelBaseModel(BaseModel): - model_config = ConfigDict( - alias_generator=to_camel, populate_by_name=True, extra="forbid", frozen=True - ) +from opossum_lib.shared.entities.camel_base_model import CamelBaseModel class Metadata(CamelBaseModel): From 508710202db744bc93c2bbb592a8dee46a03e165 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 15:05:47 +0100 Subject: [PATCH 49/58] refactor: review -- adapt test structure Goal: Match the project structure as good as possible --- ...test_opossum_file_writer.py => test_write_opossum_file.py} | 0 ...del_to_opossum_converter.py => test_convert_to_opossum.py} | 2 +- ...t_opossum_format_reader.py => test_opossum_file_reader.py} | 0 tests/input_formats/scancode/entities/services/__init__.py | 0 tests/input_formats/scancode/services/__init__.py | 4 ++++ .../test_convert_to_opossum.py} | 0 .../{entities => }/services/test_get_attribution_info.py | 0 7 files changed, 5 insertions(+), 1 deletion(-) rename tests/core/services/{test_opossum_file_writer.py => test_write_opossum_file.py} (100%) rename tests/input_formats/opossum/services/{test_opossum_file_model_to_opossum_converter.py => test_convert_to_opossum.py} (97%) rename tests/input_formats/opossum/services/{test_opossum_format_reader.py => test_opossum_file_reader.py} (100%) delete mode 100644 tests/input_formats/scancode/entities/services/__init__.py create mode 100644 tests/input_formats/scancode/services/__init__.py rename tests/input_formats/scancode/{entities/services/test_scancode_data_to_opossum_converter.py => services/test_convert_to_opossum.py} (100%) rename tests/input_formats/scancode/{entities => }/services/test_get_attribution_info.py (100%) diff --git a/tests/core/services/test_opossum_file_writer.py b/tests/core/services/test_write_opossum_file.py similarity index 100% rename from tests/core/services/test_opossum_file_writer.py rename to tests/core/services/test_write_opossum_file.py diff --git a/tests/input_formats/opossum/services/test_opossum_file_model_to_opossum_converter.py b/tests/input_formats/opossum/services/test_convert_to_opossum.py similarity index 97% rename from tests/input_formats/opossum/services/test_opossum_file_model_to_opossum_converter.py rename to tests/input_formats/opossum/services/test_convert_to_opossum.py index e2f32fa2..d0975c9e 100644 --- a/tests/input_formats/opossum/services/test_opossum_file_model_to_opossum_converter.py +++ b/tests/input_formats/opossum/services/test_convert_to_opossum.py @@ -13,7 +13,7 @@ from tests.setup.opossum_file_faker_setup import OpossumFileFaker -class TestOpossumFileModelToOpossumConverter: +class TestConvertToOpossum: def test_output_file_moved(self, opossum_file_faker: OpossumFileFaker) -> None: output_file = opossum_file_faker.output_file() input_file = opossum_file_faker.opossum_file_content(out_file=output_file) diff --git a/tests/input_formats/opossum/services/test_opossum_format_reader.py b/tests/input_formats/opossum/services/test_opossum_file_reader.py similarity index 100% rename from tests/input_formats/opossum/services/test_opossum_format_reader.py rename to tests/input_formats/opossum/services/test_opossum_file_reader.py diff --git a/tests/input_formats/scancode/entities/services/__init__.py b/tests/input_formats/scancode/entities/services/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/input_formats/scancode/services/__init__.py b/tests/input_formats/scancode/services/__init__.py new file mode 100644 index 00000000..698616e0 --- /dev/null +++ b/tests/input_formats/scancode/services/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 + diff --git a/tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py b/tests/input_formats/scancode/services/test_convert_to_opossum.py similarity index 100% rename from tests/input_formats/scancode/entities/services/test_scancode_data_to_opossum_converter.py rename to tests/input_formats/scancode/services/test_convert_to_opossum.py diff --git a/tests/input_formats/scancode/entities/services/test_get_attribution_info.py b/tests/input_formats/scancode/services/test_get_attribution_info.py similarity index 100% rename from tests/input_formats/scancode/entities/services/test_get_attribution_info.py rename to tests/input_formats/scancode/services/test_get_attribution_info.py From 938c94efeb2aa6e054a2c6c4e0ad4d9f08885be2 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Thu, 30 Jan 2025 15:51:16 +0100 Subject: [PATCH 50/58] refactor: review -- group tests in class structures --- .../core/services/test_write_opossum_file.py | 50 +-- .../services/test_opossum_file_reader.py | 30 +- .../services/test_convert_to_opossum.py | 134 ++++---- .../services/test_get_attribution_info.py | 146 ++++---- tests/test_cli.py | 312 +++++++++--------- 5 files changed, 354 insertions(+), 318 deletions(-) diff --git a/tests/core/services/test_write_opossum_file.py b/tests/core/services/test_write_opossum_file.py index ce37df31..8a8260fa 100644 --- a/tests/core/services/test_write_opossum_file.py +++ b/tests/core/services/test_write_opossum_file.py @@ -14,28 +14,28 @@ from tests.setup.opossum_file_faker_setup import OpossumFileFaker -def test_only_input_information_available_writes_only_input_information( - tmp_path: Path, opossum_file_faker: OpossumFileFaker -) -> None: - opossum_file_content = OpossumFileModel( - input_file=opossum_file_faker.opossum_file_information() - ) - output_path = tmp_path / "output.opossum" - - write_opossum_file(opossum_file_content, output_path) - - with ZipFile(output_path, "r") as zip_file: - assert zip_file.namelist() == [INPUT_JSON_NAME] - - -def test_input_and_output_information_available_writes_both( - tmp_path: Path, opossum_file_faker: OpossumFileFaker -) -> None: - opossum_file_content = opossum_file_faker.opossum_file_content() - output_path = tmp_path / "output.opossum" - - write_opossum_file(opossum_file_content, output_path) - - with ZipFile(output_path, "r") as zip_file: - assert INPUT_JSON_NAME in zip_file.namelist() - assert OUTPUT_JSON_NAME in zip_file.namelist() +class TestWriteOpossumFile: + def test_only_input_information_available_writes_only_input_information( + self, tmp_path: Path, opossum_file_faker: OpossumFileFaker + ) -> None: + opossum_file_content = OpossumFileModel( + input_file=opossum_file_faker.opossum_file_information() + ) + output_path = tmp_path / "output.opossum" + + write_opossum_file(opossum_file_content, output_path) + + with ZipFile(output_path, "r") as zip_file: + assert zip_file.namelist() == [INPUT_JSON_NAME] + + def test_input_and_output_information_available_writes_both( + self, tmp_path: Path, opossum_file_faker: OpossumFileFaker + ) -> None: + opossum_file_content = opossum_file_faker.opossum_file_content() + output_path = tmp_path / "output.opossum" + + write_opossum_file(opossum_file_content, output_path) + + with ZipFile(output_path, "r") as zip_file: + assert INPUT_JSON_NAME in zip_file.namelist() + assert OUTPUT_JSON_NAME in zip_file.namelist() diff --git a/tests/input_formats/opossum/services/test_opossum_file_reader.py b/tests/input_formats/opossum/services/test_opossum_file_reader.py index cf0d5d64..a5741c11 100644 --- a/tests/input_formats/opossum/services/test_opossum_file_reader.py +++ b/tests/input_formats/opossum/services/test_opossum_file_reader.py @@ -15,22 +15,22 @@ TEST_DATA_DIR = Path(__file__).resolve().parent.parent.parent.parent / "data" -def test_read_opossum_file_corrupted_file_exits_1(caplog: LogCaptureFixture) -> None: - input_path = TEST_DATA_DIR / "opossum_input_corrupt.opossum" - opossum_format_reader = OpossumFileReader(input_path) +class TestOpossumFileReader: + def test_read_corrupted_file_exits_1(self, caplog: LogCaptureFixture) -> None: + input_path = TEST_DATA_DIR / "opossum_input_corrupt.opossum" + opossum_format_reader = OpossumFileReader(input_path) - with pytest.raises(SystemExit) as system_exit: - opossum_format_reader.read() - assert system_exit.value.code == 1 - assert "is corrupt and does not contain 'input.json'" in caplog.messages[0] + with pytest.raises(SystemExit) as system_exit: + opossum_format_reader.read() + assert system_exit.value.code == 1 + assert "is corrupt and does not contain 'input.json'" in caplog.messages[0] + def test_read_with_output_json(self) -> None: + input_path = TEST_DATA_DIR / "opossum_input_with_result.opossum" + opossum_format_reader = OpossumFileReader(input_path) -def test_read_opossum_file_containing_output_json() -> None: - input_path = TEST_DATA_DIR / "opossum_input_with_result.opossum" - opossum_format_reader = OpossumFileReader(input_path) + result = opossum_format_reader.read() - result = opossum_format_reader.read() - - assert result is not None - assert result.scan_results is not None - assert result.review_results is not None + assert result is not None + assert result.scan_results is not None + assert result.review_results is not None diff --git a/tests/input_formats/scancode/services/test_convert_to_opossum.py b/tests/input_formats/scancode/services/test_convert_to_opossum.py index 00188081..71ac4be4 100644 --- a/tests/input_formats/scancode/services/test_convert_to_opossum.py +++ b/tests/input_formats/scancode/services/test_convert_to_opossum.py @@ -13,64 +13,76 @@ from tests.setup.scancode_faker_setup import ScanCodeFaker -def test_extract_scancode_header_produces_expected_result( - scancode_faker: ScanCodeFaker, -) -> None: - scancode_data = scancode_faker.scancode_data() - opossum = convert_to_opossum( - scancode_data, - ) - metadata = opossum.scan_results.metadata - header = scancode_data.headers[0] - assert metadata.file_creation_date == header.end_timestamp - assert metadata.project_title == "ScanCode file" - - -def test_extract_scancode_header_errors_with_missing_header( - caplog: LogCaptureFixture, scancode_faker: ScanCodeFaker -) -> None: - scancode_data = scancode_faker.scancode_data(headers=[]) - - with pytest.raises(SystemExit): - convert_to_opossum(scancode_data) - - assert "header" in caplog.messages[0].lower() - - -def test_extract_scancode_header_error_with_multiple_headers( - caplog: LogCaptureFixture, scancode_faker: ScanCodeFaker -) -> None: - header1 = scancode_faker.header() - header2 = scancode_faker.header() - scancode_data = scancode_faker.scancode_data(headers=[header1, header2]) - - with pytest.raises(SystemExit): - convert_to_opossum(scancode_data) - - assert "header" in caplog.messages[0].lower() - - -def count_resources(resource: Resource) -> int: - return 1 + sum(count_resources(child) for child in resource.children.values()) - - -def count_attributions(resource: Resource) -> int: - return len(resource.attributions) + sum( - count_attributions(child) for child in resource.children.values() - ) - - -def test_convert_scancode_produces_expected_result( - scancode_faker: ScanCodeFaker, -) -> None: - scancode_data = scancode_faker.scancode_data() - opossum_data = convert_to_opossum(scancode_data) - - assert opossum_data.review_results is None - scan_results = opossum_data.scan_results - assert sum(count_resources(res) for res in scan_results.resources) == len( - scancode_data.files - ) - num_attributions = sum(count_attributions(res) for res in scan_results.resources) - num_license_detections = sum(len(f.license_detections) for f in scancode_data.files) - assert num_attributions == num_license_detections +class TestExtractScancodeHeader: + def test_produces_expected_result( + self, + scancode_faker: ScanCodeFaker, + ) -> None: + scancode_data = scancode_faker.scancode_data() + opossum = convert_to_opossum( + scancode_data, + ) + metadata = opossum.scan_results.metadata + header = scancode_data.headers[0] + assert metadata.file_creation_date == header.end_timestamp + assert metadata.project_title == "ScanCode file" + + def test_errors_with_missing_header( + self, caplog: LogCaptureFixture, scancode_faker: ScanCodeFaker + ) -> None: + scancode_data = scancode_faker.scancode_data(headers=[]) + + with pytest.raises(SystemExit): + convert_to_opossum(scancode_data) + + assert "header" in caplog.messages[0].lower() + + def test_error_with_multiple_headers( + self, caplog: LogCaptureFixture, scancode_faker: ScanCodeFaker + ) -> None: + header1 = scancode_faker.header() + header2 = scancode_faker.header() + scancode_data = scancode_faker.scancode_data(headers=[header1, header2]) + + with pytest.raises(SystemExit): + convert_to_opossum(scancode_data) + + assert "header" in caplog.messages[0].lower() + + +class TestConvertToOpossumFull: + @staticmethod + def _count_resources(resource: Resource) -> int: + return 1 + sum( + TestConvertToOpossumFull._count_resources(child) + for child in resource.children.values() + ) + + @staticmethod + def _count_attributions(resource: Resource) -> int: + return len(resource.attributions) + sum( + TestConvertToOpossumFull._count_attributions(child) + for child in resource.children.values() + ) + + def test_convert( + self, + scancode_faker: ScanCodeFaker, + ) -> None: + scancode_data = scancode_faker.scancode_data() + opossum_data = convert_to_opossum(scancode_data) + + assert opossum_data.review_results is None + scan_results = opossum_data.scan_results + assert sum( + TestConvertToOpossumFull._count_resources(res) + for res in scan_results.resources + ) == len(scancode_data.files) + num_attributions = sum( + TestConvertToOpossumFull._count_attributions(res) + for res in scan_results.resources + ) + num_license_detections = sum( + len(f.license_detections) for f in scancode_data.files + ) + assert num_attributions == num_license_detections diff --git a/tests/input_formats/scancode/services/test_get_attribution_info.py b/tests/input_formats/scancode/services/test_get_attribution_info.py index 869fbbc8..7a30c39b 100644 --- a/tests/input_formats/scancode/services/test_get_attribution_info.py +++ b/tests/input_formats/scancode/services/test_get_attribution_info.py @@ -1,4 +1,6 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# SPDX-FileCopyrightText: TNG Technology Consulting GmbH +# # +# SPDX-License-Identifier: Apache-2.0 # # SPDX-License-Identifier: Apache-2.0 @@ -14,73 +16,81 @@ from tests.setup.scancode_faker_setup import ScanCodeFaker -def test_get_attribution_info_directory(scancode_faker: ScanCodeFaker) -> None: - folder = scancode_faker.single_folder(path="some/single/folder") - scancode_data = scancode_faker.scancode_data(files=[folder]) - opossum = convert_to_opossum(scancode_data) - assert len(opossum.scan_results.resources) == 1 - assert opossum.scan_results.resources[0].attributions == [] +class TestGetAttributionInfo: + def test_get_attribution_info_directory( + self, scancode_faker: ScanCodeFaker + ) -> None: + folder = scancode_faker.single_folder(path="some/single/folder") + scancode_data = scancode_faker.scancode_data(files=[folder]) + opossum = convert_to_opossum(scancode_data) + assert len(opossum.scan_results.resources) == 1 + assert opossum.scan_results.resources[0].attributions == [] + def test_get_attribution_info_from_file_without_detections( + self, + scancode_faker: ScanCodeFaker, + ) -> None: + file = scancode_faker.single_file( + path="some/single/file", license_detections=[] + ) + scancode_data = scancode_faker.scancode_data(files=[file]) + opossum = convert_to_opossum(scancode_data) + assert len(opossum.scan_results.resources) == 1 + assert opossum.scan_results.resources[0].attributions == [] -def test_get_attribution_info_from_file_without_detections( - scancode_faker: ScanCodeFaker, -) -> None: - file = scancode_faker.single_file(path="some/single/file", license_detections=[]) - scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = convert_to_opossum(scancode_data) - assert len(opossum.scan_results.resources) == 1 - assert opossum.scan_results.resources[0].attributions == [] + def test_get_attribution_info_file_multiple( + self, scancode_faker: ScanCodeFaker + ) -> None: + match1 = scancode_faker.match( + license_expression_spdx="Apache-2.0", + from_file="A", + score=75, + rule_relevance=50, + ) + match2 = scancode_faker.match( + license_expression_spdx="Apache-2.0", + from_file="A", + score=95, + rule_relevance=50, + ) + match3 = scancode_faker.match( + license_expression_spdx="MIT", + from_file="A", + score=50, + rule_relevance=50, + ) + license1 = scancode_faker.license_detection( + license_expression_spdx="Apache-2.0", + matches=[match1, match2], + ) + license2 = scancode_faker.license_detection( + license_expression_spdx="MIT", + matches=[match3], + ) + copyright1 = scancode_faker.copyright(copyright="Me") + copyright2 = scancode_faker.copyright(copyright="Myself") + copyright3 = scancode_faker.copyright(copyright="I") + file = scancode_faker.single_file( + path="A", + license_detections=[license1, license2], + copyrights=[copyright1, copyright2, copyright3], + ) + scancode_data = scancode_faker.scancode_data(files=[file]) + opossum = convert_to_opossum(scancode_data) + attributions = ( + opossum.to_opossum_model().input_file.external_attributions.values() + ) - -def test_get_attribution_info_file_multiple(scancode_faker: ScanCodeFaker) -> None: - match1 = scancode_faker.match( - license_expression_spdx="Apache-2.0", - from_file="A", - score=75, - rule_relevance=50, - ) - match2 = scancode_faker.match( - license_expression_spdx="Apache-2.0", - from_file="A", - score=95, - rule_relevance=50, - ) - match3 = scancode_faker.match( - license_expression_spdx="MIT", - from_file="A", - score=50, - rule_relevance=50, - ) - license1 = scancode_faker.license_detection( - license_expression_spdx="Apache-2.0", - matches=[match1, match2], - ) - license2 = scancode_faker.license_detection( - license_expression_spdx="MIT", - matches=[match3], - ) - copyright1 = scancode_faker.copyright(copyright="Me") - copyright2 = scancode_faker.copyright(copyright="Myself") - copyright3 = scancode_faker.copyright(copyright="I") - file = scancode_faker.single_file( - path="A", - license_detections=[license1, license2], - copyrights=[copyright1, copyright2, copyright3], - ) - scancode_data = scancode_faker.scancode_data(files=[file]) - opossum = convert_to_opossum(scancode_data) - attributions = opossum.to_opossum_model().input_file.external_attributions.values() - - expected1 = OpossumPackageModel( - source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), - license_name="Apache-2.0", - copyright="Me\nMyself\nI", - attribution_confidence=95, - ) - expected2 = OpossumPackageModel( - source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), - license_name="MIT", - copyright="Me\nMyself\nI", - attribution_confidence=50, - ) - assert set(attributions) == {expected1, expected2} + expected1 = OpossumPackageModel( + source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), + license_name="Apache-2.0", + copyright="Me\nMyself\nI", + attribution_confidence=95, + ) + expected2 = OpossumPackageModel( + source=SourceInfoModel(name=SCANCODE_SOURCE_NAME), + license_name="MIT", + copyright="Me\nMyself\nI", + attribution_confidence=50, + ) + assert set(attributions) == {expected1, expected2} diff --git a/tests/test_cli.py b/tests/test_cli.py index f80b161a..a513091b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import json from pathlib import Path from typing import Any @@ -22,133 +24,130 @@ test_data_path = Path(__file__).resolve().parent / "data" -def generate_valid_scan_code_argument( - filename: str = "scancode_input.json", -) -> list[str]: - return ["--scan-code-json", str(test_data_path / filename)] - - -def generate_valid_opossum_argument( - filename: str = "opossum_input.opossum", -) -> list[str]: - return ["--opossum", str(test_data_path / filename)] - - def run_with_command_line_arguments(cmd_line_arguments: list[str]) -> Result: runner = CliRunner() result = runner.invoke(generate, cmd_line_arguments) return result -def test_successful_conversion_of_input_only_opossum_file(tmp_path: Path) -> None: - output_file = str(tmp_path / "output_opossum.opossum") - result = run_with_command_line_arguments( - [ - "--opossum", - str(test_data_path / "opossum_input.opossum"), - "-o", - output_file, - ], - ) - - assert result.exit_code == 0 - expected_opossum_dict = read_json_from_file("opossum_input.json") - opossum_dict = read_input_json_from_opossum(output_file) - - # Doing individual asserts as otherwise the diff viewer does no longer work - # in case of errors - assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) - - -def test_successful_conversion_of_scancode_file(tmp_path: Path) -> None: - output_file = str(tmp_path / "output_scancode.opossum") - result = run_with_command_line_arguments( - [ - "--scan-code-json", - str(test_data_path / "scancode_input.json"), - "-o", - output_file, - ], - ) - - assert result.exit_code == 0 - expected_opossum_dict = read_json_from_file("expected_scancode.json") - opossum_dict = read_input_json_from_opossum(output_file) - - md = opossum_dict.pop("metadata") - expected_md = expected_opossum_dict.pop("metadata") - md["projectId"] = expected_md["projectId"] - assert md == expected_md - - # Python has hash salting, which means the hashes changes between sessions. - # This means that the IDs of the attributions change as they are based on hashes - # Thus we need to compare externalAttributions and resourcesToAttributions - # structurally - resources_inlined = inline_attributions_into_resources( - resources_with_ids=opossum_dict.pop("resourcesToAttributions"), - all_attributions=opossum_dict.pop("externalAttributions"), - ) - expected_resources_inlined = inline_attributions_into_resources( - resources_with_ids=expected_opossum_dict.pop("resourcesToAttributions"), - all_attributions=expected_opossum_dict.pop("externalAttributions"), - ) - assert resources_inlined == expected_resources_inlined - assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) - - -def test_successful_conversion_of_input_and_output_opossum_file(tmp_path: Path) -> None: - output_file = str(tmp_path / "output_opossum.opossum") - result = run_with_command_line_arguments( - [ - "--opossum", - str(test_data_path / "opossum_input_with_result.opossum"), - "-o", - output_file, - ], - ) - - assert result.exit_code == 0 - - # Doing individual asserts as otherwise the diff viewer does no longer work - # in case of errors - assert_input_json_matches_expectations(output_file) - assert_output_json_matches_expectations(output_file) - +class TestConvertOpossumFiles: + def test_successful_conversion_of_input_only_opossum_file( + self, tmp_path: Path + ) -> None: + output_file = str(tmp_path / "output_opossum.opossum") + result = run_with_command_line_arguments( + [ + "--opossum", + str(test_data_path / "opossum_input.opossum"), + "-o", + output_file, + ], + ) -def assert_input_json_matches_expectations(output_file: str) -> None: - expected_opossum_dict = read_json_from_file("opossum_input.json") - opossum_dict = read_input_json_from_opossum(output_file) - assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) + assert result.exit_code == 0 + expected_opossum_dict = _read_json_from_file("opossum_input.json") + opossum_dict = _read_input_json_from_opossum(output_file) + + # Doing individual asserts as otherwise the diff viewer does no longer work + # in case of errors + _assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) + + def test_successful_conversion_of_input_and_output_opossum_file( + self, tmp_path: Path + ) -> None: + output_file = str(tmp_path / "output_opossum.opossum") + result = run_with_command_line_arguments( + [ + "--opossum", + str(test_data_path / "opossum_input_with_result.opossum"), + "-o", + output_file, + ], + ) + assert result.exit_code == 0 -def assert_output_json_matches_expectations(output_file: str) -> None: - expected_opossum_dict = read_json_from_file("opossum_output.json") - opossum_dict = read_output_json_from_opossum(output_file) - assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) + # Doing individual asserts as otherwise the diff viewer does no longer work + # in case of errors + TestConvertOpossumFiles._assert_input_json_matches_expectations(output_file) + TestConvertOpossumFiles._assert_output_json_matches_expectations(output_file) + + @staticmethod + def _assert_input_json_matches_expectations(output_file: str) -> None: + expected_opossum_dict = _read_json_from_file("opossum_input.json") + opossum_dict = _read_input_json_from_opossum(output_file) + _assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) + + @staticmethod + def _assert_output_json_matches_expectations(output_file: str) -> None: + expected_opossum_dict = _read_json_from_file("opossum_output.json") + opossum_dict = _read_output_json_from_opossum(output_file) + _assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) + + +class TestConvertScancodeFiles: + def test_successful_conversion_of_scancode_file(self, tmp_path: Path) -> None: + output_file = str(tmp_path / "output_scancode.opossum") + result = run_with_command_line_arguments( + [ + "--scan-code-json", + str(test_data_path / "scancode_input.json"), + "-o", + output_file, + ], + ) + assert result.exit_code == 0 + expected_opossum_dict = _read_json_from_file("expected_scancode.json") + opossum_dict = _read_input_json_from_opossum(output_file) + + md = opossum_dict.pop("metadata") + expected_md = expected_opossum_dict.pop("metadata") + md["projectId"] = expected_md["projectId"] + assert md == expected_md + + # Python has hash salting, which means the hashes changes between sessions. + # This means that the IDs of the attributions change as they are based on hashes + # Thus we need to compare externalAttributions and resourcesToAttributions + # structurally + resources_inlined = ( + TestConvertScancodeFiles._inline_attributions_into_resources( + resources_with_ids=opossum_dict.pop("resourcesToAttributions"), + all_attributions=opossum_dict.pop("externalAttributions"), + ) + ) + expected_resources_inlined = ( + TestConvertScancodeFiles._inline_attributions_into_resources( + resources_with_ids=expected_opossum_dict.pop("resourcesToAttributions"), + all_attributions=expected_opossum_dict.pop("externalAttributions"), + ) + ) + assert resources_inlined == expected_resources_inlined + _assert_expected_file_equals_generated_file(expected_opossum_dict, opossum_dict) -def inline_attributions_into_resources( - *, resources_with_ids: dict[str, list[str]], all_attributions: dict[str, Any] -) -> dict[str, set[OpossumPackageModel]]: - resource_with_inlined_attributions = {} - for path, ids in resources_with_ids.items(): - attributions = [] - for id in ids: - attribution = OpossumPackageModel(**all_attributions[id]) - attributions.append(attribution) - resource_with_inlined_attributions[path] = set(attributions) - return resource_with_inlined_attributions + @staticmethod + def _inline_attributions_into_resources( + *, resources_with_ids: dict[str, list[str]], all_attributions: dict[str, Any] + ) -> dict[str, set[OpossumPackageModel]]: + resource_with_inlined_attributions = {} + for path, ids in resources_with_ids.items(): + attributions = [] + for id in ids: + attribution = OpossumPackageModel(**all_attributions[id]) + attributions.append(attribution) + resource_with_inlined_attributions[path] = set(attributions) + return resource_with_inlined_attributions -def read_input_json_from_opossum(output_file_path: str) -> Any: - return read_json_from_zip_file(output_file_path, INPUT_JSON_NAME) +def _read_input_json_from_opossum(output_file_path: str) -> Any: + return _read_json_from_zip_file(output_file_path, INPUT_JSON_NAME) -def read_output_json_from_opossum(output_file_path: str) -> Any: - return read_json_from_zip_file(output_file_path, OUTPUT_JSON_NAME) +def _read_output_json_from_opossum(output_file_path: str) -> Any: + return _read_json_from_zip_file(output_file_path, OUTPUT_JSON_NAME) -def read_json_from_zip_file(output_file_path: str, file_name: str) -> Any: +def _read_json_from_zip_file(output_file_path: str, file_name: str) -> Any: with ( ZipFile(output_file_path, "r") as z, z.open(file_name) as file, @@ -157,13 +156,13 @@ def read_json_from_zip_file(output_file_path: str, file_name: str) -> Any: return opossum_dict -def read_json_from_file(filename: str) -> Any: +def _read_json_from_file(filename: str) -> Any: with open(test_data_path / filename, encoding="utf-8") as file: expected_opossum_dict = json.load(file) return expected_opossum_dict -def assert_expected_file_equals_generated_file( +def _assert_expected_file_equals_generated_file( expected_opossum_dict: Any, opossum_dict: Any ) -> None: assert expected_opossum_dict.keys() == opossum_dict.keys() @@ -174,44 +173,59 @@ def assert_expected_file_equals_generated_file( assert opossum_dict.get(field, None) == expected_opossum_dict.get(field, None) -def test_cli_no_output_file_provided(opossum_file_faker: OpossumFileFaker) -> None: - runner = CliRunner() - - with runner.isolated_filesystem(): - file_path = "input.opossum" - opossum_file = opossum_file_faker.opossum_file_content() - write_opossum_file(opossum_file, Path(file_path)) - result = runner.invoke( - generate, - "--opossum " + file_path, - ) - - assert result.exit_code == 0 - assert Path.is_file(Path("output.opossum")) - - -@pytest.mark.parametrize( - "options", - [ - generate_valid_opossum_argument() + generate_valid_opossum_argument(), - generate_valid_opossum_argument() + generate_valid_scan_code_argument(), - generate_valid_scan_code_argument() + generate_valid_scan_code_argument(), - ], -) -def test_cli_with_multiple_files(caplog: LogCaptureFixture, options: list[str]) -> None: - result = run_with_command_line_arguments(options) - assert result.exit_code == 1 - - assert caplog.messages == ["Merging of multiple files not yet supported!"] - - -def test_cli_without_inputs(caplog: LogCaptureFixture) -> None: - result = run_with_command_line_arguments( +class TestCliValidations: + @staticmethod + def generate_valid_scan_code_argument( + filename: str = "scancode_input.json", + ) -> list[str]: + return ["--scan-code-json", str(test_data_path / filename)] + + @staticmethod + def generate_valid_opossum_argument( + filename: str = "opossum_input.opossum", + ) -> list[str]: + return ["--opossum", str(test_data_path / filename)] + + def test_cli_no_output_file_provided( + self, opossum_file_faker: OpossumFileFaker + ) -> None: + runner = CliRunner() + + with runner.isolated_filesystem(): + file_path = "input.opossum" + opossum_file = opossum_file_faker.opossum_file_content() + write_opossum_file(opossum_file, Path(file_path)) + result = runner.invoke( + generate, + "--opossum " + file_path, + ) + + assert result.exit_code == 0 + assert Path.is_file(Path("output.opossum")) + + @pytest.mark.parametrize( + "options", [ - "-o", - "output.opossum", + generate_valid_opossum_argument() + generate_valid_opossum_argument(), + generate_valid_opossum_argument() + generate_valid_scan_code_argument(), + generate_valid_scan_code_argument() + generate_valid_scan_code_argument(), ], ) - assert result.exit_code == 1 + def test_cli_with_multiple_files( + self, caplog: LogCaptureFixture, options: list[str] + ) -> None: + result = run_with_command_line_arguments(options) + assert result.exit_code == 1 + + assert caplog.messages == ["Merging of multiple files not yet supported!"] + + def test_cli_without_inputs(self, caplog: LogCaptureFixture) -> None: + result = run_with_command_line_arguments( + [ + "-o", + "output.opossum", + ], + ) + assert result.exit_code == 1 - assert caplog.messages == ["No input provided. Exiting."] + assert caplog.messages == ["No input provided. Exiting."] From c21de0c09782f1cdad3b520ddda68b5f9d2ae79d Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:06:14 +0100 Subject: [PATCH 51/58] refactor: review -- unify closely related classes --- .../opossum/services/opossum_file_reader.py | 58 +++++++++++++++-- .../opossum/services/read_opossum_file.py | 64 ------------------- 2 files changed, 54 insertions(+), 68 deletions(-) delete mode 100644 src/opossum_lib/input_formats/opossum/services/read_opossum_file.py diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index ba81d7b5..61d514ea 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -1,16 +1,21 @@ # SPDX-FileCopyrightText: TNG Technology Consulting GmbH # # SPDX-License-Identifier: Apache-2.0 +import json +import logging +import sys from pathlib import Path +from zipfile import ZipFile from opossum_lib.core.entities.opossum import Opossum from opossum_lib.core.services.input_reader import InputReader from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 convert_to_opossum, ) -from opossum_lib.input_formats.opossum.services.read_opossum_file import ( - read_opossum_file, -) +from opossum_lib.shared.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME +from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel +from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel +from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel class OpossumFileReader(InputReader): @@ -20,5 +25,50 @@ def __init__(self, path: Path): self.path = path def read(self) -> Opossum: - opossum_input_file = read_opossum_file(path=self.path) + opossum_input_file = self.read_opossum_file() return convert_to_opossum(opossum_input_file) + + def read_opossum_file(self) -> OpossumFileModel: + logging.info(f"Converting opossum to opossum {self.path}") + + try: + with ( + ZipFile(self.path, "r") as zip_file, + ): + OpossumFileReader._validate_zip_file_contents(zip_file) + input_file = OpossumFileReader._read_input_json_from_zip_file(zip_file) + return OpossumFileModel( + input_file=input_file, + output_file=OpossumFileReader._read_output_json_if_exists(zip_file), + ) + except Exception as e: + print(f"Error reading file {self.path}: {e}") + sys.exit(1) + + @staticmethod + def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: + with zip_file.open(INPUT_JSON_NAME) as input_json_file: + input_json = json.load(input_json_file) + input_file = OpossumInputFileModel.model_validate(input_json) + return input_file + + @staticmethod + def _read_output_json_if_exists( + input_zip_file: ZipFile, + ) -> OpossumOutputFileModel | None: + if OUTPUT_JSON_NAME in input_zip_file.namelist(): + with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: + output_json = json.load(output_json_file) + output_file = OpossumOutputFileModel.model_validate(output_json) + else: + output_file = None + return output_file + + @staticmethod + def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: + if INPUT_JSON_NAME not in input_zip_file.namelist(): + logging.error( + f"Opossum file {input_zip_file.filename} is corrupt" + f" and does not contain '{INPUT_JSON_NAME}'" + ) + sys.exit(1) diff --git a/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py b/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py deleted file mode 100644 index 535acadc..00000000 --- a/src/opossum_lib/input_formats/opossum/services/read_opossum_file.py +++ /dev/null @@ -1,64 +0,0 @@ -# SPDX-FileCopyrightText: TNG Technology Consulting GmbH -# -# SPDX-License-Identifier: Apache-2.0 - - -import json -import logging -import sys -from pathlib import Path -from zipfile import ZipFile - -from opossum_lib.shared.constants import ( - INPUT_JSON_NAME, - OUTPUT_JSON_NAME, -) -from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel -from opossum_lib.shared.entities.opossum_input_file_model import OpossumInputFileModel -from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel - - -def read_opossum_file(path: Path) -> OpossumFileModel: - logging.info(f"Converting opossum to opossum {path}") - - try: - with ( - ZipFile(path, "r") as input_zip_file, - ): - _validate_zip_file_contents(input_zip_file) - input_file = _read_input_json_from_zip_file(input_zip_file) - return OpossumFileModel( - input_file=input_file, - output_file=_read_output_json_if_exists(input_zip_file), - ) - except Exception as e: - print(f"Error reading file {path}: {e}") - sys.exit(1) - - -def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: - with zip_file.open(INPUT_JSON_NAME) as input_json_file: - input_json = json.load(input_json_file) - input_file = OpossumInputFileModel.model_validate(input_json) - return input_file - - -def _read_output_json_if_exists( - input_zip_file: ZipFile, -) -> OpossumOutputFileModel | None: - if OUTPUT_JSON_NAME in input_zip_file.namelist(): - with input_zip_file.open(OUTPUT_JSON_NAME) as output_json_file: - output_json = json.load(output_json_file) - output_file = OpossumOutputFileModel.model_validate(output_json) - else: - output_file = None - return output_file - - -def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: - if INPUT_JSON_NAME not in input_zip_file.namelist(): - logging.error( - f"Opossum file {input_zip_file.filename} is corrupt" - f" and does not contain '{INPUT_JSON_NAME}'" - ) - sys.exit(1) From 17e603d2d298228a2bf5a2c8792346e8a094fd90 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:14:54 +0100 Subject: [PATCH 52/58] refactor: review -- remove no longer necessary linter exceptions --- .../opossum/services/convert_to_opossum.py | 11 +++-------- .../opossum/services/opossum_file_reader.py | 2 +- .../scancode/services/scancode_file_reader.py | 2 +- 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py index 1712030d..76ca2efd 100644 --- a/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py +++ b/src/opossum_lib/input_formats/opossum/services/convert_to_opossum.py @@ -54,12 +54,10 @@ def _convert_to_scan_results( external_attributions=opossum_input_file_model.external_attributions, resources_to_attributions=opossum_input_file_model.resources_to_attributions, ) - # fmt: off frequent_licenses = ( - opossum_input_file_model.frequent_licenses - and _convert_frequent_licenses(opossum_input_file_model.frequent_licenses) + opossum_input_file_model.frequent_licenses + and _convert_frequent_licenses(opossum_input_file_model.frequent_licenses) ) - # fmt: on base_urls_for_sources = ( opossum_input_file_model.base_urls_for_sources @@ -70,10 +68,7 @@ def _convert_to_scan_results( file_attribution_sources = opossum_input_file_model.external_attribution_sources external_attribution_sources = { - # noqa required due to clash between linter and formatter - name: _convert_external_attribution_source( # noqa: E501 - attribution_source - ) + name: _convert_external_attribution_source(attribution_source) for name, attribution_source in file_attribution_sources.items() } diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 61d514ea..245b64ab 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -9,7 +9,7 @@ from opossum_lib.core.entities.opossum import Opossum from opossum_lib.core.services.input_reader import InputReader -from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( convert_to_opossum, ) from opossum_lib.shared.constants import INPUT_JSON_NAME, OUTPUT_JSON_NAME diff --git a/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py b/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py index 6309277a..ffc551f1 100644 --- a/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py +++ b/src/opossum_lib/input_formats/scancode/services/scancode_file_reader.py @@ -14,7 +14,7 @@ from opossum_lib.input_formats.scancode.entities.scancode_model import ( ScancodeModel, ) -from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( convert_to_opossum, ) From 47f0aade553570eba02c4e2973443d86d95fa1f3 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:15:16 +0100 Subject: [PATCH 53/58] refactor: review -- delete dead code --- .../entities/opossum_input_file_model.py | 95 ------------------- 1 file changed, 95 deletions(-) diff --git a/src/opossum_lib/shared/entities/opossum_input_file_model.py b/src/opossum_lib/shared/entities/opossum_input_file_model.py index b664e368..2572c73f 100644 --- a/src/opossum_lib/shared/entities/opossum_input_file_model.py +++ b/src/opossum_lib/shared/entities/opossum_input_file_model.py @@ -3,9 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -from copy import deepcopy from dataclasses import field -from enum import Enum, auto from typing import Literal from pydantic import ConfigDict, model_serializer @@ -88,99 +86,6 @@ class MetadataModel(CamelBaseModel): build_date: str | None = None -class ResourceTypeModel(Enum): - FILE = auto() - FOLDER = auto() - TOP_LEVEL = auto() - OTHER = auto() - - -class ResourceModel(CamelBaseModel): - type: ResourceTypeModel - children: dict[str, ResourceModel] = field(default_factory=dict) - - def add_path( - self, path_with_resource_types: list[tuple[str, ResourceTypeModel]] - ) -> ResourceModel: - resource = deepcopy(self) - if len(path_with_resource_types) == 0: - return resource - (first, resource_type), rest = ( - path_with_resource_types[0], - path_with_resource_types[1:], - ) - if self.element_exists_but_resource_type_differs(first, resource_type): - raise TypeError( - "Couldn't add path to resource: ResourceType of elements with" - " the same path differ." - ) - if first not in self.children: - resource.children[first] = ResourceModel(type=resource_type) - resource.children[first] = resource.children[first].add_path(rest) - - return resource - - def element_exists_but_resource_type_differs( - self, element: str, resource_type: ResourceTypeModel - ) -> bool: - if element in self.children: - return self.children[element].type != resource_type - return False - - def drop_element( - self, path_to_element_to_drop: list[tuple[str, ResourceTypeModel]] - ) -> ResourceModel: - paths_in_resource = self.get_paths_of_all_leaf_nodes_with_types() - if path_to_element_to_drop not in paths_in_resource: - raise ValueError( - f"Element {path_to_element_to_drop} doesn't exist in resource!" - ) - - else: - resource = ResourceModel(type=ResourceTypeModel.TOP_LEVEL) - paths_in_resource.remove(path_to_element_to_drop) - paths_in_resource.append(path_to_element_to_drop[:-1]) - - for path_to_element_to_drop in paths_in_resource: - resource = resource.add_path(path_to_element_to_drop) - - return resource - - def to_dict(self) -> ResourceInFileModel: - if not self.has_children(): - if self.type == ResourceTypeModel.FOLDER: - return {} - else: - return 1 - else: - return { - name: resource.to_dict() for name, resource in self.children.items() - } - - def get_paths_of_all_leaf_nodes_with_types( - self, - ) -> list[list[tuple[str, ResourceTypeModel]]]: - paths = [] - for name, resource in self.children.items(): - path = [(name, resource.type)] - if resource.has_children(): - paths.extend( - [ - path + element - for element in resource.get_paths_of_all_leaf_nodes_with_types() - ] - ) - else: - paths.extend([path]) - return paths - - def has_children(self) -> bool: - return len(self.children) > 0 - - def convert_to_file_resource(self) -> ResourceInFileModel: - return self.to_dict() - - class ExternalAttributionSourceModel(CamelBaseModel): model_config = ConfigDict(frozen=True) name: str From 768d9c0adcc55a8c1fb2d13ce01d24d839e72388 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:16:47 +0100 Subject: [PATCH 54/58] refactor: review -- move primary class in file to top --- .../entities/opossum_output_file_model.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/opossum_lib/shared/entities/opossum_output_file_model.py b/src/opossum_lib/shared/entities/opossum_output_file_model.py index e9f9337e..3036552d 100644 --- a/src/opossum_lib/shared/entities/opossum_output_file_model.py +++ b/src/opossum_lib/shared/entities/opossum_output_file_model.py @@ -14,6 +14,23 @@ from opossum_lib.shared.entities.camel_base_model import CamelBaseModel +class OpossumOutputFileModel(CamelBaseModel): + model_config = ConfigDict(extra="allow") + + metadata: Metadata + manual_attributions: dict[str, ManualAttributions] + resources_to_attributions: dict[str, list[str]] = Field( + ..., + description="Map from a path in the resource tree " + "(e.g. `/folder/subfolder/`, `/folder/file`, " + "note the mandatory slashes at the beginning and end) " + "to a list of attribution IDs.", + ) + resolved_external_attributions: list[str] | None = Field( + None, description="List of attribution IDs for input signals that are resolved." + ) + + class Metadata(CamelBaseModel): project_id: str = Field( ..., description="An ID for the compliance scan, copied from the input file." @@ -122,20 +139,3 @@ class ManualAttributions(CamelBaseModel): description="Indicates that the attribution had previously" " been marked as preferred.", ) - - -class OpossumOutputFileModel(CamelBaseModel): - model_config = ConfigDict(extra="allow") - - metadata: Metadata - manual_attributions: dict[str, ManualAttributions] - resources_to_attributions: dict[str, list[str]] = Field( - ..., - description="Map from a path in the resource tree " - "(e.g. `/folder/subfolder/`, `/folder/file`, " - "note the mandatory slashes at the beginning and end) " - "to a list of attribution IDs.", - ) - resolved_external_attributions: list[str] | None = Field( - None, description="List of attribution IDs for input signals that are resolved." - ) From dd4cf3883d40886eb180d9b4e19245bf46d73348 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:30:11 +0100 Subject: [PATCH 55/58] refactor: review -- rename method to match target type --- .../core/entities/base_url_for_sources.py | 2 +- .../entities/external_attribution_source.py | 2 +- .../core/entities/frequent_license.py | 2 +- src/opossum_lib/core/entities/metadata.py | 2 +- src/opossum_lib/core/entities/opossum.py | 4 ++-- .../core/entities/opossum_package.py | 4 ++-- src/opossum_lib/core/entities/resource.py | 4 ++-- src/opossum_lib/core/entities/scan_results.py | 19 ++++++++++--------- src/opossum_lib/core/entities/source_info.py | 2 +- .../core/services/generate_impl.py | 2 +- .../generators/scan_results_provider.py | 4 ++-- tests/core/entities/test_opossum.py | 11 ++++++----- .../services/test_conversion_roundtrip.py | 2 +- .../services/test_get_attribution_info.py | 2 +- 14 files changed, 32 insertions(+), 30 deletions(-) diff --git a/src/opossum_lib/core/entities/base_url_for_sources.py b/src/opossum_lib/core/entities/base_url_for_sources.py index 0e952bb6..998243fe 100644 --- a/src/opossum_lib/core/entities/base_url_for_sources.py +++ b/src/opossum_lib/core/entities/base_url_for_sources.py @@ -12,5 +12,5 @@ class BaseUrlsForSources(BaseModel): model_config = ConfigDict(frozen=True, extra="allow") - def to_opossum_model(self) -> BaseUrlsForSourcesModel: + def to_opossum_file_model(self) -> BaseUrlsForSourcesModel: return BaseUrlsForSourcesModel(**self.model_dump()) diff --git a/src/opossum_lib/core/entities/external_attribution_source.py b/src/opossum_lib/core/entities/external_attribution_source.py index 3fec8151..bd45623d 100644 --- a/src/opossum_lib/core/entities/external_attribution_source.py +++ b/src/opossum_lib/core/entities/external_attribution_source.py @@ -17,7 +17,7 @@ class ExternalAttributionSource(BaseModel): priority: int is_relevant_for_preferred: bool | None = None - def to_opossum_model(self) -> ExternalAttributionSourceModel: + def to_opossum_file_model(self) -> ExternalAttributionSourceModel: return ExternalAttributionSourceModel( name=self.name, priority=self.priority, diff --git a/src/opossum_lib/core/entities/frequent_license.py b/src/opossum_lib/core/entities/frequent_license.py index af3feea6..90cbbb24 100644 --- a/src/opossum_lib/core/entities/frequent_license.py +++ b/src/opossum_lib/core/entities/frequent_license.py @@ -15,7 +15,7 @@ class FrequentLicense(BaseModel): short_name: str default_text: str - def to_opossum_model(self) -> FrequentLicenseModel: + def to_opossum_file_model(self) -> FrequentLicenseModel: return FrequentLicenseModel( full_name=self.full_name, short_name=self.short_name, diff --git a/src/opossum_lib/core/entities/metadata.py b/src/opossum_lib/core/entities/metadata.py index 463b59a9..904f8244 100644 --- a/src/opossum_lib/core/entities/metadata.py +++ b/src/opossum_lib/core/entities/metadata.py @@ -18,5 +18,5 @@ class Metadata(BaseModel): expected_release_date: str | None = None build_date: str | None = None - def to_opossum_model(self) -> MetadataModel: + def to_opossum_file_model(self) -> MetadataModel: return MetadataModel(**self.model_dump()) diff --git a/src/opossum_lib/core/entities/opossum.py b/src/opossum_lib/core/entities/opossum.py index e5a06db9..83cb3a52 100644 --- a/src/opossum_lib/core/entities/opossum.py +++ b/src/opossum_lib/core/entities/opossum.py @@ -19,8 +19,8 @@ class Opossum(BaseModel): scan_results: ScanResults review_results: OpossumOutputFileModel | None = None - def to_opossum_model(self) -> OpossumFileModel: + def to_opossum_file_model(self) -> OpossumFileModel: return OpossumFileModel( - input_file=self.scan_results.to_opossum_model(), + input_file=self.scan_results.to_opossum_file_model(), output_file=self.review_results, ) diff --git a/src/opossum_lib/core/entities/opossum_package.py b/src/opossum_lib/core/entities/opossum_package.py index cffe9f6d..303429cf 100644 --- a/src/opossum_lib/core/entities/opossum_package.py +++ b/src/opossum_lib/core/entities/opossum_package.py @@ -35,9 +35,9 @@ class OpossumPackage(BaseModel): criticality: Literal["high"] | Literal["medium"] | None = None was_preferred: bool | None = None - def to_opossum_model(self) -> OpossumPackageModel: + def to_opossum_file_model(self) -> OpossumPackageModel: return OpossumPackageModel( - source=self.source.to_opossum_model(), + source=self.source.to_opossum_file_model(), attribution_confidence=self.attribution_confidence, comment=self.comment, package_name=self.package_name, diff --git a/src/opossum_lib/core/entities/resource.py b/src/opossum_lib/core/entities/resource.py index ef533c19..eac03776 100644 --- a/src/opossum_lib/core/entities/resource.py +++ b/src/opossum_lib/core/entities/resource.py @@ -31,12 +31,12 @@ class Resource(BaseModel): attributions: list[OpossumPackage] = [] children: dict[str, Resource] = {} - def to_opossum_model(self) -> ResourceInFileModel: + def to_opossum_file_model(self) -> ResourceInFileModel: if self.children or self.type == ResourceType.FOLDER: return { _convert_path_to_str( child.path.relative_to(self.path) - ): child.to_opossum_model() + ): child.to_opossum_file_model() for child in self.children.values() } else: diff --git a/src/opossum_lib/core/entities/scan_results.py b/src/opossum_lib/core/entities/scan_results.py index 6c7bfffe..15f1278e 100644 --- a/src/opossum_lib/core/entities/scan_results.py +++ b/src/opossum_lib/core/entities/scan_results.py @@ -45,7 +45,7 @@ class ScanResults(BaseModel): ) unassigned_attributions: list[OpossumPackage] = [] - def to_opossum_model(self) -> OpossumInputFileModel: + def to_opossum_file_model(self) -> OpossumInputFileModel: external_attributions, resources_to_attributions = ( self.create_attribution_mapping(self.resources) ) @@ -54,21 +54,22 @@ def to_opossum_model(self) -> OpossumInputFileModel: frequent_licenses = None if self.frequent_licenses: frequent_licenses = [ - license.to_opossum_model() for license in self.frequent_licenses + license.to_opossum_file_model() for license in self.frequent_licenses ] base_urls_for_sources = ( - self.base_urls_for_sources and self.base_urls_for_sources.to_opossum_model() + self.base_urls_for_sources + and self.base_urls_for_sources.to_opossum_file_model() ) external_attribution_sources = { - key: val.to_opossum_model() + key: val.to_opossum_file_model() for (key, val) in self.external_attribution_sources.items() } return OpossumInputFileModel( - metadata=self.metadata.to_opossum_model(), + metadata=self.metadata.to_opossum_file_model(), resources={ - str(resource.path): resource.to_opossum_model() + str(resource.path): resource.to_opossum_file_model() for resource in self.resources }, external_attributions=external_attributions, @@ -89,13 +90,13 @@ def _get_unassigned_attributions( if unassigned_attribution in self.attribution_to_id: package_identifier = self.attribution_to_id[unassigned_attribution] result[package_identifier] = ( - unassigned_attribution.to_opossum_model() + unassigned_attribution.to_opossum_file_model() ) else: package_identifier = str(uuid.uuid4()) self.attribution_to_id[unassigned_attribution] = package_identifier result[package_identifier] = ( - unassigned_attribution.to_opossum_model() + unassigned_attribution.to_opossum_file_model() ) return result else: @@ -122,7 +123,7 @@ def process_node(node: Resource) -> None: path = "/" + path node_attributions_by_id = { - self.get_attribution_key(a): a.to_opossum_model() + self.get_attribution_key(a): a.to_opossum_file_model() for a in node.attributions } external_attributions.update(node_attributions_by_id) diff --git a/src/opossum_lib/core/entities/source_info.py b/src/opossum_lib/core/entities/source_info.py index f83aa6ae..c4b9d390 100644 --- a/src/opossum_lib/core/entities/source_info.py +++ b/src/opossum_lib/core/entities/source_info.py @@ -15,7 +15,7 @@ class SourceInfo(BaseModel): document_confidence: int | float | None = 0 additional_name: str | None = None - def to_opossum_model(self) -> SourceInfoModel: + def to_opossum_file_model(self) -> SourceInfoModel: return SourceInfoModel( name=self.name, document_confidence=self.document_confidence, diff --git a/src/opossum_lib/core/services/generate_impl.py b/src/opossum_lib/core/services/generate_impl.py index 977927ea..a3b7cc8d 100644 --- a/src/opossum_lib/core/services/generate_impl.py +++ b/src/opossum_lib/core/services/generate_impl.py @@ -12,5 +12,5 @@ def generate_impl(input_readers: list[InputReader], output_file: Path) -> None: # for the future a merge step is planned after reading the files opossum = input_readers[0].read() - opossum_file_content = opossum.to_opossum_model() + opossum_file_content = opossum.to_opossum_file_model() write_opossum_file(opossum_file_content, output_file) diff --git a/tests/core/entities/generators/scan_results_provider.py b/tests/core/entities/generators/scan_results_provider.py index 00d2a435..9c132e48 100644 --- a/tests/core/entities/generators/scan_results_provider.py +++ b/tests/core/entities/generators/scan_results_provider.py @@ -64,7 +64,7 @@ def scan_results( random_list(self, entry_generator=lambda: self.package_provider.package()), ) if generate_attribution_to_id: - attribution_to_id = ScanResultsProvider._attribution_to_id( + attribution_to_id = self._attribution_to_id( generated_resources, generated_unassigned_attributions ) # fmt: off @@ -100,8 +100,8 @@ def scan_results( ) # fmt: on - @staticmethod def _attribution_to_id( + self, resources: list[Resource] | None, unassigned_attributions: list[OpossumPackage] | None, ) -> dict[OpossumPackage, str] | None: diff --git a/tests/core/entities/test_opossum.py b/tests/core/entities/test_opossum.py index 0a264592..7cbb06a5 100644 --- a/tests/core/entities/test_opossum.py +++ b/tests/core/entities/test_opossum.py @@ -14,7 +14,7 @@ class TestOpossumToOpossumModelConversion: def test_moves_outfile(self, opossum_faker: OpossumFaker) -> None: opossum = opossum_faker.opossum() - result = opossum.to_opossum_model() + result = opossum.to_opossum_file_model() assert result.output_file == opossum.review_results @@ -22,18 +22,19 @@ def test_roundtrip(self, opossum_faker: OpossumFaker) -> None: opossum = opossum_faker.opossum() expected_result = deepcopy(opossum) - opossum_file = opossum.to_opossum_model() + opossum_file = opossum.to_opossum_file_model() result = convert_to_opossum(opossum_file) - ## this can change due to the generation of new ids + # this workaround is necessary as model_dump fails result_json = result.model_dump_json() expected_result_json = expected_result.model_dump_json() result_dict = json.loads(result_json) expected_result_dict = json.loads(expected_result_json) + # this can change due to the generation of new ids expected_result_dict["scan_results"]["attribution_to_id"] = None result_dict["scan_results"]["attribution_to_id"] = None - # sort the lists again for comperability + # sort the lists again for comparability expected_result_dict["scan_results"]["unassigned_attributions"] = sorted( expected_result_dict["scan_results"]["unassigned_attributions"], key=lambda x: x["source"]["name"], @@ -51,7 +52,7 @@ def test_roundtrip_with_resource_ids(self, opossum_faker: OpossumFaker) -> None: ) expected_result = deepcopy(opossum) - opossum_file = opossum.to_opossum_model() + opossum_file = opossum.to_opossum_file_model() result = convert_to_opossum(opossum_file) diff --git a/tests/input_formats/opossum/services/test_conversion_roundtrip.py b/tests/input_formats/opossum/services/test_conversion_roundtrip.py index d0ac02f7..1d048886 100644 --- a/tests/input_formats/opossum/services/test_conversion_roundtrip.py +++ b/tests/input_formats/opossum/services/test_conversion_roundtrip.py @@ -31,5 +31,5 @@ def test_surplus_attributions(self, opossum_file_faker: OpossumFileFaker) -> Non @staticmethod def _check_round_trip(start_file_content: OpossumFileModel) -> None: expected_file_content = deepcopy(start_file_content) - result = convert_to_opossum(start_file_content).to_opossum_model() + result = convert_to_opossum(start_file_content).to_opossum_file_model() assert result == expected_file_content diff --git a/tests/input_formats/scancode/services/test_get_attribution_info.py b/tests/input_formats/scancode/services/test_get_attribution_info.py index 7a30c39b..f4aeae17 100644 --- a/tests/input_formats/scancode/services/test_get_attribution_info.py +++ b/tests/input_formats/scancode/services/test_get_attribution_info.py @@ -78,7 +78,7 @@ def test_get_attribution_info_file_multiple( scancode_data = scancode_faker.scancode_data(files=[file]) opossum = convert_to_opossum(scancode_data) attributions = ( - opossum.to_opossum_model().input_file.external_attributions.values() + opossum.to_opossum_file_model().input_file.external_attributions.values() ) expected1 = OpossumPackageModel( From c36ea7df95caf7f3ef61e19820ab07f1fecfc810 Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:33:19 +0100 Subject: [PATCH 56/58] refactor: review -- improve docs --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e992241d..14ab0b1d 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ uv run ruff check uv run ruff format --check uv run python -m mypy src/ tests/ uv run pytest +uv run lint-imports ``` # Build From 9d0458088c13a9b182e559e078d9052c86cf8e3d Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:43:11 +0100 Subject: [PATCH 57/58] refactor: review -- remove unused noqa --- tests/core/entities/generators/opossum_provider.py | 2 +- tests/core/entities/test_opossum.py | 2 +- .../input_formats/opossum/services/test_conversion_roundtrip.py | 2 +- tests/input_formats/opossum/services/test_convert_to_opossum.py | 2 +- .../scancode/entities/generators/generate_scancode_file.py | 2 +- .../input_formats/scancode/services/test_convert_to_opossum.py | 2 +- .../scancode/services/test_get_attribution_info.py | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/core/entities/generators/opossum_provider.py b/tests/core/entities/generators/opossum_provider.py index 8f0ab056..cf72653d 100644 --- a/tests/core/entities/generators/opossum_provider.py +++ b/tests/core/entities/generators/opossum_provider.py @@ -9,7 +9,7 @@ from opossum_lib.core.entities.scan_results import ScanResults from opossum_lib.shared.entities.opossum_output_file_model import OpossumOutputFileModel from tests.core.entities.generators.scan_results_provider import ScanResultsProvider -from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa: E501 +from tests.input_formats.opossum.entities.generators.generate_outfile_information import ( # noqa OpossumOutputFileProvider, ) diff --git a/tests/core/entities/test_opossum.py b/tests/core/entities/test_opossum.py index 7cbb06a5..d2c404bd 100644 --- a/tests/core/entities/test_opossum.py +++ b/tests/core/entities/test_opossum.py @@ -4,7 +4,7 @@ import json from copy import deepcopy -from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( convert_to_opossum, ) from tests.setup.opossum_faker_setup import OpossumFaker diff --git a/tests/input_formats/opossum/services/test_conversion_roundtrip.py b/tests/input_formats/opossum/services/test_conversion_roundtrip.py index 1d048886..87bad897 100644 --- a/tests/input_formats/opossum/services/test_conversion_roundtrip.py +++ b/tests/input_formats/opossum/services/test_conversion_roundtrip.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 from copy import deepcopy -from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( convert_to_opossum, ) from opossum_lib.shared.entities.opossum_file_model import OpossumFileModel diff --git a/tests/input_formats/opossum/services/test_convert_to_opossum.py b/tests/input_formats/opossum/services/test_convert_to_opossum.py index d0975c9e..222a262e 100644 --- a/tests/input_formats/opossum/services/test_convert_to_opossum.py +++ b/tests/input_formats/opossum/services/test_convert_to_opossum.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest -from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.opossum.services.convert_to_opossum import ( convert_to_opossum, ) from opossum_lib.shared.entities.opossum_input_file_model import ( diff --git a/tests/input_formats/scancode/entities/generators/generate_scancode_file.py b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py index 9327293a..cd629a6c 100644 --- a/tests/input_formats/scancode/entities/generators/generate_scancode_file.py +++ b/tests/input_formats/scancode/entities/generators/generate_scancode_file.py @@ -404,7 +404,7 @@ def single_file( if license_detections is not None else random_list( self, - lambda: self.license_detection(path=path), # noqa: B023 + lambda: self.license_detection(path=path), ) ) detected_license_expression = detected_license_expression or " and ".join( diff --git a/tests/input_formats/scancode/services/test_convert_to_opossum.py b/tests/input_formats/scancode/services/test_convert_to_opossum.py index 71ac4be4..7b485bd1 100644 --- a/tests/input_formats/scancode/services/test_convert_to_opossum.py +++ b/tests/input_formats/scancode/services/test_convert_to_opossum.py @@ -7,7 +7,7 @@ from _pytest.logging import LogCaptureFixture from opossum_lib.core.entities.resource import Resource -from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( convert_to_opossum, ) from tests.setup.scancode_faker_setup import ScanCodeFaker diff --git a/tests/input_formats/scancode/services/test_get_attribution_info.py b/tests/input_formats/scancode/services/test_get_attribution_info.py index f4aeae17..337743dd 100644 --- a/tests/input_formats/scancode/services/test_get_attribution_info.py +++ b/tests/input_formats/scancode/services/test_get_attribution_info.py @@ -6,7 +6,7 @@ from opossum_lib.input_formats.scancode.constants import SCANCODE_SOURCE_NAME -from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( # noqa: E501 +from opossum_lib.input_formats.scancode.services.convert_to_opossum import ( convert_to_opossum, ) from opossum_lib.shared.entities.opossum_input_file_model import ( From 15d9a59f73537cc5a07a3a68bb23b938095de50b Mon Sep 17 00:00:00 2001 From: Dominikus Hellgartner Date: Fri, 31 Jan 2025 14:59:14 +0100 Subject: [PATCH 58/58] refactor: review -- static methods to private methods --- .../opossum/services/opossum_file_reader.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py index 245b64ab..5f050eac 100644 --- a/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py +++ b/src/opossum_lib/input_formats/opossum/services/opossum_file_reader.py @@ -25,35 +25,34 @@ def __init__(self, path: Path): self.path = path def read(self) -> Opossum: - opossum_input_file = self.read_opossum_file() + opossum_input_file = self._read_opossum_file() return convert_to_opossum(opossum_input_file) - def read_opossum_file(self) -> OpossumFileModel: + def _read_opossum_file(self) -> OpossumFileModel: logging.info(f"Converting opossum to opossum {self.path}") try: with ( ZipFile(self.path, "r") as zip_file, ): - OpossumFileReader._validate_zip_file_contents(zip_file) - input_file = OpossumFileReader._read_input_json_from_zip_file(zip_file) + self._validate_zip_file_contents(zip_file) + input_file = self._read_input_json(zip_file) return OpossumFileModel( input_file=input_file, - output_file=OpossumFileReader._read_output_json_if_exists(zip_file), + output_file=self._read_output_json_if_exists(zip_file), ) except Exception as e: print(f"Error reading file {self.path}: {e}") sys.exit(1) - @staticmethod - def _read_input_json_from_zip_file(zip_file: ZipFile) -> OpossumInputFileModel: + def _read_input_json(self, zip_file: ZipFile) -> OpossumInputFileModel: with zip_file.open(INPUT_JSON_NAME) as input_json_file: input_json = json.load(input_json_file) input_file = OpossumInputFileModel.model_validate(input_json) return input_file - @staticmethod def _read_output_json_if_exists( + self, input_zip_file: ZipFile, ) -> OpossumOutputFileModel | None: if OUTPUT_JSON_NAME in input_zip_file.namelist(): @@ -64,8 +63,7 @@ def _read_output_json_if_exists( output_file = None return output_file - @staticmethod - def _validate_zip_file_contents(input_zip_file: ZipFile) -> None: + def _validate_zip_file_contents(self, input_zip_file: ZipFile) -> None: if INPUT_JSON_NAME not in input_zip_file.namelist(): logging.error( f"Opossum file {input_zip_file.filename} is corrupt"