feat: added project_manager (#694)

CMakeLists.txt

@@ -152,6 +152,7 @@ else()
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_optimizer)
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_applicator)
     execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
+    execute_process(COMMAND rm -f ${DP_LOCAL_BIN_DIR}/discopop_project_manager)
     message(STATUS "--> discopop_auto_tuner")
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_auto_tuner ${DP_LOCAL_BIN_DIR}/discopop_auto_tuner)
     message(STATUS "--> discopop_config_provider")
@@ -166,5 +167,7 @@ else()
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_patch_generator ${DP_LOCAL_BIN_DIR}/discopop_patch_generator)
     message(STATUS "--> discopop_preprocessor")
     execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_preprocessor ${DP_LOCAL_BIN_DIR}/discopop_preprocessor)
+    message(STATUS "--> discopop_project_manager")
+    execute_process(COMMAND ln -sf ${DiscoPoP_SOURCE_DIR}/venv/bin/discopop_project_manager ${DP_LOCAL_BIN_DIR}/discopop_project_manager)
 
 endif()

discopop_library/ProjectManager/ProjectManager.py

@@ -0,0 +1,53 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import json
+import os
+
+from discopop_library.ProjectManager.ProjectManagerArguments import ProjectManagerArguments
+
+import logging
+
+from discopop_library.ProjectManager.reports.full import generate_full_report
+from discopop_library.ProjectManager.utilities.CLI.listConfiguration import (
+    show_configurations_with_execution,
+    show_configurations_without_execution,
+)
+from discopop_library.ProjectManager.utilities.deriveSettingsFiles import derive_settings_files
+from discopop_library.ProjectManager.utilities.initializeDirectories import initialize_directories
+from discopop_library.ProjectManager.utilities.initializeFiles import initialize_configuration_files
+from discopop_library.ProjectManager.utilities.reset import reset_project
+
+logger = logging.getLogger("ProjectManager")
+
+
+def run(arguments: ProjectManagerArguments) -> None:
+    arguments.log_config(logger, mode="debug")
+
+    if arguments.reset:
+        reset_project(arguments)
+        return
+
+    if arguments.initialize_directory:
+        print("Initializing project directory:", arguments.dot_dp)
+        initialize_directories(arguments)
+        initialize_configuration_files(arguments)
+        return
+
+    derive_settings_files(arguments)
+
+    if arguments.list:
+        show_configurations_without_execution(arguments)
+        return
+    elif arguments.generate_report:
+        generate_full_report(arguments)
+    elif arguments.full_execute:
+        show_configurations_with_execution(arguments)
+    else:
+        show_configurations_with_execution(
+            arguments, restricted_configurations=arguments.execute_configurations.split(",")
+        )

discopop_library/ProjectManager/ProjectManagerArguments.py

@@ -0,0 +1,69 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+import json
+import os
+from pathlib import Path
+import sys
+from dataclasses import dataclass
+from typing import List, Optional
+import warnings
+
+from discopop_library.ArgumentClasses.GeneralArguments import GeneralArguments
+from logging import Logger
+
+
+@dataclass
+class ProjectManagerArguments(GeneralArguments):
+    """Container Class for the arguments passed to the discopop_project_manager"""
+
+    project_root: str
+    full_execute: bool
+    list: bool
+    execute_configurations: str
+    execute_inplace: bool
+    skip_cleanup: bool
+    generate_report: bool
+    initialize_directory: bool
+    apply_suggestions: Optional[str]
+    reset: bool
+    # derived values
+    dot_dp: str = ""
+    project_config_dir: str = ""
+
+    def __post_init__(self) -> None:
+        # save derived values
+        if self.project_root.endswith(".discopop"):
+            print(
+                "WARNING: Project root set to .discopop directory. \n-> Old project root: "
+                + self.project_root
+                + "\nFixed as follows:\n-> Project root: "
+                + str(Path(self.project_root).parent.absolute())
+                + "\n-> .discopop: "
+                + self.project_root
+            )
+            self.project_root = str(Path(self.project_root).parent.absolute())
+        self.dot_dp = os.path.join(self.project_root, ".discopop")
+        self.project_config_dir = os.path.join(self.dot_dp, "project")
+        # validate arguments
+        self.__validate()
+
+    def __validate(self) -> None:
+        """Validate the arguments passed to the discopop_configuration_manager, e.g check if given files exist"""
+        # check if .discopop folder exists
+        if not self.initialize_directory and not os.path.exists(self.dot_dp):
+            print("ERROR: folder not found: " + self.dot_dp + "\n-> Did you initialize the project using '--init' ? ")
+            sys.exit(1)
+
+    def log_config(self, logger: Logger, mode: str = "debug") -> None:
+        attributes = "\nConfiguration:\n"
+        for attribute in self.__dict__:
+            attributes += "-> " + str(attribute) + " : " + str(self.__dict__[attribute]) + "\n"
+        if mode == "info":
+            logger.info(attributes)
+        else:
+            logger.debug(attributes)

discopop_library/ProjectManager/__main__.py

@@ -0,0 +1,69 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+from argparse import ArgumentParser
+import os
+from discopop_library.GlobalLogger.setup import setup_logger
+from discopop_library.ProjectManager.ProjectManagerArguments import ProjectManagerArguments
+from discopop_library.ProjectManager.ProjectManager import run
+
+
+def parse_args() -> ProjectManagerArguments:
+    """Parse the arguments passed to the discopop_configuration_manager"""
+    parser = ArgumentParser(description="Initialize and prepare projects for the use in the DiscoPoP framework.")
+    # all flags that are not considered stable should be added to the experimental_parser
+    experimental_parser = parser.add_argument_group(
+        "EXPERIMENTAL",
+        "Arguments for configuration manager and other experimental features. These arguments are considered EXPERIMENTAL and they may or may not be removed or modified in the near future.",
+    )
+
+    # fmt: off
+    parser.add_argument("-v", "--verbose", action="store_true",
+                        help="Enable verbose output.")
+    parser.add_argument("-p", "--project", default=os.getcwd(), help="Path to the projects root folder. Important: it must be possible to create a copy of this folder and compile / execute the copy with the definitions from compile.sh, execute.sh, and settings.json. Please refer to the wiki pages (https://discopop-project.github.io/discopop/) for further details. Default: $(cwd)")
+    parser.add_argument("--init", action="store_true", help="Initialize the .discopop directory in the specified project path")
+    parser.add_argument("-x", "--execute", default="tiny", help="Comma separated list of configurations to be executed. Format: <config_name>[:<mode>] . Modes: dp,hd,seq,par. Default: tiny")
+    parser.add_argument("-xf", "--execute-full", action="store_true", help="Execute all configurations for validation purposes.")
+    parser.add_argument("-a", "--apply-suggestions", help="Comma separated list of suggestions ids to be applied before the specified execution. Use the keyword 'auto' to load the configuration determined by the autotuner (if multiple configurations exist, union will be considered).")
+    parser.add_argument('-l', '--list', action="store_true", help="Show a list of available configurations. If set, nothing else will be done.")
+    parser.add_argument("-i", "--inplace", action="store_true", help="Prevents the creation of project copies when code configurations are executed. Instead, executes the configuration in the project root folder.")
+    parser.add_argument("--skip-cleanup", action="store_true", help="Prevents the deletion of created project copies. May requires high amount of disk space.")
+    parser.add_argument("--report", action="store_true", help="Generate and show a report of the stored execution results.")
+    parser.add_argument("-r", "--reset", action="store_true", help="Reset the .discopop folder except configurations in project subdirectory.")
+
+    parser.add_argument("--log", type=str, default="WARNING", help="Specify log level: DEBUG, INFO, WARNING, ERROR, CRITICAL")
+    parser.add_argument("--write-log", action="store_true", help="Create Logfile.")
+    # EXPERIMENTAL FLAGS:
+    # fmt: on
+
+    arguments = parser.parse_args()
+
+    return ProjectManagerArguments(
+        project_root=arguments.project,
+        full_execute=arguments.execute_full,
+        list=arguments.list,
+        execute_configurations=arguments.execute,
+        execute_inplace=arguments.inplace,
+        skip_cleanup=arguments.skip_cleanup,
+        generate_report=arguments.report,
+        initialize_directory=arguments.init,
+        apply_suggestions=arguments.apply_suggestions,
+        reset=arguments.reset,
+        log_level=arguments.log.upper(),
+        write_log=arguments.write_log,
+    )
+
+
+def main() -> None:
+    arguments = parse_args()
+    setup_logger(arguments)
+    run(arguments)
+
+
+if __name__ == "__main__":
+    main()

discopop_library/ProjectManager/configurations/copying.py

@@ -0,0 +1,86 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import json
+import logging
+import os
+from pathlib import Path
+import shutil
+from typing import List, Optional
+from discopop_library.ProjectManager.ProjectManagerArguments import ProjectManagerArguments
+from discopop_library.PatchApplicator.PatchApplicatorArguments import PatchApplicatorArguments
+from discopop_library.PatchApplicator.patch_applicator import run as apply_suggestions
+
+
+PATH = str
+logger = logging.getLogger("ConfigurationManager")
+
+
+def copy_configuration(
+    arguments: ProjectManagerArguments,
+    config_path: PATH,
+    settings_path: PATH,
+    configuration_id: Optional[int] = None,
+) -> PATH:
+
+    config_name = os.path.basename(config_path)
+    settings_name = os.path.basename(settings_path)
+
+    # create project copy
+    project_copy_folder_name = config_name + "_" + settings_name + "_" + os.path.basename(arguments.project_root)
+    if configuration_id is not None:
+        project_copy_folder_name += "_" + str(configuration_id)
+    project_copy_path = os.path.join(Path(arguments.project_root).parent.absolute(), project_copy_folder_name)
+    if os.path.exists(project_copy_path):
+        shutil.rmtree(project_copy_path)
+    shutil.copytree(arguments.project_root, project_copy_path)
+    # correct paths in Filemapping.txt
+    fmap_path = os.path.join(project_copy_path, ".discopop", "FileMapping.txt")
+    if os.path.exists(fmap_path):
+        fmap_lines = []
+        with open(fmap_path, "r") as f:
+            fmap_lines = f.readlines()
+        for idx, line in enumerate(fmap_lines):
+            if arguments.project_root in line:
+                fmap_lines[idx] = fmap_lines[idx].replace(arguments.project_root, project_copy_path)
+        with open(fmap_path, "w") as f:
+            for line in fmap_lines:
+                f.write(line)
+
+    # apply the specified parallelization suggestions if requested
+    if arguments.apply_suggestions is not None:
+        dp_folder_path = os.path.join(project_copy_path, ".discopop")
+        if arguments.apply_suggestions == "auto":
+            auto_tuner_results_path = os.path.join(dp_folder_path, "auto_tuner", "results.json")
+            suggestions: List[str] = []
+            if os.path.exists(auto_tuner_results_path):
+                with open(auto_tuner_results_path, "r") as f:
+                    auto_tuner_results = json.load(f)
+                for configuration in auto_tuner_results:
+                    for sugg in auto_tuner_results[configuration]["applied_suggestions"]:
+                        if sugg not in suggestions:
+                            suggestions.append(sugg)
+        else:
+            suggestions = arguments.apply_suggestions.split(",")
+
+        pa_args = PatchApplicatorArguments(
+            verbose=False,
+            apply=suggestions,
+            rollback=[],
+            clear=False,
+            load=False,
+            list=False,
+            log_level=arguments.log_level,
+            write_log=arguments.write_log,
+        )
+        home_dir = os.getcwd()
+        os.chdir(dp_folder_path)
+        apply_suggestions(pa_args)
+        os.chdir(home_dir)
+
+    return project_copy_path

discopop_library/ProjectManager/configurations/deletion.py

@@ -0,0 +1,26 @@
+# This file is part of the DiscoPoP software (http://www.discopop.tu-darmstadt.de)
+#
+# Copyright (c) 2020, Technische Universitaet Darmstadt, Germany
+#
+# This software may be modified and distributed under the terms of
+# the 3-Clause BSD License.  See the LICENSE file in the package base
+# directory for details.
+
+import logging
+import os
+import shutil
+
+from discopop_library.ProjectManager.ProjectManagerArguments import ProjectManagerArguments
+
+PATH = str
+
+logger = logging.getLogger("ConfigurationManager")
+
+
+def delete_configuration(arguments: ProjectManagerArguments, project_copy_root_path: PATH) -> None:
+    if not os.path.exists(project_copy_root_path):
+        return
+    if arguments.project_root == project_copy_root_path:
+        return
+    shutil.rmtree(project_copy_root_path)
+    logger.debug("removed project copy: " + project_copy_root_path)