From 94e4b3b1c4f8e2112adeeedc70c5bd4e529990ec Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 17 Feb 2023 05:33:57 +0100 Subject: [PATCH 01/10] resources/templates/dev.yml.j2 new format --- resources/templates/dev.yml.j2 | 202 +++++++++++++-------------------- 1 file changed, 77 insertions(+), 125 deletions(-) diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index 7072d536c..21a8642e5 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -1,5 +1,7 @@ +# Generated with MLonMCU version {{ TODO }} --- # The MLONMCU_HOME is filled in automatically when creating the environment +version: 2 home: "{{ home_dir }}" logging: level: DEBUG @@ -68,139 +70,89 @@ repos: espidf: url: "https://github.com/espressif/esp-idf.git" ref: release/v4.4 - -# Here all supported frameworks with their specific features are defined -# Optionally disable unwanted or incomatible backends or features here -# The configured defaults are used if no backend was specified in the command line options frameworks: - default: tvm - tflm: - enabled: true - backends: - default: tflmi - tflmc: - enabled: false - tflmi: - enabled: true - features: - debug_arena: true - features: - muriscvnn: true - cmsisnn: true - tvm: - enabled: true - backends: - default: tvmaot - tvmrt: - enabled: true - features: - debug_arena: true - disable_legalize: true - autotune: true - autotuned: true - tvmaot: - enabled: true - features: - debug_arena: true - unpacked_api: true - usmp: true - disable_legalize: true - autotune: true - autotuned: true - tvmcg: - enabled: true - features: - debug_arena: true - disable_legalize: true - autotune: true - autotuned: true - features: - cmsisnnbyoc: false -# Some frontends are eperimental and therefore disabled here -# Features like packing are only available in certain environments -# + supported: &supported_frameworks + - tflm + - tvm + use: *supported_frameworks +backends: + supported: + - tflmc + - tflmi + - tvmaot + - tvmrt + - tvmcg + use: + - tvmaot frontends: - tflite: - enabled: true - features: - validate: true - visualize: true - relay: - enabled: true - features: - relayviz: true - packed: - enabled: false - features: - packing: true - packed: true - onnx: - enabled: false - # TODO: saved_model (TF->TFLITE), ipynb (IPYNB->?) + supported: &supported_frontends + - tflite + - relay + # - packed + # - onnx + use: *supported_frontends # List of supported targets in the environment targets: - default: etiss_pulpino - etiss_pulpino: - enabled: true - features: - gdbserver: true - etissdbg: true - trace: true - log_instrs: true - host_x86: - enabled: true - features: - gdbserver: true - spike: - enabled: true - features: - vext: true - cachesim: true - log_instrs: true - ovpsim: - enabled: true - features: - vext: true - corstone300: - enabled: true - features: - ethosu: false - esp32: - enabled: true - esp32c3: - enabled: true - microtvm_etissvp: - enabled: true - microtvm_espidf: - enabled: false + supported: + - etiss_pulpino # mlif + - spike # mlif + - ovpsim # mlif + - host_x86 # mlif + - corstone300 # mlif + # espidf targets do not need to be listed here + # - esp32 # espidf + # - esp32c3 # espidf + # tvm targets do not need to be listed here + # - tvm_cpu # tvm + # microtvm targets need to be listed here if they have dependencies which need to be fetched + - microtvm_zephyr # microtvm + - microtvm_host # microtvm + # - microtvm_spike # microtvm + # - microtvm_etissvp # microtvm + # - microtvm_espidf # microtvm + use: + - etiss_pulpino # Some targets/platforms support multiple toolchains toolchains: - gcc: true - llvm: true + supported: &supported_toolchains + gcc: true # TODO: split into gcc, riscv_gcc, arm_gcc + llvm: true + # use: NO EFFECT platforms: - mlif: - enabled: true - features: - debug: true - validate: true - espidf: - enabled: true - features: - debug: true - zephyr: - enabled: false - features: - debug: true + supported: &supported_platforms + - mlif + - espidf + - zephyr + # - arduino + use: *supported_platforms +features: + supported: + - gdbserver + - etissdbg + - vext + - trace + - disable_legalize + - debug_arena + - unpacked_api + - usmp + - cmsisnn + - muriscvnn + # - cmsisnnbyoc + # - packing + # - packed + # - ethosu + use: [] +# Issues: Do we need to add user-provided postprocesses to the list as well? postprocesses: - use: - # - detailed_cycles - # - average_cycles - # - filter_cols - # - features2cols - # - config2cols - # - bytes2kb - # - visualize -# This is where further options such as specific versions of dependencies can be set in the furture + supported: + - detailed_cycles + - average_cycles + - filter_cols + - features2cols + - config2cols + - bytes2kb + - visualize + use: [] vars: allow_extensions: false # tvm.make_tool: "ninja" From ec0673a70be3931756a4ea4e6d09edaf2c1674a6 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 17 Feb 2023 05:34:23 +0100 Subject: [PATCH 02/10] add support for new environment format --- mlonmcu/environment/config.py | 34 +++++++++++++++++++++++++----- mlonmcu/environment/environment.py | 6 +++--- mlonmcu/environment/loader.py | 4 ++-- 3 files changed, 34 insertions(+), 10 deletions(-) diff --git a/mlonmcu/environment/config.py b/mlonmcu/environment/config.py index 1d41de426..de2e1b60e 100644 --- a/mlonmcu/environment/config.py +++ b/mlonmcu/environment/config.py @@ -22,6 +22,7 @@ import os import xdg import logging +from collections import namedtuple from enum import Enum from pathlib import Path @@ -88,20 +89,40 @@ def __init__( log_level=logging.INFO, log_to_file=False, log_rotate=False, - default_framework=None, - default_backends={}, - default_target=None, cleanup_auto=False, cleanup_keep=100, ): self.log_level = log_level self.log_to_file = log_to_file self.log_rotate = log_rotate + self.cleanup_auto = cleanup_auto + self.cleanup_keep = cleanup_keep + + +class DefaultsConfigOld(DefaultsConfig): + # TODO: loglevels enum + + def __init__( + self, + log_level=logging.INFO, + log_to_file=False, + log_rotate=False, + cleanup_auto=False, + cleanup_keep=100, + default_framework=None, + default_backends={}, + default_target=None, + ): + super().__init__( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + cleanup_auto=False, + cleanup_keep=100, + ) self.default_framework = default_framework self.default_backends = default_backends self.default_target = default_target - self.cleanup_auto = cleanup_auto - self.cleanup_keep = cleanup_keep class PathConfig(BaseConfig): @@ -131,6 +152,9 @@ def __init__(self, url, ref=None): self.ref = ref +ComponentConfig = namedtuple("ComponentConfig", "supported used") + + class BackendConfig(BaseConfig): def __init__(self, name, enabled=True, features={}): self.name = name diff --git a/mlonmcu/environment/environment.py b/mlonmcu/environment/environment.py index ccb8a5e94..9ef74412b 100644 --- a/mlonmcu/environment/environment.py +++ b/mlonmcu/environment/environment.py @@ -19,7 +19,7 @@ import logging from .config import ( - DefaultsConfig, + DefaultsConfigOld, PathConfig, RepoConfig, FrameworkConfig, @@ -61,7 +61,7 @@ class Environment: def __init__(self): self._home = None self.alias = None - self.defaults = DefaultsConfig() + self.defaults = DefaultsConfigOld() self.paths = {} self.repos = {} self.frameworks = [] @@ -349,7 +349,7 @@ def get_default_targets(self): class DefaultEnvironment(Environment): def __init__(self): super().__init__() - self.defaults = DefaultsConfig( + self.defaults = DefaultsConfigOld( log_level=logging.DEBUG, log_to_file=False, default_framework=None, diff --git a/mlonmcu/environment/loader.py b/mlonmcu/environment/loader.py index 5ff88ad51..b950e45e2 100644 --- a/mlonmcu/environment/loader.py +++ b/mlonmcu/environment/loader.py @@ -21,7 +21,7 @@ import logging from .config import ( - DefaultsConfig, + DefaultsConfigOld, PathConfig, RepoConfig, FrameworkConfig, @@ -224,7 +224,7 @@ def load_environment_from_file(filename, base): default_flags = loaded["flags"] else: default_flags = None - defaults = DefaultsConfig( + defaults = DefaultsConfigOld( log_level=log_level, log_to_file=log_to_file, log_rotate=log_rotate, From 57e7d1cd49944c1c37026d19f19dd226437df25f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 17 Feb 2023 06:45:01 +0100 Subject: [PATCH 03/10] add script for converting environment file to new version --- mlonmcu/environment/convert.py | 173 +++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 mlonmcu/environment/convert.py diff --git a/mlonmcu/environment/convert.py b/mlonmcu/environment/convert.py new file mode 100644 index 000000000..8f68a6649 --- /dev/null +++ b/mlonmcu/environment/convert.py @@ -0,0 +1,173 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# import yaml +# import pathlib +# import logging +# +# from .config import PathConfig + + +import argparse +from mlonmcu.environment.environment import UserEnvironment +from mlonmcu.environment.environment2 import UserEnvironment2 +from mlonmcu.environment.config import ComponentConfig, DefaultsConfig + +# from mlonmcu.environment.loader import load_environment_from_file + + +def convert_v1_to_v2(env): + frameworks = {f.name: ComponentConfig(f.enabled, f.name == env.defaults.default_framework) for f in env.frameworks} + backends = { + b.name: ComponentConfig( + b.enabled, f.name == env.defaults.default_framework and b.name == env.defaults.default_backends[f.name] + ) + for f in env.frameworks + for b in f.backends + if f.enabled + } + toolchains = {t: ComponentConfig(True, False) for t in env.toolchains} + frontends = {f.name: ComponentConfig(f.enabled, f.enabled) for f in env.frontends} + platforms = {p.name: ComponentConfig(p.enabled, p.enabled) for p in env.platforms} + targets = {t.name: ComponentConfig(t.enabled, t.name == env.defaults.default_target) for t in env.targets} + + framework_features = { + f_.name: ComponentConfig(f_.supported, False) for f in env.frameworks for f_ in f.features if f.enabled + } + backend_features = { + f_.name: ComponentConfig(f_.supported, False) + for f in env.frameworks + for b in f.backends + for f_ in b.features + if b.enabled and f.enabled + } + target_features = { + f_.name: ComponentConfig(f_.supported, False) for t in env.targets for f_ in t.features if t.enabled + } + platform_features = { + f_.name: ComponentConfig(f_.supported, False) for p in env.platforms for f_ in p.features if p.enabled + } + frontend_features = { + f_.name: ComponentConfig(f_.supported, False) for f in env.frontends for f_ in f.features if f.enabled + } + other_features = {} + # print("framework_features", framework_features) + # print("backend_features", backend_features) + # print("target_features", target_features) + # print("frontend_features", frontend_features) + # print("other_features", other_features) + # print("platform_features", platform_features) + + def helper(name): + return any( + [ + name in x and x[name].supported + for x in [ + framework_features, + backend_features, + target_features, + platform_features, + frontend_features, + other_features, + ] + ] + ) + + all_feature_names = ( + set(framework_features.keys()) + .union(set(backend_features.keys())) + .union(set(target_features.keys())) + .union(set(platform_features.keys())) + .union(set(frontend_features.keys())) + .union(set(other_features.keys())) + ) + features = {name: ComponentConfig(helper(name), False) for name in all_feature_names} + # print("features", features) + # print("?") + + env2 = UserEnvironment2( + env.home, + alias=env.alias, + defaults=DefaultsConfig( + log_level=env.defaults.log_level, + log_to_file=env.defaults.log_to_file, + log_rotate=env.defaults.log_rotate, + cleanup_auto=env.defaults.cleanup_auto, + cleanup_keep=env.defaults.cleanup_keep, + ), + paths=env.paths, + repos=env.repos, + frameworks=frameworks, + backends=backends, + frontends=frontends, + platforms=platforms, + toolchains=toolchains, + targets=targets, + features=features, + # postprocesses=None, + variables=env.vars, + default_flags=env.flags, + ) + return env2 + + +def main(): + parser = argparse.ArgumentParser( + description=f"Converter for environment files/templates", + ) + parser.add_argument("input", metavar="FILE", type=str, nargs=1, help="File to process") + parser.add_argument( + "--output", + "-o", + metavar="DEST", + type=str, + default=None, + help="""Output directory/file (default: print to stdout instead)""", + ) + parser.add_argument( + "--in-version", + metavar="VER", + type=int, + default=1, + help="""Output directory/file (default: %(default)s)""", + ) + parser.add_argument( + "--out-version", + metavar="VER", + type=int, + default=2, + help="""Output directory/file (default: %(default)s)""", + ) + args = parser.parse_args() + + assert args.in_version == 1 and args.out_version == 2, "Unsupported combination of versions" + + file = args.input[0] + + env = UserEnvironment.from_file(file) + + env2 = convert_v1_to_v2(env) + + if args.output: + env2.to_file(args.output) + else: + print(env2.to_text()) + + +if __name__ == "__main__": + main() From 5097693e72b85039997ad380f3d7c86130469104 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Fri, 17 Feb 2023 06:45:35 +0100 Subject: [PATCH 04/10] add missing files for new environment format --- mlonmcu/environment/environment2.py | 532 ++++++++++++++++++++++++++++ mlonmcu/environment/loader2.py | 160 +++++++++ mlonmcu/environment/writer2.py | 88 +++++ 3 files changed, 780 insertions(+) create mode 100644 mlonmcu/environment/environment2.py create mode 100644 mlonmcu/environment/loader2.py create mode 100644 mlonmcu/environment/writer2.py diff --git a/mlonmcu/environment/environment2.py b/mlonmcu/environment/environment2.py new file mode 100644 index 000000000..406cf57d4 --- /dev/null +++ b/mlonmcu/environment/environment2.py @@ -0,0 +1,532 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import tempfile +from abc import ABC +from pathlib import Path +from typing import Dict, List, Union + +from .config import ( + DefaultsConfig, + PathConfig, + RepoConfig, + ComponentConfig, + # FrameworkConfig, + # FrameworkFeatureConfig, + # BackendConfig, + # BackendFeatureConfig, + # TargetConfig, + # TargetFeatureConfig, + # PlatformConfig, + # PlatformFeatureConfig, + # FrontendConfig, + # FrontendFeatureConfig, +) +from .loader2 import load_environment_from_file +from .writer2 import write_environment_to_file + +# from mlonmcu.feature.type import FeatureType + + +# def _feature_helper(obj, name): +# if not obj.enabled: +# return [] +# features = obj.features +# if name: +# return [feature for feature in features if feature.name == name] +# else: +# return features + + +# def _extract_names(objs): +# return [obj.name for obj in objs] + + +# def _filter_enabled(objs): +# return [obj for obj in objs if obj.enabled] + +class BaseEnvironment(ABC): + def __init__(self): + self.version = None + self._home: Path = None + + +class Environment2(BaseEnvironment): + def __init__(self): + super().__init__() + self.alias: str = None + self.version = 2 + self.defaults = DefaultsConfig() + self.paths: "Dict[str, Union[PathConfig, List[PathConfig]]]" = {} + self.repos: "Dict[str, RepoConfig]" = {} + self.frameworks: "Dict[str, ComponentConfig]" = {} + self.backends: "Dict[str, ComponentConfig]" = {} + self.features: "Dict[str, ComponentConfig]" = {} + self.toolchains: "Dict[str, ComponentConfig]" = {} + # self.frontends = [] + # self.platforms = [] + # self.toolchains = [] + # self.targets = [] + # self.postprocesses = [] + self.vars = {} + self.flags = {} + + def __str__(self): + return self.__class__.__name__ + "(" + str(vars(self)) + ")" + + @property + def home(self): + """Home directory of mlonmcu environment.""" + return self._home + + @classmethod + def from_file(cls, filename): + return load_environment_from_file(filename, base=cls) + + def to_file(self, filename): + write_environment_to_file(self, filename) + + def to_text(self): + with tempfile.TemporaryDirectory() as tmpdirname: + dest = Path(tmpdirname) / "out.yml" + self.to_file(dest) + with open(dest, "r") as handle: + content = handle.read() + assert len(content) > 0 + return content + + def lookup_path(self, name): + assert name in self.paths, f"Unable to find '{name}' path in environment config" + return self.paths[name] + + def lookup_var(self, name, default=None): + return self.vars.get(name, default) + + # def lookup_frontend_feature_configs(self, name=None, frontend=None): + # configs = [] + # if frontend: + # names = [frontend.name for frontend in self.frontends] + # index = names.index(frontend) + # assert index is not None, f"Frontend {frontend} not found in environment config" + # configs.extend(_feature_helper(self.frontends[index], name)) + # else: + # for frontend in self.frontends: + # configs.extend(_feature_helper(frontend, name)) + # return configs + + # def lookup_framework_feature_configs(self, name=None, framework=None): + # configs = [] + # if framework: + # names = [framework.name for framework in self.frameworks] + # index = names.index(framework) + # assert index is not None, f"Framework {framework} not found in environment config" + # configs.extend(_feature_helper(self.frameworks[index], name)) + # else: + # for framework in self.frameworks: + # configs.extend(_feature_helper(framework, name)) + # return configs + + # def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): + # def helper(framework, backend, name): + # backend_features = self.framework[framework].backends[backend].features + # if name: + # return [backend_features[name]] + # else: + # return backend_features.values() + + # configs = [] + # if framework: + # names = [framework.name for framework in self.frameworks] + # index = names.index(framework) + # assert index is not None, f"Framework {framework} not found in environment config" + # if backend: + # names_ = [backend.name for backend in self.frameworks[index].backends] + # index_ = names_.index(backend) + # assert index_ is not None, f"Backend {backend} not found in environment config" + # configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + # else: + # for backend in self.frameworks[index].backends: + # configs.extend(_feature_helper(backend, name)) + # else: + # for framework in self.frameworks: + # if backend: + # names_ = [backend.name for backend in framework.backends] + # index_ = names_.index(backend) + # assert index_ is not None, f"Backend {backend} not found in environment config" + # configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + # else: + # for backend in framework.backends: + # configs.extend(_feature_helper(backend, name)) + # backend = None + # return configs + + # def lookup_platform_feature_configs(self, name=None, platform=None): + # configs = [] + # if platform: + # names = [platform.name for platform in self.platforms] + # index = names.index(platform) + # assert ( + # index is not None + # ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list + # configs.extend(_feature_helper(self.platforms[index], name)) + # else: + # for platform in self.platforms: + # configs.extend(_feature_helper(platform, name)) + # return configs + + # def lookup_target_feature_configs(self, name=None, target=None): + # configs = [] + # if target: + # names = [target.name for target in self.targets] + # index = names.index(target) + # assert ( + # index is not None + # ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list + # configs.extend(_feature_helper(self.targets[index], name)) + # else: + # for target in self.targets: + # configs.extend(_feature_helper(target, name)) + # return configs + + # def lookup_feature_configs( + # self, + # name=None, + # kind=None, + # frontend=None, + # framework=None, + # backend=None, + # platform=None, + # target=None, + # ): + # configs = [] + # if kind == FeatureType.FRONTEND or kind is None: + # configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) + # if kind == FeatureType.FRAMEWORK or kind is None: + # configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) + # if kind == FeatureType.BACKEND or kind is None: + # configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) + # if kind == FeatureType.PLATFORM or kind is None: + # configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) + # if kind == FeatureType.TARGET or kind is None: + # configs.extend(self.lookup_target_feature_configs(name=name, target=target)) + + # return configs + + def supports_feature(self, name): + configs = self.lookup_feature_configs(name=name) + supported = [feature.supported for feature in configs] + return any(supported) + + def has_feature(self, name): + """An alias for supports_feature.""" + return self.supports_feature(name) + + # def lookup_backend_configs(self, backend=None, framework=None, names_only=False): + # enabled_frameworks = _filter_enabled(self.frameworks) + + # configs = [] + # for framework_config in enabled_frameworks: + # if framework is not None and framework_config.name != framework: + # continue + # enabled_backends = _filter_enabled(framework_config.backends) + # if backend is None: + # configs.extend(enabled_backends) + # else: + # for backend_config in enabled_backends: + # if backend_config.name == backend: + # return [backend_config.name if names_only else backend_config] + # return _extract_names(configs) if names_only else configs + + # def lookup_framework_configs(self, framework=None, names_only=False): + # enabled_frameworks = _filter_enabled(self.frameworks) + + # if framework is None: + # return _extract_names(enabled_frameworks) if names_only else enabled_frameworks + + # for framework_config in enabled_frameworks: + # if framework_config.name == framework: + # return [framework_config.name if names_only else framework_config] + + # return [] + + # def lookup_frontend_configs(self, frontend=None, names_only=False): + # enabled_frontends = _filter_enabled(self.frontends) + + # if frontend is None: + # return _extract_names(enabled_frontends) if names_only else enabled_frontends + + # for frontend_config in enabled_frontends: + # if frontend_config.name == frontend: + # return [frontend_config.name if names_only else frontend_config] + # return [] + + # def lookup_platform_configs(self, platform=None, names_only=False): + # enabled_platforms = _filter_enabled(self.platforms) + + # if platform is None: + # return _extract_names(enabled_platforms) if names_only else enabled_platforms + # for platform_config in enabled_platforms: + # if platform_config.name == platform: + # return [platform_config.name if names_only else platform_config] + + # return [] + + # def lookup_target_configs(self, target=None, names_only=False): + # enabled_targets = _filter_enabled(self.targets) + + # if target is None: + # return _extract_names(enabled_targets) if names_only else enabled_targets + + # for target_config in enabled_targets: + # if target_config.name == target: + # return [target_config.name if names_only else target_config] + # return [] + + def has_frontend(self, name): + configs = self.lookup_frontend_configs(frontend=name) + return len(configs) > 0 + + def has_backend(self, name): + configs = self.lookup_backend_configs(backend=name) + return len(configs) > 0 + + def has_framework(self, name): + configs = self.lookup_framework_configs(framework=name) + return len(configs) > 0 + + def has_platform(self, name): + configs = self.lookup_platform_configs(platform=name) + return len(configs) > 0 + + def has_toolchain(self, name): + return self.toolchains.get(name, False) + + def has_target(self, name): + configs = self.lookup_target_configs(target=name) + return len(configs) > 0 + + # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead + # of the supported ones in the environment + # def has_postprocess(self, name): + # configs = self.lookup_postprocess_configs(postprocess=name) + # return len(configs) > 0 + + # def get_default_backends(self, framework): + # if framework is None or framework not in self.defaults.default_backends: + # return [] + # default = self.defaults.default_backends[framework] + # # framework_names = [framework_config.name for framework_config in self.frameworks] + # # framework_config = self.frameworks[framework_names.index(framework)] + # if default is None: + # return [] + # if isinstance(default, str): + # if default == "*": # Wildcard all enabled frameworks + # default = self.get_enabled_backends() + # else: + # default = [default] + # else: + # assert isinstance(default, list), "TODO" + # return default + + # def get_default_frameworks(self): + # default = self.defaults.default_framework + # if default is None: + # return [] + # if isinstance(default, str): + # if default == "*": # Wildcard all enabled frameworks + # default = self.get_enabled_frameworks() + # else: + # default = [default] + # else: + # assert isinstance(default, list), "TODO" + # return default + + # def get_default_targets(self): + # default = self.defaults.default_target + # if default is not None: + # if isinstance(default, str): + # if default == "*": # Wildcard all enabled targets + # default = self.get_enabled_targets() + # else: + # default = [default] + # else: + # assert isinstance(default, list) + # return default + + +class DefaultEnvironment2(Environment2): + def __init__(self): + super().__init__() + self.defaults = DefaultsConfig( + log_level=logging.DEBUG, + log_to_file=False, + cleanup_auto=False, + cleanup_keep=100, + ) + self.paths = { + "deps": PathConfig("./deps"), + "logs": PathConfig("./logs"), + "results": PathConfig("./results"), + "plugins": PathConfig("./plugins"), + "temp": PathConfig("out"), + "models": [ + PathConfig("./models"), + ], + } + # TODO: refresh or remove + self.repos = { + "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), + "tflite_micro_compiler": RepoConfig( + "https://github.com/cpetig/tflite_micro_compiler.git", ref="master" + ), # TODO: freeze ref? + "tvm": RepoConfig( + "https://github.com/tum-ei-eda/tvm.git", ref="tumeda" + ), # TODO: use upstream repo with suitable commit? + "utvm_staticrt_codegen": RepoConfig( + "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git", ref="master" + ), # TODO: freeze ref? + "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? + } + self.frameworks = {} + self.backends = {} + self.features = {} + self.postprocesses = {} + self.toolchains = {} + # self.frameworks = [ + # FrameworkConfig( + # "tflm", + # enabled=True, + # backends=[ + # BackendConfig("tflmc", enabled=True, features=[]), + # BackendConfig("tflmi", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), + # ], + # ), + # FrameworkConfig( + # "utvm", + # enabled=True, + # backends=[ + # BackendConfig( + # "tvmaot", + # enabled=True, + # features=[ + # BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), + # ], + # ), + # BackendConfig("tvmrt", enabled=True, features=[]), + # BackendConfig("tvmcg", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("memplan", framework="utvm", supported=False), + # ], + # ), + # ] + # self.frontends = [ + # FrontendConfig("saved_model", enabled=False), + # FrontendConfig("ipynb", enabled=False), + # FrontendConfig( + # "tflite", + # enabled=True, + # features=[ + # FrontendFeatureConfig("packing", frontend="tflite", supported=False), + # ], + # ), + # ] + # self.vars = { + # "TEST": "abc", + # } + # self.flags = {} + # self.platforms = [ + # PlatformConfig( + # "mlif", + # enabled=True, + # features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], + # ) + # ] + # self.targets = [ + # TargetConfig( + # "etiss_pulpino", + # features=[ + # TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), + # ], + # ), + # TargetConfig( + # "host_x86", + # features=[ + # TargetFeatureConfig("debug", target="host_x86", supported=True), + # TargetFeatureConfig("attach", target="host_x86", supported=True), + # ], + # ), + # ] + + +class UserEnvironment2(DefaultEnvironment2): + def __init__( + self, + home, + alias=None, + defaults=None, + paths=None, + repos=None, + frameworks=None, + backends=None, + frontends=None, + platforms=None, + toolchains=None, + targets=None, + features=None, + postprocesses=None, + variables=None, + default_flags=None, + ): + super().__init__() + self._home = home + + if alias: + self.alias = alias + if defaults: + self.defaults = defaults + if paths: + self.paths = paths + if repos: + self.repos = repos + if frameworks: + self.frameworks = frameworks + if backends: + self.backends = backends + if frontends: + self.frontends = frontends + if platforms: + self.platforms = platforms + if toolchains: + self.toolchains = toolchains + if targets: + self.targets = targets + if features: + self.features = features + if postprocesses: + self.postprocesses = postprocesses + if variables: + self.vars = variables + if default_flags: + self.flags = default_flags diff --git a/mlonmcu/environment/loader2.py b/mlonmcu/environment/loader2.py new file mode 100644 index 000000000..104c8b8e1 --- /dev/null +++ b/mlonmcu/environment/loader2.py @@ -0,0 +1,160 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from .config import ( + DefaultsConfig, + PathConfig, + RepoConfig, + ComponentConfig, +) + +# def load_environment_from_file(filename): +# """Utility to initialize a mlonmcu environment from a YAML file.""" +# if isinstance(filename, str): +# filename = pathlib.Path(filename) +# with open(filename, encoding="utf-8") as yaml_file: +# data = yaml.safe_load(yaml_file) +# if data: +# if "home" in data: +# print(data["home"], filename.parent) +# assert os.path.realpath(data["home"]) == os.path.realpath(filename.parent) +# else: +# data["home"] = filename.parent +# env = Environment(data) +# return env +# raise RuntimeError(f"Error opening environment file: {filename}") +# return None + + +def load_environment_from_file(filename, base): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + with open(filename, encoding="utf-8") as yaml_file: + loaded = yaml.safe_load(yaml_file) + if not loaded: + raise RuntimeError("Invalid YAML contents") + if "home" in loaded: + home = loaded["home"] + else: + home = None + if "logging" in loaded: + if "level" in loaded["logging"]: + log_level = logging.getLevelName(loaded["logging"]["level"]) + else: + log_level = None + if "to_file" in loaded["logging"]: + log_to_file = bool(loaded["logging"]["to_file"]) + else: + log_to_file = None + if "rotate" in loaded["logging"]: + log_rotate = bool(loaded["logging"]["rotate"]) + else: + log_rotate = None + else: + log_level = None + log_to_file = False + log_rotate = False + if "cleanup" in loaded: + if "auto" in loaded["cleanup"]: + cleanup_auto = bool(loaded["cleanup"]["auto"]) + else: + cleanup_auto = False + if "auto" in loaded["cleanup"]: + cleanup_keep = int(loaded["cleanup"]["keep"]) + else: + cleanup_keep = 100 + else: + cleanup_auto = False + cleanup_keep = 100 + if "paths" in loaded: + paths = {} + for key in loaded["paths"]: + path = loaded["paths"][key] + if isinstance(path, list): + paths[key] = [] + for p in path: + paths[key].append(PathConfig(p, base=home)) + else: + paths[key] = PathConfig(path, base=home) + else: + paths = None + if "repos" in loaded: + repos = {} + for key in loaded["repos"]: + repo = loaded["repos"][key] + if "url" not in repo: + raise RuntimeError("Missing field 'url' in YAML file") + if "ref" in repo: + repos[key] = RepoConfig(repo["url"], ref=repo["ref"]) + else: + repos[key] = RepoConfig(repo["url"]) + else: + repos = None + + def helper(data): + if data is None: + return None + supported = data.get("supported", []) + used = data.get("use", []) + combined = list(set(supported + used)) + return {key: ComponentConfig(key in supported, key in used) for key in combined} + frameworks = helper(loaded.get("frameworks", None)) + backends = helper(loaded.get("backends", None)) + frontends = helper(loaded.get("frontends", None)) + features = helper(loaded.get("features", None)) + platforms = helper(loaded.get("platforms", None)) + targets = helper(loaded.get("targets", None)) + postprocesses = helper(loaded.get("postprocesses", None)) + toolchains = helper(loaded.get("toolchains", None)) + if "vars" in loaded: + variables = loaded["vars"] + else: + variables = None + if "flags" in loaded: + default_flags = loaded["flags"] + else: + default_flags = None + defaults = DefaultsConfig( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + cleanup_auto=cleanup_auto, + cleanup_keep=cleanup_keep, + ) + env = base( + home, + defaults=defaults, + paths=paths, + repos=repos, + frameworks=frameworks, + backends=backends, + frontends=frontends, + platforms=platforms, + toolchains=toolchains, + targets=targets, + features=features, + postprocesses=postprocesses, + variables=variables, + default_flags=default_flags, + ) + return env diff --git a/mlonmcu/environment/writer2.py b/mlonmcu/environment/writer2.py new file mode 100644 index 000000000..815c6b36f --- /dev/null +++ b/mlonmcu/environment/writer2.py @@ -0,0 +1,88 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from .config import PathConfig + + +def create_environment_dict(environment): + data = {} + data["home"] = environment.home + data["logging"] = { + "level": logging.getLevelName(environment.defaults.log_level), + "to_file": environment.defaults.log_to_file, + "rotate": environment.defaults.log_rotate, + } + data["cleanup"] = { + "auto": environment.defaults.cleanup_auto, + "keep": environment.defaults.cleanup_keep, + } + data["paths"] = { + path: str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + for path, path_config in environment.paths.items() + } # TODO: allow relative paths + data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} + data["frameworks"] = { + "supported": [name for name, value in environment.frameworks.items() if value.supported], + "use": [name for name, value in environment.frameworks.items() if value.used], + } + data["backends"] = { + "supported": [name for name, value in environment.backends.items() if value.supported], + "use": [name for name, value in environment.backends.items() if value.used], + } + data["frontends"] = { + "supported": [name for name, value in environment.frontends.items() if value.supported], + "use": [name for name, value in environment.frontends.items() if value.used], + } + data["toolchains"] = { + "supported": [name for name, value in environment.toolchains.items() if value.supported], + "use": [name for name, value in environment.toolchains.items() if value.used], + } + data["platforms"] = { + "supported": [name for name, value in environment.platforms.items() if value.supported], + "use": [name for name, value in environment.platforms.items() if value.used], + } + data["targets"] = { + "supported": [name for name, value in environment.targets.items() if value.supported], + "use": [name for name, value in environment.targets.items() if value.used], + } + data["features"] = { + "supported": [name for name, value in environment.features.items() if value.supported], + "use": [name for name, value in environment.features.items() if value.used], + } + data["postprocesses"] = { + "supported": [name for name, value in environment.postprocesses.items() if value.supported], + "use": [name for name, value in environment.postprocesses.items() if value.used], + } + data["vars"] = environment.vars + data["flags"] = environment.flags + return data + + +def write_environment_to_file(environment, filename): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + data = create_environment_dict(environment) + with open(filename, "w") as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False, sort_keys=False) From 1d16878aa8efc3b53aafa08faa5bc702c9cfbeba Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 18 Feb 2023 17:34:19 +0100 Subject: [PATCH 05/10] followup fixes --- mlonmcu/cli/helper/parse.py | 6 +- mlonmcu/context/context.py | 18 +- mlonmcu/environment/config.py | 116 +---- mlonmcu/environment/convert.py | 16 +- mlonmcu/environment/environment.py | 566 +++++++--------------- mlonmcu/environment/environment2.py | 532 -------------------- mlonmcu/environment/legacy/config.py | 132 +++++ mlonmcu/environment/legacy/environment.py | 514 ++++++++++++++++++++ mlonmcu/environment/legacy/loader.py | 252 ++++++++++ mlonmcu/environment/legacy/writer.py | 117 +++++ mlonmcu/environment/loader.py | 148 ++---- mlonmcu/environment/loader2.py | 160 ------ mlonmcu/environment/user_environment.py | 92 ++++ mlonmcu/environment/writer.py | 81 ++-- mlonmcu/environment/writer2.py | 88 ---- mlonmcu/platform/lookup.py | 2 +- 16 files changed, 1370 insertions(+), 1470 deletions(-) delete mode 100644 mlonmcu/environment/environment2.py create mode 100644 mlonmcu/environment/legacy/config.py create mode 100644 mlonmcu/environment/legacy/environment.py create mode 100644 mlonmcu/environment/legacy/loader.py create mode 100644 mlonmcu/environment/legacy/writer.py delete mode 100644 mlonmcu/environment/loader2.py create mode 100644 mlonmcu/environment/user_environment.py delete mode 100644 mlonmcu/environment/writer2.py diff --git a/mlonmcu/cli/helper/parse.py b/mlonmcu/cli/helper/parse.py index 7e455d9c2..b4941be8e 100644 --- a/mlonmcu/cli/helper/parse.py +++ b/mlonmcu/cli/helper/parse.py @@ -109,8 +109,8 @@ def extract_frontend_names(args, context=None): # No need to specify a default, because we just use the provided order in the environment.yml assert frontend_names is None, "TODO" assert context is not None, "Need context to resolve default frontends" - all_frontend_names = context.environment.lookup_frontend_configs(names_only=True) - names.extend(all_frontend_names) + default_frontend_names = context.environment.get_used_frontends() + names.extend(default_frontend_names) return names @@ -155,5 +155,5 @@ def extract_platform_names(args, context=None): assert args.platform is None if context is None: return [None] - platforms = context.environment.lookup_platform_configs(names_only=True) + platforms = context.environment.get_used_platforms() return platforms diff --git a/mlonmcu/context/context.py b/mlonmcu/context/context.py index db9277ef8..c0bdf59a4 100644 --- a/mlonmcu/context/context.py +++ b/mlonmcu/context/context.py @@ -35,7 +35,7 @@ from mlonmcu.plugins import process_extensions from mlonmcu.context.read_write_filelock import ReadFileLock, WriteFileLock, RWLockTimeout -from mlonmcu.environment.environment import Environment, UserEnvironment +from mlonmcu.environment.user_environment import UserEnvironment from mlonmcu.environment.list import get_environments_map from mlonmcu.environment.config import get_environments_dir, get_plugins_dir @@ -43,7 +43,7 @@ logger = get_logger() -def lookup_environment() -> Environment: +def lookup_environment() -> "UserEnvironment": """Helper function to automatically find a suitable environment. This function is used if neither a name nor a path of the environment was specified by the user. @@ -85,7 +85,7 @@ def lookup_environment() -> Environment: return None -def get_environment_by_path(path: Union[str, Path]) -> Environment: +def get_environment_by_path(path: Union[str, Path]) -> UserEnvironment: """Utility to find an environment file using a supplied path. Parameters @@ -95,7 +95,7 @@ def get_environment_by_path(path: Union[str, Path]) -> Environment: Returns ------- - Environment: + UserEnvironment: The environment (if the lookup was successful). """ if isinstance(path, str): @@ -108,7 +108,7 @@ def get_environment_by_path(path: Union[str, Path]) -> Environment: return None -def get_environment_by_name(name: str) -> Environment: +def get_environment_by_name(name: str) -> UserEnvironment: """Utility to find an environment file using a supplied name. Parameters @@ -149,12 +149,12 @@ def get_ids(directory: Path) -> List[int]: return sorted(ids) # TODO: sort by session datetime? -def load_recent_sessions(env: Environment, count: int = None) -> List[Session]: +def load_recent_sessions(env: UserEnvironment, count: int = None) -> List[Session]: """Get a list of recent sessions for the environment. Parameters ---------- - env : Environment + env : UserEnvironment MLonMCU environment which should be used. count : int Maximum number of sessions to return. Collect all if None. @@ -234,7 +234,7 @@ def setup_logging(environment): Attributes ---------- - environment : Environment + environment : UserEnvironment The MLonMCU Environment where paths, repos, features,... are configured. """ defaults = environment.defaults @@ -255,7 +255,7 @@ class MlonMcuContext: Attributes ---------- - environment : Environment + environment : UserEnvironment The MLonMCU Environment where paths, repos, features,... are configured. deps_lock : str ("read" or "write" default "write") Read means that the program does not write to the ./deps folder in the env folder. diff --git a/mlonmcu/environment/config.py b/mlonmcu/environment/config.py index de2e1b60e..dd1c96871 100644 --- a/mlonmcu/environment/config.py +++ b/mlonmcu/environment/config.py @@ -99,32 +99,6 @@ def __init__( self.cleanup_keep = cleanup_keep -class DefaultsConfigOld(DefaultsConfig): - # TODO: loglevels enum - - def __init__( - self, - log_level=logging.INFO, - log_to_file=False, - log_rotate=False, - cleanup_auto=False, - cleanup_keep=100, - default_framework=None, - default_backends={}, - default_target=None, - ): - super().__init__( - log_level=log_level, - log_to_file=log_to_file, - log_rotate=log_rotate, - cleanup_auto=False, - cleanup_keep=100, - ) - self.default_framework = default_framework - self.default_backends = default_backends - self.default_target = default_target - - class PathConfig(BaseConfig): def __init__(self, path, base=None): if isinstance(path, str): @@ -146,93 +120,5 @@ def __repr(self): return f"PathConfig({self.path})" -class RepoConfig(BaseConfig): - def __init__(self, url, ref=None): - self.url = url - self.ref = ref - - +RepoConfig = namedtuple("RepoConfig", "url ref") ComponentConfig = namedtuple("ComponentConfig", "supported used") - - -class BackendConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class FeatureKind(Enum): - UNKNOWN = 0 - FRAMEWORK = 1 - BACKEND = 2 - TARGET = 3 - FRONTEND = 4 - - -class FeatureConfig: - def __init__(self, name, kind=FeatureKind.UNKNOWN, supported=True): - self.name = name - self.supported = supported - - def __repr__(self): - return self.__class__.__name__ + "(" + str(vars(self)) + ")" - - -class FrameworkFeatureConfig(FeatureConfig): - def __init__(self, name, framework, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.framework = framework - - -class BackendFeatureConfig(FeatureConfig): - def __init__(self, name, backend, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.backend = backend - - -class PlatformFeatureConfig(FeatureConfig): - def __init__(self, name, platform, supported=True): - super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) - self.platform = platform - - -class TargetFeatureConfig(FeatureConfig): - def __init__(self, name, target, supported=True): - super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) - self.target = target - - -class FrontendFeatureConfig(FeatureConfig): - def __init__(self, name, frontend, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.frontend = frontend - - -class FrameworkConfig(BaseConfig): - def __init__(self, name, enabled=True, backends={}, features={}): - self.name = name - self.enabled = enabled - self.backends = backends - self.features = features - - -class FrontendConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class PlatformConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class TargetConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features diff --git a/mlonmcu/environment/convert.py b/mlonmcu/environment/convert.py index 8f68a6649..27ed42b47 100644 --- a/mlonmcu/environment/convert.py +++ b/mlonmcu/environment/convert.py @@ -24,8 +24,8 @@ import argparse -from mlonmcu.environment.environment import UserEnvironment -from mlonmcu.environment.environment2 import UserEnvironment2 +from mlonmcu.environment.legacy.environment import UserEnvironment as UserEnvironmentOld +from mlonmcu.environment.user_environment import UserEnvironment from mlonmcu.environment.config import ComponentConfig, DefaultsConfig # from mlonmcu.environment.loader import load_environment_from_file @@ -66,12 +66,6 @@ def convert_v1_to_v2(env): f_.name: ComponentConfig(f_.supported, False) for f in env.frontends for f_ in f.features if f.enabled } other_features = {} - # print("framework_features", framework_features) - # print("backend_features", backend_features) - # print("target_features", target_features) - # print("frontend_features", frontend_features) - # print("other_features", other_features) - # print("platform_features", platform_features) def helper(name): return any( @@ -97,10 +91,8 @@ def helper(name): .union(set(other_features.keys())) ) features = {name: ComponentConfig(helper(name), False) for name in all_feature_names} - # print("features", features) - # print("?") - env2 = UserEnvironment2( + env2 = UserEnvironment( env.home, alias=env.alias, defaults=DefaultsConfig( @@ -159,7 +151,7 @@ def main(): file = args.input[0] - env = UserEnvironment.from_file(file) + env = UserEnvironmentOld.from_file(file) env2 = convert_v1_to_v2(env) diff --git a/mlonmcu/environment/environment.py b/mlonmcu/environment/environment.py index 9ef74412b..9e82b76d7 100644 --- a/mlonmcu/environment/environment.py +++ b/mlonmcu/environment/environment.py @@ -17,58 +17,72 @@ # limitations under the License. # import logging +import tempfile +from abc import ABC +from pathlib import Path +from typing import Dict, List, Union from .config import ( - DefaultsConfigOld, + DefaultsConfig, PathConfig, RepoConfig, - FrameworkConfig, - FrameworkFeatureConfig, - BackendConfig, - BackendFeatureConfig, - TargetConfig, - TargetFeatureConfig, - PlatformConfig, - PlatformFeatureConfig, - FrontendConfig, - FrontendFeatureConfig, + ComponentConfig, + # FrameworkConfig, + # FrameworkFeatureConfig, + # BackendConfig, + # BackendFeatureConfig, + # TargetConfig, + # TargetFeatureConfig, + # PlatformConfig, + # PlatformFeatureConfig, + # FrontendConfig, + # FrontendFeatureConfig, ) -from .loader import load_environment_from_file -from .writer import write_environment_to_file -from mlonmcu.feature.type import FeatureType +# from mlonmcu.feature.type import FeatureType -def _feature_helper(obj, name): - if not obj.enabled: - return [] - features = obj.features - if name: - return [feature for feature in features if feature.name == name] - else: - return features +# def _feature_helper(obj, name): +# if not obj.enabled: +# return [] +# features = obj.features +# if name: +# return [feature for feature in features if feature.name == name] +# else: +# return features -def _extract_names(objs): - return [obj.name for obj in objs] +# def _extract_names(objs): +# return [obj.name for obj in objs] -def _filter_enabled(objs): - return [obj for obj in objs if obj.enabled] +# def _filter_enabled(objs): +# return [obj for obj in objs if obj.enabled] -class Environment: +class BaseEnvironment(ABC): def __init__(self): - self._home = None - self.alias = None - self.defaults = DefaultsConfigOld() - self.paths = {} - self.repos = {} - self.frameworks = [] - self.frontends = [] - self.platforms = [] - self.toolchains = [] - self.targets = [] + self.version = None + self._home: Path = None + + +class Environment(BaseEnvironment): + def __init__(self): + super().__init__() + self.alias: str = None + self.version = 2 + self.defaults = DefaultsConfig() + self.paths: "Dict[str, Union[PathConfig, List[PathConfig]]]" = {} + self.repos: "Dict[str, RepoConfig]" = {} + self.frameworks: "Dict[str, ComponentConfig]" = {} + self.backends: "Dict[str, ComponentConfig]" = {} + self.features: "Dict[str, ComponentConfig]" = {} + self.postprocesses: "Dict[str, ComponentConfig]" = {} + self.toolchains: "Dict[str, ComponentConfig]" = {} + self.frontends: "Dict[str, ComponentConfig]" = {} + self.platforms: "Dict[str, ComponentConfig]" = {} + self.toolchains: "Dict[str, ComponentConfig]" = {} + self.targets: "Dict[str, ComponentConfig]" = {} self.vars = {} self.flags = {} @@ -80,13 +94,6 @@ def home(self): """Home directory of mlonmcu environment.""" return self._home - @classmethod - def from_file(cls, filename): - return load_environment_from_file(filename, base=cls) - - def to_file(self, filename): - write_environment_to_file(self, filename) - def lookup_path(self, name): assert name in self.paths, f"Unable to find '{name}' path in environment config" return self.paths[name] @@ -94,267 +101,98 @@ def lookup_path(self, name): def lookup_var(self, name, default=None): return self.vars.get(name, default) - def lookup_frontend_feature_configs(self, name=None, frontend=None): - configs = [] - if frontend: - names = [frontend.name for frontend in self.frontends] - index = names.index(frontend) - assert index is not None, f"Frontend {frontend} not found in environment config" - configs.extend(_feature_helper(self.frontends[index], name)) - else: - for frontend in self.frontends: - configs.extend(_feature_helper(frontend, name)) - return configs - - def lookup_framework_feature_configs(self, name=None, framework=None): - configs = [] - if framework: - names = [framework.name for framework in self.frameworks] - index = names.index(framework) - assert index is not None, f"Framework {framework} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index], name)) - else: - for framework in self.frameworks: - configs.extend(_feature_helper(framework, name)) - return configs - - def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): - def helper(framework, backend, name): - backend_features = self.framework[framework].backends[backend].features - if name: - return [backend_features[name]] - else: - return backend_features.values() - - configs = [] - if framework: - names = [framework.name for framework in self.frameworks] - index = names.index(framework) - assert index is not None, f"Framework {framework} not found in environment config" - if backend: - names_ = [backend.name for backend in self.frameworks[index].backends] - index_ = names_.index(backend) - assert index_ is not None, f"Backend {backend} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - else: - for backend in self.frameworks[index].backends: - configs.extend(_feature_helper(backend, name)) - else: - for framework in self.frameworks: - if backend: - names_ = [backend.name for backend in framework.backends] - index_ = names_.index(backend) - assert index_ is not None, f"Backend {backend} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - else: - for backend in framework.backends: - configs.extend(_feature_helper(backend, name)) - backend = None - return configs - - def lookup_platform_feature_configs(self, name=None, platform=None): - configs = [] - if platform: - names = [platform.name for platform in self.platforms] - index = names.index(platform) - assert ( - index is not None - ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list - configs.extend(_feature_helper(self.platforms[index], name)) - else: - for platform in self.platforms: - configs.extend(_feature_helper(platform, name)) - return configs - - def lookup_target_feature_configs(self, name=None, target=None): - configs = [] - if target: - names = [target.name for target in self.targets] - index = names.index(target) - assert ( - index is not None - ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list - configs.extend(_feature_helper(self.targets[index], name)) - else: - for target in self.targets: - configs.extend(_feature_helper(target, name)) - return configs - - def lookup_feature_configs( - self, - name=None, - kind=None, - frontend=None, - framework=None, - backend=None, - platform=None, - target=None, - ): - configs = [] - if kind == FeatureType.FRONTEND or kind is None: - configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) - if kind == FeatureType.FRAMEWORK or kind is None: - configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) - if kind == FeatureType.BACKEND or kind is None: - configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) - if kind == FeatureType.PLATFORM or kind is None: - configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) - if kind == FeatureType.TARGET or kind is None: - configs.extend(self.lookup_target_feature_configs(name=name, target=target)) - - return configs - - def supports_feature(self, name): - configs = self.lookup_feature_configs(name=name) - supported = [feature.supported for feature in configs] - return any(supported) + def get_supported_features(self): + supported = [name for name, config in self.features.items() if config.supported] + return supported - def has_feature(self, name): - """An alias for supports_feature.""" - return self.supports_feature(name) + def get_used_features(self): + used = [name for name, config in self.features.items() if config.used] + return used - def lookup_backend_configs(self, backend=None, framework=None, names_only=False): - enabled_frameworks = _filter_enabled(self.frameworks) - - configs = [] - for framework_config in enabled_frameworks: - if framework is not None and framework_config.name != framework: - continue - enabled_backends = _filter_enabled(framework_config.backends) - if backend is None: - configs.extend(enabled_backends) - else: - for backend_config in enabled_backends: - if backend_config.name == backend: - return [backend_config.name if names_only else backend_config] - return _extract_names(configs) if names_only else configs - - def lookup_framework_configs(self, framework=None, names_only=False): - enabled_frameworks = _filter_enabled(self.frameworks) - - if framework is None: - return _extract_names(enabled_frameworks) if names_only else enabled_frameworks + def has_feature(self, name): + return name in self.get_supported_features() - for framework_config in enabled_frameworks: - if framework_config.name == framework: - return [framework_config.name if names_only else framework_config] + def get_supported_frontends(self): + supported = [name for name, config in self.frontends.items() if config.supported] + return supported - return [] + def get_used_frontends(self): + used = [name for name, config in self.frontends.items() if config.used] + return used - def lookup_frontend_configs(self, frontend=None, names_only=False): - enabled_frontends = _filter_enabled(self.frontends) + def has_frontend(self, name): + return name in self.get_supported_frontends() - if frontend is None: - return _extract_names(enabled_frontends) if names_only else enabled_frontends + def get_supported_backends(self): + supported = [name for name, config in self.backends.items() if config.supported] + return supported - for frontend_config in enabled_frontends: - if frontend_config.name == frontend: - return [frontend_config.name if names_only else frontend_config] - return [] + def get_used_backends(self): + used = [name for name, config in self.backends.items() if config.used] + return used - def lookup_platform_configs(self, platform=None, names_only=False): - enabled_platforms = _filter_enabled(self.platforms) + def has_backend(self, name): + return name in self.get_supported_backends() - if platform is None: - return _extract_names(enabled_platforms) if names_only else enabled_platforms - for platform_config in enabled_platforms: - if platform_config.name == platform: - return [platform_config.name if names_only else platform_config] + def get_supported_frameworks(self): + supported = [name for name, config in self.frameworks.items() if config.supported] + return supported - return [] + def get_used_frameworks(self): + used = [name for name, config in self.frameworks.items() if config.used] + return used - def lookup_target_configs(self, target=None, names_only=False): - enabled_targets = _filter_enabled(self.targets) + def has_framework(self, name): + return name in self.get_supported_frameworks() - if target is None: - return _extract_names(enabled_targets) if names_only else enabled_targets + def get_supported_platforms(self): + supported = [name for name, config in self.platforms.items() if config.supported] + return supported - for target_config in enabled_targets: - if target_config.name == target: - return [target_config.name if names_only else target_config] - return [] + def get_used_platforms(self): + used = [name for name, config in self.platforms.items() if config.used] + return used - def has_frontend(self, name): - configs = self.lookup_frontend_configs(frontend=name) - return len(configs) > 0 + def has_platform(self, name): + return name in self.get_supported_platforms() - def has_backend(self, name): - configs = self.lookup_backend_configs(backend=name) - return len(configs) > 0 + def get_supported_target(self): + supported = [name for name, config in self.targets.items() if config.supported] + return supported - def has_framework(self, name): - configs = self.lookup_framework_configs(framework=name) - return len(configs) > 0 + def get_used_targets(self): + used = [name for name, config in self.targets.items() if config.used] + return used - def has_platform(self, name): - configs = self.lookup_platform_configs(platform=name) - return len(configs) > 0 + def has_target(self, name): + return name in self.get_supported_targets() def has_toolchain(self, name): return self.toolchains.get(name, False) - def has_target(self, name): - configs = self.lookup_target_configs(target=name) - return len(configs) > 0 - - # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead - # of the supported ones in the environment # def has_postprocess(self, name): # configs = self.lookup_postprocess_configs(postprocess=name) # return len(configs) > 0 def get_default_backends(self, framework): - if framework is None or framework not in self.defaults.default_backends: - return [] - default = self.defaults.default_backends[framework] - # framework_names = [framework_config.name for framework_config in self.frameworks] - # framework_config = self.frameworks[framework_names.index(framework)] - if default is None: - return [] - if isinstance(default, str): - if default == "*": # Wildcard all enabled frameworks - default = self.get_enabled_backends() - else: - default = [default] - else: - assert isinstance(default, list), "TODO" - return default + # TODO: deprecate + return self.get_used_backends() def get_default_frameworks(self): - default = self.defaults.default_framework - if default is None: - return [] - if isinstance(default, str): - if default == "*": # Wildcard all enabled frameworks - default = self.get_enabled_frameworks() - else: - default = [default] - else: - assert isinstance(default, list), "TODO" - return default + # TODO: deprecate + return self.get_used_frameworks() def get_default_targets(self): - default = self.defaults.default_target - if default is not None: - if isinstance(default, str): - if default == "*": # Wildcard all enabled targets - default = self.get_enabled_targets() - else: - default = [default] - else: - assert isinstance(default, list) - return default + # TODO: deprecate + return self.get_used_targets() class DefaultEnvironment(Environment): def __init__(self): super().__init__() - self.defaults = DefaultsConfigOld( + self.defaults = DefaultsConfig( log_level=logging.DEBUG, log_to_file=False, - default_framework=None, - default_backends={}, - default_target=None, cleanup_auto=False, cleanup_keep=100, ) @@ -368,6 +206,7 @@ def __init__(self): PathConfig("./models"), ], } + # TODO: refresh or remove self.repos = { "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), "tflite_micro_compiler": RepoConfig( @@ -381,121 +220,78 @@ def __init__(self): ), # TODO: freeze ref? "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? } - self.frameworks = [ - FrameworkConfig( - "tflm", - enabled=True, - backends=[ - BackendConfig("tflmc", enabled=True, features=[]), - BackendConfig("tflmi", enabled=True, features=[]), - ], - features=[ - FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), - ], - ), - FrameworkConfig( - "utvm", - enabled=True, - backends=[ - BackendConfig( - "tvmaot", - enabled=True, - features=[ - BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), - ], - ), - BackendConfig("tvmrt", enabled=True, features=[]), - BackendConfig("tvmcg", enabled=True, features=[]), - ], - features=[ - FrameworkFeatureConfig("memplan", framework="utvm", supported=False), - ], - ), - ] - self.frontends = [ - FrontendConfig("saved_model", enabled=False), - FrontendConfig("ipynb", enabled=False), - FrontendConfig( - "tflite", - enabled=True, - features=[ - FrontendFeatureConfig("packing", frontend="tflite", supported=False), - ], - ), - ] - self.vars = { - "TEST": "abc", - } - self.flags = {} - self.platforms = [ - PlatformConfig( - "mlif", - enabled=True, - features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], - ) - ] + self.frameworks = {} + self.backends = {} + self.features = {} + self.postprocesses = {} self.toolchains = {} - self.targets = [ - TargetConfig( - "etiss_pulpino", - features=[ - TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), - TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), - TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), - ], - ), - TargetConfig( - "host_x86", - features=[ - TargetFeatureConfig("debug", target="host_x86", supported=True), - TargetFeatureConfig("attach", target="host_x86", supported=True), - ], - ), - ] - - -class UserEnvironment(DefaultEnvironment): - def __init__( - self, - home, - merge=False, - alias=None, - defaults=None, - paths=None, - repos=None, - frameworks=None, - frontends=None, - platforms=None, - toolchains=None, - targets=None, - variables=None, - default_flags=None, - ): - super().__init__() - self._home = home - - if merge: - raise NotImplementedError - - if alias: - self.alias = alias - if defaults: - self.defaults = defaults - if paths: - self.paths = paths - if repos: - self.repos = repos - if frameworks: - self.frameworks = frameworks - if frontends: - self.frontends = frontends - if platforms: - self.platforms = platforms - if toolchains: - self.toolchains = toolchains - if targets: - self.targets = targets - if variables: - self.vars = variables - if default_flags: - self.flags = default_flags + # self.frameworks = [ + # FrameworkConfig( + # "tflm", + # enabled=True, + # backends=[ + # BackendConfig("tflmc", enabled=True, features=[]), + # BackendConfig("tflmi", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), + # ], + # ), + # FrameworkConfig( + # "utvm", + # enabled=True, + # backends=[ + # BackendConfig( + # "tvmaot", + # enabled=True, + # features=[ + # BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), + # ], + # ), + # BackendConfig("tvmrt", enabled=True, features=[]), + # BackendConfig("tvmcg", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("memplan", framework="utvm", supported=False), + # ], + # ), + # ] + # self.frontends = [ + # FrontendConfig("saved_model", enabled=False), + # FrontendConfig("ipynb", enabled=False), + # FrontendConfig( + # "tflite", + # enabled=True, + # features=[ + # FrontendFeatureConfig("packing", frontend="tflite", supported=False), + # ], + # ), + # ] + # self.vars = { + # "TEST": "abc", + # } + # self.flags = {} + # self.platforms = [ + # PlatformConfig( + # "mlif", + # enabled=True, + # features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], + # ) + # ] + # self.targets = [ + # TargetConfig( + # "etiss_pulpino", + # features=[ + # TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), + # ], + # ), + # TargetConfig( + # "host_x86", + # features=[ + # TargetFeatureConfig("debug", target="host_x86", supported=True), + # TargetFeatureConfig("attach", target="host_x86", supported=True), + # ], + # ), + # ] diff --git a/mlonmcu/environment/environment2.py b/mlonmcu/environment/environment2.py deleted file mode 100644 index 406cf57d4..000000000 --- a/mlonmcu/environment/environment2.py +++ /dev/null @@ -1,532 +0,0 @@ -# -# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. -# -# This file is part of MLonMCU. -# See https://github.com/tum-ei-eda/mlonmcu.git for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import logging -import tempfile -from abc import ABC -from pathlib import Path -from typing import Dict, List, Union - -from .config import ( - DefaultsConfig, - PathConfig, - RepoConfig, - ComponentConfig, - # FrameworkConfig, - # FrameworkFeatureConfig, - # BackendConfig, - # BackendFeatureConfig, - # TargetConfig, - # TargetFeatureConfig, - # PlatformConfig, - # PlatformFeatureConfig, - # FrontendConfig, - # FrontendFeatureConfig, -) -from .loader2 import load_environment_from_file -from .writer2 import write_environment_to_file - -# from mlonmcu.feature.type import FeatureType - - -# def _feature_helper(obj, name): -# if not obj.enabled: -# return [] -# features = obj.features -# if name: -# return [feature for feature in features if feature.name == name] -# else: -# return features - - -# def _extract_names(objs): -# return [obj.name for obj in objs] - - -# def _filter_enabled(objs): -# return [obj for obj in objs if obj.enabled] - -class BaseEnvironment(ABC): - def __init__(self): - self.version = None - self._home: Path = None - - -class Environment2(BaseEnvironment): - def __init__(self): - super().__init__() - self.alias: str = None - self.version = 2 - self.defaults = DefaultsConfig() - self.paths: "Dict[str, Union[PathConfig, List[PathConfig]]]" = {} - self.repos: "Dict[str, RepoConfig]" = {} - self.frameworks: "Dict[str, ComponentConfig]" = {} - self.backends: "Dict[str, ComponentConfig]" = {} - self.features: "Dict[str, ComponentConfig]" = {} - self.toolchains: "Dict[str, ComponentConfig]" = {} - # self.frontends = [] - # self.platforms = [] - # self.toolchains = [] - # self.targets = [] - # self.postprocesses = [] - self.vars = {} - self.flags = {} - - def __str__(self): - return self.__class__.__name__ + "(" + str(vars(self)) + ")" - - @property - def home(self): - """Home directory of mlonmcu environment.""" - return self._home - - @classmethod - def from_file(cls, filename): - return load_environment_from_file(filename, base=cls) - - def to_file(self, filename): - write_environment_to_file(self, filename) - - def to_text(self): - with tempfile.TemporaryDirectory() as tmpdirname: - dest = Path(tmpdirname) / "out.yml" - self.to_file(dest) - with open(dest, "r") as handle: - content = handle.read() - assert len(content) > 0 - return content - - def lookup_path(self, name): - assert name in self.paths, f"Unable to find '{name}' path in environment config" - return self.paths[name] - - def lookup_var(self, name, default=None): - return self.vars.get(name, default) - - # def lookup_frontend_feature_configs(self, name=None, frontend=None): - # configs = [] - # if frontend: - # names = [frontend.name for frontend in self.frontends] - # index = names.index(frontend) - # assert index is not None, f"Frontend {frontend} not found in environment config" - # configs.extend(_feature_helper(self.frontends[index], name)) - # else: - # for frontend in self.frontends: - # configs.extend(_feature_helper(frontend, name)) - # return configs - - # def lookup_framework_feature_configs(self, name=None, framework=None): - # configs = [] - # if framework: - # names = [framework.name for framework in self.frameworks] - # index = names.index(framework) - # assert index is not None, f"Framework {framework} not found in environment config" - # configs.extend(_feature_helper(self.frameworks[index], name)) - # else: - # for framework in self.frameworks: - # configs.extend(_feature_helper(framework, name)) - # return configs - - # def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): - # def helper(framework, backend, name): - # backend_features = self.framework[framework].backends[backend].features - # if name: - # return [backend_features[name]] - # else: - # return backend_features.values() - - # configs = [] - # if framework: - # names = [framework.name for framework in self.frameworks] - # index = names.index(framework) - # assert index is not None, f"Framework {framework} not found in environment config" - # if backend: - # names_ = [backend.name for backend in self.frameworks[index].backends] - # index_ = names_.index(backend) - # assert index_ is not None, f"Backend {backend} not found in environment config" - # configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - # else: - # for backend in self.frameworks[index].backends: - # configs.extend(_feature_helper(backend, name)) - # else: - # for framework in self.frameworks: - # if backend: - # names_ = [backend.name for backend in framework.backends] - # index_ = names_.index(backend) - # assert index_ is not None, f"Backend {backend} not found in environment config" - # configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - # else: - # for backend in framework.backends: - # configs.extend(_feature_helper(backend, name)) - # backend = None - # return configs - - # def lookup_platform_feature_configs(self, name=None, platform=None): - # configs = [] - # if platform: - # names = [platform.name for platform in self.platforms] - # index = names.index(platform) - # assert ( - # index is not None - # ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list - # configs.extend(_feature_helper(self.platforms[index], name)) - # else: - # for platform in self.platforms: - # configs.extend(_feature_helper(platform, name)) - # return configs - - # def lookup_target_feature_configs(self, name=None, target=None): - # configs = [] - # if target: - # names = [target.name for target in self.targets] - # index = names.index(target) - # assert ( - # index is not None - # ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list - # configs.extend(_feature_helper(self.targets[index], name)) - # else: - # for target in self.targets: - # configs.extend(_feature_helper(target, name)) - # return configs - - # def lookup_feature_configs( - # self, - # name=None, - # kind=None, - # frontend=None, - # framework=None, - # backend=None, - # platform=None, - # target=None, - # ): - # configs = [] - # if kind == FeatureType.FRONTEND or kind is None: - # configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) - # if kind == FeatureType.FRAMEWORK or kind is None: - # configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) - # if kind == FeatureType.BACKEND or kind is None: - # configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) - # if kind == FeatureType.PLATFORM or kind is None: - # configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) - # if kind == FeatureType.TARGET or kind is None: - # configs.extend(self.lookup_target_feature_configs(name=name, target=target)) - - # return configs - - def supports_feature(self, name): - configs = self.lookup_feature_configs(name=name) - supported = [feature.supported for feature in configs] - return any(supported) - - def has_feature(self, name): - """An alias for supports_feature.""" - return self.supports_feature(name) - - # def lookup_backend_configs(self, backend=None, framework=None, names_only=False): - # enabled_frameworks = _filter_enabled(self.frameworks) - - # configs = [] - # for framework_config in enabled_frameworks: - # if framework is not None and framework_config.name != framework: - # continue - # enabled_backends = _filter_enabled(framework_config.backends) - # if backend is None: - # configs.extend(enabled_backends) - # else: - # for backend_config in enabled_backends: - # if backend_config.name == backend: - # return [backend_config.name if names_only else backend_config] - # return _extract_names(configs) if names_only else configs - - # def lookup_framework_configs(self, framework=None, names_only=False): - # enabled_frameworks = _filter_enabled(self.frameworks) - - # if framework is None: - # return _extract_names(enabled_frameworks) if names_only else enabled_frameworks - - # for framework_config in enabled_frameworks: - # if framework_config.name == framework: - # return [framework_config.name if names_only else framework_config] - - # return [] - - # def lookup_frontend_configs(self, frontend=None, names_only=False): - # enabled_frontends = _filter_enabled(self.frontends) - - # if frontend is None: - # return _extract_names(enabled_frontends) if names_only else enabled_frontends - - # for frontend_config in enabled_frontends: - # if frontend_config.name == frontend: - # return [frontend_config.name if names_only else frontend_config] - # return [] - - # def lookup_platform_configs(self, platform=None, names_only=False): - # enabled_platforms = _filter_enabled(self.platforms) - - # if platform is None: - # return _extract_names(enabled_platforms) if names_only else enabled_platforms - # for platform_config in enabled_platforms: - # if platform_config.name == platform: - # return [platform_config.name if names_only else platform_config] - - # return [] - - # def lookup_target_configs(self, target=None, names_only=False): - # enabled_targets = _filter_enabled(self.targets) - - # if target is None: - # return _extract_names(enabled_targets) if names_only else enabled_targets - - # for target_config in enabled_targets: - # if target_config.name == target: - # return [target_config.name if names_only else target_config] - # return [] - - def has_frontend(self, name): - configs = self.lookup_frontend_configs(frontend=name) - return len(configs) > 0 - - def has_backend(self, name): - configs = self.lookup_backend_configs(backend=name) - return len(configs) > 0 - - def has_framework(self, name): - configs = self.lookup_framework_configs(framework=name) - return len(configs) > 0 - - def has_platform(self, name): - configs = self.lookup_platform_configs(platform=name) - return len(configs) > 0 - - def has_toolchain(self, name): - return self.toolchains.get(name, False) - - def has_target(self, name): - configs = self.lookup_target_configs(target=name) - return len(configs) > 0 - - # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead - # of the supported ones in the environment - # def has_postprocess(self, name): - # configs = self.lookup_postprocess_configs(postprocess=name) - # return len(configs) > 0 - - # def get_default_backends(self, framework): - # if framework is None or framework not in self.defaults.default_backends: - # return [] - # default = self.defaults.default_backends[framework] - # # framework_names = [framework_config.name for framework_config in self.frameworks] - # # framework_config = self.frameworks[framework_names.index(framework)] - # if default is None: - # return [] - # if isinstance(default, str): - # if default == "*": # Wildcard all enabled frameworks - # default = self.get_enabled_backends() - # else: - # default = [default] - # else: - # assert isinstance(default, list), "TODO" - # return default - - # def get_default_frameworks(self): - # default = self.defaults.default_framework - # if default is None: - # return [] - # if isinstance(default, str): - # if default == "*": # Wildcard all enabled frameworks - # default = self.get_enabled_frameworks() - # else: - # default = [default] - # else: - # assert isinstance(default, list), "TODO" - # return default - - # def get_default_targets(self): - # default = self.defaults.default_target - # if default is not None: - # if isinstance(default, str): - # if default == "*": # Wildcard all enabled targets - # default = self.get_enabled_targets() - # else: - # default = [default] - # else: - # assert isinstance(default, list) - # return default - - -class DefaultEnvironment2(Environment2): - def __init__(self): - super().__init__() - self.defaults = DefaultsConfig( - log_level=logging.DEBUG, - log_to_file=False, - cleanup_auto=False, - cleanup_keep=100, - ) - self.paths = { - "deps": PathConfig("./deps"), - "logs": PathConfig("./logs"), - "results": PathConfig("./results"), - "plugins": PathConfig("./plugins"), - "temp": PathConfig("out"), - "models": [ - PathConfig("./models"), - ], - } - # TODO: refresh or remove - self.repos = { - "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), - "tflite_micro_compiler": RepoConfig( - "https://github.com/cpetig/tflite_micro_compiler.git", ref="master" - ), # TODO: freeze ref? - "tvm": RepoConfig( - "https://github.com/tum-ei-eda/tvm.git", ref="tumeda" - ), # TODO: use upstream repo with suitable commit? - "utvm_staticrt_codegen": RepoConfig( - "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git", ref="master" - ), # TODO: freeze ref? - "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? - } - self.frameworks = {} - self.backends = {} - self.features = {} - self.postprocesses = {} - self.toolchains = {} - # self.frameworks = [ - # FrameworkConfig( - # "tflm", - # enabled=True, - # backends=[ - # BackendConfig("tflmc", enabled=True, features=[]), - # BackendConfig("tflmi", enabled=True, features=[]), - # ], - # features=[ - # FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), - # ], - # ), - # FrameworkConfig( - # "utvm", - # enabled=True, - # backends=[ - # BackendConfig( - # "tvmaot", - # enabled=True, - # features=[ - # BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), - # ], - # ), - # BackendConfig("tvmrt", enabled=True, features=[]), - # BackendConfig("tvmcg", enabled=True, features=[]), - # ], - # features=[ - # FrameworkFeatureConfig("memplan", framework="utvm", supported=False), - # ], - # ), - # ] - # self.frontends = [ - # FrontendConfig("saved_model", enabled=False), - # FrontendConfig("ipynb", enabled=False), - # FrontendConfig( - # "tflite", - # enabled=True, - # features=[ - # FrontendFeatureConfig("packing", frontend="tflite", supported=False), - # ], - # ), - # ] - # self.vars = { - # "TEST": "abc", - # } - # self.flags = {} - # self.platforms = [ - # PlatformConfig( - # "mlif", - # enabled=True, - # features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], - # ) - # ] - # self.targets = [ - # TargetConfig( - # "etiss_pulpino", - # features=[ - # TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), - # TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), - # TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), - # ], - # ), - # TargetConfig( - # "host_x86", - # features=[ - # TargetFeatureConfig("debug", target="host_x86", supported=True), - # TargetFeatureConfig("attach", target="host_x86", supported=True), - # ], - # ), - # ] - - -class UserEnvironment2(DefaultEnvironment2): - def __init__( - self, - home, - alias=None, - defaults=None, - paths=None, - repos=None, - frameworks=None, - backends=None, - frontends=None, - platforms=None, - toolchains=None, - targets=None, - features=None, - postprocesses=None, - variables=None, - default_flags=None, - ): - super().__init__() - self._home = home - - if alias: - self.alias = alias - if defaults: - self.defaults = defaults - if paths: - self.paths = paths - if repos: - self.repos = repos - if frameworks: - self.frameworks = frameworks - if backends: - self.backends = backends - if frontends: - self.frontends = frontends - if platforms: - self.platforms = platforms - if toolchains: - self.toolchains = toolchains - if targets: - self.targets = targets - if features: - self.features = features - if postprocesses: - self.postprocesses = postprocesses - if variables: - self.vars = variables - if default_flags: - self.flags = default_flags diff --git a/mlonmcu/environment/legacy/config.py b/mlonmcu/environment/legacy/config.py new file mode 100644 index 000000000..5a0b5adc7 --- /dev/null +++ b/mlonmcu/environment/legacy/config.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO: rename to paths.py or user.py? + +import logging +from enum import Enum + +from ..config import DefaultsConfig, BaseConfig + + +class DefaultsConfigOld(DefaultsConfig): + def __init__( + self, + log_level=logging.INFO, + log_to_file=False, + log_rotate=False, + cleanup_auto=False, + cleanup_keep=100, + default_framework=None, + default_backends={}, + default_target=None, + ): + super().__init__( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + cleanup_auto=False, + cleanup_keep=100, + ) + self.default_framework = default_framework + self.default_backends = default_backends + self.default_target = default_target + + +class BackendConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class FeatureKind(Enum): + UNKNOWN = 0 + FRAMEWORK = 1 + BACKEND = 2 + TARGET = 3 + FRONTEND = 4 + + +class FeatureConfig: + def __init__(self, name, kind=FeatureKind.UNKNOWN, supported=True): + self.name = name + self.supported = supported + + def __repr__(self): + return self.__class__.__name__ + "(" + str(vars(self)) + ")" + + +class FrameworkFeatureConfig(FeatureConfig): + def __init__(self, name, framework, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.framework = framework + + +class BackendFeatureConfig(FeatureConfig): + def __init__(self, name, backend, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.backend = backend + + +class PlatformFeatureConfig(FeatureConfig): + def __init__(self, name, platform, supported=True): + super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) + self.platform = platform + + +class TargetFeatureConfig(FeatureConfig): + def __init__(self, name, target, supported=True): + super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) + self.target = target + + +class FrontendFeatureConfig(FeatureConfig): + def __init__(self, name, frontend, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.frontend = frontend + + +class FrameworkConfig(BaseConfig): + def __init__(self, name, enabled=True, backends={}, features={}): + self.name = name + self.enabled = enabled + self.backends = backends + self.features = features + + +class FrontendConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class PlatformConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class TargetConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features diff --git a/mlonmcu/environment/legacy/environment.py b/mlonmcu/environment/legacy/environment.py new file mode 100644 index 000000000..db251b70c --- /dev/null +++ b/mlonmcu/environment/legacy/environment.py @@ -0,0 +1,514 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +from abc import ABC +from pathlib import Path + +from ..config import ( + PathConfig, + RepoConfig, +) +from .config import ( + DefaultsConfigOld, + FrameworkConfig, + FrameworkFeatureConfig, + BackendConfig, + BackendFeatureConfig, + TargetConfig, + TargetFeatureConfig, + PlatformConfig, + PlatformFeatureConfig, + FrontendConfig, + FrontendFeatureConfig, +) +from .loader import load_environment_from_file +from .writer import write_environment_to_file + +from mlonmcu.feature.type import FeatureType + + +def _feature_helper(obj, name): + if not obj.enabled: + return [] + features = obj.features + if name: + return [feature for feature in features if feature.name == name] + else: + return features + + +def _extract_names(objs): + return [obj.name for obj in objs] + + +def _filter_enabled(objs): + return [obj for obj in objs if obj.enabled] + + +class BaseEnvironment(ABC): + def __init__(self): + self.version = None + self._home: Path = None + + +class Environment(BaseEnvironment): + def __init__(self): + super().__init__() + self.alias: str = None + self.version = 1 + self._home = None + self.alias = None + self.defaults = DefaultsConfigOld() + self.paths = {} + self.repos = {} + self.frameworks = [] + self.frontends = [] + self.platforms = [] + self.toolchains = [] + self.targets = [] + self.vars = {} + self.flags = {} + + def __str__(self): + return self.__class__.__name__ + "(" + str(vars(self)) + ")" + + @property + def home(self): + """Home directory of mlonmcu environment.""" + return self._home + + @classmethod + def from_file(cls, filename): + return load_environment_from_file(filename, base=cls) + + def to_file(self, filename): + write_environment_to_file(self, filename) + + def lookup_path(self, name): + assert name in self.paths, f"Unable to find '{name}' path in environment config" + return self.paths[name] + + def lookup_var(self, name, default=None): + return self.vars.get(name, default) + + def lookup_frontend_feature_configs(self, name=None, frontend=None): + configs = [] + if frontend: + names = [frontend.name for frontend in self.frontends] + index = names.index(frontend) + assert index is not None, f"Frontend {frontend} not found in environment config" + configs.extend(_feature_helper(self.frontends[index], name)) + else: + for frontend in self.frontends: + configs.extend(_feature_helper(frontend, name)) + return configs + + def lookup_framework_feature_configs(self, name=None, framework=None): + configs = [] + if framework: + names = [framework.name for framework in self.frameworks] + index = names.index(framework) + assert index is not None, f"Framework {framework} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index], name)) + else: + for framework in self.frameworks: + configs.extend(_feature_helper(framework, name)) + return configs + + def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): + def helper(framework, backend, name): + backend_features = self.framework[framework].backends[backend].features + if name: + return [backend_features[name]] + else: + return backend_features.values() + + configs = [] + if framework: + names = [framework.name for framework in self.frameworks] + index = names.index(framework) + assert index is not None, f"Framework {framework} not found in environment config" + if backend: + names_ = [backend.name for backend in self.frameworks[index].backends] + index_ = names_.index(backend) + assert index_ is not None, f"Backend {backend} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + else: + for backend in self.frameworks[index].backends: + configs.extend(_feature_helper(backend, name)) + else: + for framework in self.frameworks: + if backend: + names_ = [backend.name for backend in framework.backends] + index_ = names_.index(backend) + assert index_ is not None, f"Backend {backend} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + else: + for backend in framework.backends: + configs.extend(_feature_helper(backend, name)) + backend = None + return configs + + def lookup_platform_feature_configs(self, name=None, platform=None): + configs = [] + if platform: + names = [platform.name for platform in self.platforms] + index = names.index(platform) + assert ( + index is not None + ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list + configs.extend(_feature_helper(self.platforms[index], name)) + else: + for platform in self.platforms: + configs.extend(_feature_helper(platform, name)) + return configs + + def lookup_target_feature_configs(self, name=None, target=None): + configs = [] + if target: + names = [target.name for target in self.targets] + index = names.index(target) + assert ( + index is not None + ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list + configs.extend(_feature_helper(self.targets[index], name)) + else: + for target in self.targets: + configs.extend(_feature_helper(target, name)) + return configs + + def lookup_feature_configs( + self, + name=None, + kind=None, + frontend=None, + framework=None, + backend=None, + platform=None, + target=None, + ): + configs = [] + if kind == FeatureType.FRONTEND or kind is None: + configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) + if kind == FeatureType.FRAMEWORK or kind is None: + configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) + if kind == FeatureType.BACKEND or kind is None: + configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) + if kind == FeatureType.PLATFORM or kind is None: + configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) + if kind == FeatureType.TARGET or kind is None: + configs.extend(self.lookup_target_feature_configs(name=name, target=target)) + + return configs + + def supports_feature(self, name): + configs = self.lookup_feature_configs(name=name) + supported = [feature.supported for feature in configs] + return any(supported) + + def has_feature(self, name): + """An alias for supports_feature.""" + return self.supports_feature(name) + + def lookup_backend_configs(self, backend=None, framework=None, names_only=False): + enabled_frameworks = _filter_enabled(self.frameworks) + + configs = [] + for framework_config in enabled_frameworks: + if framework is not None and framework_config.name != framework: + continue + enabled_backends = _filter_enabled(framework_config.backends) + if backend is None: + configs.extend(enabled_backends) + else: + for backend_config in enabled_backends: + if backend_config.name == backend: + return [backend_config.name if names_only else backend_config] + return _extract_names(configs) if names_only else configs + + def lookup_framework_configs(self, framework=None, names_only=False): + enabled_frameworks = _filter_enabled(self.frameworks) + + if framework is None: + return _extract_names(enabled_frameworks) if names_only else enabled_frameworks + + for framework_config in enabled_frameworks: + if framework_config.name == framework: + return [framework_config.name if names_only else framework_config] + + return [] + + def lookup_frontend_configs(self, frontend=None, names_only=False): + enabled_frontends = _filter_enabled(self.frontends) + + if frontend is None: + return _extract_names(enabled_frontends) if names_only else enabled_frontends + + for frontend_config in enabled_frontends: + if frontend_config.name == frontend: + return [frontend_config.name if names_only else frontend_config] + return [] + + def lookup_platform_configs(self, platform=None, names_only=False): + enabled_platforms = _filter_enabled(self.platforms) + + if platform is None: + return _extract_names(enabled_platforms) if names_only else enabled_platforms + for platform_config in enabled_platforms: + if platform_config.name == platform: + return [platform_config.name if names_only else platform_config] + + return [] + + def lookup_target_configs(self, target=None, names_only=False): + enabled_targets = _filter_enabled(self.targets) + + if target is None: + return _extract_names(enabled_targets) if names_only else enabled_targets + + for target_config in enabled_targets: + if target_config.name == target: + return [target_config.name if names_only else target_config] + return [] + + def has_frontend(self, name): + configs = self.lookup_frontend_configs(frontend=name) + return len(configs) > 0 + + def has_backend(self, name): + configs = self.lookup_backend_configs(backend=name) + return len(configs) > 0 + + def has_framework(self, name): + configs = self.lookup_framework_configs(framework=name) + return len(configs) > 0 + + def has_platform(self, name): + configs = self.lookup_platform_configs(platform=name) + return len(configs) > 0 + + def has_toolchain(self, name): + return self.toolchains.get(name, False) + + def has_target(self, name): + configs = self.lookup_target_configs(target=name) + return len(configs) > 0 + + # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead + # of the supported ones in the environment + # def has_postprocess(self, name): + # configs = self.lookup_postprocess_configs(postprocess=name) + # return len(configs) > 0 + + def get_default_backends(self, framework): + if framework is None or framework not in self.defaults.default_backends: + return [] + default = self.defaults.default_backends[framework] + # framework_names = [framework_config.name for framework_config in self.frameworks] + # framework_config = self.frameworks[framework_names.index(framework)] + if default is None: + return [] + if isinstance(default, str): + if default == "*": # Wildcard all enabled frameworks + default = self.get_enabled_backends() + else: + default = [default] + else: + assert isinstance(default, list), "TODO" + return default + + def get_default_frameworks(self): + default = self.defaults.default_framework + if default is None: + return [] + if isinstance(default, str): + if default == "*": # Wildcard all enabled frameworks + default = self.get_enabled_frameworks() + else: + default = [default] + else: + assert isinstance(default, list), "TODO" + return default + + def get_default_targets(self): + default = self.defaults.default_target + if default is not None: + if isinstance(default, str): + if default == "*": # Wildcard all enabled targets + default = self.get_enabled_targets() + else: + default = [default] + else: + assert isinstance(default, list) + return default + + +class DefaultEnvironment(Environment): + def __init__(self): + super().__init__() + self.defaults = DefaultsConfigOld( + log_level=logging.DEBUG, + log_to_file=False, + default_framework=None, + default_backends={}, + default_target=None, + cleanup_auto=False, + cleanup_keep=100, + ) + self.paths = { + "deps": PathConfig("./deps"), + "logs": PathConfig("./logs"), + "results": PathConfig("./results"), + "plugins": PathConfig("./plugins"), + "temp": PathConfig("out"), + "models": [ + PathConfig("./models"), + ], + } + self.repos = { + "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), + "tflite_micro_compiler": RepoConfig( + "https://github.com/cpetig/tflite_micro_compiler.git", ref="master" + ), # TODO: freeze ref? + "tvm": RepoConfig( + "https://github.com/tum-ei-eda/tvm.git", ref="tumeda" + ), # TODO: use upstream repo with suitable commit? + "utvm_staticrt_codegen": RepoConfig( + "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git", ref="master" + ), # TODO: freeze ref? + "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? + } + self.frameworks = [ + FrameworkConfig( + "tflm", + enabled=True, + backends=[ + BackendConfig("tflmc", enabled=True, features=[]), + BackendConfig("tflmi", enabled=True, features=[]), + ], + features=[ + FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), + ], + ), + FrameworkConfig( + "utvm", + enabled=True, + backends=[ + BackendConfig( + "tvmaot", + enabled=True, + features=[ + BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), + ], + ), + BackendConfig("tvmrt", enabled=True, features=[]), + BackendConfig("tvmcg", enabled=True, features=[]), + ], + features=[ + FrameworkFeatureConfig("memplan", framework="utvm", supported=False), + ], + ), + ] + self.frontends = [ + FrontendConfig("saved_model", enabled=False), + FrontendConfig("ipynb", enabled=False), + FrontendConfig( + "tflite", + enabled=True, + features=[ + FrontendFeatureConfig("packing", frontend="tflite", supported=False), + ], + ), + ] + self.vars = { + "TEST": "abc", + } + self.flags = {} + self.platforms = [ + PlatformConfig( + "mlif", + enabled=True, + features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], + ) + ] + self.toolchains = {} + self.targets = [ + TargetConfig( + "etiss_pulpino", + features=[ + TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), + TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), + TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), + ], + ), + TargetConfig( + "host_x86", + features=[ + TargetFeatureConfig("debug", target="host_x86", supported=True), + TargetFeatureConfig("attach", target="host_x86", supported=True), + ], + ), + ] + + +class UserEnvironment(DefaultEnvironment): + def __init__( + self, + home, + merge=False, + alias=None, + defaults=None, + paths=None, + repos=None, + frameworks=None, + frontends=None, + platforms=None, + toolchains=None, + targets=None, + variables=None, + default_flags=None, + ): + super().__init__() + self._home = home + + if merge: + raise NotImplementedError + + if alias: + self.alias = alias + if defaults: + self.defaults = defaults + if paths: + self.paths = paths + if repos: + self.repos = repos + if frameworks: + self.frameworks = frameworks + if frontends: + self.frontends = frontends + if platforms: + self.platforms = platforms + if toolchains: + self.toolchains = toolchains + if targets: + self.targets = targets + if variables: + self.vars = variables + if default_flags: + self.flags = default_flags diff --git a/mlonmcu/environment/legacy/loader.py b/mlonmcu/environment/legacy/loader.py new file mode 100644 index 000000000..67f84a031 --- /dev/null +++ b/mlonmcu/environment/legacy/loader.py @@ -0,0 +1,252 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from ..config import ( + PathConfig, + RepoConfig, +) +from .config import ( + DefaultsConfigOld, + FrameworkConfig, + FrameworkFeatureConfig, + BackendConfig, + BackendFeatureConfig, + TargetConfig, + TargetFeatureConfig, + PlatformConfig, + PlatformFeatureConfig, + FrontendConfig, + FrontendFeatureConfig, +) + +# def load_environment_from_file(filename): +# """Utility to initialize a mlonmcu environment from a YAML file.""" +# if isinstance(filename, str): +# filename = pathlib.Path(filename) +# with open(filename, encoding="utf-8") as yaml_file: +# data = yaml.safe_load(yaml_file) +# if data: +# if "home" in data: +# print(data["home"], filename.parent) +# assert os.path.realpath(data["home"]) == os.path.realpath(filename.parent) +# else: +# data["home"] = filename.parent +# env = Environment(data) +# return env +# raise RuntimeError(f"Error opening environment file: {filename}") +# return None + + +def load_environment_from_file(filename, base): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + with open(filename, encoding="utf-8") as yaml_file: + loaded = yaml.safe_load(yaml_file) + if not loaded: + raise RuntimeError("Invalid YAML contents") + if "home" in loaded: + home = loaded["home"] + else: + home = None + if "logging" in loaded: + if "level" in loaded["logging"]: + log_level = logging.getLevelName(loaded["logging"]["level"]) + else: + log_level = None + if "to_file" in loaded["logging"]: + log_to_file = bool(loaded["logging"]["to_file"]) + else: + log_to_file = None + if "rotate" in loaded["logging"]: + log_rotate = bool(loaded["logging"]["rotate"]) + else: + log_rotate = None + else: + log_level = None + log_to_file = False + log_rotate = False + if "cleanup" in loaded: + if "auto" in loaded["cleanup"]: + cleanup_auto = bool(loaded["cleanup"]["auto"]) + else: + cleanup_auto = False + if "auto" in loaded["cleanup"]: + cleanup_keep = int(loaded["cleanup"]["keep"]) + else: + cleanup_keep = 100 + else: + cleanup_auto = False + cleanup_keep = 100 + if "paths" in loaded: + paths = {} + for key in loaded["paths"]: + path = loaded["paths"][key] + if isinstance(path, list): + paths[key] = [] + for p in path: + paths[key].append(PathConfig(p, base=home)) + else: + paths[key] = PathConfig(path, base=home) + else: + paths = None + if "repos" in loaded: + repos = {} + for key in loaded["repos"]: + repo = loaded["repos"][key] + if "url" not in repo: + raise RuntimeError("Missing field 'url' in YAML file") + if "ref" in repo: + repos[key] = RepoConfig(repo["url"], ref=repo["ref"]) + else: + repos[key] = RepoConfig(repo["url"]) + else: + repos = None + default_framework = None + default_backends = {} + if "frameworks" in loaded: + frameworks = [] + default_framework = loaded["frameworks"].pop("default", None) + for key in loaded["frameworks"]: + framework = loaded["frameworks"][key] + if "enabled" in framework: + enabled = bool(framework["enabled"]) + else: + enabled = False + backends = [] + if "backends" in framework: + default_backend = framework["backends"].pop("default", None) + default_backends[key] = default_backend + for key2 in framework["backends"]: + backend = framework["backends"][key2] + if "enabled" in backend: + enabled2 = bool(backend["enabled"]) + else: + enabled2 = True + backend_features = [] + if "features" in backend: + for key3 in backend["features"]: + supported = bool(backend["features"][key3]) + backend_features.append(BackendFeatureConfig(key3, backend=key2, supported=supported)) + backends.append(BackendConfig(key2, enabled=enabled2, features=backend_features)) + framework_features = [] + if "features" in framework: + for key2 in framework["features"]: + supported = bool(framework["features"][key2]) + framework_features.append(FrameworkFeatureConfig(key2, framework=key, supported=supported)) + frameworks.append( + FrameworkConfig( + key, + enabled=enabled, + backends=backends, + features=framework_features, + ) + ) + else: + frameworks = None + if "frontends" in loaded: + frontends = [] + for key in loaded["frontends"]: + frontend = loaded["frontends"][key] + + if "enabled" in frontend: + enabled = frontend["enabled"] + else: + enabled = True + frontend_features = [] + if "features" in frontend: + for key2 in frontend["features"]: + supported = bool(frontend["features"][key2]) + frontend_features.append(FrontendFeatureConfig(key2, frontend=key, supported=supported)) + frontends.append(FrontendConfig(key, enabled=enabled, features=frontend_features)) + else: + frontends = None + if "platforms" in loaded: + platforms = [] + for key in loaded["platforms"]: + platform = loaded["platforms"][key] + if "enabled" in platform: + enabled = platform["enabled"] + else: + enabled = True + platform_features = [] + if "features" in platform: + for key2 in platform["features"]: + supported = bool(platform["features"][key2]) + platform_features.append(PlatformFeatureConfig(key2, platform=key, supported=supported)) + platforms.append(PlatformConfig(key, enabled=enabled, features=platform_features)) + else: + platforms = None + if "toolchains" in loaded: + toolchains = loaded["toolchains"] + else: + toolchains = None + default_target = None + if "targets" in loaded: + targets = [] + default_target = loaded["targets"].pop("default", None) + for key in loaded["targets"]: + target = loaded["targets"][key] + if "enabled" in target: + enabled = target["enabled"] + else: + enabled = True + target_features = [] + if "features" in target: + for key2 in target["features"]: + supported = bool(target["features"][key2]) + target_features.append(TargetFeatureConfig(key2, target=key, supported=supported)) + targets.append(TargetConfig(key, enabled=enabled, features=target_features)) + else: + targets = None + if "vars" in loaded: + variables = loaded["vars"] + else: + variables = None + if "flags" in loaded: + default_flags = loaded["flags"] + else: + default_flags = None + defaults = DefaultsConfigOld( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + default_framework=default_framework, + default_backends=default_backends, + default_target=default_target, + cleanup_auto=cleanup_auto, + cleanup_keep=cleanup_keep, + ) + env = base( + home, + defaults=defaults, + paths=paths, + repos=repos, + frameworks=frameworks, + frontends=frontends, + platforms=platforms, + toolchains=toolchains, + targets=targets, + variables=variables, + default_flags=default_flags, + ) + return env diff --git a/mlonmcu/environment/legacy/writer.py b/mlonmcu/environment/legacy/writer.py new file mode 100644 index 000000000..9d97e06a0 --- /dev/null +++ b/mlonmcu/environment/legacy/writer.py @@ -0,0 +1,117 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from ..config import PathConfig + + +def create_environment_dict(environment): + data = {} + data["home"] = environment.home + data["logging"] = { + "level": logging.getLevelName(environment.defaults.log_level), + "to_file": environment.defaults.log_to_file, + "rotate": environment.defaults.log_rotate, + } + data["cleanup"] = { + "auto": environment.defaults.cleanup_auto, + "keep": environment.defaults.cleanup_keep, + } + data["paths"] = { + path: str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + for path, path_config in environment.paths.items() + } # TODO: allow relative paths + data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} + data["frameworks"] = { + "default": environment.defaults.default_framework if environment.defaults.default_framework else None, + **{ + framework.name: { + "enabled": framework.enabled, + "backends": { + "default": environment.defaults.default_backends[framework.name] + if environment.defaults.default_backends and environment.defaults.default_backends[framework.name] + else None, + **{ + backend.name: { + "enabled": backend.enabled, + "features": { + backend_feature.name: backend_feature.supported for backend_feature in backend.features + }, + } + for backend in framework.backends + }, + }, + "features": { + framework_feature.name: framework_feature.supported for framework_feature in framework.features + }, + } + for framework in environment.frameworks + }, + } + data["frontends"] = { + # "default": None, # unimplemented? + **{ + frontend.name: { + "enabled": frontend.enabled, + "features": { + frontend_feature.name: frontend_feature.supported for frontend_feature in frontend.features + }, + } + for frontend in environment.frontends + }, + } + data["platforms"] = { + # "default": None, # unimplemented? + **{ + platform.name: { + "enabled": platform.enabled, + "features": { + platform_feature.name: platform_feature.supported for platform_feature in platform.features + }, + } + for platform in environment.platforms + }, + } + data["toolchains"] = environment.toolchains + data["targets"] = { + "default": environment.defaults.default_target, + **{ + target.name: { + "enabled": target.enabled, + "features": {target_feature.name: target_feature.supported for target_feature in target.features}, + } + for target in environment.targets + }, + } + data["vars"] = environment.vars + data["flags"] = environment.flags + return data + + +def write_environment_to_file(environment, filename): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + data = create_environment_dict(environment) + with open(filename, "w") as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False, sort_keys=False) diff --git a/mlonmcu/environment/loader.py b/mlonmcu/environment/loader.py index b950e45e2..064f4c15d 100644 --- a/mlonmcu/environment/loader.py +++ b/mlonmcu/environment/loader.py @@ -20,21 +20,17 @@ import pathlib import logging +from mlonmcu.logging import get_logger from .config import ( - DefaultsConfigOld, + DefaultsConfig, PathConfig, RepoConfig, - FrameworkConfig, - FrameworkFeatureConfig, - BackendConfig, - BackendFeatureConfig, - TargetConfig, - TargetFeatureConfig, - PlatformConfig, - PlatformFeatureConfig, - FrontendConfig, - FrontendFeatureConfig, + ComponentConfig, ) +from .legacy.loader import load_environment_from_file as load_legacy_environment_from_file +from .legacy.environment import UserEnvironment as UserEnvironmentOld + +logger = get_logger() # def load_environment_from_file(filename): # """Utility to initialize a mlonmcu environment from a YAML file.""" @@ -62,6 +58,16 @@ def load_environment_from_file(filename, base): loaded = yaml.safe_load(yaml_file) if not loaded: raise RuntimeError("Invalid YAML contents") + if "version" in loaded: + version = int(loaded["version"]) + else: + version = 1 + if version == 1: + # fallback + logger.warning("Environment format v1 is deprecated. Please convert environment files to v2.") + from .convert import convert_v1_to_v2 + return convert_v1_to_v2(load_legacy_environment_from_file(filename, UserEnvironmentOld)) + assert version == 2, "Unsupported version of environment file" if "home" in loaded: home = loaded["home"] else: @@ -119,103 +125,23 @@ def load_environment_from_file(filename, base): repos[key] = RepoConfig(repo["url"]) else: repos = None - default_framework = None - default_backends = {} - if "frameworks" in loaded: - frameworks = [] - default_framework = loaded["frameworks"].pop("default", None) - for key in loaded["frameworks"]: - framework = loaded["frameworks"][key] - if "enabled" in framework: - enabled = bool(framework["enabled"]) - else: - enabled = False - backends = [] - if "backends" in framework: - default_backend = framework["backends"].pop("default", None) - default_backends[key] = default_backend - for key2 in framework["backends"]: - backend = framework["backends"][key2] - if "enabled" in backend: - enabled2 = bool(backend["enabled"]) - else: - enabled2 = True - backend_features = [] - if "features" in backend: - for key3 in backend["features"]: - supported = bool(backend["features"][key3]) - backend_features.append(BackendFeatureConfig(key3, backend=key2, supported=supported)) - backends.append(BackendConfig(key2, enabled=enabled2, features=backend_features)) - framework_features = [] - if "features" in framework: - for key2 in framework["features"]: - supported = bool(framework["features"][key2]) - framework_features.append(FrameworkFeatureConfig(key2, framework=key, supported=supported)) - frameworks.append( - FrameworkConfig( - key, - enabled=enabled, - backends=backends, - features=framework_features, - ) - ) - else: - frameworks = None - if "frontends" in loaded: - frontends = [] - for key in loaded["frontends"]: - frontend = loaded["frontends"][key] - if "enabled" in frontend: - enabled = frontend["enabled"] - else: - enabled = True - frontend_features = [] - if "features" in frontend: - for key2 in frontend["features"]: - supported = bool(frontend["features"][key2]) - frontend_features.append(FrontendFeatureConfig(key2, frontend=key, supported=supported)) - frontends.append(FrontendConfig(key, enabled=enabled, features=frontend_features)) - else: - frontends = None - if "platforms" in loaded: - platforms = [] - for key in loaded["platforms"]: - platform = loaded["platforms"][key] - if "enabled" in platform: - enabled = platform["enabled"] - else: - enabled = True - platform_features = [] - if "features" in platform: - for key2 in platform["features"]: - supported = bool(platform["features"][key2]) - platform_features.append(PlatformFeatureConfig(key2, platform=key, supported=supported)) - platforms.append(PlatformConfig(key, enabled=enabled, features=platform_features)) - else: - platforms = None - if "toolchains" in loaded: - toolchains = loaded["toolchains"] - else: - toolchains = None - default_target = None - if "targets" in loaded: - targets = [] - default_target = loaded["targets"].pop("default", None) - for key in loaded["targets"]: - target = loaded["targets"][key] - if "enabled" in target: - enabled = target["enabled"] - else: - enabled = True - target_features = [] - if "features" in target: - for key2 in target["features"]: - supported = bool(target["features"][key2]) - target_features.append(TargetFeatureConfig(key2, target=key, supported=supported)) - targets.append(TargetConfig(key, enabled=enabled, features=target_features)) - else: - targets = None + def helper(data): + if data is None: + return None + supported = data.get("supported", []) + used = data.get("use", []) + combined = list(set(supported + used)) + return {key: ComponentConfig(key in supported, key in used) for key in combined} + + frameworks = helper(loaded.get("frameworks", None)) + backends = helper(loaded.get("backends", None)) + frontends = helper(loaded.get("frontends", None)) + features = helper(loaded.get("features", None)) + platforms = helper(loaded.get("platforms", None)) + targets = helper(loaded.get("targets", None)) + postprocesses = helper(loaded.get("postprocesses", None)) + toolchains = helper(loaded.get("toolchains", None)) if "vars" in loaded: variables = loaded["vars"] else: @@ -224,13 +150,10 @@ def load_environment_from_file(filename, base): default_flags = loaded["flags"] else: default_flags = None - defaults = DefaultsConfigOld( + defaults = DefaultsConfig( log_level=log_level, log_to_file=log_to_file, log_rotate=log_rotate, - default_framework=default_framework, - default_backends=default_backends, - default_target=default_target, cleanup_auto=cleanup_auto, cleanup_keep=cleanup_keep, ) @@ -240,10 +163,13 @@ def load_environment_from_file(filename, base): paths=paths, repos=repos, frameworks=frameworks, + backends=backends, frontends=frontends, platforms=platforms, toolchains=toolchains, targets=targets, + features=features, + postprocesses=postprocesses, variables=variables, default_flags=default_flags, ) diff --git a/mlonmcu/environment/loader2.py b/mlonmcu/environment/loader2.py deleted file mode 100644 index 104c8b8e1..000000000 --- a/mlonmcu/environment/loader2.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. -# -# This file is part of MLonMCU. -# See https://github.com/tum-ei-eda/mlonmcu.git for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import yaml -import pathlib -import logging - -from .config import ( - DefaultsConfig, - PathConfig, - RepoConfig, - ComponentConfig, -) - -# def load_environment_from_file(filename): -# """Utility to initialize a mlonmcu environment from a YAML file.""" -# if isinstance(filename, str): -# filename = pathlib.Path(filename) -# with open(filename, encoding="utf-8") as yaml_file: -# data = yaml.safe_load(yaml_file) -# if data: -# if "home" in data: -# print(data["home"], filename.parent) -# assert os.path.realpath(data["home"]) == os.path.realpath(filename.parent) -# else: -# data["home"] = filename.parent -# env = Environment(data) -# return env -# raise RuntimeError(f"Error opening environment file: {filename}") -# return None - - -def load_environment_from_file(filename, base): - """Utility to initialize a mlonmcu environment from a YAML file.""" - if isinstance(filename, str): - filename = pathlib.Path(filename) - with open(filename, encoding="utf-8") as yaml_file: - loaded = yaml.safe_load(yaml_file) - if not loaded: - raise RuntimeError("Invalid YAML contents") - if "home" in loaded: - home = loaded["home"] - else: - home = None - if "logging" in loaded: - if "level" in loaded["logging"]: - log_level = logging.getLevelName(loaded["logging"]["level"]) - else: - log_level = None - if "to_file" in loaded["logging"]: - log_to_file = bool(loaded["logging"]["to_file"]) - else: - log_to_file = None - if "rotate" in loaded["logging"]: - log_rotate = bool(loaded["logging"]["rotate"]) - else: - log_rotate = None - else: - log_level = None - log_to_file = False - log_rotate = False - if "cleanup" in loaded: - if "auto" in loaded["cleanup"]: - cleanup_auto = bool(loaded["cleanup"]["auto"]) - else: - cleanup_auto = False - if "auto" in loaded["cleanup"]: - cleanup_keep = int(loaded["cleanup"]["keep"]) - else: - cleanup_keep = 100 - else: - cleanup_auto = False - cleanup_keep = 100 - if "paths" in loaded: - paths = {} - for key in loaded["paths"]: - path = loaded["paths"][key] - if isinstance(path, list): - paths[key] = [] - for p in path: - paths[key].append(PathConfig(p, base=home)) - else: - paths[key] = PathConfig(path, base=home) - else: - paths = None - if "repos" in loaded: - repos = {} - for key in loaded["repos"]: - repo = loaded["repos"][key] - if "url" not in repo: - raise RuntimeError("Missing field 'url' in YAML file") - if "ref" in repo: - repos[key] = RepoConfig(repo["url"], ref=repo["ref"]) - else: - repos[key] = RepoConfig(repo["url"]) - else: - repos = None - - def helper(data): - if data is None: - return None - supported = data.get("supported", []) - used = data.get("use", []) - combined = list(set(supported + used)) - return {key: ComponentConfig(key in supported, key in used) for key in combined} - frameworks = helper(loaded.get("frameworks", None)) - backends = helper(loaded.get("backends", None)) - frontends = helper(loaded.get("frontends", None)) - features = helper(loaded.get("features", None)) - platforms = helper(loaded.get("platforms", None)) - targets = helper(loaded.get("targets", None)) - postprocesses = helper(loaded.get("postprocesses", None)) - toolchains = helper(loaded.get("toolchains", None)) - if "vars" in loaded: - variables = loaded["vars"] - else: - variables = None - if "flags" in loaded: - default_flags = loaded["flags"] - else: - default_flags = None - defaults = DefaultsConfig( - log_level=log_level, - log_to_file=log_to_file, - log_rotate=log_rotate, - cleanup_auto=cleanup_auto, - cleanup_keep=cleanup_keep, - ) - env = base( - home, - defaults=defaults, - paths=paths, - repos=repos, - frameworks=frameworks, - backends=backends, - frontends=frontends, - platforms=platforms, - toolchains=toolchains, - targets=targets, - features=features, - postprocesses=postprocesses, - variables=variables, - default_flags=default_flags, - ) - return env diff --git a/mlonmcu/environment/user_environment.py b/mlonmcu/environment/user_environment.py new file mode 100644 index 000000000..884eeddb3 --- /dev/null +++ b/mlonmcu/environment/user_environment.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tempfile +from pathlib import Path + +from .loader import load_environment_from_file +from .writer import write_environment_to_file +from .environment import DefaultEnvironment + + +class UserEnvironment(DefaultEnvironment): + def __init__( + self, + home, + alias=None, + defaults=None, + paths=None, + repos=None, + frameworks=None, + backends=None, + frontends=None, + platforms=None, + toolchains=None, + targets=None, + features=None, + postprocesses=None, + variables=None, + default_flags=None, + ): + super().__init__() + self._home = home + + if alias: + self.alias = alias + if defaults: + self.defaults = defaults + if paths: + self.paths = paths + if repos: + self.repos = repos + if frameworks: + self.frameworks = frameworks + if backends: + self.backends = backends + if frontends: + self.frontends = frontends + if platforms: + self.platforms = platforms + if toolchains: + self.toolchains = toolchains + if targets: + self.targets = targets + if features: + self.features = features + if postprocesses: + self.postprocesses = postprocesses + if variables: + self.vars = variables + if default_flags: + self.flags = default_flags + + @classmethod + def from_file(cls, filename): + return load_environment_from_file(filename, base=cls) + + def to_file(self, filename): + write_environment_to_file(self, filename) + + def to_text(self): + with tempfile.TemporaryDirectory() as tmpdirname: + dest = Path(tmpdirname) / "out.yml" + self.to_file(dest) + with open(dest, "r") as handle: + content = handle.read() + assert len(content) > 0 + return content diff --git a/mlonmcu/environment/writer.py b/mlonmcu/environment/writer.py index 94175da79..7ba71408c 100644 --- a/mlonmcu/environment/writer.py +++ b/mlonmcu/environment/writer.py @@ -41,67 +41,40 @@ def create_environment_dict(environment): else [str(config.path) for config in path_config] for path, path_config in environment.paths.items() } # TODO: allow relative paths - data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} + data["repos"] = { + repo: {"url": repo_config.url, "ref": repo_config.ref} for repo, repo_config in environment.repos.items() + } data["frameworks"] = { - "default": environment.defaults.default_framework if environment.defaults.default_framework else None, - **{ - framework.name: { - "enabled": framework.enabled, - "backends": { - "default": environment.defaults.default_backends[framework.name] - if environment.defaults.default_backends and environment.defaults.default_backends[framework.name] - else None, - **{ - backend.name: { - "enabled": backend.enabled, - "features": { - backend_feature.name: backend_feature.supported for backend_feature in backend.features - }, - } - for backend in framework.backends - }, - }, - "features": { - framework_feature.name: framework_feature.supported for framework_feature in framework.features - }, - } - for framework in environment.frameworks - }, + "supported": [name for name, value in environment.frameworks.items() if value.supported], + "use": [name for name, value in environment.frameworks.items() if value.used], + } + data["backends"] = { + "supported": [name for name, value in environment.backends.items() if value.supported], + "use": [name for name, value in environment.backends.items() if value.used], } data["frontends"] = { - # "default": None, # unimplemented? - **{ - frontend.name: { - "enabled": frontend.enabled, - "features": { - frontend_feature.name: frontend_feature.supported for frontend_feature in frontend.features - }, - } - for frontend in environment.frontends - }, + "supported": [name for name, value in environment.frontends.items() if value.supported], + "use": [name for name, value in environment.frontends.items() if value.used], + } + data["toolchains"] = { + "supported": [name for name, value in environment.toolchains.items() if value.supported], + "use": [name for name, value in environment.toolchains.items() if value.used], } data["platforms"] = { - # "default": None, # unimplemented? - **{ - platform.name: { - "enabled": platform.enabled, - "features": { - platform_feature.name: platform_feature.supported for platform_feature in platform.features - }, - } - for platform in environment.platforms - }, + "supported": [name for name, value in environment.platforms.items() if value.supported], + "use": [name for name, value in environment.platforms.items() if value.used], } - data["toolchains"] = environment.toolchains data["targets"] = { - "default": environment.defaults.default_target, - **{ - target.name: { - "enabled": target.enabled, - "features": {target_feature.name: target_feature.supported for target_feature in target.features}, - } - for target in environment.targets - }, + "supported": [name for name, value in environment.targets.items() if value.supported], + "use": [name for name, value in environment.targets.items() if value.used], + } + data["features"] = { + "supported": [name for name, value in environment.features.items() if value.supported], + "use": [name for name, value in environment.features.items() if value.used], + } + data["postprocesses"] = { + "supported": [name for name, value in environment.postprocesses.items() if value.supported], + "use": [name for name, value in environment.postprocesses.items() if value.used], } data["vars"] = environment.vars data["flags"] = environment.flags diff --git a/mlonmcu/environment/writer2.py b/mlonmcu/environment/writer2.py deleted file mode 100644 index 815c6b36f..000000000 --- a/mlonmcu/environment/writer2.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. -# -# This file is part of MLonMCU. -# See https://github.com/tum-ei-eda/mlonmcu.git for further info. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import yaml -import pathlib -import logging - -from .config import PathConfig - - -def create_environment_dict(environment): - data = {} - data["home"] = environment.home - data["logging"] = { - "level": logging.getLevelName(environment.defaults.log_level), - "to_file": environment.defaults.log_to_file, - "rotate": environment.defaults.log_rotate, - } - data["cleanup"] = { - "auto": environment.defaults.cleanup_auto, - "keep": environment.defaults.cleanup_keep, - } - data["paths"] = { - path: str(path_config.path) - if isinstance(path_config, PathConfig) - else [str(config.path) for config in path_config] - for path, path_config in environment.paths.items() - } # TODO: allow relative paths - data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} - data["frameworks"] = { - "supported": [name for name, value in environment.frameworks.items() if value.supported], - "use": [name for name, value in environment.frameworks.items() if value.used], - } - data["backends"] = { - "supported": [name for name, value in environment.backends.items() if value.supported], - "use": [name for name, value in environment.backends.items() if value.used], - } - data["frontends"] = { - "supported": [name for name, value in environment.frontends.items() if value.supported], - "use": [name for name, value in environment.frontends.items() if value.used], - } - data["toolchains"] = { - "supported": [name for name, value in environment.toolchains.items() if value.supported], - "use": [name for name, value in environment.toolchains.items() if value.used], - } - data["platforms"] = { - "supported": [name for name, value in environment.platforms.items() if value.supported], - "use": [name for name, value in environment.platforms.items() if value.used], - } - data["targets"] = { - "supported": [name for name, value in environment.targets.items() if value.supported], - "use": [name for name, value in environment.targets.items() if value.used], - } - data["features"] = { - "supported": [name for name, value in environment.features.items() if value.supported], - "use": [name for name, value in environment.features.items() if value.used], - } - data["postprocesses"] = { - "supported": [name for name, value in environment.postprocesses.items() if value.supported], - "use": [name for name, value in environment.postprocesses.items() if value.used], - } - data["vars"] = environment.vars - data["flags"] = environment.flags - return data - - -def write_environment_to_file(environment, filename): - """Utility to initialize a mlonmcu environment from a YAML file.""" - if isinstance(filename, str): - filename = pathlib.Path(filename) - data = create_environment_dict(environment) - with open(filename, "w") as yaml_file: - yaml.dump(data, yaml_file, default_flow_style=False, sort_keys=False) diff --git a/mlonmcu/platform/lookup.py b/mlonmcu/platform/lookup.py index 8ffcab76a..15a43c8d5 100644 --- a/mlonmcu/platform/lookup.py +++ b/mlonmcu/platform/lookup.py @@ -23,7 +23,7 @@ def get_platform_names(context): - return context.environment.lookup_platform_configs(names_only=True) + return context.environment.get_supported_platforms() def get_platforms_targets(context, config=None): From 10b5a2c281ba674b2abb554761a5a3ebc148a26c Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 18 Feb 2023 17:41:00 +0100 Subject: [PATCH 06/10] followup fixes 2 --- mlonmcu/environment/environment.py | 2 +- mlonmcu/setup/tasks/muriscvnn.py | 6 +++--- mlonmcu/setup/tasks/spike.py | 4 ++-- mlonmcu/setup/tasks/tflmc.py | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/mlonmcu/environment/environment.py b/mlonmcu/environment/environment.py index 9e82b76d7..29d974428 100644 --- a/mlonmcu/environment/environment.py +++ b/mlonmcu/environment/environment.py @@ -156,7 +156,7 @@ def get_used_platforms(self): def has_platform(self, name): return name in self.get_supported_platforms() - def get_supported_target(self): + def get_supported_targets(self): supported = [name for name, config in self.targets.items() if config.supported] return supported diff --git a/mlonmcu/setup/tasks/muriscvnn.py b/mlonmcu/setup/tasks/muriscvnn.py index 64ccc9a28..f23854294 100644 --- a/mlonmcu/setup/tasks/muriscvnn.py +++ b/mlonmcu/setup/tasks/muriscvnn.py @@ -34,7 +34,7 @@ def _validate_muriscvnn(context: MlonMcuContext, params=None): - if not context.environment.supports_feature("muriscvnn"): + if not context.environment.has_feature("muriscvnn"): return False user_vars = context.environment.vars if "muriscvnn.src_dir" not in user_vars: @@ -46,13 +46,13 @@ def _validate_muriscvnn(context: MlonMcuContext, params=None): target_arch = params.get("target_arch", "riscv") if target_arch == "riscv": if params.get("vext", False): - if not context.environment.supports_feature("vext"): + if not context.environment.has_feature("vext"): return False if params.get("pext", False): if toolchain == "llvm": # Unsupported return False - if not context.environment.supports_feature("pext"): + if not context.environment.has_feature("pext"): return False if params.get("vext", False) and params.get("pext", False): # Either pext or vext! diff --git a/mlonmcu/setup/tasks/spike.py b/mlonmcu/setup/tasks/spike.py index 5f5615f64..e797d59e2 100644 --- a/mlonmcu/setup/tasks/spike.py +++ b/mlonmcu/setup/tasks/spike.py @@ -40,12 +40,12 @@ def _validate_spike(context: MlonMcuContext, params=None): if params.get("vext", False): if params.get("pext", False): return False # Can not use booth at a time - if not context.environment.supports_feature("vext"): + if not context.environment.has_feature("vext"): return False if params.get("pext", False): if params.get("vext", False): return False # Can not use booth at a time - if not context.environment.supports_feature("pext"): + if not context.environment.has_feature("pext"): return False user_vars = context.environment.vars if "spike.pk" not in user_vars: # TODO: also check command line flags? diff --git a/mlonmcu/setup/tasks/tflmc.py b/mlonmcu/setup/tasks/tflmc.py index a45a82128..07a0c487f 100644 --- a/mlonmcu/setup/tasks/tflmc.py +++ b/mlonmcu/setup/tasks/tflmc.py @@ -65,10 +65,10 @@ def _validate_build_tflite_micro_compiler(context: MlonMcuContext, params=None): # Not allowed return False elif muriscvnn: - if not context.environment.supports_feature("muriscvnn"): + if not context.environment.has_feature("muriscvnn"): return False elif cmsisnn: - if not context.environment.supports_feature("cmsisnn"): + if not context.environment.has_feature("cmsisnn"): return False return _validate_tflite_micro_compiler(context, params=params) From 0c06947e6d8b288c668ddfd494a35f72b4d9608f Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 18 Feb 2023 17:58:11 +0100 Subject: [PATCH 07/10] fix failing test --- mlonmcu/environment/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mlonmcu/environment/config.py b/mlonmcu/environment/config.py index dd1c96871..23a2f3d76 100644 --- a/mlonmcu/environment/config.py +++ b/mlonmcu/environment/config.py @@ -120,5 +120,5 @@ def __repr(self): return f"PathConfig({self.path})" -RepoConfig = namedtuple("RepoConfig", "url ref") -ComponentConfig = namedtuple("ComponentConfig", "supported used") +RepoConfig = namedtuple("RepoConfig", "url ref", defaults=[None, None]) +ComponentConfig = namedtuple("ComponentConfig", "supported used", defaults=[False, False]) From 03b4a383f4021305da0ce22a90c80915c4236a79 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 18 Feb 2023 20:12:31 +0100 Subject: [PATCH 08/10] lint --- mlonmcu/environment/legacy/writer.py | 21 ++++++++++++++------- mlonmcu/environment/loader.py | 1 + mlonmcu/environment/writer.py | 8 +++++--- mlonmcu/feature/features.py | 2 +- mlonmcu/session/run.py | 24 +++++++++++++++--------- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/mlonmcu/environment/legacy/writer.py b/mlonmcu/environment/legacy/writer.py index 9d97e06a0..100cd54f3 100644 --- a/mlonmcu/environment/legacy/writer.py +++ b/mlonmcu/environment/legacy/writer.py @@ -36,21 +36,28 @@ def create_environment_dict(environment): "keep": environment.defaults.cleanup_keep, } data["paths"] = { - path: str(path_config.path) - if isinstance(path_config, PathConfig) - else [str(config.path) for config in path_config] + path: ( + str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + ) for path, path_config in environment.paths.items() } # TODO: allow relative paths - data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} + data["repos"] = { + repo: {"url": repo_config.url, "ref": repo_config.ref} for repo, repo_config in environment.repos.items() + } data["frameworks"] = { "default": environment.defaults.default_framework if environment.defaults.default_framework else None, **{ framework.name: { "enabled": framework.enabled, "backends": { - "default": environment.defaults.default_backends[framework.name] - if environment.defaults.default_backends and environment.defaults.default_backends[framework.name] - else None, + "default": ( + environment.defaults.default_backends[framework.name] + if environment.defaults.default_backends + and environment.defaults.default_backends[framework.name] + else None + ), **{ backend.name: { "enabled": backend.enabled, diff --git a/mlonmcu/environment/loader.py b/mlonmcu/environment/loader.py index 064f4c15d..24a418aa7 100644 --- a/mlonmcu/environment/loader.py +++ b/mlonmcu/environment/loader.py @@ -66,6 +66,7 @@ def load_environment_from_file(filename, base): # fallback logger.warning("Environment format v1 is deprecated. Please convert environment files to v2.") from .convert import convert_v1_to_v2 + return convert_v1_to_v2(load_legacy_environment_from_file(filename, UserEnvironmentOld)) assert version == 2, "Unsupported version of environment file" if "home" in loaded: diff --git a/mlonmcu/environment/writer.py b/mlonmcu/environment/writer.py index 7ba71408c..b99eb179f 100644 --- a/mlonmcu/environment/writer.py +++ b/mlonmcu/environment/writer.py @@ -36,9 +36,11 @@ def create_environment_dict(environment): "keep": environment.defaults.cleanup_keep, } data["paths"] = { - path: str(path_config.path) - if isinstance(path_config, PathConfig) - else [str(config.path) for config in path_config] + path: ( + str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + ) for path, path_config in environment.paths.items() } # TODO: allow relative paths data["repos"] = { diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index ad643d299..24f1f04bf 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -1317,7 +1317,7 @@ def benchmark_callback(stdout, metrics, artifacts): if self.total: data.update( { - f"Total {key}": ((value * self.num_runs) if self.num_runs > 1 else value) + f"Total {key}": (value * self.num_runs) if self.num_runs > 1 else value for key, value in data_[-1].items() if "cycle" in key.lower() or "time" in key.lower() } diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 85c3a3798..9d14e9f3d 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -598,9 +598,11 @@ def _merge_dicts_of_lists(*args): if isinstance(artifacts, dict): new.update( { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } ) @@ -632,9 +634,11 @@ def run(self): artifacts = self.target.artifacts if isinstance(artifacts, dict): new = { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } else: @@ -650,9 +654,11 @@ def run(self): artifacts = self.target.artifacts if isinstance(artifacts, dict): new = { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } else: From 9584357331552f4e190a190a7f6f5a20e6f78d14 Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sat, 18 Feb 2023 20:13:18 +0100 Subject: [PATCH 09/10] fix failing test --- requirements.txt | 1 + tests/unit-tests/environment/test_environment_writer.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7c45de684..1939f10be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ pyyaml pandas numpy psutil +Pillow # Sessions tqdm diff --git a/tests/unit-tests/environment/test_environment_writer.py b/tests/unit-tests/environment/test_environment_writer.py index 2a0e5dd57..cae0e633c 100644 --- a/tests/unit-tests/environment/test_environment_writer.py +++ b/tests/unit-tests/environment/test_environment_writer.py @@ -1,9 +1,10 @@ import os import logging -from mlonmcu.environment.environment import Environment -from mlonmcu.environment.config import DefaultsConfig, PathConfig, RepoConfig -from mlonmcu.environment.writer import create_environment_dict +from mlonmcu.environment.legacy.environment import Environment +from mlonmcu.environment.config import PathConfig, RepoConfig +from mlonmcu.environment.legacy.config import DefaultsConfigOld as DefaultsConfig +from mlonmcu.environment.legacy.writer import create_environment_dict class MyEnvironment(Environment): From 7f00790af5862f10ec8786f4c9a38e316dfaf5fa Mon Sep 17 00:00:00 2001 From: Philipp van Kempen Date: Sun, 19 Feb 2023 11:52:50 +0100 Subject: [PATCH 10/10] remove unit tests --- tests/unit-tests/platform/test_platform_lookup.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/unit-tests/platform/test_platform_lookup.py b/tests/unit-tests/platform/test_platform_lookup.py index b33fae097..b43af9060 100644 --- a/tests/unit-tests/platform/test_platform_lookup.py +++ b/tests/unit-tests/platform/test_platform_lookup.py @@ -184,13 +184,3 @@ def test_platform_registry(): assert len(get_platforms()) == before + 2 register_platform("foo", MyPlatformA, override=True) assert len(get_platforms()) == before + 2 - - -def test_platform_get_platform_names(fake_context): - def lookup_platform_configs(names_only=True): - assert names_only - return ["foo", "bar"] - - fake_context.environment.lookup_platform_configs = lookup_platform_configs - res = mlonmcu.platform.lookup.get_platform_names(fake_context) - assert res == ["foo", "bar"]