diff --git a/mlonmcu/cli/helper/parse.py b/mlonmcu/cli/helper/parse.py index 7e455d9c2..b4941be8e 100644 --- a/mlonmcu/cli/helper/parse.py +++ b/mlonmcu/cli/helper/parse.py @@ -109,8 +109,8 @@ def extract_frontend_names(args, context=None): # No need to specify a default, because we just use the provided order in the environment.yml assert frontend_names is None, "TODO" assert context is not None, "Need context to resolve default frontends" - all_frontend_names = context.environment.lookup_frontend_configs(names_only=True) - names.extend(all_frontend_names) + default_frontend_names = context.environment.get_used_frontends() + names.extend(default_frontend_names) return names @@ -155,5 +155,5 @@ def extract_platform_names(args, context=None): assert args.platform is None if context is None: return [None] - platforms = context.environment.lookup_platform_configs(names_only=True) + platforms = context.environment.get_used_platforms() return platforms diff --git a/mlonmcu/context/context.py b/mlonmcu/context/context.py index db9277ef8..c0bdf59a4 100644 --- a/mlonmcu/context/context.py +++ b/mlonmcu/context/context.py @@ -35,7 +35,7 @@ from mlonmcu.plugins import process_extensions from mlonmcu.context.read_write_filelock import ReadFileLock, WriteFileLock, RWLockTimeout -from mlonmcu.environment.environment import Environment, UserEnvironment +from mlonmcu.environment.user_environment import UserEnvironment from mlonmcu.environment.list import get_environments_map from mlonmcu.environment.config import get_environments_dir, get_plugins_dir @@ -43,7 +43,7 @@ logger = get_logger() -def lookup_environment() -> Environment: +def lookup_environment() -> "UserEnvironment": """Helper function to automatically find a suitable environment. This function is used if neither a name nor a path of the environment was specified by the user. @@ -85,7 +85,7 @@ def lookup_environment() -> Environment: return None -def get_environment_by_path(path: Union[str, Path]) -> Environment: +def get_environment_by_path(path: Union[str, Path]) -> UserEnvironment: """Utility to find an environment file using a supplied path. Parameters @@ -95,7 +95,7 @@ def get_environment_by_path(path: Union[str, Path]) -> Environment: Returns ------- - Environment: + UserEnvironment: The environment (if the lookup was successful). """ if isinstance(path, str): @@ -108,7 +108,7 @@ def get_environment_by_path(path: Union[str, Path]) -> Environment: return None -def get_environment_by_name(name: str) -> Environment: +def get_environment_by_name(name: str) -> UserEnvironment: """Utility to find an environment file using a supplied name. Parameters @@ -149,12 +149,12 @@ def get_ids(directory: Path) -> List[int]: return sorted(ids) # TODO: sort by session datetime? -def load_recent_sessions(env: Environment, count: int = None) -> List[Session]: +def load_recent_sessions(env: UserEnvironment, count: int = None) -> List[Session]: """Get a list of recent sessions for the environment. Parameters ---------- - env : Environment + env : UserEnvironment MLonMCU environment which should be used. count : int Maximum number of sessions to return. Collect all if None. @@ -234,7 +234,7 @@ def setup_logging(environment): Attributes ---------- - environment : Environment + environment : UserEnvironment The MLonMCU Environment where paths, repos, features,... are configured. """ defaults = environment.defaults @@ -255,7 +255,7 @@ class MlonMcuContext: Attributes ---------- - environment : Environment + environment : UserEnvironment The MLonMCU Environment where paths, repos, features,... are configured. deps_lock : str ("read" or "write" default "write") Read means that the program does not write to the ./deps folder in the env folder. diff --git a/mlonmcu/environment/config.py b/mlonmcu/environment/config.py index 1d41de426..23a2f3d76 100644 --- a/mlonmcu/environment/config.py +++ b/mlonmcu/environment/config.py @@ -22,6 +22,7 @@ import os import xdg import logging +from collections import namedtuple from enum import Enum from pathlib import Path @@ -88,18 +89,12 @@ def __init__( log_level=logging.INFO, log_to_file=False, log_rotate=False, - default_framework=None, - default_backends={}, - default_target=None, cleanup_auto=False, cleanup_keep=100, ): self.log_level = log_level self.log_to_file = log_to_file self.log_rotate = log_rotate - self.default_framework = default_framework - self.default_backends = default_backends - self.default_target = default_target self.cleanup_auto = cleanup_auto self.cleanup_keep = cleanup_keep @@ -125,90 +120,5 @@ def __repr(self): return f"PathConfig({self.path})" -class RepoConfig(BaseConfig): - def __init__(self, url, ref=None): - self.url = url - self.ref = ref - - -class BackendConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class FeatureKind(Enum): - UNKNOWN = 0 - FRAMEWORK = 1 - BACKEND = 2 - TARGET = 3 - FRONTEND = 4 - - -class FeatureConfig: - def __init__(self, name, kind=FeatureKind.UNKNOWN, supported=True): - self.name = name - self.supported = supported - - def __repr__(self): - return self.__class__.__name__ + "(" + str(vars(self)) + ")" - - -class FrameworkFeatureConfig(FeatureConfig): - def __init__(self, name, framework, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.framework = framework - - -class BackendFeatureConfig(FeatureConfig): - def __init__(self, name, backend, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.backend = backend - - -class PlatformFeatureConfig(FeatureConfig): - def __init__(self, name, platform, supported=True): - super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) - self.platform = platform - - -class TargetFeatureConfig(FeatureConfig): - def __init__(self, name, target, supported=True): - super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) - self.target = target - - -class FrontendFeatureConfig(FeatureConfig): - def __init__(self, name, frontend, supported=True): - super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) - self.frontend = frontend - - -class FrameworkConfig(BaseConfig): - def __init__(self, name, enabled=True, backends={}, features={}): - self.name = name - self.enabled = enabled - self.backends = backends - self.features = features - - -class FrontendConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class PlatformConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features - - -class TargetConfig(BaseConfig): - def __init__(self, name, enabled=True, features={}): - self.name = name - self.enabled = enabled - self.features = features +RepoConfig = namedtuple("RepoConfig", "url ref", defaults=[None, None]) +ComponentConfig = namedtuple("ComponentConfig", "supported used", defaults=[False, False]) diff --git a/mlonmcu/environment/convert.py b/mlonmcu/environment/convert.py new file mode 100644 index 000000000..27ed42b47 --- /dev/null +++ b/mlonmcu/environment/convert.py @@ -0,0 +1,165 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# import yaml +# import pathlib +# import logging +# +# from .config import PathConfig + + +import argparse +from mlonmcu.environment.legacy.environment import UserEnvironment as UserEnvironmentOld +from mlonmcu.environment.user_environment import UserEnvironment +from mlonmcu.environment.config import ComponentConfig, DefaultsConfig + +# from mlonmcu.environment.loader import load_environment_from_file + + +def convert_v1_to_v2(env): + frameworks = {f.name: ComponentConfig(f.enabled, f.name == env.defaults.default_framework) for f in env.frameworks} + backends = { + b.name: ComponentConfig( + b.enabled, f.name == env.defaults.default_framework and b.name == env.defaults.default_backends[f.name] + ) + for f in env.frameworks + for b in f.backends + if f.enabled + } + toolchains = {t: ComponentConfig(True, False) for t in env.toolchains} + frontends = {f.name: ComponentConfig(f.enabled, f.enabled) for f in env.frontends} + platforms = {p.name: ComponentConfig(p.enabled, p.enabled) for p in env.platforms} + targets = {t.name: ComponentConfig(t.enabled, t.name == env.defaults.default_target) for t in env.targets} + + framework_features = { + f_.name: ComponentConfig(f_.supported, False) for f in env.frameworks for f_ in f.features if f.enabled + } + backend_features = { + f_.name: ComponentConfig(f_.supported, False) + for f in env.frameworks + for b in f.backends + for f_ in b.features + if b.enabled and f.enabled + } + target_features = { + f_.name: ComponentConfig(f_.supported, False) for t in env.targets for f_ in t.features if t.enabled + } + platform_features = { + f_.name: ComponentConfig(f_.supported, False) for p in env.platforms for f_ in p.features if p.enabled + } + frontend_features = { + f_.name: ComponentConfig(f_.supported, False) for f in env.frontends for f_ in f.features if f.enabled + } + other_features = {} + + def helper(name): + return any( + [ + name in x and x[name].supported + for x in [ + framework_features, + backend_features, + target_features, + platform_features, + frontend_features, + other_features, + ] + ] + ) + + all_feature_names = ( + set(framework_features.keys()) + .union(set(backend_features.keys())) + .union(set(target_features.keys())) + .union(set(platform_features.keys())) + .union(set(frontend_features.keys())) + .union(set(other_features.keys())) + ) + features = {name: ComponentConfig(helper(name), False) for name in all_feature_names} + + env2 = UserEnvironment( + env.home, + alias=env.alias, + defaults=DefaultsConfig( + log_level=env.defaults.log_level, + log_to_file=env.defaults.log_to_file, + log_rotate=env.defaults.log_rotate, + cleanup_auto=env.defaults.cleanup_auto, + cleanup_keep=env.defaults.cleanup_keep, + ), + paths=env.paths, + repos=env.repos, + frameworks=frameworks, + backends=backends, + frontends=frontends, + platforms=platforms, + toolchains=toolchains, + targets=targets, + features=features, + # postprocesses=None, + variables=env.vars, + default_flags=env.flags, + ) + return env2 + + +def main(): + parser = argparse.ArgumentParser( + description=f"Converter for environment files/templates", + ) + parser.add_argument("input", metavar="FILE", type=str, nargs=1, help="File to process") + parser.add_argument( + "--output", + "-o", + metavar="DEST", + type=str, + default=None, + help="""Output directory/file (default: print to stdout instead)""", + ) + parser.add_argument( + "--in-version", + metavar="VER", + type=int, + default=1, + help="""Output directory/file (default: %(default)s)""", + ) + parser.add_argument( + "--out-version", + metavar="VER", + type=int, + default=2, + help="""Output directory/file (default: %(default)s)""", + ) + args = parser.parse_args() + + assert args.in_version == 1 and args.out_version == 2, "Unsupported combination of versions" + + file = args.input[0] + + env = UserEnvironmentOld.from_file(file) + + env2 = convert_v1_to_v2(env) + + if args.output: + env2.to_file(args.output) + else: + print(env2.to_text()) + + +if __name__ == "__main__": + main() diff --git a/mlonmcu/environment/environment.py b/mlonmcu/environment/environment.py index ccb8a5e94..29d974428 100644 --- a/mlonmcu/environment/environment.py +++ b/mlonmcu/environment/environment.py @@ -17,58 +17,72 @@ # limitations under the License. # import logging +import tempfile +from abc import ABC +from pathlib import Path +from typing import Dict, List, Union from .config import ( DefaultsConfig, PathConfig, RepoConfig, - FrameworkConfig, - FrameworkFeatureConfig, - BackendConfig, - BackendFeatureConfig, - TargetConfig, - TargetFeatureConfig, - PlatformConfig, - PlatformFeatureConfig, - FrontendConfig, - FrontendFeatureConfig, + ComponentConfig, + # FrameworkConfig, + # FrameworkFeatureConfig, + # BackendConfig, + # BackendFeatureConfig, + # TargetConfig, + # TargetFeatureConfig, + # PlatformConfig, + # PlatformFeatureConfig, + # FrontendConfig, + # FrontendFeatureConfig, ) -from .loader import load_environment_from_file -from .writer import write_environment_to_file -from mlonmcu.feature.type import FeatureType +# from mlonmcu.feature.type import FeatureType -def _feature_helper(obj, name): - if not obj.enabled: - return [] - features = obj.features - if name: - return [feature for feature in features if feature.name == name] - else: - return features +# def _feature_helper(obj, name): +# if not obj.enabled: +# return [] +# features = obj.features +# if name: +# return [feature for feature in features if feature.name == name] +# else: +# return features -def _extract_names(objs): - return [obj.name for obj in objs] +# def _extract_names(objs): +# return [obj.name for obj in objs] -def _filter_enabled(objs): - return [obj for obj in objs if obj.enabled] +# def _filter_enabled(objs): +# return [obj for obj in objs if obj.enabled] -class Environment: +class BaseEnvironment(ABC): def __init__(self): - self._home = None - self.alias = None + self.version = None + self._home: Path = None + + +class Environment(BaseEnvironment): + def __init__(self): + super().__init__() + self.alias: str = None + self.version = 2 self.defaults = DefaultsConfig() - self.paths = {} - self.repos = {} - self.frameworks = [] - self.frontends = [] - self.platforms = [] - self.toolchains = [] - self.targets = [] + self.paths: "Dict[str, Union[PathConfig, List[PathConfig]]]" = {} + self.repos: "Dict[str, RepoConfig]" = {} + self.frameworks: "Dict[str, ComponentConfig]" = {} + self.backends: "Dict[str, ComponentConfig]" = {} + self.features: "Dict[str, ComponentConfig]" = {} + self.postprocesses: "Dict[str, ComponentConfig]" = {} + self.toolchains: "Dict[str, ComponentConfig]" = {} + self.frontends: "Dict[str, ComponentConfig]" = {} + self.platforms: "Dict[str, ComponentConfig]" = {} + self.toolchains: "Dict[str, ComponentConfig]" = {} + self.targets: "Dict[str, ComponentConfig]" = {} self.vars = {} self.flags = {} @@ -80,13 +94,6 @@ def home(self): """Home directory of mlonmcu environment.""" return self._home - @classmethod - def from_file(cls, filename): - return load_environment_from_file(filename, base=cls) - - def to_file(self, filename): - write_environment_to_file(self, filename) - def lookup_path(self, name): assert name in self.paths, f"Unable to find '{name}' path in environment config" return self.paths[name] @@ -94,256 +101,90 @@ def lookup_path(self, name): def lookup_var(self, name, default=None): return self.vars.get(name, default) - def lookup_frontend_feature_configs(self, name=None, frontend=None): - configs = [] - if frontend: - names = [frontend.name for frontend in self.frontends] - index = names.index(frontend) - assert index is not None, f"Frontend {frontend} not found in environment config" - configs.extend(_feature_helper(self.frontends[index], name)) - else: - for frontend in self.frontends: - configs.extend(_feature_helper(frontend, name)) - return configs - - def lookup_framework_feature_configs(self, name=None, framework=None): - configs = [] - if framework: - names = [framework.name for framework in self.frameworks] - index = names.index(framework) - assert index is not None, f"Framework {framework} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index], name)) - else: - for framework in self.frameworks: - configs.extend(_feature_helper(framework, name)) - return configs - - def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): - def helper(framework, backend, name): - backend_features = self.framework[framework].backends[backend].features - if name: - return [backend_features[name]] - else: - return backend_features.values() - - configs = [] - if framework: - names = [framework.name for framework in self.frameworks] - index = names.index(framework) - assert index is not None, f"Framework {framework} not found in environment config" - if backend: - names_ = [backend.name for backend in self.frameworks[index].backends] - index_ = names_.index(backend) - assert index_ is not None, f"Backend {backend} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - else: - for backend in self.frameworks[index].backends: - configs.extend(_feature_helper(backend, name)) - else: - for framework in self.frameworks: - if backend: - names_ = [backend.name for backend in framework.backends] - index_ = names_.index(backend) - assert index_ is not None, f"Backend {backend} not found in environment config" - configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) - else: - for backend in framework.backends: - configs.extend(_feature_helper(backend, name)) - backend = None - return configs - - def lookup_platform_feature_configs(self, name=None, platform=None): - configs = [] - if platform: - names = [platform.name for platform in self.platforms] - index = names.index(platform) - assert ( - index is not None - ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list - configs.extend(_feature_helper(self.platforms[index], name)) - else: - for platform in self.platforms: - configs.extend(_feature_helper(platform, name)) - return configs - - def lookup_target_feature_configs(self, name=None, target=None): - configs = [] - if target: - names = [target.name for target in self.targets] - index = names.index(target) - assert ( - index is not None - ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list - configs.extend(_feature_helper(self.targets[index], name)) - else: - for target in self.targets: - configs.extend(_feature_helper(target, name)) - return configs - - def lookup_feature_configs( - self, - name=None, - kind=None, - frontend=None, - framework=None, - backend=None, - platform=None, - target=None, - ): - configs = [] - if kind == FeatureType.FRONTEND or kind is None: - configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) - if kind == FeatureType.FRAMEWORK or kind is None: - configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) - if kind == FeatureType.BACKEND or kind is None: - configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) - if kind == FeatureType.PLATFORM or kind is None: - configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) - if kind == FeatureType.TARGET or kind is None: - configs.extend(self.lookup_target_feature_configs(name=name, target=target)) - - return configs - - def supports_feature(self, name): - configs = self.lookup_feature_configs(name=name) - supported = [feature.supported for feature in configs] - return any(supported) + def get_supported_features(self): + supported = [name for name, config in self.features.items() if config.supported] + return supported - def has_feature(self, name): - """An alias for supports_feature.""" - return self.supports_feature(name) + def get_used_features(self): + used = [name for name, config in self.features.items() if config.used] + return used - def lookup_backend_configs(self, backend=None, framework=None, names_only=False): - enabled_frameworks = _filter_enabled(self.frameworks) - - configs = [] - for framework_config in enabled_frameworks: - if framework is not None and framework_config.name != framework: - continue - enabled_backends = _filter_enabled(framework_config.backends) - if backend is None: - configs.extend(enabled_backends) - else: - for backend_config in enabled_backends: - if backend_config.name == backend: - return [backend_config.name if names_only else backend_config] - return _extract_names(configs) if names_only else configs - - def lookup_framework_configs(self, framework=None, names_only=False): - enabled_frameworks = _filter_enabled(self.frameworks) - - if framework is None: - return _extract_names(enabled_frameworks) if names_only else enabled_frameworks + def has_feature(self, name): + return name in self.get_supported_features() - for framework_config in enabled_frameworks: - if framework_config.name == framework: - return [framework_config.name if names_only else framework_config] + def get_supported_frontends(self): + supported = [name for name, config in self.frontends.items() if config.supported] + return supported - return [] + def get_used_frontends(self): + used = [name for name, config in self.frontends.items() if config.used] + return used - def lookup_frontend_configs(self, frontend=None, names_only=False): - enabled_frontends = _filter_enabled(self.frontends) + def has_frontend(self, name): + return name in self.get_supported_frontends() - if frontend is None: - return _extract_names(enabled_frontends) if names_only else enabled_frontends + def get_supported_backends(self): + supported = [name for name, config in self.backends.items() if config.supported] + return supported - for frontend_config in enabled_frontends: - if frontend_config.name == frontend: - return [frontend_config.name if names_only else frontend_config] - return [] + def get_used_backends(self): + used = [name for name, config in self.backends.items() if config.used] + return used - def lookup_platform_configs(self, platform=None, names_only=False): - enabled_platforms = _filter_enabled(self.platforms) + def has_backend(self, name): + return name in self.get_supported_backends() - if platform is None: - return _extract_names(enabled_platforms) if names_only else enabled_platforms - for platform_config in enabled_platforms: - if platform_config.name == platform: - return [platform_config.name if names_only else platform_config] + def get_supported_frameworks(self): + supported = [name for name, config in self.frameworks.items() if config.supported] + return supported - return [] + def get_used_frameworks(self): + used = [name for name, config in self.frameworks.items() if config.used] + return used - def lookup_target_configs(self, target=None, names_only=False): - enabled_targets = _filter_enabled(self.targets) + def has_framework(self, name): + return name in self.get_supported_frameworks() - if target is None: - return _extract_names(enabled_targets) if names_only else enabled_targets + def get_supported_platforms(self): + supported = [name for name, config in self.platforms.items() if config.supported] + return supported - for target_config in enabled_targets: - if target_config.name == target: - return [target_config.name if names_only else target_config] - return [] + def get_used_platforms(self): + used = [name for name, config in self.platforms.items() if config.used] + return used - def has_frontend(self, name): - configs = self.lookup_frontend_configs(frontend=name) - return len(configs) > 0 + def has_platform(self, name): + return name in self.get_supported_platforms() - def has_backend(self, name): - configs = self.lookup_backend_configs(backend=name) - return len(configs) > 0 + def get_supported_targets(self): + supported = [name for name, config in self.targets.items() if config.supported] + return supported - def has_framework(self, name): - configs = self.lookup_framework_configs(framework=name) - return len(configs) > 0 + def get_used_targets(self): + used = [name for name, config in self.targets.items() if config.used] + return used - def has_platform(self, name): - configs = self.lookup_platform_configs(platform=name) - return len(configs) > 0 + def has_target(self, name): + return name in self.get_supported_targets() def has_toolchain(self, name): return self.toolchains.get(name, False) - def has_target(self, name): - configs = self.lookup_target_configs(target=name) - return len(configs) > 0 - - # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead - # of the supported ones in the environment # def has_postprocess(self, name): # configs = self.lookup_postprocess_configs(postprocess=name) # return len(configs) > 0 def get_default_backends(self, framework): - if framework is None or framework not in self.defaults.default_backends: - return [] - default = self.defaults.default_backends[framework] - # framework_names = [framework_config.name for framework_config in self.frameworks] - # framework_config = self.frameworks[framework_names.index(framework)] - if default is None: - return [] - if isinstance(default, str): - if default == "*": # Wildcard all enabled frameworks - default = self.get_enabled_backends() - else: - default = [default] - else: - assert isinstance(default, list), "TODO" - return default + # TODO: deprecate + return self.get_used_backends() def get_default_frameworks(self): - default = self.defaults.default_framework - if default is None: - return [] - if isinstance(default, str): - if default == "*": # Wildcard all enabled frameworks - default = self.get_enabled_frameworks() - else: - default = [default] - else: - assert isinstance(default, list), "TODO" - return default + # TODO: deprecate + return self.get_used_frameworks() def get_default_targets(self): - default = self.defaults.default_target - if default is not None: - if isinstance(default, str): - if default == "*": # Wildcard all enabled targets - default = self.get_enabled_targets() - else: - default = [default] - else: - assert isinstance(default, list) - return default + # TODO: deprecate + return self.get_used_targets() class DefaultEnvironment(Environment): @@ -352,9 +193,6 @@ def __init__(self): self.defaults = DefaultsConfig( log_level=logging.DEBUG, log_to_file=False, - default_framework=None, - default_backends={}, - default_target=None, cleanup_auto=False, cleanup_keep=100, ) @@ -368,6 +206,7 @@ def __init__(self): PathConfig("./models"), ], } + # TODO: refresh or remove self.repos = { "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), "tflite_micro_compiler": RepoConfig( @@ -381,121 +220,78 @@ def __init__(self): ), # TODO: freeze ref? "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? } - self.frameworks = [ - FrameworkConfig( - "tflm", - enabled=True, - backends=[ - BackendConfig("tflmc", enabled=True, features=[]), - BackendConfig("tflmi", enabled=True, features=[]), - ], - features=[ - FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), - ], - ), - FrameworkConfig( - "utvm", - enabled=True, - backends=[ - BackendConfig( - "tvmaot", - enabled=True, - features=[ - BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), - ], - ), - BackendConfig("tvmrt", enabled=True, features=[]), - BackendConfig("tvmcg", enabled=True, features=[]), - ], - features=[ - FrameworkFeatureConfig("memplan", framework="utvm", supported=False), - ], - ), - ] - self.frontends = [ - FrontendConfig("saved_model", enabled=False), - FrontendConfig("ipynb", enabled=False), - FrontendConfig( - "tflite", - enabled=True, - features=[ - FrontendFeatureConfig("packing", frontend="tflite", supported=False), - ], - ), - ] - self.vars = { - "TEST": "abc", - } - self.flags = {} - self.platforms = [ - PlatformConfig( - "mlif", - enabled=True, - features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], - ) - ] + self.frameworks = {} + self.backends = {} + self.features = {} + self.postprocesses = {} self.toolchains = {} - self.targets = [ - TargetConfig( - "etiss_pulpino", - features=[ - TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), - TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), - TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), - ], - ), - TargetConfig( - "host_x86", - features=[ - TargetFeatureConfig("debug", target="host_x86", supported=True), - TargetFeatureConfig("attach", target="host_x86", supported=True), - ], - ), - ] - - -class UserEnvironment(DefaultEnvironment): - def __init__( - self, - home, - merge=False, - alias=None, - defaults=None, - paths=None, - repos=None, - frameworks=None, - frontends=None, - platforms=None, - toolchains=None, - targets=None, - variables=None, - default_flags=None, - ): - super().__init__() - self._home = home - - if merge: - raise NotImplementedError - - if alias: - self.alias = alias - if defaults: - self.defaults = defaults - if paths: - self.paths = paths - if repos: - self.repos = repos - if frameworks: - self.frameworks = frameworks - if frontends: - self.frontends = frontends - if platforms: - self.platforms = platforms - if toolchains: - self.toolchains = toolchains - if targets: - self.targets = targets - if variables: - self.vars = variables - if default_flags: - self.flags = default_flags + # self.frameworks = [ + # FrameworkConfig( + # "tflm", + # enabled=True, + # backends=[ + # BackendConfig("tflmc", enabled=True, features=[]), + # BackendConfig("tflmi", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), + # ], + # ), + # FrameworkConfig( + # "utvm", + # enabled=True, + # backends=[ + # BackendConfig( + # "tvmaot", + # enabled=True, + # features=[ + # BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), + # ], + # ), + # BackendConfig("tvmrt", enabled=True, features=[]), + # BackendConfig("tvmcg", enabled=True, features=[]), + # ], + # features=[ + # FrameworkFeatureConfig("memplan", framework="utvm", supported=False), + # ], + # ), + # ] + # self.frontends = [ + # FrontendConfig("saved_model", enabled=False), + # FrontendConfig("ipynb", enabled=False), + # FrontendConfig( + # "tflite", + # enabled=True, + # features=[ + # FrontendFeatureConfig("packing", frontend="tflite", supported=False), + # ], + # ), + # ] + # self.vars = { + # "TEST": "abc", + # } + # self.flags = {} + # self.platforms = [ + # PlatformConfig( + # "mlif", + # enabled=True, + # features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], + # ) + # ] + # self.targets = [ + # TargetConfig( + # "etiss_pulpino", + # features=[ + # TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), + # TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), + # ], + # ), + # TargetConfig( + # "host_x86", + # features=[ + # TargetFeatureConfig("debug", target="host_x86", supported=True), + # TargetFeatureConfig("attach", target="host_x86", supported=True), + # ], + # ), + # ] diff --git a/mlonmcu/environment/legacy/config.py b/mlonmcu/environment/legacy/config.py new file mode 100644 index 000000000..5a0b5adc7 --- /dev/null +++ b/mlonmcu/environment/legacy/config.py @@ -0,0 +1,132 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# TODO: rename to paths.py or user.py? + +import logging +from enum import Enum + +from ..config import DefaultsConfig, BaseConfig + + +class DefaultsConfigOld(DefaultsConfig): + def __init__( + self, + log_level=logging.INFO, + log_to_file=False, + log_rotate=False, + cleanup_auto=False, + cleanup_keep=100, + default_framework=None, + default_backends={}, + default_target=None, + ): + super().__init__( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + cleanup_auto=False, + cleanup_keep=100, + ) + self.default_framework = default_framework + self.default_backends = default_backends + self.default_target = default_target + + +class BackendConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class FeatureKind(Enum): + UNKNOWN = 0 + FRAMEWORK = 1 + BACKEND = 2 + TARGET = 3 + FRONTEND = 4 + + +class FeatureConfig: + def __init__(self, name, kind=FeatureKind.UNKNOWN, supported=True): + self.name = name + self.supported = supported + + def __repr__(self): + return self.__class__.__name__ + "(" + str(vars(self)) + ")" + + +class FrameworkFeatureConfig(FeatureConfig): + def __init__(self, name, framework, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.framework = framework + + +class BackendFeatureConfig(FeatureConfig): + def __init__(self, name, backend, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.backend = backend + + +class PlatformFeatureConfig(FeatureConfig): + def __init__(self, name, platform, supported=True): + super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) + self.platform = platform + + +class TargetFeatureConfig(FeatureConfig): + def __init__(self, name, target, supported=True): + super().__init__(name=name, kind=FeatureKind.TARGET, supported=supported) + self.target = target + + +class FrontendFeatureConfig(FeatureConfig): + def __init__(self, name, frontend, supported=True): + super().__init__(name=name, kind=FeatureKind.FRONTEND, supported=supported) + self.frontend = frontend + + +class FrameworkConfig(BaseConfig): + def __init__(self, name, enabled=True, backends={}, features={}): + self.name = name + self.enabled = enabled + self.backends = backends + self.features = features + + +class FrontendConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class PlatformConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features + + +class TargetConfig(BaseConfig): + def __init__(self, name, enabled=True, features={}): + self.name = name + self.enabled = enabled + self.features = features diff --git a/mlonmcu/environment/legacy/environment.py b/mlonmcu/environment/legacy/environment.py new file mode 100644 index 000000000..db251b70c --- /dev/null +++ b/mlonmcu/environment/legacy/environment.py @@ -0,0 +1,514 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +from abc import ABC +from pathlib import Path + +from ..config import ( + PathConfig, + RepoConfig, +) +from .config import ( + DefaultsConfigOld, + FrameworkConfig, + FrameworkFeatureConfig, + BackendConfig, + BackendFeatureConfig, + TargetConfig, + TargetFeatureConfig, + PlatformConfig, + PlatformFeatureConfig, + FrontendConfig, + FrontendFeatureConfig, +) +from .loader import load_environment_from_file +from .writer import write_environment_to_file + +from mlonmcu.feature.type import FeatureType + + +def _feature_helper(obj, name): + if not obj.enabled: + return [] + features = obj.features + if name: + return [feature for feature in features if feature.name == name] + else: + return features + + +def _extract_names(objs): + return [obj.name for obj in objs] + + +def _filter_enabled(objs): + return [obj for obj in objs if obj.enabled] + + +class BaseEnvironment(ABC): + def __init__(self): + self.version = None + self._home: Path = None + + +class Environment(BaseEnvironment): + def __init__(self): + super().__init__() + self.alias: str = None + self.version = 1 + self._home = None + self.alias = None + self.defaults = DefaultsConfigOld() + self.paths = {} + self.repos = {} + self.frameworks = [] + self.frontends = [] + self.platforms = [] + self.toolchains = [] + self.targets = [] + self.vars = {} + self.flags = {} + + def __str__(self): + return self.__class__.__name__ + "(" + str(vars(self)) + ")" + + @property + def home(self): + """Home directory of mlonmcu environment.""" + return self._home + + @classmethod + def from_file(cls, filename): + return load_environment_from_file(filename, base=cls) + + def to_file(self, filename): + write_environment_to_file(self, filename) + + def lookup_path(self, name): + assert name in self.paths, f"Unable to find '{name}' path in environment config" + return self.paths[name] + + def lookup_var(self, name, default=None): + return self.vars.get(name, default) + + def lookup_frontend_feature_configs(self, name=None, frontend=None): + configs = [] + if frontend: + names = [frontend.name for frontend in self.frontends] + index = names.index(frontend) + assert index is not None, f"Frontend {frontend} not found in environment config" + configs.extend(_feature_helper(self.frontends[index], name)) + else: + for frontend in self.frontends: + configs.extend(_feature_helper(frontend, name)) + return configs + + def lookup_framework_feature_configs(self, name=None, framework=None): + configs = [] + if framework: + names = [framework.name for framework in self.frameworks] + index = names.index(framework) + assert index is not None, f"Framework {framework} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index], name)) + else: + for framework in self.frameworks: + configs.extend(_feature_helper(framework, name)) + return configs + + def lookup_backend_feature_configs(self, name=None, framework=None, backend=None): + def helper(framework, backend, name): + backend_features = self.framework[framework].backends[backend].features + if name: + return [backend_features[name]] + else: + return backend_features.values() + + configs = [] + if framework: + names = [framework.name for framework in self.frameworks] + index = names.index(framework) + assert index is not None, f"Framework {framework} not found in environment config" + if backend: + names_ = [backend.name for backend in self.frameworks[index].backends] + index_ = names_.index(backend) + assert index_ is not None, f"Backend {backend} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + else: + for backend in self.frameworks[index].backends: + configs.extend(_feature_helper(backend, name)) + else: + for framework in self.frameworks: + if backend: + names_ = [backend.name for backend in framework.backends] + index_ = names_.index(backend) + assert index_ is not None, f"Backend {backend} not found in environment config" + configs.extend(_feature_helper(self.frameworks[index].backends[index], name)) + else: + for backend in framework.backends: + configs.extend(_feature_helper(backend, name)) + backend = None + return configs + + def lookup_platform_feature_configs(self, name=None, platform=None): + configs = [] + if platform: + names = [platform.name for platform in self.platforms] + index = names.index(platform) + assert ( + index is not None + ), f"Platform {platform} not found in environment config" # TODO: do not fail, just return empty list + configs.extend(_feature_helper(self.platforms[index], name)) + else: + for platform in self.platforms: + configs.extend(_feature_helper(platform, name)) + return configs + + def lookup_target_feature_configs(self, name=None, target=None): + configs = [] + if target: + names = [target.name for target in self.targets] + index = names.index(target) + assert ( + index is not None + ), f"Target {target} not found in environment config" # TODO: do not fail, just return empty list + configs.extend(_feature_helper(self.targets[index], name)) + else: + for target in self.targets: + configs.extend(_feature_helper(target, name)) + return configs + + def lookup_feature_configs( + self, + name=None, + kind=None, + frontend=None, + framework=None, + backend=None, + platform=None, + target=None, + ): + configs = [] + if kind == FeatureType.FRONTEND or kind is None: + configs.extend(self.lookup_frontend_feature_configs(name=name, frontend=frontend)) + if kind == FeatureType.FRAMEWORK or kind is None: + configs.extend(self.lookup_framework_feature_configs(name=name, framework=framework)) + if kind == FeatureType.BACKEND or kind is None: + configs.extend(self.lookup_backend_feature_configs(name=name, framework=framework, backend=backend)) + if kind == FeatureType.PLATFORM or kind is None: + configs.extend(self.lookup_platform_feature_configs(name=name, platform=platform)) + if kind == FeatureType.TARGET or kind is None: + configs.extend(self.lookup_target_feature_configs(name=name, target=target)) + + return configs + + def supports_feature(self, name): + configs = self.lookup_feature_configs(name=name) + supported = [feature.supported for feature in configs] + return any(supported) + + def has_feature(self, name): + """An alias for supports_feature.""" + return self.supports_feature(name) + + def lookup_backend_configs(self, backend=None, framework=None, names_only=False): + enabled_frameworks = _filter_enabled(self.frameworks) + + configs = [] + for framework_config in enabled_frameworks: + if framework is not None and framework_config.name != framework: + continue + enabled_backends = _filter_enabled(framework_config.backends) + if backend is None: + configs.extend(enabled_backends) + else: + for backend_config in enabled_backends: + if backend_config.name == backend: + return [backend_config.name if names_only else backend_config] + return _extract_names(configs) if names_only else configs + + def lookup_framework_configs(self, framework=None, names_only=False): + enabled_frameworks = _filter_enabled(self.frameworks) + + if framework is None: + return _extract_names(enabled_frameworks) if names_only else enabled_frameworks + + for framework_config in enabled_frameworks: + if framework_config.name == framework: + return [framework_config.name if names_only else framework_config] + + return [] + + def lookup_frontend_configs(self, frontend=None, names_only=False): + enabled_frontends = _filter_enabled(self.frontends) + + if frontend is None: + return _extract_names(enabled_frontends) if names_only else enabled_frontends + + for frontend_config in enabled_frontends: + if frontend_config.name == frontend: + return [frontend_config.name if names_only else frontend_config] + return [] + + def lookup_platform_configs(self, platform=None, names_only=False): + enabled_platforms = _filter_enabled(self.platforms) + + if platform is None: + return _extract_names(enabled_platforms) if names_only else enabled_platforms + for platform_config in enabled_platforms: + if platform_config.name == platform: + return [platform_config.name if names_only else platform_config] + + return [] + + def lookup_target_configs(self, target=None, names_only=False): + enabled_targets = _filter_enabled(self.targets) + + if target is None: + return _extract_names(enabled_targets) if names_only else enabled_targets + + for target_config in enabled_targets: + if target_config.name == target: + return [target_config.name if names_only else target_config] + return [] + + def has_frontend(self, name): + configs = self.lookup_frontend_configs(frontend=name) + return len(configs) > 0 + + def has_backend(self, name): + configs = self.lookup_backend_configs(backend=name) + return len(configs) > 0 + + def has_framework(self, name): + configs = self.lookup_framework_configs(framework=name) + return len(configs) > 0 + + def has_platform(self, name): + configs = self.lookup_platform_configs(platform=name) + return len(configs) > 0 + + def has_toolchain(self, name): + return self.toolchains.get(name, False) + + def has_target(self, name): + configs = self.lookup_target_configs(target=name) + return len(configs) > 0 + + # TODO: actually we do not need to explicitly enable those? environment.yml list the default enabled ones instead + # of the supported ones in the environment + # def has_postprocess(self, name): + # configs = self.lookup_postprocess_configs(postprocess=name) + # return len(configs) > 0 + + def get_default_backends(self, framework): + if framework is None or framework not in self.defaults.default_backends: + return [] + default = self.defaults.default_backends[framework] + # framework_names = [framework_config.name for framework_config in self.frameworks] + # framework_config = self.frameworks[framework_names.index(framework)] + if default is None: + return [] + if isinstance(default, str): + if default == "*": # Wildcard all enabled frameworks + default = self.get_enabled_backends() + else: + default = [default] + else: + assert isinstance(default, list), "TODO" + return default + + def get_default_frameworks(self): + default = self.defaults.default_framework + if default is None: + return [] + if isinstance(default, str): + if default == "*": # Wildcard all enabled frameworks + default = self.get_enabled_frameworks() + else: + default = [default] + else: + assert isinstance(default, list), "TODO" + return default + + def get_default_targets(self): + default = self.defaults.default_target + if default is not None: + if isinstance(default, str): + if default == "*": # Wildcard all enabled targets + default = self.get_enabled_targets() + else: + default = [default] + else: + assert isinstance(default, list) + return default + + +class DefaultEnvironment(Environment): + def __init__(self): + super().__init__() + self.defaults = DefaultsConfigOld( + log_level=logging.DEBUG, + log_to_file=False, + default_framework=None, + default_backends={}, + default_target=None, + cleanup_auto=False, + cleanup_keep=100, + ) + self.paths = { + "deps": PathConfig("./deps"), + "logs": PathConfig("./logs"), + "results": PathConfig("./results"), + "plugins": PathConfig("./plugins"), + "temp": PathConfig("out"), + "models": [ + PathConfig("./models"), + ], + } + self.repos = { + "tensorflow": RepoConfig("https://github.com/tensorflow/tensorflow.git", ref="v2.5.2"), + "tflite_micro_compiler": RepoConfig( + "https://github.com/cpetig/tflite_micro_compiler.git", ref="master" + ), # TODO: freeze ref? + "tvm": RepoConfig( + "https://github.com/tum-ei-eda/tvm.git", ref="tumeda" + ), # TODO: use upstream repo with suitable commit? + "utvm_staticrt_codegen": RepoConfig( + "https://github.com/tum-ei-eda/utvm_staticrt_codegen.git", ref="master" + ), # TODO: freeze ref? + "etiss": RepoConfig("https://github.com/tum-ei-eda/etiss.git", ref="master"), # TODO: freeze ref? + } + self.frameworks = [ + FrameworkConfig( + "tflm", + enabled=True, + backends=[ + BackendConfig("tflmc", enabled=True, features=[]), + BackendConfig("tflmi", enabled=True, features=[]), + ], + features=[ + FrameworkFeatureConfig("muriscvnn", framework="tflm", supported=False), + ], + ), + FrameworkConfig( + "utvm", + enabled=True, + backends=[ + BackendConfig( + "tvmaot", + enabled=True, + features=[ + BackendFeatureConfig("unpacked_api", backend="tvmaot", supported=True), + ], + ), + BackendConfig("tvmrt", enabled=True, features=[]), + BackendConfig("tvmcg", enabled=True, features=[]), + ], + features=[ + FrameworkFeatureConfig("memplan", framework="utvm", supported=False), + ], + ), + ] + self.frontends = [ + FrontendConfig("saved_model", enabled=False), + FrontendConfig("ipynb", enabled=False), + FrontendConfig( + "tflite", + enabled=True, + features=[ + FrontendFeatureConfig("packing", frontend="tflite", supported=False), + ], + ), + ] + self.vars = { + "TEST": "abc", + } + self.flags = {} + self.platforms = [ + PlatformConfig( + "mlif", + enabled=True, + features=[PlatformFeatureConfig("debug", platform="mlif", supported=True)], + ) + ] + self.toolchains = {} + self.targets = [ + TargetConfig( + "etiss_pulpino", + features=[ + TargetFeatureConfig("debug", target="etiss_pulpino", supported=True), + TargetFeatureConfig("attach", target="etiss_pulpino", supported=True), + TargetFeatureConfig("trace", target="etiss_pulpino", supported=True), + ], + ), + TargetConfig( + "host_x86", + features=[ + TargetFeatureConfig("debug", target="host_x86", supported=True), + TargetFeatureConfig("attach", target="host_x86", supported=True), + ], + ), + ] + + +class UserEnvironment(DefaultEnvironment): + def __init__( + self, + home, + merge=False, + alias=None, + defaults=None, + paths=None, + repos=None, + frameworks=None, + frontends=None, + platforms=None, + toolchains=None, + targets=None, + variables=None, + default_flags=None, + ): + super().__init__() + self._home = home + + if merge: + raise NotImplementedError + + if alias: + self.alias = alias + if defaults: + self.defaults = defaults + if paths: + self.paths = paths + if repos: + self.repos = repos + if frameworks: + self.frameworks = frameworks + if frontends: + self.frontends = frontends + if platforms: + self.platforms = platforms + if toolchains: + self.toolchains = toolchains + if targets: + self.targets = targets + if variables: + self.vars = variables + if default_flags: + self.flags = default_flags diff --git a/mlonmcu/environment/legacy/loader.py b/mlonmcu/environment/legacy/loader.py new file mode 100644 index 000000000..67f84a031 --- /dev/null +++ b/mlonmcu/environment/legacy/loader.py @@ -0,0 +1,252 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from ..config import ( + PathConfig, + RepoConfig, +) +from .config import ( + DefaultsConfigOld, + FrameworkConfig, + FrameworkFeatureConfig, + BackendConfig, + BackendFeatureConfig, + TargetConfig, + TargetFeatureConfig, + PlatformConfig, + PlatformFeatureConfig, + FrontendConfig, + FrontendFeatureConfig, +) + +# def load_environment_from_file(filename): +# """Utility to initialize a mlonmcu environment from a YAML file.""" +# if isinstance(filename, str): +# filename = pathlib.Path(filename) +# with open(filename, encoding="utf-8") as yaml_file: +# data = yaml.safe_load(yaml_file) +# if data: +# if "home" in data: +# print(data["home"], filename.parent) +# assert os.path.realpath(data["home"]) == os.path.realpath(filename.parent) +# else: +# data["home"] = filename.parent +# env = Environment(data) +# return env +# raise RuntimeError(f"Error opening environment file: {filename}") +# return None + + +def load_environment_from_file(filename, base): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + with open(filename, encoding="utf-8") as yaml_file: + loaded = yaml.safe_load(yaml_file) + if not loaded: + raise RuntimeError("Invalid YAML contents") + if "home" in loaded: + home = loaded["home"] + else: + home = None + if "logging" in loaded: + if "level" in loaded["logging"]: + log_level = logging.getLevelName(loaded["logging"]["level"]) + else: + log_level = None + if "to_file" in loaded["logging"]: + log_to_file = bool(loaded["logging"]["to_file"]) + else: + log_to_file = None + if "rotate" in loaded["logging"]: + log_rotate = bool(loaded["logging"]["rotate"]) + else: + log_rotate = None + else: + log_level = None + log_to_file = False + log_rotate = False + if "cleanup" in loaded: + if "auto" in loaded["cleanup"]: + cleanup_auto = bool(loaded["cleanup"]["auto"]) + else: + cleanup_auto = False + if "auto" in loaded["cleanup"]: + cleanup_keep = int(loaded["cleanup"]["keep"]) + else: + cleanup_keep = 100 + else: + cleanup_auto = False + cleanup_keep = 100 + if "paths" in loaded: + paths = {} + for key in loaded["paths"]: + path = loaded["paths"][key] + if isinstance(path, list): + paths[key] = [] + for p in path: + paths[key].append(PathConfig(p, base=home)) + else: + paths[key] = PathConfig(path, base=home) + else: + paths = None + if "repos" in loaded: + repos = {} + for key in loaded["repos"]: + repo = loaded["repos"][key] + if "url" not in repo: + raise RuntimeError("Missing field 'url' in YAML file") + if "ref" in repo: + repos[key] = RepoConfig(repo["url"], ref=repo["ref"]) + else: + repos[key] = RepoConfig(repo["url"]) + else: + repos = None + default_framework = None + default_backends = {} + if "frameworks" in loaded: + frameworks = [] + default_framework = loaded["frameworks"].pop("default", None) + for key in loaded["frameworks"]: + framework = loaded["frameworks"][key] + if "enabled" in framework: + enabled = bool(framework["enabled"]) + else: + enabled = False + backends = [] + if "backends" in framework: + default_backend = framework["backends"].pop("default", None) + default_backends[key] = default_backend + for key2 in framework["backends"]: + backend = framework["backends"][key2] + if "enabled" in backend: + enabled2 = bool(backend["enabled"]) + else: + enabled2 = True + backend_features = [] + if "features" in backend: + for key3 in backend["features"]: + supported = bool(backend["features"][key3]) + backend_features.append(BackendFeatureConfig(key3, backend=key2, supported=supported)) + backends.append(BackendConfig(key2, enabled=enabled2, features=backend_features)) + framework_features = [] + if "features" in framework: + for key2 in framework["features"]: + supported = bool(framework["features"][key2]) + framework_features.append(FrameworkFeatureConfig(key2, framework=key, supported=supported)) + frameworks.append( + FrameworkConfig( + key, + enabled=enabled, + backends=backends, + features=framework_features, + ) + ) + else: + frameworks = None + if "frontends" in loaded: + frontends = [] + for key in loaded["frontends"]: + frontend = loaded["frontends"][key] + + if "enabled" in frontend: + enabled = frontend["enabled"] + else: + enabled = True + frontend_features = [] + if "features" in frontend: + for key2 in frontend["features"]: + supported = bool(frontend["features"][key2]) + frontend_features.append(FrontendFeatureConfig(key2, frontend=key, supported=supported)) + frontends.append(FrontendConfig(key, enabled=enabled, features=frontend_features)) + else: + frontends = None + if "platforms" in loaded: + platforms = [] + for key in loaded["platforms"]: + platform = loaded["platforms"][key] + if "enabled" in platform: + enabled = platform["enabled"] + else: + enabled = True + platform_features = [] + if "features" in platform: + for key2 in platform["features"]: + supported = bool(platform["features"][key2]) + platform_features.append(PlatformFeatureConfig(key2, platform=key, supported=supported)) + platforms.append(PlatformConfig(key, enabled=enabled, features=platform_features)) + else: + platforms = None + if "toolchains" in loaded: + toolchains = loaded["toolchains"] + else: + toolchains = None + default_target = None + if "targets" in loaded: + targets = [] + default_target = loaded["targets"].pop("default", None) + for key in loaded["targets"]: + target = loaded["targets"][key] + if "enabled" in target: + enabled = target["enabled"] + else: + enabled = True + target_features = [] + if "features" in target: + for key2 in target["features"]: + supported = bool(target["features"][key2]) + target_features.append(TargetFeatureConfig(key2, target=key, supported=supported)) + targets.append(TargetConfig(key, enabled=enabled, features=target_features)) + else: + targets = None + if "vars" in loaded: + variables = loaded["vars"] + else: + variables = None + if "flags" in loaded: + default_flags = loaded["flags"] + else: + default_flags = None + defaults = DefaultsConfigOld( + log_level=log_level, + log_to_file=log_to_file, + log_rotate=log_rotate, + default_framework=default_framework, + default_backends=default_backends, + default_target=default_target, + cleanup_auto=cleanup_auto, + cleanup_keep=cleanup_keep, + ) + env = base( + home, + defaults=defaults, + paths=paths, + repos=repos, + frameworks=frameworks, + frontends=frontends, + platforms=platforms, + toolchains=toolchains, + targets=targets, + variables=variables, + default_flags=default_flags, + ) + return env diff --git a/mlonmcu/environment/legacy/writer.py b/mlonmcu/environment/legacy/writer.py new file mode 100644 index 000000000..100cd54f3 --- /dev/null +++ b/mlonmcu/environment/legacy/writer.py @@ -0,0 +1,124 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import yaml +import pathlib +import logging + +from ..config import PathConfig + + +def create_environment_dict(environment): + data = {} + data["home"] = environment.home + data["logging"] = { + "level": logging.getLevelName(environment.defaults.log_level), + "to_file": environment.defaults.log_to_file, + "rotate": environment.defaults.log_rotate, + } + data["cleanup"] = { + "auto": environment.defaults.cleanup_auto, + "keep": environment.defaults.cleanup_keep, + } + data["paths"] = { + path: ( + str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + ) + for path, path_config in environment.paths.items() + } # TODO: allow relative paths + data["repos"] = { + repo: {"url": repo_config.url, "ref": repo_config.ref} for repo, repo_config in environment.repos.items() + } + data["frameworks"] = { + "default": environment.defaults.default_framework if environment.defaults.default_framework else None, + **{ + framework.name: { + "enabled": framework.enabled, + "backends": { + "default": ( + environment.defaults.default_backends[framework.name] + if environment.defaults.default_backends + and environment.defaults.default_backends[framework.name] + else None + ), + **{ + backend.name: { + "enabled": backend.enabled, + "features": { + backend_feature.name: backend_feature.supported for backend_feature in backend.features + }, + } + for backend in framework.backends + }, + }, + "features": { + framework_feature.name: framework_feature.supported for framework_feature in framework.features + }, + } + for framework in environment.frameworks + }, + } + data["frontends"] = { + # "default": None, # unimplemented? + **{ + frontend.name: { + "enabled": frontend.enabled, + "features": { + frontend_feature.name: frontend_feature.supported for frontend_feature in frontend.features + }, + } + for frontend in environment.frontends + }, + } + data["platforms"] = { + # "default": None, # unimplemented? + **{ + platform.name: { + "enabled": platform.enabled, + "features": { + platform_feature.name: platform_feature.supported for platform_feature in platform.features + }, + } + for platform in environment.platforms + }, + } + data["toolchains"] = environment.toolchains + data["targets"] = { + "default": environment.defaults.default_target, + **{ + target.name: { + "enabled": target.enabled, + "features": {target_feature.name: target_feature.supported for target_feature in target.features}, + } + for target in environment.targets + }, + } + data["vars"] = environment.vars + data["flags"] = environment.flags + return data + + +def write_environment_to_file(environment, filename): + """Utility to initialize a mlonmcu environment from a YAML file.""" + if isinstance(filename, str): + filename = pathlib.Path(filename) + data = create_environment_dict(environment) + with open(filename, "w") as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False, sort_keys=False) diff --git a/mlonmcu/environment/loader.py b/mlonmcu/environment/loader.py index 5ff88ad51..24a418aa7 100644 --- a/mlonmcu/environment/loader.py +++ b/mlonmcu/environment/loader.py @@ -20,21 +20,17 @@ import pathlib import logging +from mlonmcu.logging import get_logger from .config import ( DefaultsConfig, PathConfig, RepoConfig, - FrameworkConfig, - FrameworkFeatureConfig, - BackendConfig, - BackendFeatureConfig, - TargetConfig, - TargetFeatureConfig, - PlatformConfig, - PlatformFeatureConfig, - FrontendConfig, - FrontendFeatureConfig, + ComponentConfig, ) +from .legacy.loader import load_environment_from_file as load_legacy_environment_from_file +from .legacy.environment import UserEnvironment as UserEnvironmentOld + +logger = get_logger() # def load_environment_from_file(filename): # """Utility to initialize a mlonmcu environment from a YAML file.""" @@ -62,6 +58,17 @@ def load_environment_from_file(filename, base): loaded = yaml.safe_load(yaml_file) if not loaded: raise RuntimeError("Invalid YAML contents") + if "version" in loaded: + version = int(loaded["version"]) + else: + version = 1 + if version == 1: + # fallback + logger.warning("Environment format v1 is deprecated. Please convert environment files to v2.") + from .convert import convert_v1_to_v2 + + return convert_v1_to_v2(load_legacy_environment_from_file(filename, UserEnvironmentOld)) + assert version == 2, "Unsupported version of environment file" if "home" in loaded: home = loaded["home"] else: @@ -119,103 +126,23 @@ def load_environment_from_file(filename, base): repos[key] = RepoConfig(repo["url"]) else: repos = None - default_framework = None - default_backends = {} - if "frameworks" in loaded: - frameworks = [] - default_framework = loaded["frameworks"].pop("default", None) - for key in loaded["frameworks"]: - framework = loaded["frameworks"][key] - if "enabled" in framework: - enabled = bool(framework["enabled"]) - else: - enabled = False - backends = [] - if "backends" in framework: - default_backend = framework["backends"].pop("default", None) - default_backends[key] = default_backend - for key2 in framework["backends"]: - backend = framework["backends"][key2] - if "enabled" in backend: - enabled2 = bool(backend["enabled"]) - else: - enabled2 = True - backend_features = [] - if "features" in backend: - for key3 in backend["features"]: - supported = bool(backend["features"][key3]) - backend_features.append(BackendFeatureConfig(key3, backend=key2, supported=supported)) - backends.append(BackendConfig(key2, enabled=enabled2, features=backend_features)) - framework_features = [] - if "features" in framework: - for key2 in framework["features"]: - supported = bool(framework["features"][key2]) - framework_features.append(FrameworkFeatureConfig(key2, framework=key, supported=supported)) - frameworks.append( - FrameworkConfig( - key, - enabled=enabled, - backends=backends, - features=framework_features, - ) - ) - else: - frameworks = None - if "frontends" in loaded: - frontends = [] - for key in loaded["frontends"]: - frontend = loaded["frontends"][key] - if "enabled" in frontend: - enabled = frontend["enabled"] - else: - enabled = True - frontend_features = [] - if "features" in frontend: - for key2 in frontend["features"]: - supported = bool(frontend["features"][key2]) - frontend_features.append(FrontendFeatureConfig(key2, frontend=key, supported=supported)) - frontends.append(FrontendConfig(key, enabled=enabled, features=frontend_features)) - else: - frontends = None - if "platforms" in loaded: - platforms = [] - for key in loaded["platforms"]: - platform = loaded["platforms"][key] - if "enabled" in platform: - enabled = platform["enabled"] - else: - enabled = True - platform_features = [] - if "features" in platform: - for key2 in platform["features"]: - supported = bool(platform["features"][key2]) - platform_features.append(PlatformFeatureConfig(key2, platform=key, supported=supported)) - platforms.append(PlatformConfig(key, enabled=enabled, features=platform_features)) - else: - platforms = None - if "toolchains" in loaded: - toolchains = loaded["toolchains"] - else: - toolchains = None - default_target = None - if "targets" in loaded: - targets = [] - default_target = loaded["targets"].pop("default", None) - for key in loaded["targets"]: - target = loaded["targets"][key] - if "enabled" in target: - enabled = target["enabled"] - else: - enabled = True - target_features = [] - if "features" in target: - for key2 in target["features"]: - supported = bool(target["features"][key2]) - target_features.append(TargetFeatureConfig(key2, target=key, supported=supported)) - targets.append(TargetConfig(key, enabled=enabled, features=target_features)) - else: - targets = None + def helper(data): + if data is None: + return None + supported = data.get("supported", []) + used = data.get("use", []) + combined = list(set(supported + used)) + return {key: ComponentConfig(key in supported, key in used) for key in combined} + + frameworks = helper(loaded.get("frameworks", None)) + backends = helper(loaded.get("backends", None)) + frontends = helper(loaded.get("frontends", None)) + features = helper(loaded.get("features", None)) + platforms = helper(loaded.get("platforms", None)) + targets = helper(loaded.get("targets", None)) + postprocesses = helper(loaded.get("postprocesses", None)) + toolchains = helper(loaded.get("toolchains", None)) if "vars" in loaded: variables = loaded["vars"] else: @@ -228,9 +155,6 @@ def load_environment_from_file(filename, base): log_level=log_level, log_to_file=log_to_file, log_rotate=log_rotate, - default_framework=default_framework, - default_backends=default_backends, - default_target=default_target, cleanup_auto=cleanup_auto, cleanup_keep=cleanup_keep, ) @@ -240,10 +164,13 @@ def load_environment_from_file(filename, base): paths=paths, repos=repos, frameworks=frameworks, + backends=backends, frontends=frontends, platforms=platforms, toolchains=toolchains, targets=targets, + features=features, + postprocesses=postprocesses, variables=variables, default_flags=default_flags, ) diff --git a/mlonmcu/environment/user_environment.py b/mlonmcu/environment/user_environment.py new file mode 100644 index 000000000..884eeddb3 --- /dev/null +++ b/mlonmcu/environment/user_environment.py @@ -0,0 +1,92 @@ +# +# Copyright (c) 2022 TUM Department of Electrical and Computer Engineering. +# +# This file is part of MLonMCU. +# See https://github.com/tum-ei-eda/mlonmcu.git for further info. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import tempfile +from pathlib import Path + +from .loader import load_environment_from_file +from .writer import write_environment_to_file +from .environment import DefaultEnvironment + + +class UserEnvironment(DefaultEnvironment): + def __init__( + self, + home, + alias=None, + defaults=None, + paths=None, + repos=None, + frameworks=None, + backends=None, + frontends=None, + platforms=None, + toolchains=None, + targets=None, + features=None, + postprocesses=None, + variables=None, + default_flags=None, + ): + super().__init__() + self._home = home + + if alias: + self.alias = alias + if defaults: + self.defaults = defaults + if paths: + self.paths = paths + if repos: + self.repos = repos + if frameworks: + self.frameworks = frameworks + if backends: + self.backends = backends + if frontends: + self.frontends = frontends + if platforms: + self.platforms = platforms + if toolchains: + self.toolchains = toolchains + if targets: + self.targets = targets + if features: + self.features = features + if postprocesses: + self.postprocesses = postprocesses + if variables: + self.vars = variables + if default_flags: + self.flags = default_flags + + @classmethod + def from_file(cls, filename): + return load_environment_from_file(filename, base=cls) + + def to_file(self, filename): + write_environment_to_file(self, filename) + + def to_text(self): + with tempfile.TemporaryDirectory() as tmpdirname: + dest = Path(tmpdirname) / "out.yml" + self.to_file(dest) + with open(dest, "r") as handle: + content = handle.read() + assert len(content) > 0 + return content diff --git a/mlonmcu/environment/writer.py b/mlonmcu/environment/writer.py index 94175da79..b99eb179f 100644 --- a/mlonmcu/environment/writer.py +++ b/mlonmcu/environment/writer.py @@ -36,72 +36,47 @@ def create_environment_dict(environment): "keep": environment.defaults.cleanup_keep, } data["paths"] = { - path: str(path_config.path) - if isinstance(path_config, PathConfig) - else [str(config.path) for config in path_config] + path: ( + str(path_config.path) + if isinstance(path_config, PathConfig) + else [str(config.path) for config in path_config] + ) for path, path_config in environment.paths.items() } # TODO: allow relative paths - data["repos"] = {repo: vars(repo_config) for repo, repo_config in environment.repos.items()} + data["repos"] = { + repo: {"url": repo_config.url, "ref": repo_config.ref} for repo, repo_config in environment.repos.items() + } data["frameworks"] = { - "default": environment.defaults.default_framework if environment.defaults.default_framework else None, - **{ - framework.name: { - "enabled": framework.enabled, - "backends": { - "default": environment.defaults.default_backends[framework.name] - if environment.defaults.default_backends and environment.defaults.default_backends[framework.name] - else None, - **{ - backend.name: { - "enabled": backend.enabled, - "features": { - backend_feature.name: backend_feature.supported for backend_feature in backend.features - }, - } - for backend in framework.backends - }, - }, - "features": { - framework_feature.name: framework_feature.supported for framework_feature in framework.features - }, - } - for framework in environment.frameworks - }, + "supported": [name for name, value in environment.frameworks.items() if value.supported], + "use": [name for name, value in environment.frameworks.items() if value.used], + } + data["backends"] = { + "supported": [name for name, value in environment.backends.items() if value.supported], + "use": [name for name, value in environment.backends.items() if value.used], } data["frontends"] = { - # "default": None, # unimplemented? - **{ - frontend.name: { - "enabled": frontend.enabled, - "features": { - frontend_feature.name: frontend_feature.supported for frontend_feature in frontend.features - }, - } - for frontend in environment.frontends - }, + "supported": [name for name, value in environment.frontends.items() if value.supported], + "use": [name for name, value in environment.frontends.items() if value.used], + } + data["toolchains"] = { + "supported": [name for name, value in environment.toolchains.items() if value.supported], + "use": [name for name, value in environment.toolchains.items() if value.used], } data["platforms"] = { - # "default": None, # unimplemented? - **{ - platform.name: { - "enabled": platform.enabled, - "features": { - platform_feature.name: platform_feature.supported for platform_feature in platform.features - }, - } - for platform in environment.platforms - }, + "supported": [name for name, value in environment.platforms.items() if value.supported], + "use": [name for name, value in environment.platforms.items() if value.used], } - data["toolchains"] = environment.toolchains data["targets"] = { - "default": environment.defaults.default_target, - **{ - target.name: { - "enabled": target.enabled, - "features": {target_feature.name: target_feature.supported for target_feature in target.features}, - } - for target in environment.targets - }, + "supported": [name for name, value in environment.targets.items() if value.supported], + "use": [name for name, value in environment.targets.items() if value.used], + } + data["features"] = { + "supported": [name for name, value in environment.features.items() if value.supported], + "use": [name for name, value in environment.features.items() if value.used], + } + data["postprocesses"] = { + "supported": [name for name, value in environment.postprocesses.items() if value.supported], + "use": [name for name, value in environment.postprocesses.items() if value.used], } data["vars"] = environment.vars data["flags"] = environment.flags diff --git a/mlonmcu/feature/features.py b/mlonmcu/feature/features.py index ad643d299..24f1f04bf 100644 --- a/mlonmcu/feature/features.py +++ b/mlonmcu/feature/features.py @@ -1317,7 +1317,7 @@ def benchmark_callback(stdout, metrics, artifacts): if self.total: data.update( { - f"Total {key}": ((value * self.num_runs) if self.num_runs > 1 else value) + f"Total {key}": (value * self.num_runs) if self.num_runs > 1 else value for key, value in data_[-1].items() if "cycle" in key.lower() or "time" in key.lower() } diff --git a/mlonmcu/platform/lookup.py b/mlonmcu/platform/lookup.py index 8ffcab76a..15a43c8d5 100644 --- a/mlonmcu/platform/lookup.py +++ b/mlonmcu/platform/lookup.py @@ -23,7 +23,7 @@ def get_platform_names(context): - return context.environment.lookup_platform_configs(names_only=True) + return context.environment.get_supported_platforms() def get_platforms_targets(context, config=None): diff --git a/mlonmcu/session/run.py b/mlonmcu/session/run.py index 85c3a3798..9d14e9f3d 100644 --- a/mlonmcu/session/run.py +++ b/mlonmcu/session/run.py @@ -598,9 +598,11 @@ def _merge_dicts_of_lists(*args): if isinstance(artifacts, dict): new.update( { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } ) @@ -632,9 +634,11 @@ def run(self): artifacts = self.target.artifacts if isinstance(artifacts, dict): new = { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } else: @@ -650,9 +654,11 @@ def run(self): artifacts = self.target.artifacts if isinstance(artifacts, dict): new = { - key - if name in ["", "default"] - else (f"{name}_{key}" if key not in ["", "default"] else name): value + ( + key + if name in ["", "default"] + else (f"{name}_{key}" if key not in ["", "default"] else name) + ): value for key, value in artifacts.items() } else: diff --git a/mlonmcu/setup/tasks/muriscvnn.py b/mlonmcu/setup/tasks/muriscvnn.py index 64ccc9a28..f23854294 100644 --- a/mlonmcu/setup/tasks/muriscvnn.py +++ b/mlonmcu/setup/tasks/muriscvnn.py @@ -34,7 +34,7 @@ def _validate_muriscvnn(context: MlonMcuContext, params=None): - if not context.environment.supports_feature("muriscvnn"): + if not context.environment.has_feature("muriscvnn"): return False user_vars = context.environment.vars if "muriscvnn.src_dir" not in user_vars: @@ -46,13 +46,13 @@ def _validate_muriscvnn(context: MlonMcuContext, params=None): target_arch = params.get("target_arch", "riscv") if target_arch == "riscv": if params.get("vext", False): - if not context.environment.supports_feature("vext"): + if not context.environment.has_feature("vext"): return False if params.get("pext", False): if toolchain == "llvm": # Unsupported return False - if not context.environment.supports_feature("pext"): + if not context.environment.has_feature("pext"): return False if params.get("vext", False) and params.get("pext", False): # Either pext or vext! diff --git a/mlonmcu/setup/tasks/spike.py b/mlonmcu/setup/tasks/spike.py index 5f5615f64..e797d59e2 100644 --- a/mlonmcu/setup/tasks/spike.py +++ b/mlonmcu/setup/tasks/spike.py @@ -40,12 +40,12 @@ def _validate_spike(context: MlonMcuContext, params=None): if params.get("vext", False): if params.get("pext", False): return False # Can not use booth at a time - if not context.environment.supports_feature("vext"): + if not context.environment.has_feature("vext"): return False if params.get("pext", False): if params.get("vext", False): return False # Can not use booth at a time - if not context.environment.supports_feature("pext"): + if not context.environment.has_feature("pext"): return False user_vars = context.environment.vars if "spike.pk" not in user_vars: # TODO: also check command line flags? diff --git a/mlonmcu/setup/tasks/tflmc.py b/mlonmcu/setup/tasks/tflmc.py index a45a82128..07a0c487f 100644 --- a/mlonmcu/setup/tasks/tflmc.py +++ b/mlonmcu/setup/tasks/tflmc.py @@ -65,10 +65,10 @@ def _validate_build_tflite_micro_compiler(context: MlonMcuContext, params=None): # Not allowed return False elif muriscvnn: - if not context.environment.supports_feature("muriscvnn"): + if not context.environment.has_feature("muriscvnn"): return False elif cmsisnn: - if not context.environment.supports_feature("cmsisnn"): + if not context.environment.has_feature("cmsisnn"): return False return _validate_tflite_micro_compiler(context, params=params) diff --git a/requirements.txt b/requirements.txt index 7c45de684..1939f10be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,6 +17,7 @@ pyyaml pandas numpy psutil +Pillow # Sessions tqdm diff --git a/resources/templates/dev.yml.j2 b/resources/templates/dev.yml.j2 index 7072d536c..21a8642e5 100644 --- a/resources/templates/dev.yml.j2 +++ b/resources/templates/dev.yml.j2 @@ -1,5 +1,7 @@ +# Generated with MLonMCU version {{ TODO }} --- # The MLONMCU_HOME is filled in automatically when creating the environment +version: 2 home: "{{ home_dir }}" logging: level: DEBUG @@ -68,139 +70,89 @@ repos: espidf: url: "https://github.com/espressif/esp-idf.git" ref: release/v4.4 - -# Here all supported frameworks with their specific features are defined -# Optionally disable unwanted or incomatible backends or features here -# The configured defaults are used if no backend was specified in the command line options frameworks: - default: tvm - tflm: - enabled: true - backends: - default: tflmi - tflmc: - enabled: false - tflmi: - enabled: true - features: - debug_arena: true - features: - muriscvnn: true - cmsisnn: true - tvm: - enabled: true - backends: - default: tvmaot - tvmrt: - enabled: true - features: - debug_arena: true - disable_legalize: true - autotune: true - autotuned: true - tvmaot: - enabled: true - features: - debug_arena: true - unpacked_api: true - usmp: true - disable_legalize: true - autotune: true - autotuned: true - tvmcg: - enabled: true - features: - debug_arena: true - disable_legalize: true - autotune: true - autotuned: true - features: - cmsisnnbyoc: false -# Some frontends are eperimental and therefore disabled here -# Features like packing are only available in certain environments -# + supported: &supported_frameworks + - tflm + - tvm + use: *supported_frameworks +backends: + supported: + - tflmc + - tflmi + - tvmaot + - tvmrt + - tvmcg + use: + - tvmaot frontends: - tflite: - enabled: true - features: - validate: true - visualize: true - relay: - enabled: true - features: - relayviz: true - packed: - enabled: false - features: - packing: true - packed: true - onnx: - enabled: false - # TODO: saved_model (TF->TFLITE), ipynb (IPYNB->?) + supported: &supported_frontends + - tflite + - relay + # - packed + # - onnx + use: *supported_frontends # List of supported targets in the environment targets: - default: etiss_pulpino - etiss_pulpino: - enabled: true - features: - gdbserver: true - etissdbg: true - trace: true - log_instrs: true - host_x86: - enabled: true - features: - gdbserver: true - spike: - enabled: true - features: - vext: true - cachesim: true - log_instrs: true - ovpsim: - enabled: true - features: - vext: true - corstone300: - enabled: true - features: - ethosu: false - esp32: - enabled: true - esp32c3: - enabled: true - microtvm_etissvp: - enabled: true - microtvm_espidf: - enabled: false + supported: + - etiss_pulpino # mlif + - spike # mlif + - ovpsim # mlif + - host_x86 # mlif + - corstone300 # mlif + # espidf targets do not need to be listed here + # - esp32 # espidf + # - esp32c3 # espidf + # tvm targets do not need to be listed here + # - tvm_cpu # tvm + # microtvm targets need to be listed here if they have dependencies which need to be fetched + - microtvm_zephyr # microtvm + - microtvm_host # microtvm + # - microtvm_spike # microtvm + # - microtvm_etissvp # microtvm + # - microtvm_espidf # microtvm + use: + - etiss_pulpino # Some targets/platforms support multiple toolchains toolchains: - gcc: true - llvm: true + supported: &supported_toolchains + gcc: true # TODO: split into gcc, riscv_gcc, arm_gcc + llvm: true + # use: NO EFFECT platforms: - mlif: - enabled: true - features: - debug: true - validate: true - espidf: - enabled: true - features: - debug: true - zephyr: - enabled: false - features: - debug: true + supported: &supported_platforms + - mlif + - espidf + - zephyr + # - arduino + use: *supported_platforms +features: + supported: + - gdbserver + - etissdbg + - vext + - trace + - disable_legalize + - debug_arena + - unpacked_api + - usmp + - cmsisnn + - muriscvnn + # - cmsisnnbyoc + # - packing + # - packed + # - ethosu + use: [] +# Issues: Do we need to add user-provided postprocesses to the list as well? postprocesses: - use: - # - detailed_cycles - # - average_cycles - # - filter_cols - # - features2cols - # - config2cols - # - bytes2kb - # - visualize -# This is where further options such as specific versions of dependencies can be set in the furture + supported: + - detailed_cycles + - average_cycles + - filter_cols + - features2cols + - config2cols + - bytes2kb + - visualize + use: [] vars: allow_extensions: false # tvm.make_tool: "ninja" diff --git a/tests/unit-tests/environment/test_environment_writer.py b/tests/unit-tests/environment/test_environment_writer.py index 2a0e5dd57..cae0e633c 100644 --- a/tests/unit-tests/environment/test_environment_writer.py +++ b/tests/unit-tests/environment/test_environment_writer.py @@ -1,9 +1,10 @@ import os import logging -from mlonmcu.environment.environment import Environment -from mlonmcu.environment.config import DefaultsConfig, PathConfig, RepoConfig -from mlonmcu.environment.writer import create_environment_dict +from mlonmcu.environment.legacy.environment import Environment +from mlonmcu.environment.config import PathConfig, RepoConfig +from mlonmcu.environment.legacy.config import DefaultsConfigOld as DefaultsConfig +from mlonmcu.environment.legacy.writer import create_environment_dict class MyEnvironment(Environment): diff --git a/tests/unit-tests/platform/test_platform_lookup.py b/tests/unit-tests/platform/test_platform_lookup.py index b33fae097..b43af9060 100644 --- a/tests/unit-tests/platform/test_platform_lookup.py +++ b/tests/unit-tests/platform/test_platform_lookup.py @@ -184,13 +184,3 @@ def test_platform_registry(): assert len(get_platforms()) == before + 2 register_platform("foo", MyPlatformA, override=True) assert len(get_platforms()) == before + 2 - - -def test_platform_get_platform_names(fake_context): - def lookup_platform_configs(names_only=True): - assert names_only - return ["foo", "bar"] - - fake_context.environment.lookup_platform_configs = lookup_platform_configs - res = mlonmcu.platform.lookup.get_platform_names(fake_context) - assert res == ["foo", "bar"]