diff --git a/sebs.py b/sebs.py index 80fb11ed..62615adb 100755 --- a/sebs.py +++ b/sebs.py @@ -334,6 +334,51 @@ def process(**kwargs): out_f.write(sebs.utils.serialize(experiments)) sebs_client.logging.info("Save results to {}".format(output_file)) +@benchmark.command() +@click.argument("benchmark", type=str) # , help="Benchmark to be used.") +@click.option( + "--function-name", + default=None, + type=str, + help="Override function name for random generation.", +) +@click.option( + "--image-tag-prefix", + default=None, + type=str, + help="Attach prefix to generated Docker image tag.", +) +@common_params +def package( + benchmark, + function_name, + image_tag_prefix, + **kwargs, +): + + ( + config, + output_dir, + logging_filename, + sebs_client, + deployment_client, + ) = parse_common_params(**kwargs) + if image_tag_prefix is not None: + sebs_client.config.image_tag_prefix = image_tag_prefix + + experiment_config = sebs_client.get_experiment_config(config["experiments"]) + update_nested_dict(config, ["experiments", "benchmark"], benchmark) + benchmark_obj = sebs_client.get_benchmark( + benchmark, + deployment_client, + experiment_config, + logging_filename=logging_filename, + ) + + func = deployment_client.build_function( + benchmark_obj, + function_name if function_name else deployment_client.default_function_name(benchmark_obj) + ) @benchmark.command() @click.argument( @@ -390,7 +435,12 @@ def storage(): @click.argument("storage", type=click.Choice(["object", "nosql", "all"])) @click.argument("config", type=click.Path(dir_okay=False, readable=True)) @click.option("--output-json", type=click.Path(dir_okay=False, writable=True), default=None) -def storage_start(storage, config, output_json): +@click.option( + "--remove-containers/--no-remove-containers", + default=True, + help="Remove containers after stopping.", +) +def storage_start(storage, config, output_json, remove_containers): import docker @@ -405,6 +455,7 @@ def storage_start(storage, config, output_json): storage_type = sebs.SeBS.get_storage_implementation(storage_type_enum) storage_config = sebs.SeBS.get_storage_config_implementation(storage_type_enum) config = storage_config.deserialize(user_storage_config["object"][storage_type_name]) + config.remove_containers = remove_containers storage_instance = storage_type(docker.from_env(), None, None, True) storage_instance.config = config @@ -423,6 +474,7 @@ def storage_start(storage, config, output_json): storage_type = sebs.SeBS.get_nosql_implementation(storage_type_enum) storage_config = sebs.SeBS.get_nosql_config_implementation(storage_type_enum) config = storage_config.deserialize(user_storage_config["nosql"][storage_type_name]) + config.remove_containers = remove_containers storage_instance = storage_type(docker.from_env(), None, config) diff --git a/sebs/cache.py b/sebs/cache.py index f690e747..8e70a247 100644 --- a/sebs/cache.py +++ b/sebs/cache.py @@ -319,6 +319,8 @@ def add_code_package( config = cached_config with open(os.path.join(benchmark_dir, "config.json"), "w") as fp: json.dump(config, fp, indent=2) + + self.logging.info(f"Updating cached code package {cached_location}") else: # TODO: update raise RuntimeError( @@ -389,6 +391,7 @@ def update_code_package( with open(os.path.join(benchmark_dir, "config.json"), "w") as fp: json.dump(config, fp, indent=2) + self.logging.info(f"Updated cached code package {cached_location}") else: self.add_code_package(deployment_name, code_package) diff --git a/sebs/faas/system.py b/sebs/faas/system.py index 9fbe0e27..e31338ff 100644 --- a/sebs/faas/system.py +++ b/sebs/faas/system.py @@ -233,6 +233,23 @@ def update_function( """ pass + def build_function(self, code_package: Benchmark, func_name: Optional[str] = None): + + if code_package.language_version not in self.system_config.supported_language_versions( + self.name(), code_package.language_name + ): + raise Exception( + "Unsupported {language} version {version} in {system}!".format( + language=code_package.language_name, + version=code_package.language_version, + system=self.name(), + ) + ) + + if not func_name: + func_name = self.default_function_name(code_package) + code_package.build(self.package_code) + """ a) if a cached function with given name is present and code has not changed, then just return function name diff --git a/sebs/storage/config.py b/sebs/storage/config.py index cd47df39..2e6660c2 100644 --- a/sebs/storage/config.py +++ b/sebs/storage/config.py @@ -30,6 +30,7 @@ class MinioConfig(PersistentStorageConfig): version: str = "" data_volume: str = "" type: str = "minio" + remove_containers: bool = False def update_cache(self, path: List[str], cache: Cache): @@ -79,6 +80,7 @@ class ScyllaDBConfig(NoSQLStorageConfig): memory: int = -1 version: str = "" data_volume: str = "" + remove_containers: bool = False def update_cache(self, path: List[str], cache: Cache): diff --git a/sebs/storage/minio.py b/sebs/storage/minio.py index 757d2025..b258d116 100644 --- a/sebs/storage/minio.py +++ b/sebs/storage/minio.py @@ -100,7 +100,7 @@ def start(self): "MINIO_SECRET_KEY": self._cfg.secret_key, }, volumes=volumes, - remove=True, + remove=self.config.remove_containers, stdout=True, stderr=True, detach=True, diff --git a/sebs/storage/scylladb.py b/sebs/storage/scylladb.py index aae97815..e59e0cb2 100644 --- a/sebs/storage/scylladb.py +++ b/sebs/storage/scylladb.py @@ -93,7 +93,7 @@ def start(self): network_mode="bridge", volumes=volumes, ports={"8000": str(self._cfg.mapped_port)}, - remove=True, + remove=self.config.remove_containers, stdout=True, stderr=True, detach=True, diff --git a/tools/update_docker_images.py b/tools/update_docker_images.py new file mode 100755 index 00000000..48156116 --- /dev/null +++ b/tools/update_docker_images.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +import argparse +import docker +import json +import os +from typing import Optional + +PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), os.path.pardir) +DOCKER_DIR = os.path.join(PROJECT_DIR, "dockerfiles") + +parser = argparse.ArgumentParser(description="Run local app experiments.") +parser.add_argument( + "--deployment", default=None, choices=["local", "aws", "azure", "gcp", "openwhisk"], action="store" +) +parser.add_argument("--type", default=None, choices=["build", "run", "manage", "function"], action="store") +parser.add_argument("--language", default=None, choices=["python", "nodejs"], action="store") +parser.add_argument("--language-version", default=None, type=str, action="store") +args = parser.parse_args() +config = json.load(open(os.path.join(PROJECT_DIR, "config", "systems.json"), "r")) +repository = config["general"]["docker_repository"] +client = docker.from_env() + +def pull(image_name): + + image_name = f'{repository}:{image_name}' + previous_sha: Optional[str] + try: + img = client.images.get(image_name) + previous_sha = img.attrs['Id'] + except docker.errors.ImageNotFound: + previous_sha = None + print(f"Ignoring not present image: {image_name}") + return + + img = client.images.pull(image_name) + + if img.attrs['Id'] != previous_sha: + print(f"Updated image: {image_name}") + else: + print(f"Image up-to-date: {image_name}") + +def generic_pull(image_type, system, language=None, version=None): + + if language is not None and version is not None: + image_name = f"{image_type}.{system}.{language}.{version}" + else: + image_name = f"{image_type}.{system}" + + pull(image_name) + +benchmarks = { + "python": [ + "110.dynamic-html", + "120.uploader", + "210.thumbnailer", + "220.video-processing", + "311.compression", + "411.image-recognition", + "501.graph-pagerank", + "502.graph-mst", + "503.graph-bfs", + "504.dna-visualisation", + ], + "nodejs": [ + "110.dynamic-html", + "120.uploader", + "210.thumbnailer" + ] +} + +def pull_function(image_type, system, language, language_version): + + for bench in benchmarks[language]: + image_name = f"{image_type}.{system}.{bench}.{language}-{language_version}" + pull(image_name) + +def pull_language(system, language, language_config): + configs = [] + if "base_images" in language_config: + for version, base_image in language_config["base_images"].items(): + if args.language_version is not None and args.language_version == version: + configs.append([version, base_image]) + elif args.language_version is None: + configs.append([version, base_image]) + else: + configs.append([None, None]) + + for image in configs: + if args.type is None: + for image_type in language_config["images"]: + + if image_type == "function": + pull_function(image_type, system, language, image[0]) + else: + generic_pull(image_type, system, language, image[0]) + else: + generic_pull(args.type, system, language, image[0]) + + +def pull_systems(system, system_config): + + if args.type == "manage": + if "images" in system_config: + generic_pull(args.type, system) + else: + print(f"Skipping manage image for {system}") + else: + if args.language: + pull_language(system, args.language, system_config["languages"][args.language]) + else: + for language, language_dict in system_config["languages"].items(): + pull_language(system, language, language_dict) + # Build additional types + if "images" in system_config: + for image_type, image_config in system_config["images"].items(): + generic_pull(image_type, system) + + +if args.deployment is None: + for system, system_dict in config.items(): + if system == "general": + continue + pull_systems(system, system_dict) +else: + pull_systems(args.deployment, config[args.deployment])