From a60b5502b2ecb144c4f2478940baeba0d5135356 Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Thu, 5 Mar 2026 13:20:50 +0400 Subject: [PATCH 01/15] added context execution --- fedot/core/context.py | 74 ++++++ .../initializer_industrial_models.py | 225 ++++++------------ fedot/industrial/industrial_extension.py | 63 +++++ 3 files changed, 205 insertions(+), 157 deletions(-) create mode 100644 fedot/core/context.py create mode 100644 fedot/industrial/industrial_extension.py diff --git a/fedot/core/context.py b/fedot/core/context.py new file mode 100644 index 0000000000..c91ad1275b --- /dev/null +++ b/fedot/core/context.py @@ -0,0 +1,74 @@ +from fedot.core.data.merge.data_merger import ImageDataMerger, TSDataMerger, DataMerger +from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( + TopologicalFeaturesImplementation, +) +from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation, + TsSmoothingImplementation, +) +from fedot.core.operations.operation import Operation +from fedot.core.optimisers.objective import PipelineObjectiveEvaluate +from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter +from fedot.core.pipelines.pipeline import Pipeline +from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace +from fedot.core.pipelines.verification import class_rules, ts_rules, common_rules +from fedot.core.repository.operation_types_repository import OperationTypesRepository +from fedot.core.data.data_split import _split_any, _split_time_series +from fedot.api.api_utils.api_params_repository import ApiParamsRepository +from fedot.api.api_utils.api_composer import ApiComposer +from golem.core.tuning.optuna_tuner import OptunaTuner +from golem.core.optimisers.genetic.operators.reproduction import ReproductionController + +import fedot.core.data.data_split as fedot_data_split +import golem.core.tuning.optuna_tuner as OptunaImpl + + +class ExecutionContext: + def __init__(self) -> None: + self.operation_registry = OperationTypesRepository() + self.evaluator_evaluate = PipelineObjectiveEvaluate + self.search_space_get_parameters_dict = PipelineSearchSpace + self.api_params_repository__get_default_mutations = ApiParamsRepository + self.merger_find_main_output = DataMerger + self.merger_get = DataMerger + self.merger_merge_predicts = DataMerger + self.image_merger_preprocess_predicts = ImageDataMerger + self.image_merger_merge_predicts = ImageDataMerger + self.ts_merger_merge_predicts = TSDataMerger + self.ts_merger_merge_targets = TSDataMerger + self.ts_merger_postprocess_predicts = TSDataMerger + self.ts_merger_preprocess_predicts = TSDataMerger + self.data_source_splitter_build = DataSourceSplitter + self.data_split__split_any = fedot_data_split._split_any + self.data_split__split_time_series = fedot_data_split._split_time_series + self.operation__predict = Operation + self.operation_predict = Operation + self.operation_predict_for_fit = Operation + self.lagged__update_column_types = LaggedImplementation + self.lagged_transform = LaggedImplementation + self.lagged_transform_for_fit = LaggedImplementation + self.lagged__check_and_correct_window_size = LaggedImplementation + self.topo_features_fit = TopologicalFeaturesImplementation + self.topo_features_transform = TopologicalFeaturesImplementation + self.ts_smoothing_transform = TsSmoothingImplementation + self.optuna_optuna_tuner = OptunaImpl.OptunaTuner + self.api_composer_tune_final_pipeline = ApiComposer + self.reproduction_reproduce = ReproductionController + self.reproduction_reproduce_uncontrolled = ReproductionController + self.class_rules = class_rules.copy() + self.ts_rules = ts_rules.copy() + self.common_rules = common_rules.copy() + + def __getstate__(self): + return { + "industrial": True, + "class_rules": self.class_rules, + "ts_rules": self.ts_rules, + "common_rules": self.common_rules, + } + + def __setstate__(self, state): + self.__init__() + self.class_rules = state["class_rules"] + self.ts_rules = state["ts_rules"] + self.common_rules = state["common_rules"] \ No newline at end of file diff --git a/fedot/industrial/core/repository/initializer_industrial_models.py b/fedot/industrial/core/repository/initializer_industrial_models.py index eeb00a35d3..4e9d5a36c9 100644 --- a/fedot/industrial/core/repository/initializer_industrial_models.py +++ b/fedot/industrial/core/repository/initializer_industrial_models.py @@ -39,71 +39,8 @@ from fedot.industrial.core.repository.model_repository import overload_model_implementation from fedot.industrial.core.tuning.search_space import get_industrial_search_space -FEDOT_METHOD_TO_REPLACE = [(PipelineObjectiveEvaluate, "evaluate"), - (PipelineSearchSpace, "get_parameters_dict"), - (ApiParamsRepository, "_get_default_mutations"), - (DataMerger, "find_main_output"), - (DataMerger, "get"), - (DataMerger, "merge_predicts"), - (ImageDataMerger, "preprocess_predicts"), - (ImageDataMerger, "merge_predicts"), - (TSDataMerger, "merge_predicts"), - (TSDataMerger, "merge_targets"), - (TSDataMerger, 'postprocess_predicts'), - (TSDataMerger, 'preprocess_predicts'), - (DataSourceSplitter, "build"), - (fedot_data_split, "_split_any"), - (fedot_data_split, "_split_time_series"), - (Operation, "_predict"), - (Operation, "predict"), - (Operation, "predict_for_fit"), - (LaggedImplementation, '_update_column_types'), - (LaggedImplementation, 'transform'), - (TopologicalFeaturesImplementation, 'fit'), - (TopologicalFeaturesImplementation, 'transform'), - (LaggedImplementation, 'transform_for_fit'), - (LaggedImplementation, '_check_and_correct_window_size'), - (TsSmoothingImplementation, 'transform'), - (OptunaImpl, 'OptunaTuner'), - (ApiComposer, 'tune_final_pipeline'), - (ReproductionController, 'reproduce_uncontrolled'), - (ReproductionController, 'reproduce')] -INDUSTRIAL_REPLACE_METHODS = [industrial_evaluate_pipeline, - get_industrial_search_space, - _get_default_industrial_mutations, - find_main_output_industrial, - get_merger_industrial, - merge_industrial_predicts, - preprocess_industrial_predicts, - merge_industrial_predicts, - merge_industrial_predicts, - merge_industrial_targets, - postprocess_industrial_predicts, - preprocess_industrial_predicts, - build_industrial, - split_any_industrial, - split_time_series_industrial, - predict_operation_industrial, - predict_industrial, - predict_for_fit_industrial, - update_column_types_industrial, - transform_lagged_industrial, - fit_topo_extractor_industrial, - transform_topo_extractor_industrial, - transform_lagged_for_fit_industrial, - _check_and_correct_window_size_industrial, - transform_smoothing_industrial, - DaskOptunaTuner, - tune_pipeline_industrial, - reproduce_controlled_industrial, - reproduce_industrial] - -DEFAULT_METHODS = [getattr(class_impl[0], class_impl[1]) - for class_impl in FEDOT_METHOD_TO_REPLACE] -DEFAULT_MODELS_TO_REPLACE = [(MODEL_REPO, 'SKLEARN_REG_MODELS'), - (MODEL_REPO, 'SKLEARN_CLF_MODELS'), - (MODEL_REPO, 'FEDOT_PREPROC_MODEL')] - +from fedot.core.context import ExecutionContext +from fedot.industrial.industrial_extension import IndustrialExtension def has_no_resample(pipeline: Pipeline): """ @@ -113,111 +50,85 @@ def has_no_resample(pipeline: Pipeline): """ for node in pipeline.nodes: if node.name == 'resample': - raise ValueError( - f'Pipeline can not have resample operation') + raise ValueError("Pipeline can not have resample operation") return True -class IndustrialModels: - def __init__(self): +def initialize_industrial_context(backend: str = "default") -> ExecutionContext: + context = ExecutionContext() + extension = IndustrialExtension(backend=backend) + extension.apply(context) + return context + +class IndustrialModels: + def __init__(self, backend: str = "default"): self.industrial_data_operation_path = IND_DATA_OPERATION_PATH self.industrial_model_path = IND_MODEL_OPERATION_PATH - self.base_data_operation_path = DEFAULT_DATA_OPERATION_PATH self.base_model_path = DEFAULT_MODEL_OPERATION_PATH - def _replace_operation(self, to_industrial=True, backend: str = 'default'): - method = INDUSTRIAL_REPLACE_METHODS if to_industrial else DEFAULT_METHODS - for class_impl, method_to_replace in zip(FEDOT_METHOD_TO_REPLACE, method): - setattr(class_impl[0], class_impl[1], method_to_replace) - if backend.__contains__('dask'): - model_to_overload = [SKLEARN_REG_MODELS, SKLEARN_CLF_MODELS, FEDOT_PREPROC_MODEL] - overloaded_model = overload_model_implementation(model_to_overload, backend=backend) - for model_impl, new_backend_impl in zip(DEFAULT_MODELS_TO_REPLACE, overloaded_model): - setattr(model_impl[0], model_impl[1], new_backend_impl) - - def setup_repository(self, backend: str = 'default'): - OperationTypesRepository.__repository_dict__.update( - {'data_operation': {'file': self.industrial_data_operation_path, - 'initialized_repo': True, - 'default_tags': []}}) - - OperationTypesRepository.assign_repo( - 'data_operation', self.industrial_data_operation_path) - - OperationTypesRepository.__repository_dict__.update( - {'model': {'file': self.industrial_model_path, - 'initialized_repo': True, - 'default_tags': []}}) - OperationTypesRepository.assign_repo( - 'model', self.industrial_model_path) - # replace mutations - self._replace_operation(to_industrial=True, backend=backend) - - class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) - ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) + self.backend = backend + self.extension = IndustrialExtension(backend=backend) + self.context: ExecutionContext | None = None + + def get_industrial_context(self) -> ExecutionContext: + self.context = ExecutionContext() + self.extension.apply(self.context) + return self.context + + def setup_repository(self) -> OperationTypesRepository: + OperationTypesRepository.__repository_dict__.update({ + 'data_operation': { + 'file': self.industrial_data_operation_path, + 'initialized_repo': True, + 'default_tags': [] + } + }) + OperationTypesRepository.assign_repo('data_operation', self.industrial_data_operation_path) + + OperationTypesRepository.__repository_dict__.update({ + 'model': { + 'file': self.industrial_model_path, + 'initialized_repo': True, + 'default_tags': [] + } + }) + OperationTypesRepository.assign_repo('model', self.industrial_model_path) + + self.get_industrial_context() + self.context.class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) + self.context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) + return OperationTypesRepository - def setup_default_repository(self, backend: str = 'default'): - """ - Switching to fedot models. - """ - OperationTypesRepository.__repository_dict__.update( - {'data_operation': {'file': self.base_data_operation_path, - 'initialized_repo': None, - 'default_tags': [ - OperationTypesRepository.DEFAULT_DATA_OPERATION_TAGS]}}) - OperationTypesRepository.assign_repo( - 'data_operation', self.base_data_operation_path) - - OperationTypesRepository.__repository_dict__.update( - {'model': {'file': self.base_model_path, - 'initialized_repo': None, - 'default_tags': []}}) + def setup_default_repository(self) -> OperationTypesRepository: + OperationTypesRepository.__repository_dict__.update({ + 'data_operation': { + 'file': self.base_data_operation_path, + 'initialized_repo': None, + 'default_tags': [OperationTypesRepository.DEFAULT_DATA_OPERATION_TAGS] + } + }) + OperationTypesRepository.assign_repo('data_operation', self.base_data_operation_path) + + OperationTypesRepository.__repository_dict__.update({ + 'model': { + 'file': self.base_model_path, + 'initialized_repo': None, + 'default_tags': [] + } + }) OperationTypesRepository.assign_repo('model', self.base_model_path) - self._replace_operation(to_industrial=False, backend=backend) - common_rules.append(has_no_resample) + self.context = ExecutionContext() + self.context.common_rules.append(has_no_resample) + return OperationTypesRepository - def __enter__(self): - """ - Switching to industrial models - """ - OperationTypesRepository.__repository_dict__.update( - {'data_operation': {'file': self.industrial_data_operation_path, - 'initialized_repo': True, - 'default_tags': []}}) - - OperationTypesRepository.assign_repo( - 'data_operation', self.industrial_data_operation_path) - - OperationTypesRepository.__repository_dict__.update( - {'model': {'file': self.industrial_model_path, - 'initialized_repo': True, - 'default_tags': []}}) - OperationTypesRepository.assign_repo( - 'model', self.industrial_model_path) - - setattr(PipelineSearchSpace, "get_parameters_dict", - get_industrial_search_space) - setattr(ApiComposer, "_get_default_mutations", - _get_default_industrial_mutations) + def __enter__(self) -> ExecutionContext: + self.setup_repository() + return self.context def __exit__(self, exc_type, exc_val, exc_tb): - """ - Switching to fedot models. - """ - OperationTypesRepository.__repository_dict__.update( - {'data_operation': {'file': self.base_data_operation_path, - 'initialized_repo': None, - 'default_tags': [ - OperationTypesRepository.DEFAULT_DATA_OPERATION_TAGS]}}) - OperationTypesRepository.assign_repo( - 'data_operation', self.base_data_operation_path) - - OperationTypesRepository.__repository_dict__.update( - {'model': {'file': self.base_model_path, - 'initialized_repo': None, - 'default_tags': []}}) - OperationTypesRepository.assign_repo('model', self.base_model_path) + self.setup_default_repository() + self.context = None diff --git a/fedot/industrial/industrial_extension.py b/fedot/industrial/industrial_extension.py new file mode 100644 index 0000000000..e955845e90 --- /dev/null +++ b/fedot/industrial/industrial_extension.py @@ -0,0 +1,63 @@ +import fedot.industrial.core.repository.model_repository as MODEL_REPO +from fedot.industrial.core.metrics.pipeline import industrial_evaluate_pipeline +from fedot.industrial.core.repository.constanst_repository import IND_DATA_OPERATION_PATH, IND_MODEL_OPERATION_PATH, DEFAULT_DATA_OPERATION_PATH, DEFAULT_MODEL_OPERATION_PATH +from fedot.industrial.core.repository.industrial_implementations.abstract import ( + preprocess_industrial_predicts, merge_industrial_predicts, merge_industrial_targets, + build_industrial, postprocess_industrial_predicts, split_any_industrial, + split_time_series_industrial, predict_operation_industrial, predict_industrial, + predict_for_fit_industrial, update_column_types_industrial, fit_topo_extractor_industrial, + transform_topo_extractor_industrial, find_main_output_industrial, get_merger_industrial +) +from fedot.industrial.core.repository.industrial_implementations.data_transformation import ( + transform_lagged_industrial, transform_lagged_for_fit_industrial, + _check_and_correct_window_size_industrial, transform_smoothing_industrial +) +from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import ( + DaskOptunaTuner, tune_pipeline_industrial +) +from fedot.industrial.core.repository.industrial_implementations.optimisation import ( + _get_default_industrial_mutations, has_no_lagged_conflicts_in_ts_pipeline, + reproduce_controlled_industrial, reproduce_industrial, + has_no_data_flow_conflicts_in_industrial_pipeline +) +from fedot.industrial.core.tuning.search_space import get_industrial_search_space + +from fedot.core.context import ExecutionContext + +class IndustrialExtension: + def __init__(self, backend: str = "default") -> None: + self.backend = backend + + def apply(self, context: ExecutionContext) -> None: + context.optuna_optuna_tuner = DaskOptunaTuner if "dask" in self.backend else OptunaTuner + context.evaluator_evaluate = industrial_evaluate_pipeline + context.search_space_get_parameters_dict = get_industrial_search_space + context.api_params_repository__get_default_mutations = _get_default_industrial_mutations + context.merger_find_main_output = find_main_output_industrial + context.merger_get = get_merger_industrial + context.merger_merge_predicts = merge_industrial_predicts + context.image_merger_preprocess_predicts = preprocess_industrial_predicts + context.image_merger_merge_predicts = merge_industrial_predicts + context.ts_merger_merge_predicts = merge_industrial_predicts + context.ts_merger_merge_targets = merge_industrial_targets + context.ts_merger_postprocess_predicts = postprocess_industrial_predicts + context.ts_merger_preprocess_predicts = preprocess_industrial_predicts + context.data_source_splitter_build = build_industrial + context.data_split__split_any = split_any_industrial + context.data_split__split_time_series = split_time_series_industrial + context.operation__predict = predict_operation_industrial + context.operation_predict = predict_industrial + context.operation_predict_for_fit = predict_for_fit_industrial + context.lagged__update_column_types = update_column_types_industrial + context.lagged_transform = transform_lagged_industrial + context.lagged_transform_for_fit = transform_lagged_for_fit_industrial + context.lagged__check_and_correct_window_size = _check_and_correct_window_size_industrial + context.topo_features_fit = fit_topo_extractor_industrial + context.topo_features_transform = transform_topo_extractor_industrial + context.ts_smoothing_transform = transform_smoothing_industrial + context.optuna_optuna_tuner = DaskOptunaTuner + context.api_composer_tune_final_pipeline = tune_pipeline_industrial + context.reproduction_reproduce = reproduce_industrial + context.reproduction_reproduce_uncontrolled = reproduce_controlled_industrial + context.class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) + context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) \ No newline at end of file From ed7ac9b280b5dc209bea62f899a03763abb0a307 Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Thu, 5 Mar 2026 13:48:52 +0400 Subject: [PATCH 02/15] udate context --- fedot/core/context.py | 52 +++++++++++++++++++++---------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/fedot/core/context.py b/fedot/core/context.py index c91ad1275b..259acd54ef 100644 --- a/fedot/core/context.py +++ b/fedot/core/context.py @@ -26,35 +26,35 @@ class ExecutionContext: def __init__(self) -> None: self.operation_registry = OperationTypesRepository() - self.evaluator_evaluate = PipelineObjectiveEvaluate - self.search_space_get_parameters_dict = PipelineSearchSpace - self.api_params_repository__get_default_mutations = ApiParamsRepository - self.merger_find_main_output = DataMerger - self.merger_get = DataMerger - self.merger_merge_predicts = DataMerger - self.image_merger_preprocess_predicts = ImageDataMerger - self.image_merger_merge_predicts = ImageDataMerger - self.ts_merger_merge_predicts = TSDataMerger - self.ts_merger_merge_targets = TSDataMerger - self.ts_merger_postprocess_predicts = TSDataMerger - self.ts_merger_preprocess_predicts = TSDataMerger - self.data_source_splitter_build = DataSourceSplitter + self.evaluator_evaluate = PipelineObjectiveEvaluate().evaluate + self.search_space_get_parameters_dict = PipelineSearchSpace().get_parameters_dict + self.api_params_repository__get_default_mutations = ApiParamsRepository()._get_default_mutations + self.merger_find_main_output = DataMerger().find_main_output + self.merger_get = DataMerger().get + self.merger_merge_predicts = DataMerger().merge_predicts + self.image_merger_preprocess_predicts = ImageDataMerger().preprocess_predicts + self.image_merger_merge_predicts = ImageDataMerger().merge_predicts + self.ts_merger_merge_predicts = TSDataMerger().merge_predicts + self.ts_merger_merge_targets = TSDataMerger().merge_targets + self.ts_merger_postprocess_predicts = TSDataMerger().postprocess_predicts + self.ts_merger_preprocess_predicts = TSDataMerger().preprocess_predicts + self.data_source_splitter_build = DataSourceSplitter().build self.data_split__split_any = fedot_data_split._split_any self.data_split__split_time_series = fedot_data_split._split_time_series - self.operation__predict = Operation - self.operation_predict = Operation - self.operation_predict_for_fit = Operation - self.lagged__update_column_types = LaggedImplementation - self.lagged_transform = LaggedImplementation - self.lagged_transform_for_fit = LaggedImplementation - self.lagged__check_and_correct_window_size = LaggedImplementation - self.topo_features_fit = TopologicalFeaturesImplementation - self.topo_features_transform = TopologicalFeaturesImplementation - self.ts_smoothing_transform = TsSmoothingImplementation + self.operation__predict = Operation()._predict + self.operation_predict = Operation().predict + self.operation_predict_for_fit = Operation().predict_for_fit + self.lagged__update_column_types = LaggedImplementation()._update_column_types + self.lagged_transform = LaggedImplementation().transform + self.lagged_transform_for_fit = LaggedImplementation().transform_for_fit + self.lagged__check_and_correct_window_size = LaggedImplementation()._check_and_correct_window_size + self.topo_features_fit = TopologicalFeaturesImplementation().fit + self.topo_features_transform = TopologicalFeaturesImplementation().transform + self.ts_smoothing_transform = TsSmoothingImplementation().transform self.optuna_optuna_tuner = OptunaImpl.OptunaTuner - self.api_composer_tune_final_pipeline = ApiComposer - self.reproduction_reproduce = ReproductionController - self.reproduction_reproduce_uncontrolled = ReproductionController + self.api_composer_tune_final_pipeline = ApiComposer().tune_final_pipeline + self.reproduction_reproduce = ReproductionController().reproduce + self.reproduction_reproduce_uncontrolled = ReproductionController().reproduce_uncontrolled self.class_rules = class_rules.copy() self.ts_rules = ts_rules.copy() self.common_rules = common_rules.copy() From bf06bdfcba10294a95043052aaddb99e01e08593 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 5 Mar 2026 15:25:36 +0000 Subject: [PATCH 03/15] Automated autopep8 fixes --- fedot/core/context.py | 5 ++--- .../core/repository/initializer_industrial_models.py | 1 + fedot/industrial/industrial_extension.py | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/fedot/core/context.py b/fedot/core/context.py index 259acd54ef..3fc89c2b95 100644 --- a/fedot/core/context.py +++ b/fedot/core/context.py @@ -1,7 +1,6 @@ from fedot.core.data.merge.data_merger import ImageDataMerger, TSDataMerger, DataMerger from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( - TopologicalFeaturesImplementation, -) + TopologicalFeaturesImplementation, ) from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( LaggedImplementation, TsSmoothingImplementation, @@ -71,4 +70,4 @@ def __setstate__(self, state): self.__init__() self.class_rules = state["class_rules"] self.ts_rules = state["ts_rules"] - self.common_rules = state["common_rules"] \ No newline at end of file + self.common_rules = state["common_rules"] diff --git a/fedot/industrial/core/repository/initializer_industrial_models.py b/fedot/industrial/core/repository/initializer_industrial_models.py index 4e9d5a36c9..2f2c737b12 100644 --- a/fedot/industrial/core/repository/initializer_industrial_models.py +++ b/fedot/industrial/core/repository/initializer_industrial_models.py @@ -42,6 +42,7 @@ from fedot.core.context import ExecutionContext from fedot.industrial.industrial_extension import IndustrialExtension + def has_no_resample(pipeline: Pipeline): """ Pipeline can have only one resample operation located in start of the pipeline diff --git a/fedot/industrial/industrial_extension.py b/fedot/industrial/industrial_extension.py index e955845e90..a2a94403c5 100644 --- a/fedot/industrial/industrial_extension.py +++ b/fedot/industrial/industrial_extension.py @@ -24,6 +24,7 @@ from fedot.core.context import ExecutionContext + class IndustrialExtension: def __init__(self, backend: str = "default") -> None: self.backend = backend @@ -60,4 +61,4 @@ def apply(self, context: ExecutionContext) -> None: context.reproduction_reproduce = reproduce_industrial context.reproduction_reproduce_uncontrolled = reproduce_controlled_industrial context.class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) - context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) \ No newline at end of file + context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) From 60c10d3fee02cf6a45b42c64ffa20ddc8a0b4dde Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Thu, 19 Mar 2026 17:35:31 +0400 Subject: [PATCH 04/15] forwared context flow --- fedot/api/api_utils/api_composer.py | 10 +- fedot/api/api_utils/api_params_repository.py | 4 +- fedot/api/main.py | 8 +- fedot/core/composer/composer_builder.py | 13 ++- .../core/composer/gp_composer/gp_composer.py | 14 ++- fedot/core/context.py | 91 +++++++++++++------ fedot/core/data/data_split.py | 24 +++-- fedot/core/data/merge/data_merger.py | 11 ++- .../data_operations/ts_transformations.py | 11 ++- .../objective/data_objective_eval.py | 3 +- fedot/industrial/industrial_extension.py | 17 +++- .../pipelines/tuning/test_pipeline_tuning.py | 3 +- test/unit/data/test_supplementary_data.py | 9 +- 13 files changed, 160 insertions(+), 58 deletions(-) diff --git a/fedot/api/api_utils/api_composer.py b/fedot/api/api_utils/api_composer.py index cfb7d6ee49..02b5463191 100644 --- a/fedot/api/api_utils/api_composer.py +++ b/fedot/api/api_utils/api_composer.py @@ -13,6 +13,7 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache from fedot.core.caching.predictions_cache import PredictionsCache +from fedot.core.context import ExecutionContext from fedot.core.composer.composer_builder import ComposerBuilder from fedot.core.composer.gp_composer.gp_composer import GPComposer from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER @@ -25,7 +26,10 @@ class ApiComposer: - def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]]): + def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]], context: Optional[ExecutionContext] = None): + + self.context = context or ExecutionContext() # fallback + self.log = default_log(self) self.params = api_params self.metrics = metrics @@ -86,7 +90,8 @@ def obtain_model(self, train_data: InputData) -> Tuple[Pipeline, Sequence[Pipeli if with_tuning: with fedot_composer_timer.launch_tuning('composing'): - best_pipeline = self.tune_final_pipeline(train_data, best_pipeline) + # best_pipeline = self.tune_final_pipeline(train_data, best_pipeline) + best_pipeline = self.context.api_composer_tune_final_pipeline(self, train_data, best_pipeline) if gp_composer.history: adapter = self.params.graph_generation_params.adapter @@ -133,6 +138,7 @@ def compose_pipeline(self, train_data: InputData, initial_assumption: Sequence[P fitted_assumption: Pipeline) -> Tuple[Pipeline, List[Pipeline], GPComposer]: gp_composer: GPComposer = (ComposerBuilder(task=self.params.task) + .with_context(self.context) .with_requirements(self.params.composer_requirements) .with_initial_pipelines(initial_assumption) .with_optimizer(self.params.get('optimizer')) diff --git a/fedot/api/api_utils/api_params_repository.py b/fedot/api/api_utils/api_params_repository.py index e8e0baf5bb..a57cc047ba 100644 --- a/fedot/api/api_utils/api_params_repository.py +++ b/fedot/api/api_utils/api_params_repository.py @@ -120,7 +120,9 @@ def get_params_for_gp_algorithm_params(self, params: dict) -> dict: if params.get('genetic_scheme') == 'steady_state': gp_algorithm_params['genetic_scheme_type'] = GeneticSchemeTypesEnum.steady_state - gp_algorithm_params['mutation_types'] = ApiParamsRepository._get_default_mutations(self.task_type, params) + # gp_algorithm_params['mutation_types'] = ApiParamsRepository._get_default_mutations(self.task_type, params) + gp_algorithm_params['mutation_types'] = context.api_params_repository__get_default_mutations(self.task_type, + params) gp_algorithm_params['seed'] = params['seed'] return gp_algorithm_params diff --git a/fedot/api/main.py b/fedot/api/main.py index f9a998f789..1a54526db1 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -38,6 +38,9 @@ from fedot.utilities.memory import MemoryAnalytics from fedot.utilities.project_import_export import export_project_to_zip, import_project_from_zip +from fedot.core.context import ExecutionContext +from fedot.industrial.industrial_extension import IndustrialContext + NOT_FITTED_ERR_MSG = 'Model not fitted yet' @@ -88,9 +91,12 @@ def __init__(self, logging_level: int = logging.ERROR, safe_mode: bool = False, n_jobs: int = -1, + context: Optional[ExecutionContext] = None, **composer_tuner_params ): + self.context = context or ExecutionContext() # fallback + set_random_seed(seed) self.log = self._init_logger(logging_level) @@ -101,7 +107,7 @@ def __init__(self, passed_metrics = self.params.get('metric') self.metrics = ensure_wrapped_in_sequence(passed_metrics) if passed_metrics else default_metrics - self.api_composer = ApiComposer(self.params, self.metrics) + self.api_composer = ApiComposer(self.params, self.metrics, self.context) # Initialize data processors for data preprocessing and preliminary data analysis self.data_processor = ApiDataProcessor(task=self.params.task, diff --git a/fedot/core/composer/composer_builder.py b/fedot/core/composer/composer_builder.py index e4c37bcbb0..51f868fb5a 100644 --- a/fedot/core/composer/composer_builder.py +++ b/fedot/core/composer/composer_builder.py @@ -13,6 +13,7 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache from fedot.core.caching.predictions_cache import PredictionsCache +from fedot.core.context import ExecutionContext from fedot.core.composer.composer import Composer from fedot.core.composer.gp_composer.gp_composer import GPComposer from fedot.core.optimisers.objective.metrics_objective import MetricsObjective @@ -58,6 +59,12 @@ def __init__(self, task: Task): self.preprocessing_cache: Optional[PreprocessingCache] = None self.predictions_cache: Optional[PredictionsCache] = None + self.context: Optional[ExecutionContext] = None + + def with_context(self, context: ExecutionContext): + self.context = context or ExecutionContext() + return self + def with_composer(self, composer_cls: Optional[Type[Composer]]): if composer_cls is not None: self.composer_cls = composer_cls @@ -111,7 +118,8 @@ def with_cache(self, @staticmethod def _get_default_composer_params(task: Task) -> PipelineComposerRequirements: # Get all available operations for task - operations = get_operations_for_task(task=task, mode='all') + # operations = get_operations_for_task(task=task, mode='all') + operations = self.context.operation_registry.get_operation_for_task(task=task, mode='all') return PipelineComposerRequirements(primary=operations, secondary=operations) def _get_default_graph_generation_params(self) -> GraphGenerationParams: @@ -161,6 +169,7 @@ def build(self) -> Composer: self.composer_requirements, self.operations_cache, self.preprocessing_cache, - self.predictions_cache) + self.predictions_cache, + self.context) return composer diff --git a/fedot/core/composer/gp_composer/gp_composer.py b/fedot/core/composer/gp_composer/gp_composer.py index b4b7a7421a..2a8297bf39 100644 --- a/fedot/core/composer/gp_composer/gp_composer.py +++ b/fedot/core/composer/gp_composer/gp_composer.py @@ -11,6 +11,7 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.predictions_cache import PredictionsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache +from fedot.core.context import ExecutionContext from fedot.core.composer.composer import Composer from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData @@ -24,6 +25,7 @@ ) from fedot.core.utils import default_fedot_data_dir +from functools import partial class GPComposer(Composer): """ @@ -40,7 +42,8 @@ def __init__(self, optimizer: GraphOptimizer, composer_requirements: PipelineComposerRequirements, operations_cache: Optional[OperationsCache] = None, preprocessing_cache: Optional[PreprocessingCache] = None, - predictions_cache: Optional[PredictionsCache] = None): + predictions_cache: Optional[PredictionsCache] = None, + context: Optional[ExectuionContext] = None): super().__init__(optimizer, composer_requirements) self.composer_requirements = composer_requirements self.operations_cache: Optional[OperationsCache] = operations_cache @@ -49,11 +52,14 @@ def __init__(self, optimizer: GraphOptimizer, self.best_models: Collection[Pipeline] = () + self.context = context or ExecutionContext() + def compose_pipeline(self, data: Union[InputData, MultiModalData]) -> Union[Pipeline, Sequence[Pipeline]]: # Define data source data_splitter = DataSourceSplitter(self.composer_requirements.cv_folds, shuffle=True) - data_producer = data_splitter.build(data) + + data_producer = self.context.data_source_splitter_build(data_splitter, data) parallelization_mode = self.composer_requirements.parallelization_mode if parallelization_mode == 'populational': @@ -72,7 +78,9 @@ def compose_pipeline(self, data: Union[InputData, MultiModalData]) -> Union[Pipe predictions_cache=self.predictions_cache, validation_blocks=data_splitter.validation_blocks, eval_n_jobs=n_jobs_for_evaluation) - objective_function = objective_evaluator.evaluate + + # objective_function = objective_evaluator.evaluate + objective_function = partial(self.context.evaluator_evaluate, objective_evaluator) # Define callback for computing intermediate metrics if needed if self.composer_requirements.collect_intermediate_metric: diff --git a/fedot/core/context.py b/fedot/core/context.py index 259acd54ef..a0c6ccda31 100644 --- a/fedot/core/context.py +++ b/fedot/core/context.py @@ -22,53 +22,84 @@ import fedot.core.data.data_split as fedot_data_split import golem.core.tuning.optuna_tuner as OptunaImpl - class ExecutionContext: def __init__(self) -> None: + """Initializes ExecutionContext with default configuration.""" + self._init_defaults() + + def _init_defaults(self): + """Sets default implementations for all pipeline components. + + Initializes: + - operation registry + - evaluators + - splitters + - merge strategies + - feature transforms + - rules + """ + self.extension = None + self.backend = "default" + self.operation_registry = OperationTypesRepository() - self.evaluator_evaluate = PipelineObjectiveEvaluate().evaluate - self.search_space_get_parameters_dict = PipelineSearchSpace().get_parameters_dict - self.api_params_repository__get_default_mutations = ApiParamsRepository()._get_default_mutations - self.merger_find_main_output = DataMerger().find_main_output - self.merger_get = DataMerger().get - self.merger_merge_predicts = DataMerger().merge_predicts - self.image_merger_preprocess_predicts = ImageDataMerger().preprocess_predicts - self.image_merger_merge_predicts = ImageDataMerger().merge_predicts - self.ts_merger_merge_predicts = TSDataMerger().merge_predicts - self.ts_merger_merge_targets = TSDataMerger().merge_targets - self.ts_merger_postprocess_predicts = TSDataMerger().postprocess_predicts - self.ts_merger_preprocess_predicts = TSDataMerger().preprocess_predicts - self.data_source_splitter_build = DataSourceSplitter().build + self.evaluator_evaluate = PipelineObjectiveEvaluate.evaluate + self.search_space_get_parameters_dict = PipelineSearchSpace.get_parameters_dict + self.api_params_repository__get_default_mutations = ApiParamsRepository._get_default_mutations + self.merger_find_main_output = DataMerger.find_main_output + self.merger_get = DataMerger.get + self.merger_merge_predicts = DataMerger.merge_predicts + self.image_merger_preprocess_predicts = ImageDataMerger.preprocess_predicts + self.image_merger_merge_predicts = ImageDataMerger.merge_predicts + self.ts_merger_merge_predicts = TSDataMerger.merge_predicts + self.ts_merger_merge_targets = TSDataMerger.merge_targets + self.ts_merger_postprocess_predicts = TSDataMerger.postprocess_predicts + self.ts_merger_preprocess_predicts = TSDataMerger.preprocess_predicts + self.data_source_splitter_build = DataSourceSplitter.build self.data_split__split_any = fedot_data_split._split_any self.data_split__split_time_series = fedot_data_split._split_time_series - self.operation__predict = Operation()._predict - self.operation_predict = Operation().predict - self.operation_predict_for_fit = Operation().predict_for_fit - self.lagged__update_column_types = LaggedImplementation()._update_column_types - self.lagged_transform = LaggedImplementation().transform - self.lagged_transform_for_fit = LaggedImplementation().transform_for_fit - self.lagged__check_and_correct_window_size = LaggedImplementation()._check_and_correct_window_size - self.topo_features_fit = TopologicalFeaturesImplementation().fit - self.topo_features_transform = TopologicalFeaturesImplementation().transform - self.ts_smoothing_transform = TsSmoothingImplementation().transform + self.operation__predict = Operation._predict + self.operation_predict = Operation.predict + self.operation_predict_for_fit = Operation.predict_for_fit + self.lagged__update_column_types = LaggedImplementation._update_column_types + self.lagged_transform = LaggedImplementation.transform + self.lagged_transform_for_fit = LaggedImplementation.transform_for_fit + self.lagged__check_and_correct_window_size = LaggedImplementation._check_and_correct_window_size + self.topo_features_fit = TopologicalFeaturesImplementation.fit + self.topo_features_transform = TopologicalFeaturesImplementation.transform + self.ts_smoothing_transform = TsSmoothingImplementation.transform self.optuna_optuna_tuner = OptunaImpl.OptunaTuner - self.api_composer_tune_final_pipeline = ApiComposer().tune_final_pipeline - self.reproduction_reproduce = ReproductionController().reproduce - self.reproduction_reproduce_uncontrolled = ReproductionController().reproduce_uncontrolled + self.api_composer_tune_final_pipeline = ApiComposer.tune_final_pipeline + self.reproduction_reproduce = ReproductionController.reproduce + self.reproduction_reproduce_uncontrolled = ReproductionController.reproduce_uncontrolled self.class_rules = class_rules.copy() self.ts_rules = ts_rules.copy() self.common_rules = common_rules.copy() def __getstate__(self): return { - "industrial": True, + "backend": self.backend, "class_rules": self.class_rules, "ts_rules": self.ts_rules, "common_rules": self.common_rules, + "extension_state": getattr(self.extension, "get_state", lambda: None)(), + "has_extension": self.extension is not None, } def __setstate__(self, state): - self.__init__() + self.__dict__.clear() + + self._init_defaults() + + self.backend = state.get("backend", "default") self.class_rules = state["class_rules"] self.ts_rules = state["ts_rules"] - self.common_rules = state["common_rules"] \ No newline at end of file + self.common_rules = state["common_rules"] + + if state.get("has_extension", False): + self.extension = IndustrialExtension(backend=self.backend) + + ext_state = state.get("extension_state") + if ext_state and hasattr(self.extension, "set_state"): + self.extension.set_state(ext_state) + + self.extension.apply(self) diff --git a/fedot/core/data/data_split.py b/fedot/core/data/data_split.py index a000c6e46b..8b48223982 100644 --- a/fedot/core/data/data_split.py +++ b/fedot/core/data/data_split.py @@ -6,6 +6,7 @@ from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData +from fedot.core.context import ExecutionContext from fedot.core.repository.dataset_types import DataTypesEnum from fedot.core.repository.tasks import TaskTypesEnum @@ -173,8 +174,8 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], shuffle_flag: bool = False, stratify: bool = True, random_seed: int = 42, - validation_blocks: Optional[int] = None) -> Tuple[Union[InputData, MultiModalData], - Union[InputData, MultiModalData]]: + validation_blocks: Optional[int] = None, + context: Optional[ExecutionContext] = None) -> Tuple[Union[InputData, MultiModalData], Union[InputData, MultiModalData]]: """ Function for train and test split for both InputData and MultiModalData :param data: InputData object to split @@ -184,10 +185,13 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], :param stratify: make stratified sample or not :param random_seed: random_seed for shuffle :param validation_blocks: validation blocks are used for test + :param context: FEDOT Core or Fedot.Industrial funcs :return: data for train, data for validation """ + context = context or ExecutionContext() + # for backward compatibility shuffle |= shuffle_flag # check that stratification may be done @@ -203,11 +207,17 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], 'random_seed': random_seed, 'validation_blocks': validation_blocks} if isinstance(data, InputData): - split_func_dict = {DataTypesEnum.multi_ts: _split_time_series, - DataTypesEnum.ts: _split_time_series, - DataTypesEnum.table: _split_any, - DataTypesEnum.image: _split_any, - DataTypesEnum.text: _split_any} + # split_func_dict = {DataTypesEnum.multi_ts: _split_time_series, + # DataTypesEnum.ts: _split_time_series, + # DataTypesEnum.table: _split_any, + # DataTypesEnum.image: _split_any, + # DataTypesEnum.text: _split_any} + + split_func_dict = {DataTypesEnum.multi_ts: context.data_split__split_time_series, + DataTypesEnum.ts: context.data_split__split_time_series, + DataTypesEnum.table: context.data_split__split_any, + DataTypesEnum.image: context.data_split__split_any, + DataTypesEnum.text: context.data_split__split_any} if data.data_type not in split_func_dict: raise TypeError((f'Unknown data type {type(data)}. Supported data types:' diff --git a/fedot/core/data/merge/data_merger.py b/fedot/core/data/merge/data_merger.py index a1dc312f0b..064075c594 100644 --- a/fedot/core/data/merge/data_merger.py +++ b/fedot/core/data/merge/data_merger.py @@ -8,6 +8,7 @@ from fedot.core.data.array_utilities import find_common_elements, atleast_2d, atleast_4d, flatten_extra_dim from fedot.core.data.data import OutputData, InputData from fedot.core.data.merge.supplementary_data_merger import SupplementaryDataMerger +from fedot.core.context import ExecutionContext from fedot.core.repository.dataset_types import DataTypesEnum @@ -23,10 +24,12 @@ class DataMerger: :param outputs: list with OutputData from parent nodes for merging """ - def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None): + def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None, + context: Optional[ExecutionContext] = None): self.log = default_log(self) self.outputs = outputs self.data_type = data_type or DataMerger.get_datatype_for_merge(output.data_type for output in outputs) + self.context = context or ExecutionContext() # Ensure outputs are of equal length, find common index if it is not idx_list = [np.asarray(output.idx) for output in outputs] @@ -35,7 +38,8 @@ def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None) raise ValueError('There are no common indices for outputs') # Find first output with the main target & resulting task - self.main_output = DataMerger.find_main_output(outputs) + # self.main_output = DataMerger.find_main_output(outputs) + self.main_output = self.context.merger_find_main_output(outputs) @staticmethod def get(outputs: List['OutputData']) -> 'DataMerger': @@ -71,7 +75,8 @@ def merge(self) -> 'InputData': common_predicts = self.find_common_predicts() mergeable_predicts = self.preprocess_predicts(common_predicts) - merged_features = self.merge_predicts(mergeable_predicts) + # merged_features = self.merge_predicts(mergeable_predicts) + merged_features = self.context.merger_merge_predicts(mergeable_predicts) merged_features = self.postprocess_predicts(merged_features) updated_metadata = SupplementaryDataMerger(self.outputs, self.main_output).merge() diff --git a/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py b/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py index 7222872be4..bd141cb1df 100644 --- a/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py +++ b/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py @@ -11,6 +11,7 @@ from sklearn.decomposition import TruncatedSVD from fedot.core.data.data import InputData, OutputData +from fedot.core.context import ExecutionContext from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ( DataOperationImplementation ) @@ -20,9 +21,11 @@ class LaggedImplementation(DataOperationImplementation): - def __init__(self, params: Optional[OperationParameters]): + def __init__(self, params: Optional[OperationParameters], context: Optional[ExecutionContext] = None): super().__init__(params) + self.context = context or ExecutionContext() + self.window_size_minimum = None self.sparse_transform = False self.use_svd = False @@ -72,7 +75,8 @@ def transform(self, input_data: InputData) -> OutputData: output_data = self._convert_to_output(new_input_data, self.features_columns, data_type=DataTypesEnum.table) - self._update_column_types(output_data) + # self._update_column_types(output_data) + self.context.lagged__update_column_types(self, output_data) return output_data def transform_for_fit(self, input_data: InputData) -> OutputData: @@ -103,7 +107,8 @@ def transform_for_fit(self, input_data: InputData) -> OutputData: output_data = self._convert_to_output(new_input_data, self.features_columns, data_type=DataTypesEnum.table) - self._update_column_types(output_data) + # self._update_column_types(output_data) + self.context.lagged__update_column_types(self, output_data) return output_data def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): diff --git a/fedot/core/optimisers/objective/data_objective_eval.py b/fedot/core/optimisers/objective/data_objective_eval.py index ddfe9cca30..4c5b6871e1 100644 --- a/fedot/core/optimisers/objective/data_objective_eval.py +++ b/fedot/core/optimisers/objective/data_objective_eval.py @@ -44,7 +44,8 @@ def __init__(self, preprocessing_cache: Optional[PreprocessingCache] = None, predictions_cache: Optional[PredictionsCache] = None, eval_n_jobs: int = 1, - do_unfit: bool = True): + do_unfit: bool = True, + context: Optional[ExecutionContext] = None): super().__init__(objective, eval_n_jobs=eval_n_jobs) self._data_producer = data_producer self._time_constraint = time_constraint diff --git a/fedot/industrial/industrial_extension.py b/fedot/industrial/industrial_extension.py index e955845e90..03c57f77b5 100644 --- a/fedot/industrial/industrial_extension.py +++ b/fedot/industrial/industrial_extension.py @@ -25,10 +25,12 @@ from fedot.core.context import ExecutionContext class IndustrialExtension: + """Overrides ExecutionContext with industrial implementations.""" def __init__(self, backend: str = "default") -> None: self.backend = backend def apply(self, context: ExecutionContext) -> None: + """Mutates context with industrial implementations.""" context.optuna_optuna_tuner = DaskOptunaTuner if "dask" in self.backend else OptunaTuner context.evaluator_evaluate = industrial_evaluate_pipeline context.search_space_get_parameters_dict = get_industrial_search_space @@ -55,9 +57,20 @@ def apply(self, context: ExecutionContext) -> None: context.topo_features_fit = fit_topo_extractor_industrial context.topo_features_transform = transform_topo_extractor_industrial context.ts_smoothing_transform = transform_smoothing_industrial - context.optuna_optuna_tuner = DaskOptunaTuner context.api_composer_tune_final_pipeline = tune_pipeline_industrial context.reproduction_reproduce = reproduce_industrial context.reproduction_reproduce_uncontrolled = reproduce_controlled_industrial context.class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) - context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) \ No newline at end of file + context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) + +class IndustrialContext(ExecutionContext): + """Fedot.Industrial execution context""" + def __init__(self, backend: str = "default") -> None: + super().__init__() + + self.operation_registry = OperationTypesRepository() + self.operation_registry.load_operations(IND_DATA_OPERATION_PATH, 'data_operation') + self.operation_registry.load_operations(IND_MODEL_OPERATION_PATH, 'model') + + self.extension = IndustrialExtension(backend=backend) + self.extension.apply(self) diff --git a/test/integration/pipelines/tuning/test_pipeline_tuning.py b/test/integration/pipelines/tuning/test_pipeline_tuning.py index 3fcc228243..d88098273d 100644 --- a/test/integration/pipelines/tuning/test_pipeline_tuning.py +++ b/test/integration/pipelines/tuning/test_pipeline_tuning.py @@ -232,7 +232,8 @@ def run_pipeline_tuner(train_data, else: search_space.custom_search_space['lagged'] = ssp search_space.replace_default_search_space = True - search_space.parameters_per_operation = search_space.get_parameters_dict() + # search_space.parameters_per_operation = search_space.get_parameters_dict() + search_space.parameters_per_operation = context.search_space_get_parameters_dict(search_space) # Pipeline tuning pipeline_tuner = TunerBuilder(train_data.task) \ diff --git a/test/unit/data/test_supplementary_data.py b/test/unit/data/test_supplementary_data.py index 0a4f9beaa1..732a1859ca 100644 --- a/test/unit/data/test_supplementary_data.py +++ b/test/unit/data/test_supplementary_data.py @@ -5,6 +5,7 @@ from fedot.core.data.merge.data_merger import DataMerger from fedot.core.data.merge.supplementary_data_merger import SupplementaryDataMerger from fedot.core.data.supplementary_data import SupplementaryData +from fedot.core.context import ExecutionContext from fedot.core.pipelines.node import PipelineNode from fedot.core.pipelines.pipeline import Pipeline from fedot.core.repository.dataset_types import DataTypesEnum @@ -46,12 +47,16 @@ def generate_straight_pipeline(): return pipeline -def test_parent_mask_correct(unequal_outputs_table): # noqa, fixture +def test_parent_mask_correct(unequal_outputs_table, context: Optional[ExecutionContext] = None): # noqa, fixture """ Test correctness of function for tables mask generation """ + + context = context or ExecutrionContext() + correct_parent_mask = {'input_ids': [0, 1], 'flow_lens': [1, 0]} # Calculate parent mask from outputs - main_output = DataMerger.find_main_output(unequal_outputs_table) + # main_output = DataMerger.find_main_output(unequal_outputs_table) + main_output = context.merger_find_main_output(unequal_outputs_table) p_mask = SupplementaryDataMerger(unequal_outputs_table, main_output).prepare_parent_mask() assert tuple(p_mask['input_ids']) == tuple(correct_parent_mask['input_ids']) From 9fcb7acba72949e96a7d6985d4197ea4e206d59a Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Tue, 21 Apr 2026 15:52:55 +0400 Subject: [PATCH 05/15] introduced industrial context manifest & protocols --- fedot/api/main.py | 9 +- fedot/core/context/__init__.py | 0 fedot/core/context/context.py | 169 ++++++++++++++++++++ fedot/core/context/factories.py | 60 ++++++++ fedot/core/context/industrial_backend.py | 159 +++++++++++++++++++ fedot/core/context/industrial_manifest.py | 34 ++++ fedot/core/protocols/__init__.py | 0 fedot/core/protocols/protocols.py | 109 +++++++++++++ fedot/extensions/contracts.py | 53 +++++++ fedot/extensions/registry.py | 180 ++++++++++++++++++++++ 10 files changed, 768 insertions(+), 5 deletions(-) create mode 100644 fedot/core/context/__init__.py create mode 100644 fedot/core/context/context.py create mode 100644 fedot/core/context/factories.py create mode 100644 fedot/core/context/industrial_backend.py create mode 100644 fedot/core/context/industrial_manifest.py create mode 100644 fedot/core/protocols/__init__.py create mode 100644 fedot/core/protocols/protocols.py create mode 100644 fedot/extensions/contracts.py create mode 100644 fedot/extensions/registry.py diff --git a/fedot/api/main.py b/fedot/api/main.py index 1a54526db1..9df24d0da5 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -38,8 +38,7 @@ from fedot.utilities.memory import MemoryAnalytics from fedot.utilities.project_import_export import export_project_to_zip, import_project_from_zip -from fedot.core.context import ExecutionContext -from fedot.industrial.industrial_extension import IndustrialContext +from fedot.core.context.context import resolve_context NOT_FITTED_ERR_MSG = 'Model not fitted yet' @@ -91,15 +90,15 @@ def __init__(self, logging_level: int = logging.ERROR, safe_mode: bool = False, n_jobs: int = -1, - context: Optional[ExecutionContext] = None, + context: Optional[str] = None, **composer_tuner_params ): - self.context = context or ExecutionContext() # fallback - set_random_seed(seed) self.log = self._init_logger(logging_level) + self.context = resolve_context(context) + # Attributes for dealing with metrics, data sources and hyperparameters self.params = ApiParams(composer_tuner_params, problem, task_params, n_jobs, timeout, seed) diff --git a/fedot/core/context/__init__.py b/fedot/core/context/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fedot/core/context/context.py b/fedot/core/context/context.py new file mode 100644 index 0000000000..786f07559b --- /dev/null +++ b/fedot/core/context/context.py @@ -0,0 +1,169 @@ +from fedot.core.data.merge.data_merger import ImageDataMerger, TSDataMerger, DataMerger +from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( + TopologicalFeaturesImplementation, ) +from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation, + TsSmoothingImplementation, +) +from fedot.core.operations.operation import Operation +from fedot.core.optimisers.objective import PipelineObjectiveEvaluate +from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter +from fedot.core.pipelines.pipeline import Pipeline +from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace +from fedot.core.pipelines.verification import class_rules, ts_rules, common_rules +from fedot.core.repository.operation_types_repository import OperationTypesRepository +from fedot.core.data.data_split import _split_any, _split_time_series +from fedot.api.api_utils.api_params_repository import ApiParamsRepository +from fedot.api.api_utils.api_composer import ApiComposer +from golem.core.tuning.optuna_tuner import OptunaTuner +from golem.core.optimisers.genetic.operators.reproduction import ReproductionController + +import fedot.core.data.data_split as fedot_data_split +import golem.core.tuning.optuna_tuner as OptunaImpl + +def resolve_context(context_name: str = "core", backend: str = "default") -> ExecutionContext: + if context_name == "core": + return ExecutionContext(backend=backend) + + from fedot.extensions.registry import get_registered_extension, get_registered_extensions + + ext = get_registered_extension(context_name) + if ext is not None: + factory = ext.value.manifest.protocols.get("context_factory") + if factory: + return factory(backend=backend) + + raise ValueError(f"Unknown context: {context_name}") + +class ExecutionContext: + def __init__(self, backend: str = "default") -> None: + """Initializes ExecutionContext with default configuration.""" + self.backend = backend + self._init_defaults() + self._apply_protocols() + + def _init_defaults(self): + """Sets default implementations for all pipeline components.""" + self.evaluator_evaluate = PipelineObjectiveEvaluate.evaluate + self.search_space_get_parameters_dict = PipelineSearchSpace.get_parameters_dict + self.api_params_repository__get_default_mutations = ApiParamsRepository._get_default_mutations + self.merger_find_main_output = DataMerger.find_main_output + self.merger_get = DataMerger.get + self.merger_merge_predicts = DataMerger.merge_predicts + self.image_merger_preprocess_predicts = ImageDataMerger.preprocess_predicts + self.image_merger_merge_predicts = ImageDataMerger.merge_predicts + self.ts_merger_merge_predicts = TSDataMerger.merge_predicts + self.ts_merger_merge_targets = TSDataMerger.merge_targets + self.ts_merger_postprocess_predicts = TSDataMerger.postprocess_predicts + self.ts_merger_preprocess_predicts = TSDataMerger.preprocess_predicts + self.data_source_splitter_build = DataSourceSplitter.build + self.data_split__split_any = fedot_data_split._split_any + self.data_split__split_time_series = fedot_data_split._split_time_series + self.operation__predict = Operation._predict + self.operation_predict = Operation.predict + self.operation_predict_for_fit = Operation.predict_for_fit + self.lagged__update_column_types = LaggedImplementation._update_column_types + self.lagged_transform = LaggedImplementation.transform + self.lagged_transform_for_fit = LaggedImplementation.transform_for_fit + self.lagged__check_and_correct_window_size = LaggedImplementation._check_and_correct_window_size + self.topo_features_fit = TopologicalFeaturesImplementation.fit + self.topo_features_transform = TopologicalFeaturesImplementation.transform + self.ts_smoothing_transform = TsSmoothingImplementation.transform + self.optuna_optuna_tuner = OptunaImpl.OptunaTuner + self.api_composer_tune_final_pipeline = ApiComposer.tune_final_pipeline + self.reproduction_reproduce = ReproductionController.reproduce + self.reproduction_reproduce_uncontrolled = ReproductionController.reproduce_uncontrolled + self.class_rules = class_rules.copy() + self.ts_rules = ts_rules.copy() + self.common_rules = common_rules.copy() + + def _apply_protocols(self): + # Splitters + splitters = resolve_protocol_instance("splitters", backend=self.backend) + if splitters: + self.data_split__split_any = splitters.split_any + self.data_split__split_time_series = splitters.split_time_series + + # Mergers + mergers = resolve_protocol_instance("mergers", backend=self.backend) + if mergers: + self.merger_find_main_output = mergers.find_main_output + self.merger_get = mergers.get + self.merger_merge_predicts = mergers.merge_predicts + if hasattr(mergers, 'preprocess_predicts'): + self.image_merger_preprocess_predicts = mergers.preprocess_predicts + self.ts_merger_preprocess_predicts = mergers.preprocess_predicts + if hasattr(mergers, 'postprocess_predicts'): + self.ts_merger_postprocess_predicts = mergers.postprocess_predicts + if hasattr(mergers, 'merge_targets'): + self.ts_merger_merge_targets = mergers.merge_targets + # Image merge обычно совпадает с основным + self.image_merger_merge_predicts = mergers.merge_predicts + + # DataSourceSplitter + splitter_builder = resolve_protocol_instance("data_source_splitter", backend=self.backend) + if splitter_builder: + self.data_source_splitter_build = splitter_builder.build + + # Tuner class + tuner_class = resolve_protocol_instance("tuner_class", backend=self.backend) + if tuner_class: + self.optuna_optuna_tuner = tuner_class + + # Reproduction + reproduction = resolve_protocol_instance("reproduction", backend=self.backend) + if reproduction: + self.reproduction_reproduce = reproduction.reproduce + if hasattr(reproduction, 'reproduce_uncontrolled'): + self.reproduction_reproduce_uncontrolled = reproduction.reproduce_uncontrolled + + # Evaluator + evaluator = resolve_protocol_instance("evaluator", backend=self.backend) + if evaluator: + self.evaluator_evaluate = evaluator.evaluate + + # Search space + search_space = resolve_protocol_instance("search_space", backend=self.backend) + if search_space: + self.search_space_get_parameters_dict = search_space.get_parameters_dict + + # Mutations + mutations = resolve_protocol_instance("default_mutations", backend=self.backend) + if mutations: + self.api_params_repository__get_default_mutations = mutations + + # Operation predict + op_predict = resolve_protocol_instance("operation_predict", backend=self.backend) + if op_predict: + self.operation_predict = op_predict.predict + self.operation_predict_for_fit = op_predict.predict_for_fit + if hasattr(op_predict, '_predict'): + self.operation__predict = op_predict._predict + + # Lagged transformer + lagged = resolve_protocol_instance("lagged_transformer", backend=self.backend) + if lagged: + self.lagged__update_column_types = lagged._update_column_types + self.lagged_transform = lagged.transform + self.lagged_transform_for_fit = lagged.transform_for_fit + self.lagged__check_and_correct_window_size = lagged._check_and_correct_window_size + + # Topological features + topo = resolve_protocol_instance("topological_features", backend=self.backend) + if topo: + self.topo_features_fit = topo.fit + self.topo_features_transform = topo.transform + + # TS Smoothing + smoothing = resolve_protocol_instance("ts_smoothing", backend=self.backend) + if smoothing: + self.ts_smoothing_transform = smoothing.transform + + # ApiComposer tune + tune = resolve_protocol_instance("api_composer_tune", backend=self.backend) + if tune: + self.api_composer_tune_final_pipeline = tune + + @cached_property + def set_operation_registry(self) -> OperationTypesRepository: + return OperationTypesRepository() \ No newline at end of file diff --git a/fedot/core/context/factories.py b/fedot/core/context/factories.py new file mode 100644 index 0000000000..3ed22ca720 --- /dev/null +++ b/fedot/core/context/factories.py @@ -0,0 +1,60 @@ +from fedot.core.context.industrial_backend import (IndustrialSplitter, IndustrialDataMerger, IndustrialImageMerger, + IndustrialTSMerger, IndustrialTextMerger, + IndustrialDataSourceSplitterBuilder, IndustrialTunerClass, + IndustrialReproduction, IndustrialEvaluator, IndustrialSearchSpace, + IndustrialDefaultMutations,IndustrialOperationPredict, + IndustrialLaggedTransformer, IndustrialTopologicalFeatures, + IndustrialTsSmoothing, IndustrialApiComposerTune) + + + +def industrial_context_factory(backend: str = "default"): + return IndustrialContext(backend=backend) + +def splitters_factory(): + return IndustrialSplitter() + +def data_merger_factory(): + return IndustrialDataMerger() + +def image_merger_factory(): + return IndustrialImageMerger() + +def ts_merger_factory(): + return IndustrialTSMerger() + +def text_merger_factory(): + return IndustrialTextMerger() + +def data_source_splitter_factory(): + return IndustrialDataSourceSplitterBuilder() + +def tuner_class_factory(backend: str = "default"): + return IndustrialTunerClass(backend) + +def reproduction_factory(): + return IndustrialReproduction() + +def evaluator_factory(): + return IndustrialEvaluator() + +def search_space_factory(): + return IndustrialSearchSpace() + +def mutations_factory(): + return IndustrialDefaultMutations() + +def operation_predict_factory(): + return IndustrialOperationPredict() + +def lagged_transformer_factory(): + return IndustrialLaggedTransformer() + +def topo_features_factory(): + return IndustrialTopologicalFeatures() + +def ts_smoothing_factory(): + return IndustrialTsSmoothing() + +def api_composer_tune_factory(): + return IndustrialApiComposerTune() \ No newline at end of file diff --git a/fedot/core/context/industrial_backend.py b/fedot/core/context/industrial_backend.py new file mode 100644 index 0000000000..07f0280c68 --- /dev/null +++ b/fedot/core/context/industrial_backend.py @@ -0,0 +1,159 @@ +from fedot.industrial.core.metrics.pipeline import industrial_evaluate_pipeline +from fedot.industrial.core.repository.constanst_repository import IND_DATA_OPERATION_PATH, IND_MODEL_OPERATION_PATH, DEFAULT_DATA_OPERATION_PATH, DEFAULT_MODEL_OPERATION_PATH +from fedot.industrial.core.repository.industrial_implementations.abstract import ( + preprocess_industrial_predicts, merge_industrial_predicts, merge_industrial_targets, + build_industrial, postprocess_industrial_predicts, split_any_industrial, + split_time_series_industrial, predict_operation_industrial, predict_industrial, + predict_for_fit_industrial, update_column_types_industrial, fit_topo_extractor_industrial, + transform_topo_extractor_industrial, find_main_output_industrial, get_merger_industrial +) +from fedot.industrial.core.repository.industrial_implementations.data_transformation import ( + transform_lagged_industrial, transform_lagged_for_fit_industrial, + _check_and_correct_window_size_industrial, transform_smoothing_industrial +) +from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import ( + DaskOptunaTuner, tune_pipeline_industrial +) +from fedot.industrial.core.repository.industrial_implementations.optimisation import ( + _get_default_industrial_mutations, has_no_lagged_conflicts_in_ts_pipeline, + reproduce_controlled_industrial, reproduce_industrial, + has_no_data_flow_conflicts_in_industrial_pipeline +) +from fedot.industrial.core.tuning.search_space import get_industrial_search_space + + +class IndustrialSplitter: + def split_any(self, data: InputData, split_ratio: float, shuffle: bool, + stratify: bool, random_seed: int, **kwargs): + return split_any_industrial(data, split_ratio, shuffle, stratify, random_seed, **kwargs) + + def split_time_series(self, data: InputData, validation_blocks: Optional[int] = None, **kwargs): + return split_time_series_industrial(data, validation_blocks, **kwargs) + + +class IndustrialDataMerger: + @staticmethod + def get(outputs: List[OutputData]): + return get_merger_industrial(outputs) + + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + return merge_industrial_predicts(predicts) + + @staticmethod + def find_main_output(outputs: List[OutputData]) -> OutputData: + return find_main_output_industrial(outputs) + + +class IndustrialImageMerger: + def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: + return image_preprocess(predicts) + + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + return merge_industrial_predicts(predicts) + + +class IndustrialTSMerger: + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + return merge_industrial_predicts(predicts) + + def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: + return merge_industrial_targets(targets) + + def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: + return image_preprocess(predicts) # или своя ts_preprocess + + def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + return postprocess_industrial_predicts(merged) + + +class IndustrialTextMerger: + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + return merge_industrial_predicts(predicts) + + def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + return merged + + +class IndustrialDataSourceSplitterBuilder: + def build(self, data: Union[InputData, 'MultiModalData']): + return build_industrial(data) + + +class IndustrialTunerClass: + def __init__(self, backend: str = "default"): + self.backend = backend + + def __call__(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): + if "dask" in self.backend: + return DaskOptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) + else: + return OptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) + + +class IndustrialReproduction: + def reproduce(self, population, evaluator, **kwargs): + return reproduce_industrial(population, evaluator, **kwargs) + + def reproduce_uncontrolled(self, population, **kwargs): + return reproduce_controlled_industrial(population, **kwargs) + + +class IndustrialEvaluator: + def evaluate(self, graph: Pipeline) -> Fitness: + return industrial_evaluate_pipeline(graph) + + +class IndustrialSearchSpace: + def get_parameters_dict(self): + return get_industrial_search_space() + + +class IndustrialDefaultMutations: + @staticmethod + def __call__(task_type: TaskTypesEnum, params): + return _get_default_industrial_mutations(task_type, params) + + +class IndustrialOperationPredict: + def predict(self, fitted_operation, data: InputData, params=None, output_mode='default'): + return predict_industrial(fitted_operation, data, params, output_mode) + + def predict_for_fit(self, fitted_operation, data: InputData, params=None, output_mode='default'): + return predict_for_fit_industrial(fitted_operation, data, params, output_mode) + + def _predict(self, fitted_operation, data: InputData, params=None, output_mode='default', + is_fit_stage=False, predictions_cache=None, fold_id=None, descriptive_id=None): + return predict_operation_industrial(fitted_operation, data, params, output_mode, + is_fit_stage, predictions_cache, fold_id, descriptive_id) + + +class IndustrialLaggedTransformer: + def _update_column_types(self, output_data: OutputData): + update_column_types_industrial(output_data) + + def transform(self, input_data: InputData) -> OutputData: + return transform_lagged_industrial(input_data) + + def transform_for_fit(self, input_data: InputData) -> OutputData: + return transform_lagged_for_fit_industrial(input_data) + + def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): + _check_and_correct_window_size_industrial(time_series, forecast_length) + + +class IndustrialTopologicalFeatures: + def fit(self, input_data: InputData): + return fit_topo_extractor_industrial(input_data) + + def transform(self, input_data: InputData) -> np.ndarray: + return transform_topo_extractor_industrial(input_data) + + +class IndustrialTsSmoothing: + def transform(self, input_data: InputData) -> OutputData: + return transform_smoothing_industrial(input_data) + + +class IndustrialApiComposerTune: + def tune_pipeline_industrial(self, train_data: InputData, pipeline: Pipeline, execution_plan=None) -> Pipeline: + return tune_pipeline_industrial(train_data, pipeline, execution_plan) \ No newline at end of file diff --git a/fedot/core/context/industrial_manifest.py b/fedot/core/context/industrial_manifest.py new file mode 100644 index 0000000000..3738091169 --- /dev/null +++ b/fedot/core/context/industrial_manifest.py @@ -0,0 +1,34 @@ +from fedot.extensions.contracts import ExtensionManifest +from fedot.core.context.factories import ( industrial_context_factory, splitters_factory, + data_merger_factory, image_merger_factory, ts_merger_factory, + text_merger_factory, data_source_splitter_factory, tuner_class_factory, + reproduction_factory, evaluator_factory, search_space_factory, + mutations_factory, operation_predict_factory, lagged_transformer_factory, + topo_features_factory, ts_smoothing_factory, api_composer_tune_factory) + + +FEDOT_INDUSTRIAL_MANIFEST = ExtensionManifest( + name="industrial", + version="1.0.0", + models=(), + description="Industrial extension for FEDOT with Dask support and optimized operations.", + protocols={ + "context_factory": industrial_context_factory, + "splitters": splitters_factory, + "data_merger": data_merger_factory, + "image_merger": image_merger_factory, + "ts_merger": ts_merger_factory, + "text_merger": text_merger_factory, + "data_source_splitter": data_source_splitter_factory, + "tuner_class": tuner_class_factory, + "reproduction": reproduction_factory, + "evaluator": evaluator_factory, + "search_space": search_space_factory, + "default_mutations": mutations_factory, + "operation_predict": operation_predict_factory, + "lagged_transformer": lagged_transformer_factory, + "topological_features": topo_features_factory, + "ts_smoothing": ts_smoothing_factory, + "api_composer_tune": api_composer_tune_factory, + } +) \ No newline at end of file diff --git a/fedot/core/protocols/__init__.py b/fedot/core/protocols/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/fedot/core/protocols/protocols.py b/fedot/core/protocols/protocols.py new file mode 100644 index 0000000000..ee103dfc96 --- /dev/null +++ b/fedot/core/protocols/protocols.py @@ -0,0 +1,109 @@ +import Protocol + +class EvaluatorProtocol(Protocol): + def evaluate(self, graph: Pipeline) -> Fitness: + ... + +class SearchSpaceProtocol(Protocol): + def get_parameters_dict(self) -> dict: + ... + +class DefaultMutationsProtocol(Protocol): + @staticmethod + def __call__(task_type: TaskTypesEnum, params: Any) -> Sequence[Any]: + ... + +class MergerProtocol(Protocol): + @staticmethod + def get(outputs: List[OutputData]) -> 'DataMerger': + ... + + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + ... + + def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: + ... + + @staticmethod + def find_main_output(outputs: List[OutputData]) -> OutputData: + ... + + def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: + ... + + def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + ... + +class DataSourceSplitterProtocol(Protocol): + def build(self, data: Union[InputData, 'MultiModalData']) -> Callable: + ... + +class SplitterProtocol(Protocol): + def split_any(self, data: InputData, split_ratio: float, shuffle: bool, + stratify: bool, random_seed: int, **kwargs) -> Tuple[InputData, InputData]: + ... + + def split_time_series(self, data: InputData, validation_blocks: Optional[int] = None, + **kwargs) -> Tuple[InputData, InputData]: + ... + +class OperationPredictProtocol(Protocol): + def predict(self, fitted_operation, data: InputData, + params: Optional[Any] = None, output_mode: str = 'default') -> OutputData: + ... + + def predict_for_fit(self, fitted_operation, data: InputData, + params: Optional[Any] = None, output_mode: str = 'default') -> OutputData: + ... + + def _predict(self, fitted_operation, data: InputData, params: Optional[Any] = None, + output_mode: str = 'default', is_fit_stage: bool = False, + predictions_cache: Optional[Any] = None, fold_id: Optional[int] = None, + descriptive_id: Optional[str] = None) -> OutputData: + ... + +class LaggedTransformerProtocol(Protocol): + def _update_column_types(self, output_data: OutputData) -> None: + ... + + def transform(self, input_data: InputData) -> OutputData: + ... + + def transform_for_fit(self, input_data: InputData) -> OutputData: + ... + + def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int) -> None: + ... + +class TopologicalFeaturesProtocol(Protocol): + def fit(self, input_data: InputData) -> Any: + ... + + def transform(self, input_data: InputData) -> np.ndarray: + ... + +class TsSmoothingProtocol(Protocol): + def transform(self, input_data: InputData) -> OutputData: + ... + +class TunerClassProtocol(Protocol): + def __call__(self, objective_evaluate: Any, task: Any, iterations: int, + max_lead_time: Optional[float] = None, **kwargs) -> BaseTuner: + ... + +class ApiComposerTuneProtocol(Protocol): + def __call__(self, train_data: InputData, pipeline: Pipeline, + execution_plan: Optional[Any] = None) -> Pipeline: + ... + +class ReproductionProtocol(Protocol): + def reproduce(self, population: List[Any], evaluator: Any, **kwargs) -> List[Any]: + ... + + def reproduce_uncontrolled(self, population: List[Any], **kwargs) -> List[Any]: + ... + +class VerificationRulesProtocol(Protocol): + class_rules: List[Callable] + ts_rules: List[Callable] + common_rules: List[Callable] \ No newline at end of file diff --git a/fedot/extensions/contracts.py b/fedot/extensions/contracts.py new file mode 100644 index 0000000000..ab10c38418 --- /dev/null +++ b/fedot/extensions/contracts.py @@ -0,0 +1,53 @@ +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Optional, Tuple + +from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.repository.tasks import TaskTypesEnum + +ModelFactory = Callable[[Optional[Dict[str, Any]]], Any] + + +@dataclass(frozen=True) +class ModelHyperparamsSchema: + required: Tuple[str, ...] = () + optional: Tuple[str, ...] = () + defaults: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ModelCapabilities: + tasks: Tuple[TaskTypesEnum, ...] + data_types: Tuple[DataTypesEnum, ...] + tags: Tuple[str, ...] = () + supports_multimodal: bool = False + + +@dataclass(frozen=True) +class ExternalModelSpec: + name: str + factory: ModelFactory + capabilities: ModelCapabilities + hyperparams_schema: ModelHyperparamsSchema = field(default_factory=ModelHyperparamsSchema) + description: str = '' + + +@dataclass(frozen=True) +class ExtensionManifest: + name: str + version: str + models: Tuple[ExternalModelSpec, ...] + module: Optional[str] = None + protocols: Optional[Dict[str, Callable[..., Any]]] = None + description: str = '' + + +@dataclass(frozen=True) +class ExtensionError: + code: str + message: str + details: Dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class RegisteredExtension: + manifest: ExtensionManifest diff --git a/fedot/extensions/registry.py b/fedot/extensions/registry.py new file mode 100644 index 0000000000..89df08d4bc --- /dev/null +++ b/fedot/extensions/registry.py @@ -0,0 +1,180 @@ +import importlib +import inspect +from typing import Any, Dict, Iterable, Tuple + +from pymonad.either import Left, Right +from pymonad.maybe import Just, Nothing + +from fedot.extensions.contracts import ( + ExtensionError, + ExtensionManifest, + ExternalModelSpec, + RegisteredExtension, +) + +_REGISTERED_EXTENSIONS: Dict[str, ExtensionManifest] = {} + + +def validate_extension_manifest(manifest: Any): + if not isinstance(manifest, ExtensionManifest): + return Left(ExtensionError(code='invalid_manifest_type', + message='Extension manifest must be an ExtensionManifest instance.')) + + if not manifest.name.strip(): + return Left(ExtensionError(code='empty_extension_name', + message='Extension manifest name must be non-empty.')) + + if not manifest.version.strip(): + return Left(ExtensionError(code='empty_extension_version', + message='Extension manifest version must be non-empty.')) + + if not manifest.models: + return Left(ExtensionError(code='empty_models', + message='Extension manifest must expose at least one model.')) + + seen_names = set() + for model in manifest.models: + model_validation = validate_external_model_spec(model) + if model_validation.is_left(): + return model_validation + if model.name in seen_names: + return Left(ExtensionError(code='duplicate_model_name', + message=f'Duplicate model name "{model.name}" in extension manifest.', + details={'extension': manifest.name})) + seen_names.add(model.name) + + return Right(manifest) + + +def validate_external_model_spec(model: Any): + if not isinstance(model, ExternalModelSpec): + return Left(ExtensionError(code='invalid_model_spec_type', + message='External model spec must be an ExternalModelSpec instance.')) + + if not model.name.strip(): + return Left(ExtensionError(code='empty_model_name', + message='External model name must be non-empty.')) + + if not callable(model.factory): + return Left(ExtensionError(code='invalid_model_factory', + message=f'Factory for model "{model.name}" must be callable.')) + + if not model.capabilities.tasks: + return Left(ExtensionError(code='empty_model_tasks', + message=f'Model "{model.name}" must declare supported tasks.')) + + if not model.capabilities.data_types: + return Left(ExtensionError(code='empty_model_data_types', + message=f'Model "{model.name}" must declare supported data types.')) + + return Right(model) + + +def register_extension(manifest: ExtensionManifest): + validation = validate_extension_manifest(manifest) + if validation.is_left(): + return validation + + if manifest.name in _REGISTERED_EXTENSIONS: + return Left(ExtensionError(code='duplicate_extension', + message=f'Extension "{manifest.name}" is already registered.')) + + _REGISTERED_EXTENSIONS[manifest.name] = manifest + return Right(RegisteredExtension(manifest=manifest)) + + +def get_registered_extensions() -> Tuple[RegisteredExtension, ...]: + return tuple(RegisteredExtension(manifest=manifest) for manifest in _REGISTERED_EXTENSIONS.values()) + + +def get_registered_extension(extension_name: str): + manifest = _REGISTERED_EXTENSIONS.get(extension_name) + if manifest is None: + return Nothing + return Just(RegisteredExtension(manifest=manifest)) + + +def clear_extension_registry() -> None: + _REGISTERED_EXTENSIONS.clear() + + +def load_extension_manifest(module_name: str): + try: + module = importlib.import_module(module_name) + except Exception as ex: + return Left(ExtensionError(code='module_import_failed', + message=f'Unable to import extension module "{module_name}".', + details={'exception': str(ex)})) + + manifest = getattr(module, 'FEDOT_EXTENSION_MANIFEST', None) + if manifest is None: + return Left(ExtensionError(code='manifest_not_found', + message=f'Extension module "{module_name}" must expose FEDOT_EXTENSION_MANIFEST.')) + + if manifest.module is None: + manifest = ExtensionManifest(name=manifest.name, + version=manifest.version, + models=manifest.models, + module=module_name, + description=manifest.description) + return validate_extension_manifest(manifest) + + +def discover_extensions(module_names: Iterable[str]): + manifests = [] + for module_name in module_names: + loaded = load_extension_manifest(module_name) + if loaded.is_left(): + return loaded + manifests.append(loaded.value) + return Right(tuple(manifests)) + + +def smoke_test_extension(manifest: ExtensionManifest): + validation = validate_extension_manifest(manifest) + if validation.is_left(): + return validation + + for model in manifest.models: + signature = inspect.signature(model.factory) + positional_required = [ + parameter for parameter in signature.parameters.values() + if parameter.default is inspect._empty + and parameter.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + ] + if len(positional_required) > 1: + return Left(ExtensionError( + code='invalid_factory_signature', + message=f'Factory for model "{model.name}" must accept zero or one positional argument.', + details={'required_args': [parameter.name for parameter in positional_required]}, + )) + + try: + instance = model.factory(None) + except TypeError: + instance = model.factory() + except Exception as ex: + return Left(ExtensionError(code='factory_smoke_test_failed', + message=f'Factory smoke test failed for model "{model.name}".', + details={'exception': str(ex)})) + + if instance is None: + return Left(ExtensionError(code='factory_returned_none', + message=f'Factory for model "{model.name}" returned None.')) + + return Right(manifest) + +def get_protocol_factory(protocol_group: str) -> Optional[Callable[..., Any]]: + from fedot.extensions.registry import get_registered_extensions + for ext in get_registered_extensions(): + factory = ext.manifest.protocols.get(protocol_group) + if factory is not None: + return factory + return None + +def resolve_protocol_instance(protocol_group: str, backend: str = "default", **kwargs) -> Optional[Any]: + factory = get_protocol_factory(protocol_group) + if factory: + return factory(backend=backend, **kwargs) + return None + From 7b565f52a6ba916942bbdfd967c9c53095179de9 Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Thu, 23 Apr 2026 14:08:40 +0300 Subject: [PATCH 06/15] Delete fedot/core/context.py --- fedot/core/context.py | 104 ------------------------------------------ 1 file changed, 104 deletions(-) delete mode 100644 fedot/core/context.py diff --git a/fedot/core/context.py b/fedot/core/context.py deleted file mode 100644 index 2c8ab091f4..0000000000 --- a/fedot/core/context.py +++ /dev/null @@ -1,104 +0,0 @@ -from fedot.core.data.merge.data_merger import ImageDataMerger, TSDataMerger, DataMerger -from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( - TopologicalFeaturesImplementation, ) -from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation, - TsSmoothingImplementation, -) -from fedot.core.operations.operation import Operation -from fedot.core.optimisers.objective import PipelineObjectiveEvaluate -from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace -from fedot.core.pipelines.verification import class_rules, ts_rules, common_rules -from fedot.core.repository.operation_types_repository import OperationTypesRepository -from fedot.core.data.data_split import _split_any, _split_time_series -from fedot.api.api_utils.api_params_repository import ApiParamsRepository -from fedot.api.api_utils.api_composer import ApiComposer -from golem.core.tuning.optuna_tuner import OptunaTuner -from golem.core.optimisers.genetic.operators.reproduction import ReproductionController - -import fedot.core.data.data_split as fedot_data_split -import golem.core.tuning.optuna_tuner as OptunaImpl - -class ExecutionContext: - def __init__(self) -> None: - """Initializes ExecutionContext with default configuration.""" - self._init_defaults() - - def _init_defaults(self): - """Sets default implementations for all pipeline components. - - Initializes: - - operation registry - - evaluators - - splitters - - merge strategies - - feature transforms - - rules - """ - self.extension = None - self.backend = "default" - - self.operation_registry = OperationTypesRepository() - self.evaluator_evaluate = PipelineObjectiveEvaluate.evaluate - self.search_space_get_parameters_dict = PipelineSearchSpace.get_parameters_dict - self.api_params_repository__get_default_mutations = ApiParamsRepository._get_default_mutations - self.merger_find_main_output = DataMerger.find_main_output - self.merger_get = DataMerger.get - self.merger_merge_predicts = DataMerger.merge_predicts - self.image_merger_preprocess_predicts = ImageDataMerger.preprocess_predicts - self.image_merger_merge_predicts = ImageDataMerger.merge_predicts - self.ts_merger_merge_predicts = TSDataMerger.merge_predicts - self.ts_merger_merge_targets = TSDataMerger.merge_targets - self.ts_merger_postprocess_predicts = TSDataMerger.postprocess_predicts - self.ts_merger_preprocess_predicts = TSDataMerger.preprocess_predicts - self.data_source_splitter_build = DataSourceSplitter.build - self.data_split__split_any = fedot_data_split._split_any - self.data_split__split_time_series = fedot_data_split._split_time_series - self.operation__predict = Operation._predict - self.operation_predict = Operation.predict - self.operation_predict_for_fit = Operation.predict_for_fit - self.lagged__update_column_types = LaggedImplementation._update_column_types - self.lagged_transform = LaggedImplementation.transform - self.lagged_transform_for_fit = LaggedImplementation.transform_for_fit - self.lagged__check_and_correct_window_size = LaggedImplementation._check_and_correct_window_size - self.topo_features_fit = TopologicalFeaturesImplementation.fit - self.topo_features_transform = TopologicalFeaturesImplementation.transform - self.ts_smoothing_transform = TsSmoothingImplementation.transform - self.optuna_optuna_tuner = OptunaImpl.OptunaTuner - self.api_composer_tune_final_pipeline = ApiComposer.tune_final_pipeline - self.reproduction_reproduce = ReproductionController.reproduce - self.reproduction_reproduce_uncontrolled = ReproductionController.reproduce_uncontrolled - self.class_rules = class_rules.copy() - self.ts_rules = ts_rules.copy() - self.common_rules = common_rules.copy() - - def __getstate__(self): - return { - "backend": self.backend, - "class_rules": self.class_rules, - "ts_rules": self.ts_rules, - "common_rules": self.common_rules, - "extension_state": getattr(self.extension, "get_state", lambda: None)(), - "has_extension": self.extension is not None, - } - - def __setstate__(self, state): - self.__dict__.clear() - - self._init_defaults() - - self.backend = state.get("backend", "default") - self.class_rules = state["class_rules"] - self.ts_rules = state["ts_rules"] - self.common_rules = state["common_rules"] - - if state.get("has_extension", False): - self.extension = IndustrialExtension(backend=self.backend) - - ext_state = state.get("extension_state") - if ext_state and hasattr(self.extension, "set_state"): - self.extension.set_state(ext_state) - - self.extension.apply(self) From 23502698fa5be8a8e3d19dcf6212ca10d016bd0a Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 15 May 2026 03:07:42 +0400 Subject: [PATCH 07/15] fixed execution context --- fedot/api/main.py | 10 +- .../core/composer/gp_composer/gp_composer.py | 4 +- fedot/core/context/__init__.py | 1 + fedot/core/context/context.py | 315 ++++++++---------- fedot/core/context/default_backend.py | 207 ++++++++++++ fedot/core/context/industrial_backend.py | 164 +++++---- fedot/core/context/industrial_manifest.py | 62 ++-- .../objective/data_objective_eval.py | 2 + fedot/core/protocols/protocols.py | 67 +++- fedot/extensions/registry.py | 14 - test/unit/context/__init__.py | 0 test/unit/context/test_initialization.py | 90 +++++ 12 files changed, 658 insertions(+), 278 deletions(-) create mode 100644 fedot/core/context/default_backend.py create mode 100644 test/unit/context/__init__.py create mode 100644 test/unit/context/test_initialization.py diff --git a/fedot/api/main.py b/fedot/api/main.py index 1300ca1fb7..37b46e8fb0 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -51,8 +51,7 @@ from fedot.utilities.define_metric_by_task import MetricByTask from fedot.utilities.memory import MemoryAnalytics from fedot.utilities.project_import_export import export_project_to_zip, import_project_from_zip - -from fedot.core.context.context import resolve_context +from fedot.core.context.context import ExecutionContext NOT_FITTED_ERR_MSG = 'Model not fitted yet' @@ -111,7 +110,12 @@ def __init__(self, set_random_seed(seed) self.log = self._init_logger(logging_level) - self.context = resolve_context(context) + if isinstance(context, str): + if context == "core" or None: + self.context = ExecutionContext() + elif context == "industrial": + from fedot.core.context.context import create_context + self.context = create_context(context) # Attributes for dealing with metrics, data sources and hyperparameters self.params = ApiParams(composer_tuner_params, problem, task_params, n_jobs, timeout, seed) diff --git a/fedot/core/composer/gp_composer/gp_composer.py b/fedot/core/composer/gp_composer/gp_composer.py index 2a8297bf39..ada295b0ed 100644 --- a/fedot/core/composer/gp_composer/gp_composer.py +++ b/fedot/core/composer/gp_composer/gp_composer.py @@ -11,7 +11,7 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.predictions_cache import PredictionsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache -from fedot.core.context import ExecutionContext +from fedot.core.context.context import ExecutionContext from fedot.core.composer.composer import Composer from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData @@ -43,7 +43,7 @@ def __init__(self, optimizer: GraphOptimizer, operations_cache: Optional[OperationsCache] = None, preprocessing_cache: Optional[PreprocessingCache] = None, predictions_cache: Optional[PredictionsCache] = None, - context: Optional[ExectuionContext] = None): + context: Optional[ExecutionContext] = None): super().__init__(optimizer, composer_requirements) self.composer_requirements = composer_requirements self.operations_cache: Optional[OperationsCache] = operations_cache diff --git a/fedot/core/context/__init__.py b/fedot/core/context/__init__.py index e69de29bb2..367791304e 100644 --- a/fedot/core/context/__init__.py +++ b/fedot/core/context/__init__.py @@ -0,0 +1 @@ +from .context import ExecutionContext \ No newline at end of file diff --git a/fedot/core/context/context.py b/fedot/core/context/context.py index 786f07559b..2443f346b0 100644 --- a/fedot/core/context/context.py +++ b/fedot/core/context/context.py @@ -1,169 +1,150 @@ -from fedot.core.data.merge.data_merger import ImageDataMerger, TSDataMerger, DataMerger -from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( - TopologicalFeaturesImplementation, ) -from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation, - TsSmoothingImplementation, -) -from fedot.core.operations.operation import Operation -from fedot.core.optimisers.objective import PipelineObjectiveEvaluate -from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace -from fedot.core.pipelines.verification import class_rules, ts_rules, common_rules -from fedot.core.repository.operation_types_repository import OperationTypesRepository -from fedot.core.data.data_split import _split_any, _split_time_series -from fedot.api.api_utils.api_params_repository import ApiParamsRepository -from fedot.api.api_utils.api_composer import ApiComposer -from golem.core.tuning.optuna_tuner import OptunaTuner -from golem.core.optimisers.genetic.operators.reproduction import ReproductionController - -import fedot.core.data.data_split as fedot_data_split -import golem.core.tuning.optuna_tuner as OptunaImpl - -def resolve_context(context_name: str = "core", backend: str = "default") -> ExecutionContext: - if context_name == "core": - return ExecutionContext(backend=backend) - - from fedot.extensions.registry import get_registered_extension, get_registered_extensions - - ext = get_registered_extension(context_name) - if ext is not None: - factory = ext.value.manifest.protocols.get("context_factory") - if factory: - return factory(backend=backend) - - raise ValueError(f"Unknown context: {context_name}") +from typing import Dict, Any, Optional, Callable +from fedot.extensions.registry import get_registered_extension, register_extension +from fedot.core.context.industrial_manifest import FEDOT_INDUSTRIAL_MANIFEST + +register_extension(FEDOT_INDUSTRIAL_MANIFEST) class ExecutionContext: - def __init__(self, backend: str = "default") -> None: - """Initializes ExecutionContext with default configuration.""" - self.backend = backend - self._init_defaults() - self._apply_protocols() - - def _init_defaults(self): - """Sets default implementations for all pipeline components.""" - self.evaluator_evaluate = PipelineObjectiveEvaluate.evaluate - self.search_space_get_parameters_dict = PipelineSearchSpace.get_parameters_dict - self.api_params_repository__get_default_mutations = ApiParamsRepository._get_default_mutations - self.merger_find_main_output = DataMerger.find_main_output - self.merger_get = DataMerger.get - self.merger_merge_predicts = DataMerger.merge_predicts - self.image_merger_preprocess_predicts = ImageDataMerger.preprocess_predicts - self.image_merger_merge_predicts = ImageDataMerger.merge_predicts - self.ts_merger_merge_predicts = TSDataMerger.merge_predicts - self.ts_merger_merge_targets = TSDataMerger.merge_targets - self.ts_merger_postprocess_predicts = TSDataMerger.postprocess_predicts - self.ts_merger_preprocess_predicts = TSDataMerger.preprocess_predicts - self.data_source_splitter_build = DataSourceSplitter.build - self.data_split__split_any = fedot_data_split._split_any - self.data_split__split_time_series = fedot_data_split._split_time_series - self.operation__predict = Operation._predict - self.operation_predict = Operation.predict - self.operation_predict_for_fit = Operation.predict_for_fit - self.lagged__update_column_types = LaggedImplementation._update_column_types - self.lagged_transform = LaggedImplementation.transform - self.lagged_transform_for_fit = LaggedImplementation.transform_for_fit - self.lagged__check_and_correct_window_size = LaggedImplementation._check_and_correct_window_size - self.topo_features_fit = TopologicalFeaturesImplementation.fit - self.topo_features_transform = TopologicalFeaturesImplementation.transform - self.ts_smoothing_transform = TsSmoothingImplementation.transform - self.optuna_optuna_tuner = OptunaImpl.OptunaTuner - self.api_composer_tune_final_pipeline = ApiComposer.tune_final_pipeline - self.reproduction_reproduce = ReproductionController.reproduce - self.reproduction_reproduce_uncontrolled = ReproductionController.reproduce_uncontrolled - self.class_rules = class_rules.copy() - self.ts_rules = ts_rules.copy() - self.common_rules = common_rules.copy() - - def _apply_protocols(self): - # Splitters - splitters = resolve_protocol_instance("splitters", backend=self.backend) - if splitters: - self.data_split__split_any = splitters.split_any - self.data_split__split_time_series = splitters.split_time_series - - # Mergers - mergers = resolve_protocol_instance("mergers", backend=self.backend) - if mergers: - self.merger_find_main_output = mergers.find_main_output - self.merger_get = mergers.get - self.merger_merge_predicts = mergers.merge_predicts - if hasattr(mergers, 'preprocess_predicts'): - self.image_merger_preprocess_predicts = mergers.preprocess_predicts - self.ts_merger_preprocess_predicts = mergers.preprocess_predicts - if hasattr(mergers, 'postprocess_predicts'): - self.ts_merger_postprocess_predicts = mergers.postprocess_predicts - if hasattr(mergers, 'merge_targets'): - self.ts_merger_merge_targets = mergers.merge_targets - # Image merge обычно совпадает с основным - self.image_merger_merge_predicts = mergers.merge_predicts - - # DataSourceSplitter - splitter_builder = resolve_protocol_instance("data_source_splitter", backend=self.backend) - if splitter_builder: - self.data_source_splitter_build = splitter_builder.build - - # Tuner class - tuner_class = resolve_protocol_instance("tuner_class", backend=self.backend) - if tuner_class: - self.optuna_optuna_tuner = tuner_class - - # Reproduction - reproduction = resolve_protocol_instance("reproduction", backend=self.backend) - if reproduction: - self.reproduction_reproduce = reproduction.reproduce - if hasattr(reproduction, 'reproduce_uncontrolled'): - self.reproduction_reproduce_uncontrolled = reproduction.reproduce_uncontrolled - - # Evaluator - evaluator = resolve_protocol_instance("evaluator", backend=self.backend) - if evaluator: - self.evaluator_evaluate = evaluator.evaluate - - # Search space - search_space = resolve_protocol_instance("search_space", backend=self.backend) - if search_space: - self.search_space_get_parameters_dict = search_space.get_parameters_dict - - # Mutations - mutations = resolve_protocol_instance("default_mutations", backend=self.backend) - if mutations: - self.api_params_repository__get_default_mutations = mutations - - # Operation predict - op_predict = resolve_protocol_instance("operation_predict", backend=self.backend) - if op_predict: - self.operation_predict = op_predict.predict - self.operation_predict_for_fit = op_predict.predict_for_fit - if hasattr(op_predict, '_predict'): - self.operation__predict = op_predict._predict - - # Lagged transformer - lagged = resolve_protocol_instance("lagged_transformer", backend=self.backend) - if lagged: - self.lagged__update_column_types = lagged._update_column_types - self.lagged_transform = lagged.transform - self.lagged_transform_for_fit = lagged.transform_for_fit - self.lagged__check_and_correct_window_size = lagged._check_and_correct_window_size - - # Topological features - topo = resolve_protocol_instance("topological_features", backend=self.backend) - if topo: - self.topo_features_fit = topo.fit - self.topo_features_transform = topo.transform - - # TS Smoothing - smoothing = resolve_protocol_instance("ts_smoothing", backend=self.backend) - if smoothing: - self.ts_smoothing_transform = smoothing.transform - - # ApiComposer tune - tune = resolve_protocol_instance("api_composer_tune", backend=self.backend) - if tune: - self.api_composer_tune_final_pipeline = tune - - @cached_property - def set_operation_registry(self) -> OperationTypesRepository: - return OperationTypesRepository() \ No newline at end of file + def __init__(self, extension_name: str = "core", extra_params: Optional[Dict[str, Any]] = None): + self.extension_name = extension_name + self.extra_params = extra_params or {} + self._instances: Dict[str, Any] = {} + self._overridden: Dict[str, Any] = {} + + self._manifest = None + if extension_name != "core": + from fedot.extensions.registry import _REGISTERED_EXTENSIONS + manifest = _REGISTERED_EXTENSIONS.get(extension_name) + if manifest is None: + raise ValueError(f"Extension '{extension_name}' not registered") + self._manifest = manifest + + self._core_implementations = self._get_core_implementations() + + self._protocol_classes = self._core_implementations.copy() + if self._manifest and self._manifest.protocols: + self._protocol_classes.update(self._manifest.protocols) + + def _get_core_implementations(self) -> Dict[str, Callable]: + from fedot.core.context.default_backend import ( + CoreSplitter, CoreDataMerger, CoreImageMerger, + CoreTSMerger, CoreTextMerger, CoreTuner, + CoreDataSourceSplitter, CoreOperationPredict, + CoreLaggedTransformer, CoreTopologicalFeatures, + CoreTsSmoothing, CoreApiComposerTune, CoreReproduction, + CoreSearchSpace, CoreDefaultMutations, CoreEvaluator + ) + return { + "splitter": CoreSplitter, + "data_merger": CoreDataMerger, + "image_merger": CoreImageMerger, + "ts_merger": CoreTSMerger, + "text_merger": CoreTextMerger, + "tuner_class": CoreTuner, + "data_source_splitter": CoreDataSourceSplitter, + "operation_predict": CoreOperationPredict, + "lagged_transformer": CoreLaggedTransformer, + "topological_features": CoreTopologicalFeatures, + "ts_smoothing": CoreTsSmoothing, + "api_composer_tune": CoreApiComposerTune, + "reproduction": CoreReproduction, + "search_space": CoreSearchSpace, + "default_mutations": CoreDefaultMutations, + "evaluator": CoreEvaluator, + } + + def _get_protocol_class(self, protocol_name: str) -> Callable: + if self._manifest and self._manifest.protocols: + if protocol_name in self._manifest.protocols: + return self._manifest.protocols[protocol_name] + + if protocol_name in self._core_implementations: + return self._core_implementations[protocol_name] + + raise ValueError(f"No implementation for protocol '{protocol_name}'") + + def _get_instance(self, protocol_name: str) -> Any: + if protocol_name not in self._instances: + protocol_class = self._get_protocol_class(protocol_name) + self._instances[protocol_name] = protocol_class(**self.extra_params) + return self._instances[protocol_name] + + @property + def splitter(self): + return self._get_instance("splitter") + + @property + def data_merger(self): + return self._get_instance("data_merger") + + @property + def image_merger(self): + return self._get_instance("image_merger") + + @property + def ts_merger(self): + return self._get_instance("ts_merger") + + @property + def text_merger(self): + return self._get_instance("text_merger") + + @property + def tuner_class(self): + return self._get_instance("tuner_class") + + @property + def data_source_splitter(self): + return self._get_instance("data_source_splitter") + + @property + def operation_predict(self): + return self._get_instance("operation_predict") + + @property + def lagged_transformer(self): + return self._get_instance("lagged_transformer") + + @property + def topological_features(self): + return self._get_instance("topological_features") + + @property + def ts_smoothing(self): + return self._get_instance("ts_smoothing") + + @property + def api_composer_tune(self): + return self._get_instance("api_composer_tune") + + @property + def reproduction(self): + return self._get_instance("reproduction") + + @property + def search_space(self): + return self._get_instance("search_space") + + @property + def default_mutations(self): + return self._get_instance("default_mutations") + + @property + def evaluator(self): + return self._get_instance("evaluator") + + def __setattr__(self, name: str, value: Any) -> None: + if name in ('extra_params', '_instances', '_overridden', '_protocol_classes', + '_manifest', '_core_implementations', 'extension_name'): + super().__setattr__(name, value) + else: + self._overridden[name] = value + + def __getattr__(self, name: str): + if name in self._overridden: + return self._overridden[name] + + if name in ('_protocol_classes', '_instances', '_manifest', '_core_implementations'): + return super().__getattribute__(name) + + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") \ No newline at end of file diff --git a/fedot/core/context/default_backend.py b/fedot/core/context/default_backend.py new file mode 100644 index 0000000000..d6796a8e87 --- /dev/null +++ b/fedot/core/context/default_backend.py @@ -0,0 +1,207 @@ +from fedot.core.protocols.protocols import ( + SplitterProtocol, + DataMergerProtocol, + ImageMergerProtocol, + TSMergerProtocol, + TextMergerProtocol, + DataSourceSplitterProtocol, + TunerClassProtocol, + ReproductionProtocol, + EvaluatorProtocol, + SearchSpaceProtocol, + DefaultMutationsProtocol, + OperationPredictProtocol, + LaggedTransformerProtocol, + TopologicalFeaturesProtocol, + TsSmoothingProtocol, + ApiComposerTuneProtocol, +) + + +class CoreEvaluator(EvaluatorProtocol): + def evaluate(self, graph): + from fedot.core.optimisers.objective import PipelineObjectiveEvaluate + # from golem.core.optimisers.fitness import Fitness + return PipelineObjectiveEvaluate.evaluate(graph) + + +class CoreSearchSpace(SearchSpaceProtocol): + def get_parameters_dict(self) -> dict: + from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace + return PipelineSearchSpace.get_parameters_dict() + + +class CoreDefaultMutations(DefaultMutationsProtocol): + @staticmethod + def __call__(task_type, params): + from fedot.api.api_utils.api_params_repository import ApiParamsRepository + # from typing import Sequence + return ApiParamsRepository._get_default_mutations(task_type, params) + + +class CoreDataMerger(DataMergerProtocol): + @staticmethod + def get(outputs): + from fedot.core.data.merge.data_merger import DataMerger + return DataMerger.get(outputs) + + def merge_predicts(self, predicts): + from fedot.core.data.merge.data_merger import DataMerger + return DataMerger.merge_predicts(predicts) + + @staticmethod + def find_main_output(outputs): + from fedot.core.data.merge.data_merger import DataMerger + return DataMerger.find_main_output(outputs) + + def preprocess_predicts(self, predicts): + return predicts + + def postprocess_predicts(self, merged): + return merged + + +class CoreImageMerger(ImageMergerProtocol): + def preprocess_predicts(self, predicts): + from fedot.core.data.merge.data_merger import ImageDataMerger + return ImageDataMerger.preprocess_predicts(predicts) + + def merge_predicts(self, predicts): + from fedot.core.data.merge.data_merger import ImageDataMerger + return ImageDataMerger.merge_predicts(predicts) + + +class CoreTSMerger(TSMergerProtocol): + def merge_predicts(self, predicts): + from fedot.core.data.merge.data_merger import TSDataMerger + return TSDataMerger.merge_predicts(predicts) + + def merge_targets(self, targets): + from fedot.core.data.merge.data_merger import TSDataMerger + return TSDataMerger.merge_targets(targets) + + def preprocess_predicts(self, predicts): + from fedot.core.data.merge.data_merger import TSDataMerger + return TSDataMerger.preprocess_predicts(predicts) + + def postprocess_predicts(self, merged): + from fedot.core.data.merge.data_merger import TSDataMerger + return TSDataMerger.postprocess_predicts(merged) + + +class CoreTextMerger(TextMergerProtocol): + def merge_predicts(self, predicts): + from fedot.core.data.merge.data_merger import TextDataMerger + return TextDataMerger.merge_predicts(predicts) + + def postprocess_predicts(self, merged): + return merged + + +class CoreDataSourceSplitter(DataSourceSplitterProtocol): + def build(self, data): + from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter + return DataSourceSplitter.build(data) + + +class CoreSplitter(SplitterProtocol): + def split_any(self, data, split_ratio, shuffle, stratify, random_seed, **kwargs): + from fedot.core.data.data_split import _split_any + return _split_any(data, split_ratio, shuffle, stratify, random_seed, **kwargs) + + def split_time_series(self, data, validation_blocks=None, **kwargs): + from fedot.core.data.data_split import _split_time_series + return _split_time_series(data, validation_blocks, **kwargs) + + +class CoreOperationPredict(OperationPredictProtocol): + def predict(self, fitted_operation, data, params=None, output_mode='default'): + from fedot.core.operations.operation import Operation + return Operation.predict(fitted_operation, data, params, output_mode) + + def predict_for_fit(self, fitted_operation, data, params=None, output_mode='default'): + from fedot.core.operations.operation import Operation + return Operation.predict_for_fit(fitted_operation, data, params, output_mode) + + def _predict(self, fitted_operation, data, params=None, output_mode='default', + is_fit_stage=False, predictions_cache=None, fold_id=None, descriptive_id=None): + from fedot.core.operations.operation import Operation + return Operation._predict(fitted_operation, data, params, output_mode, + is_fit_stage, predictions_cache, fold_id, descriptive_id) + + +class CoreLaggedTransformer(LaggedTransformerProtocol): + def _update_column_types(self, output_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation + ) + return LaggedImplementation._update_column_types(output_data) + + def transform(self, input_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation + ) + return LaggedImplementation.transform(input_data) + + def transform_for_fit(self, input_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation + ) + return LaggedImplementation.transform_for_fit(input_data) + + def _check_and_correct_window_size(self, time_series, forecast_length): + from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + LaggedImplementation + ) + return LaggedImplementation._check_and_correct_window_size(time_series, forecast_length) + + +class CoreTopologicalFeatures(TopologicalFeaturesProtocol): + def fit(self, input_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( + TopologicalFeaturesImplementation + ) + return TopologicalFeaturesImplementation.fit(input_data) + + def transform(self, input_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( + TopologicalFeaturesImplementation + ) + return TopologicalFeaturesImplementation.transform(input_data) + + +class CoreTsSmoothing(TsSmoothingProtocol): + def transform(self, input_data): + from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( + TsSmoothingImplementation + ) + return TsSmoothingImplementation.transform(input_data) + + +class CoreTuner(TunerClassProtocol): + def __init__(self, **kwargs): + self.backend = kwargs.get("backend", "default") + + def __call__(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): + from golem.core.tuning.optuna_tuner import OptunaTuner, DaskOptunaTuner + # from fedot.core.pipelines.tuning.tuner import BaseTuner + + if "dask" in self.backend: + return DaskOptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) + return OptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) + + +class CoreApiComposerTune(ApiComposerTuneProtocol): + def __call__(self, train_data, pipeline, execution_plan=None): + from fedot.api.api_utils.api_composer import ApiComposer + return ApiComposer.tune_final_pipeline(train_data, pipeline, execution_plan) + + +class CoreReproduction(ReproductionProtocol): + def reproduce(self, population, evaluator, **kwargs): + from golem.core.optimisers.genetic.operators.reproduction import ReproductionController + return ReproductionController.reproduce(population, evaluator, **kwargs) + + def reproduce_uncontrolled(self, population, **kwargs): + from golem.core.optimisers.genetic.operators.reproduction import ReproductionController + return ReproductionController.reproduce_uncontrolled(population, **kwargs) \ No newline at end of file diff --git a/fedot/core/context/industrial_backend.py b/fedot/core/context/industrial_backend.py index 07f0280c68..79737b8620 100644 --- a/fedot/core/context/industrial_backend.py +++ b/fedot/core/context/industrial_backend.py @@ -1,159 +1,207 @@ -from fedot.industrial.core.metrics.pipeline import industrial_evaluate_pipeline -from fedot.industrial.core.repository.constanst_repository import IND_DATA_OPERATION_PATH, IND_MODEL_OPERATION_PATH, DEFAULT_DATA_OPERATION_PATH, DEFAULT_MODEL_OPERATION_PATH -from fedot.industrial.core.repository.industrial_implementations.abstract import ( - preprocess_industrial_predicts, merge_industrial_predicts, merge_industrial_targets, - build_industrial, postprocess_industrial_predicts, split_any_industrial, - split_time_series_industrial, predict_operation_industrial, predict_industrial, - predict_for_fit_industrial, update_column_types_industrial, fit_topo_extractor_industrial, - transform_topo_extractor_industrial, find_main_output_industrial, get_merger_industrial +from typing import List, Optional, Union, Any, TYPE_CHECKING +import numpy as np + +if TYPE_CHECKING: + from fedot.core.data.data import InputData, OutputData + from fedot.core.data.multi_modal import MultiModalData + from fedot.core.pipelines.pipeline import Pipeline + from golem.core.optimisers.fitness import Fitness + from golem.core.tuning.optuna_tuner import OptunaTuner + from fedot.core.repository.tasks import TaskTypesEnum + from fedot.core.pipelines.tuning.tuner import BaseTuner + +from fedot.core.protocols.protocols import ( + SplitterProtocol, + DataMergerProtocol, + ImageMergerProtocol, + TSMergerProtocol, + TextMergerProtocol, + DataSourceSplitterProtocol, + TunerClassProtocol, + ReproductionProtocol, + EvaluatorProtocol, + SearchSpaceProtocol, + DefaultMutationsProtocol, + OperationPredictProtocol, + LaggedTransformerProtocol, + TopologicalFeaturesProtocol, + TsSmoothingProtocol, + ApiComposerTuneProtocol, ) -from fedot.industrial.core.repository.industrial_implementations.data_transformation import ( - transform_lagged_industrial, transform_lagged_for_fit_industrial, - _check_and_correct_window_size_industrial, transform_smoothing_industrial -) -from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import ( - DaskOptunaTuner, tune_pipeline_industrial -) -from fedot.industrial.core.repository.industrial_implementations.optimisation import ( - _get_default_industrial_mutations, has_no_lagged_conflicts_in_ts_pipeline, - reproduce_controlled_industrial, reproduce_industrial, - has_no_data_flow_conflicts_in_industrial_pipeline -) -from fedot.industrial.core.tuning.search_space import get_industrial_search_space -class IndustrialSplitter: - def split_any(self, data: InputData, split_ratio: float, shuffle: bool, +class IndustrialSplitter(SplitterProtocol): + def split_any(self, data: 'InputData', split_ratio: float, shuffle: bool, stratify: bool, random_seed: int, **kwargs): + from fedot.industrial.core.repository.industrial_implementations.abstract import split_any_industrial return split_any_industrial(data, split_ratio, shuffle, stratify, random_seed, **kwargs) - def split_time_series(self, data: InputData, validation_blocks: Optional[int] = None, **kwargs): + def split_time_series(self, data: 'InputData', validation_blocks: Optional[int] = None, **kwargs): + from fedot.industrial.core.repository.industrial_implementations.abstract import split_time_series_industrial return split_time_series_industrial(data, validation_blocks, **kwargs) -class IndustrialDataMerger: +class IndustrialDataMerger(DataMergerProtocol): @staticmethod - def get(outputs: List[OutputData]): + def get(outputs: List['OutputData']): + from fedot.industrial.core.repository.industrial_implementations.abstract import get_merger_industrial return get_merger_industrial(outputs) def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts return merge_industrial_predicts(predicts) @staticmethod - def find_main_output(outputs: List[OutputData]) -> OutputData: + def find_main_output(outputs: List['OutputData']) -> 'OutputData': + from fedot.industrial.core.repository.industrial_implementations.abstract import find_main_output_industrial return find_main_output_industrial(outputs) -class IndustrialImageMerger: +class IndustrialImageMerger(ImageMergerProtocol): def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: - return image_preprocess(predicts) + from fedot.industrial.core.repository.industrial_implementations.abstract import preprocess_industrial_predicts + return preprocess_industrial_predicts(predicts) def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts return merge_industrial_predicts(predicts) -class IndustrialTSMerger: +class IndustrialTSMerger(TSMergerProtocol): def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts return merge_industrial_predicts(predicts) def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_targets return merge_industrial_targets(targets) def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: - return image_preprocess(predicts) # или своя ts_preprocess + from fedot.industrial.core.repository.industrial_implementations.abstract import preprocess_industrial_predicts + return preprocess_industrial_predicts(predicts) def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import postprocess_industrial_predicts return postprocess_industrial_predicts(merged) -class IndustrialTextMerger: +class IndustrialTextMerger(TextMergerProtocol): def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts return merge_industrial_predicts(predicts) def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: return merged -class IndustrialDataSourceSplitterBuilder: - def build(self, data: Union[InputData, 'MultiModalData']): +class IndustrialDataSourceSplitterBuilder(DataSourceSplitterProtocol): + def build(self, data: Union['InputData', 'MultiModalData']): + from fedot.industrial.core.repository.industrial_implementations.abstract import build_industrial return build_industrial(data) -class IndustrialTunerClass: - def __init__(self, backend: str = "default"): - self.backend = backend +class IndustrialTunerClass(TunerClassProtocol): + def __init__(self, **kwargs): + self.backend = kwargs.get("backend", "default") def __call__(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): + from golem.core.tuning.optuna_tuner import OptunaTuner + from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import DaskOptunaTuner + if "dask" in self.backend: return DaskOptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) else: return OptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) -class IndustrialReproduction: +class IndustrialReproduction(ReproductionProtocol): def reproduce(self, population, evaluator, **kwargs): + from fedot.industrial.core.repository.industrial_implementations.optimisation import reproduce_industrial return reproduce_industrial(population, evaluator, **kwargs) def reproduce_uncontrolled(self, population, **kwargs): + from fedot.industrial.core.repository.industrial_implementations.optimisation import \ + reproduce_controlled_industrial return reproduce_controlled_industrial(population, **kwargs) -class IndustrialEvaluator: - def evaluate(self, graph: Pipeline) -> Fitness: +class IndustrialEvaluator(EvaluatorProtocol): + def evaluate(self, graph: 'Pipeline') -> 'Fitness': + from fedot.industrial.core.metrics.pipeline import industrial_evaluate_pipeline return industrial_evaluate_pipeline(graph) -class IndustrialSearchSpace: +class IndustrialSearchSpace(SearchSpaceProtocol): def get_parameters_dict(self): + from fedot.industrial.core.tuning.search_space import get_industrial_search_space return get_industrial_search_space() -class IndustrialDefaultMutations: +class IndustrialDefaultMutations(DefaultMutationsProtocol): @staticmethod - def __call__(task_type: TaskTypesEnum, params): + def __call__(task_type: 'TaskTypesEnum', params): + from fedot.industrial.core.repository.industrial_implementations.optimisation import \ + _get_default_industrial_mutations return _get_default_industrial_mutations(task_type, params) -class IndustrialOperationPredict: - def predict(self, fitted_operation, data: InputData, params=None, output_mode='default'): +class IndustrialOperationPredict(OperationPredictProtocol): + def predict(self, fitted_operation, data: 'InputData', params=None, output_mode='default'): + from fedot.industrial.core.repository.industrial_implementations.abstract import predict_industrial return predict_industrial(fitted_operation, data, params, output_mode) - def predict_for_fit(self, fitted_operation, data: InputData, params=None, output_mode='default'): + def predict_for_fit(self, fitted_operation, data: 'InputData', params=None, output_mode='default'): + from fedot.industrial.core.repository.industrial_implementations.abstract import predict_for_fit_industrial return predict_for_fit_industrial(fitted_operation, data, params, output_mode) - def _predict(self, fitted_operation, data: InputData, params=None, output_mode='default', + def _predict(self, fitted_operation, data: 'InputData', params=None, output_mode='default', is_fit_stage=False, predictions_cache=None, fold_id=None, descriptive_id=None): + from fedot.industrial.core.repository.industrial_implementations.abstract import predict_operation_industrial return predict_operation_industrial(fitted_operation, data, params, output_mode, is_fit_stage, predictions_cache, fold_id, descriptive_id) -class IndustrialLaggedTransformer: - def _update_column_types(self, output_data: OutputData): +class IndustrialLaggedTransformer(LaggedTransformerProtocol): + def _update_column_types(self, output_data: 'OutputData'): + from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ + update_column_types_industrial update_column_types_industrial(output_data) - def transform(self, input_data: InputData) -> OutputData: + def transform(self, input_data: 'InputData') -> 'OutputData': + from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ + transform_lagged_industrial return transform_lagged_industrial(input_data) - def transform_for_fit(self, input_data: InputData) -> OutputData: + def transform_for_fit(self, input_data: 'InputData') -> 'OutputData': + from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ + transform_lagged_for_fit_industrial return transform_lagged_for_fit_industrial(input_data) def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): + from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ + _check_and_correct_window_size_industrial _check_and_correct_window_size_industrial(time_series, forecast_length) -class IndustrialTopologicalFeatures: - def fit(self, input_data: InputData): +class IndustrialTopologicalFeatures(TopologicalFeaturesProtocol): + def fit(self, input_data: 'InputData'): + from fedot.industrial.core.repository.industrial_implementations.abstract import fit_topo_extractor_industrial return fit_topo_extractor_industrial(input_data) - def transform(self, input_data: InputData) -> np.ndarray: + def transform(self, input_data: 'InputData') -> np.ndarray: + from fedot.industrial.core.repository.industrial_implementations.abstract import \ + transform_topo_extractor_industrial return transform_topo_extractor_industrial(input_data) -class IndustrialTsSmoothing: - def transform(self, input_data: InputData) -> OutputData: +class IndustrialTsSmoothing(TsSmoothingProtocol): + def transform(self, input_data: 'InputData') -> 'OutputData': + from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ + transform_smoothing_industrial return transform_smoothing_industrial(input_data) -class IndustrialApiComposerTune: - def tune_pipeline_industrial(self, train_data: InputData, pipeline: Pipeline, execution_plan=None) -> Pipeline: +class IndustrialApiComposerTune(ApiComposerTuneProtocol): + def __call__(self, train_data: 'InputData', pipeline: 'Pipeline', execution_plan=None) -> 'Pipeline': + from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import tune_pipeline_industrial return tune_pipeline_industrial(train_data, pipeline, execution_plan) \ No newline at end of file diff --git a/fedot/core/context/industrial_manifest.py b/fedot/core/context/industrial_manifest.py index 3738091169..9bf32c5835 100644 --- a/fedot/core/context/industrial_manifest.py +++ b/fedot/core/context/industrial_manifest.py @@ -1,34 +1,44 @@ from fedot.extensions.contracts import ExtensionManifest -from fedot.core.context.factories import ( industrial_context_factory, splitters_factory, - data_merger_factory, image_merger_factory, ts_merger_factory, - text_merger_factory, data_source_splitter_factory, tuner_class_factory, - reproduction_factory, evaluator_factory, search_space_factory, - mutations_factory, operation_predict_factory, lagged_transformer_factory, - topo_features_factory, ts_smoothing_factory, api_composer_tune_factory) - +from fedot.core.context.industrial_backend import ( + IndustrialSplitter, + IndustrialDataMerger, + IndustrialImageMerger, + IndustrialTSMerger, + IndustrialTextMerger, + IndustrialDataSourceSplitterBuilder, + IndustrialTunerClass, + IndustrialReproduction, + IndustrialEvaluator, + IndustrialSearchSpace, + IndustrialDefaultMutations, + IndustrialOperationPredict, + IndustrialLaggedTransformer, + IndustrialTopologicalFeatures, + IndustrialTsSmoothing, + IndustrialApiComposerTune, +) FEDOT_INDUSTRIAL_MANIFEST = ExtensionManifest( name="industrial", version="1.0.0", models=(), - description="Industrial extension for FEDOT with Dask support and optimized operations.", + description="Industrial extension for FEDOT.", protocols={ - "context_factory": industrial_context_factory, - "splitters": splitters_factory, - "data_merger": data_merger_factory, - "image_merger": image_merger_factory, - "ts_merger": ts_merger_factory, - "text_merger": text_merger_factory, - "data_source_splitter": data_source_splitter_factory, - "tuner_class": tuner_class_factory, - "reproduction": reproduction_factory, - "evaluator": evaluator_factory, - "search_space": search_space_factory, - "default_mutations": mutations_factory, - "operation_predict": operation_predict_factory, - "lagged_transformer": lagged_transformer_factory, - "topological_features": topo_features_factory, - "ts_smoothing": ts_smoothing_factory, - "api_composer_tune": api_composer_tune_factory, + "splitter": IndustrialSplitter, + "data_merger": IndustrialDataMerger, + "image_merger": IndustrialImageMerger, + "ts_merger": IndustrialTSMerger, + "text_merger": IndustrialTextMerger, + "data_source_splitter": IndustrialDataSourceSplitterBuilder, + "tuner_class": IndustrialTunerClass, + "reproduction": IndustrialReproduction, + "evaluator": IndustrialEvaluator, + "search_space": IndustrialSearchSpace, + "default_mutations": IndustrialDefaultMutations, + "operation_predict": IndustrialOperationPredict, + "lagged_transformer": IndustrialLaggedTransformer, + "topological_features": IndustrialTopologicalFeatures, + "ts_smoothing": IndustrialTsSmoothing, + "api_composer_tune": IndustrialApiComposerTune, } -) \ No newline at end of file +) diff --git a/fedot/core/optimisers/objective/data_objective_eval.py b/fedot/core/optimisers/objective/data_objective_eval.py index c123f4abec..47fa1cc131 100644 --- a/fedot/core/optimisers/objective/data_objective_eval.py +++ b/fedot/core/optimisers/objective/data_objective_eval.py @@ -16,6 +16,8 @@ from fedot.core.pipelines.pipeline import Pipeline from fedot.utilities.debug import is_recording_mode, save_debug_info_for_pipeline +from fedot.core.context.context import ExecutionContext + DataSource = Callable[[], Iterable[Tuple[InputData, InputData]]] diff --git a/fedot/core/protocols/protocols.py b/fedot/core/protocols/protocols.py index ee103dfc96..726d16ccd0 100644 --- a/fedot/core/protocols/protocols.py +++ b/fedot/core/protocols/protocols.py @@ -1,19 +1,33 @@ -import Protocol +from typing import Any, List, Protocol, Optional, Callable, Union, Tuple, Sequence, TYPE_CHECKING +import numpy as np + +from fedot.core.data.data import InputData, OutputData +from fedot.core.repository.tasks import TaskTypesEnum +from golem.core.optimisers.fitness import Fitness +from golem.core.tuning.tuner_interface import BaseTuner + +if TYPE_CHECKING: + from fedot.core.pipelines.pipeline import Pipeline + from fedot.core.data.multi_modal import MultiModalData + from fedot.core.data.merge.data_merger import DataMerger class EvaluatorProtocol(Protocol): - def evaluate(self, graph: Pipeline) -> Fitness: + def evaluate(self, graph: 'Pipeline') -> Fitness: ... + class SearchSpaceProtocol(Protocol): def get_parameters_dict(self) -> dict: ... + class DefaultMutationsProtocol(Protocol): @staticmethod def __call__(task_type: TaskTypesEnum, params: Any) -> Sequence[Any]: ... -class MergerProtocol(Protocol): + +class DataMergerProtocol(Protocol): @staticmethod def get(outputs: List[OutputData]) -> 'DataMerger': ... @@ -21,9 +35,6 @@ def get(outputs: List[OutputData]) -> 'DataMerger': def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: ... - def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: - ... - @staticmethod def find_main_output(outputs: List[OutputData]) -> OutputData: ... @@ -34,10 +45,42 @@ def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: ... + +class ImageMergerProtocol(Protocol): + def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: + ... + + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + ... + + +class TSMergerProtocol(Protocol): + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + ... + + def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: + ... + + def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: + ... + + def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + ... + + +class TextMergerProtocol(Protocol): + def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: + ... + + def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: + ... + + class DataSourceSplitterProtocol(Protocol): def build(self, data: Union[InputData, 'MultiModalData']) -> Callable: ... + class SplitterProtocol(Protocol): def split_any(self, data: InputData, split_ratio: float, shuffle: bool, stratify: bool, random_seed: int, **kwargs) -> Tuple[InputData, InputData]: @@ -47,6 +90,7 @@ def split_time_series(self, data: InputData, validation_blocks: Optional[int] = **kwargs) -> Tuple[InputData, InputData]: ... + class OperationPredictProtocol(Protocol): def predict(self, fitted_operation, data: InputData, params: Optional[Any] = None, output_mode: str = 'default') -> OutputData: @@ -62,6 +106,7 @@ def _predict(self, fitted_operation, data: InputData, params: Optional[Any] = No descriptive_id: Optional[str] = None) -> OutputData: ... + class LaggedTransformerProtocol(Protocol): def _update_column_types(self, output_data: OutputData) -> None: ... @@ -75,6 +120,7 @@ def transform_for_fit(self, input_data: InputData) -> OutputData: def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int) -> None: ... + class TopologicalFeaturesProtocol(Protocol): def fit(self, input_data: InputData) -> Any: ... @@ -82,20 +128,24 @@ def fit(self, input_data: InputData) -> Any: def transform(self, input_data: InputData) -> np.ndarray: ... + class TsSmoothingProtocol(Protocol): def transform(self, input_data: InputData) -> OutputData: ... + class TunerClassProtocol(Protocol): def __call__(self, objective_evaluate: Any, task: Any, iterations: int, max_lead_time: Optional[float] = None, **kwargs) -> BaseTuner: ... + class ApiComposerTuneProtocol(Protocol): - def __call__(self, train_data: InputData, pipeline: Pipeline, - execution_plan: Optional[Any] = None) -> Pipeline: + def __call__(self, train_data: InputData, pipeline: 'Pipeline', + execution_plan: Optional[Any] = None) -> 'Pipeline': ... + class ReproductionProtocol(Protocol): def reproduce(self, population: List[Any], evaluator: Any, **kwargs) -> List[Any]: ... @@ -103,6 +153,7 @@ def reproduce(self, population: List[Any], evaluator: Any, **kwargs) -> List[Any def reproduce_uncontrolled(self, population: List[Any], **kwargs) -> List[Any]: ... + class VerificationRulesProtocol(Protocol): class_rules: List[Callable] ts_rules: List[Callable] diff --git a/fedot/extensions/registry.py b/fedot/extensions/registry.py index 89df08d4bc..63c19f858d 100644 --- a/fedot/extensions/registry.py +++ b/fedot/extensions/registry.py @@ -164,17 +164,3 @@ def smoke_test_extension(manifest: ExtensionManifest): return Right(manifest) -def get_protocol_factory(protocol_group: str) -> Optional[Callable[..., Any]]: - from fedot.extensions.registry import get_registered_extensions - for ext in get_registered_extensions(): - factory = ext.manifest.protocols.get(protocol_group) - if factory is not None: - return factory - return None - -def resolve_protocol_instance(protocol_group: str, backend: str = "default", **kwargs) -> Optional[Any]: - factory = get_protocol_factory(protocol_group) - if factory: - return factory(backend=backend, **kwargs) - return None - diff --git a/test/unit/context/__init__.py b/test/unit/context/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/unit/context/test_initialization.py b/test/unit/context/test_initialization.py new file mode 100644 index 0000000000..d9ce940c82 --- /dev/null +++ b/test/unit/context/test_initialization.py @@ -0,0 +1,90 @@ +import pytest +from unittest.mock import Mock +from fedot.core.context.context import ExecutionContext + +@pytest.fixture +def ctx(): + return ExecutionContext() + + +def test_default_initialization(ctx): + assert ctx.extra_params == {} + assert ctx._instances == {} + assert ctx._overridden == {} + assert "splitter" in ctx._protocol_classes + assert "evaluator" in ctx._protocol_classes + + +def test_extra_params_stored(): + extra = {"backend": "dask", "timeout": 30} + ctx = ExecutionContext(extra_params=extra) + assert ctx.extra_params == extra + + +def test_lazy_instantiation(ctx): + assert "splitter" not in ctx._instances + splitter1 = ctx.splitter + splitter2 = ctx.splitter + assert splitter1 is splitter2 + assert "splitter" in ctx._instances + + +def test_core_implementations_are_used_by_default(ctx): + from fedot.core.context.default_backend import CoreSplitter, CoreEvaluator, CoreDataMerger + + assert isinstance(ctx.splitter, CoreSplitter) + assert isinstance(ctx.evaluator, CoreEvaluator) + assert isinstance(ctx.data_merger, CoreDataMerger) + + +def test_method_override(): + ctx_ind = ExecutionContext(extension_name="industrial") + + from fedot.core.context.industrial_backend import IndustrialSplitter + assert isinstance(ctx_ind.splitter, IndustrialSplitter) + +def test_missing(ctx): + with pytest.raises(ValueError, match="No implementation for protocol 'unknown'"): + ctx._get_protocol_class("unknown") + + +def test_attribute_override(ctx): + ctx.custom_attr = 123 + assert ctx._overridden["custom_attr"] == 123 + assert ctx.custom_attr == 123 + +def test_multiple_contexts_independence(): + ctx1 = ExecutionContext(extra_params={"p": 1}) + ctx2 = ExecutionContext(extra_params={"p": 2}) + assert ctx1.extra_params["p"] == 1 + assert ctx2.extra_params["p"] == 2 + + +def test_method_called_with_parameters(ctx): + from unittest.mock import patch + from fedot.core.data.data import InputData + + with patch('fedot.core.context.default_backend.CoreSplitter.split_any') as mock_split: + mock_split.return_value = ("train_data", "test_data") + + mock_data = Mock(spec=InputData) + + splitter = ctx.splitter + result = splitter.split_any( + data=mock_data, + split_ratio=0.75, + shuffle=True, + stratify=True, + random_seed=42, + extra_arg="custom_value" + ) + + mock_split.assert_called_once() + + args, kwargs = mock_split.call_args + assert kwargs['data'] == mock_data + assert kwargs['split_ratio'] == 0.75 + assert kwargs['shuffle'] is True + assert kwargs['stratify'] is True + assert kwargs['random_seed'] == 42 + assert kwargs['extra_arg'] == "custom_value" \ No newline at end of file From 8036b2aca3434ac5830144c1b6353949c9fe30db Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 15 May 2026 11:52:36 +0400 Subject: [PATCH 08/15] fixed --- fedot/api/main.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/fedot/api/main.py b/fedot/api/main.py index 37b46e8fb0..b7b7c955e2 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -110,12 +110,7 @@ def __init__(self, set_random_seed(seed) self.log = self._init_logger(logging_level) - if isinstance(context, str): - if context == "core" or None: - self.context = ExecutionContext() - elif context == "industrial": - from fedot.core.context.context import create_context - self.context = create_context(context) + self.context = ExecutionContext(extension_name=context) # Attributes for dealing with metrics, data sources and hyperparameters self.params = ApiParams(composer_tuner_params, problem, task_params, n_jobs, timeout, seed) From 2a4018b2cbdf379f54a300c21776a1286c129445 Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:38:25 +0300 Subject: [PATCH 09/15] removed default_backend --- fedot/api/api_utils/api_composer.py | 435 +++++++++--------- fedot/api/api_utils/api_params_repository.py | 18 +- fedot/api/main.py | 13 +- fedot/core/composer/composer_builder.py | 13 +- .../core/composer/gp_composer/gp_composer.py | 19 +- fedot/core/context/context.py | 72 +-- fedot/core/context/factories.py | 60 --- fedot/core/context/industrial_backend.py | 30 +- fedot/core/context/industrial_manifest.py | 3 + fedot/core/data/data_split.py | 31 +- fedot/core/data/merge/data_merger.py | 49 +- .../topological/fast_topological_extractor.py | 19 +- .../data_operations/ts_transformations.py | 54 ++- fedot/core/operations/operation.py | 51 +- .../objective/data_objective_eval.py | 13 +- .../objective/data_source_splitter.py | 10 +- fedot/core/pipelines/tuning/search_space.py | 12 +- fedot/core/pipelines/tuning/tuner_builder.py | 10 +- 18 files changed, 480 insertions(+), 432 deletions(-) delete mode 100644 fedot/core/context/factories.py diff --git a/fedot/api/api_utils/api_composer.py b/fedot/api/api_utils/api_composer.py index 5b7708945d..0d295ce258 100644 --- a/fedot/api/api_utils/api_composer.py +++ b/fedot/api/api_utils/api_composer.py @@ -1,215 +1,220 @@ -import datetime -import gc -from copy import deepcopy -from typing import List, Optional, Sequence, Tuple, Union - -from golem.core.log import default_log -from golem.core.optimisers.opt_history_objects.opt_history import OptHistory -from golem.core.tuning.simultaneous import SimultaneousTuner - -from fedot.api.api_utils.api_composer_rules import build_cache_init_plan, build_tuner_plan -from fedot.api.api_utils.api_run_planner import build_composer_execution_plan -from fedot.api.api_utils.assumptions.assumptions_handler import AssumptionsHandler -from fedot.api.api_utils.params import ApiParams -from fedot.api.time import ApiTime -from fedot.core.caching.operations_cache import OperationsCache -from fedot.core.caching.preprocessing_cache import PreprocessingCache -from fedot.core.caching.predictions_cache import PredictionsCache -from fedot.core.composer.composer_builder import ComposerBuilder -from fedot.core.composer.gp_composer.gp_composer import GPComposer -from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER -from fedot.core.data.data import InputData -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.metrics_repository import MetricIDType -from fedot.utilities.composer_timer import fedot_composer_timer - - -class ApiComposer: - - def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]]): - self.log = default_log(self) - self.params = api_params - self.metrics = metrics - self.operations_cache: Optional[OperationsCache] = None - self.preprocessing_cache: Optional[PreprocessingCache] = None - self.predictions_cache: Optional[PredictionsCache] = None - self.timer = None - # status flag indicating that composer step was applied - self.was_optimised = False - # status flag indicating that tuner step was applied` - self.was_tuned = False - self.init_cache() - - def init_cache(self): - cache_plan = build_cache_init_plan( - use_operations_cache=self.params.get('use_operations_cache'), - use_preprocessing_cache=self.params.get('use_preprocessing_cache'), - use_predictions_cache=self.params.get('use_predictions_cache'), - use_input_preprocessing=self.params.get('use_input_preprocessing'), - cache_dir=self.params.get('cache_dir'), - use_stats=self.params.get('use_stats'), - ) - - if cache_plan.use_operations_cache: - self.operations_cache = OperationsCache(cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) - self.operations_cache.reset() - if cache_plan.use_preprocessing_cache: - self.preprocessing_cache = PreprocessingCache( - cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) - self.preprocessing_cache.reset() - if cache_plan.use_predictions_cache: - self.predictions_cache = PredictionsCache(cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) - self.predictions_cache.reset() - - def obtain_model(self, train_data: InputData) -> Tuple[Pipeline, Sequence[Pipeline], OptHistory]: - """ Function for composing FEDOT pipeline model """ - - with fedot_composer_timer.launch_composing(): - timeout: float = self.params.timeout - with_tuning = self.params.get('with_tuning') - - self.timer = ApiTime(time_for_automl=timeout, with_tuning=with_tuning) - - initial_assumption, fitted_assumption = self.propose_and_fit_initial_assumption(train_data) - - multi_objective = len(self.metrics) > 1 - self.params.init_params_for_composing(self.timer.timedelta_composing, multi_objective) - - self.log.message(f"AutoML configured." - f" Parameters tuning: {with_tuning}." - f" Time limit: {timeout} min." - f" Set of candidate models: {self.params.get('available_operations')}.") - - best_pipeline, best_pipeline_candidates, gp_composer = self.compose_pipeline( - train_data, - initial_assumption, - fitted_assumption - ) - - timeout_for_tuning = abs(self.timer.determine_resources_for_tuning()) / 60 - execution_plan = build_composer_execution_plan( - with_tuning=with_tuning, - have_time_for_composing=self.was_optimised, - have_time_for_tuning=self.timer.have_time_for_tuning(), - tuning_timeout_minutes=timeout_for_tuning, - ) - - if execution_plan.should_tune: - with fedot_composer_timer.launch_tuning('composing'): - best_pipeline = self.tune_final_pipeline(train_data, best_pipeline, execution_plan) - elif with_tuning: - self.log.message( - f'Time for pipeline composing was {str(self.timer.composing_spend_time)}.\n' - f'The remaining {max(0, round(execution_plan.tuning_timeout_minutes, 1))} seconds are not enough ' - f'to tune the hyperparameters.') - self.log.message('Composed pipeline returned without tuning.') - self.was_tuned = False - - if gp_composer.history: - adapter = self.params.graph_generation_params.adapter - gp_composer.history.tuning_result = adapter.adapt(best_pipeline) - gc.collect() - - self.log.message('Model generation finished') - return best_pipeline, best_pipeline_candidates, gp_composer.history - - def propose_and_fit_initial_assumption(self, train_data: InputData) -> Tuple[Sequence[Pipeline], Pipeline]: - """ Method for obtaining and fitting initial assumption""" - available_operations = self.params.get('available_operations') - - preset = self.params.get('preset') - - assumption_handler = AssumptionsHandler(train_data) - - initial_assumption = assumption_handler.propose_assumptions(self.params.get('initial_assumption'), - available_operations, - use_input_preprocessing=self.params.get( - 'use_input_preprocessing')) - - with self.timer.launch_assumption_fit(n_folds=self.params.data['cv_folds']): - fitted_assumption = \ - assumption_handler.fit_assumption_and_check_correctness(deepcopy(initial_assumption[0]), - operations_cache=self.operations_cache, - preprocessing_cache=self.preprocessing_cache, - eval_n_jobs=self.params.n_jobs) - - self.log.message( - f'Initial pipeline was fitted in ' - f'{round(self.timer.assumption_fit_spend_time_single_fold.total_seconds(), 1)} sec.') - - self.log.message( - f'Taking into account n_folds={self.params.data["cv_folds"]}, estimated fit time for initial assumption ' - f'is {round(self.timer.assumption_fit_spend_time.total_seconds(), 1)} sec.') - - self.params.update(preset=assumption_handler.propose_preset(preset, self.timer, n_jobs=self.params.n_jobs)) - - return initial_assumption, fitted_assumption - - def compose_pipeline(self, train_data: InputData, initial_assumption: Sequence[Pipeline], - fitted_assumption: Pipeline) -> Tuple[Pipeline, List[Pipeline], GPComposer]: - - gp_composer: GPComposer = (ComposerBuilder(task=self.params.task) - .with_requirements(self.params.composer_requirements) - .with_initial_pipelines(initial_assumption) - .with_optimizer(self.params.get('optimizer')) - .with_optimizer_params(parameters=self.params.optimizer_params) - .with_metrics(self.metrics) - .with_cache(self.operations_cache, self.preprocessing_cache, self.predictions_cache) - .with_graph_generation_param(self.params.graph_generation_params) - .build()) - - have_time_for_composing = self.timer.have_time_for_composing(self.params.get('pop_size'), self.params.n_jobs) - execution_plan = build_composer_execution_plan( - with_tuning=self.params.get('with_tuning'), - have_time_for_composing=have_time_for_composing, - have_time_for_tuning=False, - tuning_timeout_minutes=0, - ) - - if execution_plan.should_compose: - with self.timer.launch_composing(): - self.log.message('Pipeline composition started.') - self.was_optimised = False - best_pipelines = gp_composer.compose_pipeline(data=train_data) - best_pipeline_candidates = gp_composer.best_models - self.was_optimised = True - else: - self.log.message(f'Timeout is too small for composing and is skipped ' - f'because fit_time is {self.timer.assumption_fit_spend_time.total_seconds()} sec.') - best_pipelines = fitted_assumption - best_pipeline_candidates = [fitted_assumption] - self.was_optimised = False - - for pipeline in best_pipeline_candidates: - pipeline.log = self.log - best_pipeline = best_pipelines[0] if isinstance(best_pipelines, Sequence) else best_pipelines - return best_pipeline, best_pipeline_candidates, gp_composer - - def tune_final_pipeline(self, train_data: InputData, - pipeline_gp_composed: Pipeline, - execution_plan=None) -> Pipeline: - """ Launch tuning procedure for obtained pipeline by composer """ - timeout_for_tuning = execution_plan.tuning_timeout_minutes if execution_plan else abs( - self.timer.determine_resources_for_tuning()) / 60 - tuner_plan = build_tuner_plan( - metrics=self.metrics, - timeout_minutes=timeout_for_tuning, - iterations=DEFAULT_TUNING_ITERATIONS_NUMBER, - ) - tuner = (TunerBuilder(self.params.task) - .with_tuner(SimultaneousTuner) - .with_metric(tuner_plan.metric) - .with_iterations(tuner_plan.iterations) - .with_timeout(datetime.timedelta(minutes=tuner_plan.timeout_minutes)) - .with_eval_time_constraint(self.params.composer_requirements.max_graph_fit_time) - .with_requirements(self.params.composer_requirements) - .build(train_data)) - - with self.timer.launch_tuning(): - self.was_tuned = False - self.log.message(f'Hyperparameters tuning started with {round(tuner_plan.timeout_minutes)} min. timeout') - tuned_pipeline = tuner.tune(pipeline_gp_composed) - self.log.message('Hyperparameters tuning finished') - self.was_tuned = tuner.was_tuned - return tuned_pipeline +import datetime +import gc +from copy import deepcopy +from typing import List, Optional, Sequence, Tuple, Union + +from golem.core.log import default_log +from golem.core.optimisers.opt_history_objects.opt_history import OptHistory +from golem.core.tuning.simultaneous import SimultaneousTuner + +from fedot.api.api_utils.api_composer_rules import build_cache_init_plan, build_tuner_plan +from fedot.api.api_utils.api_run_planner import build_composer_execution_plan +from fedot.api.api_utils.assumptions.assumptions_handler import AssumptionsHandler +from fedot.api.api_utils.params import ApiParams +from fedot.api.time import ApiTime +from fedot.core.caching.operations_cache import OperationsCache +from fedot.core.caching.preprocessing_cache import PreprocessingCache +from fedot.core.caching.predictions_cache import PredictionsCache +from fedot.core.composer.composer_builder import ComposerBuilder +from fedot.core.composer.gp_composer.gp_composer import GPComposer +from fedot.core.constants import DEFAULT_TUNING_ITERATIONS_NUMBER +from fedot.core.data.data import InputData +from fedot.core.pipelines.pipeline import Pipeline +from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder +from fedot.core.repository.metrics_repository import MetricIDType +from fedot.utilities.composer_timer import fedot_composer_timer +from fedot.core.context.context import ExecutionContext + + +class ApiComposer: + + def __init__(self, api_params: ApiParams, metrics: Union[MetricIDType, Sequence[MetricIDType]], + context: Optional[ExecutionContext] = None): + self.log = default_log(self) + self.params = api_params + self.metrics = metrics + self.operations_cache: Optional[OperationsCache] = None + self.preprocessing_cache: Optional[PreprocessingCache] = None + self.predictions_cache: Optional[PredictionsCache] = None + self.timer = None + # status flag indicating that composer step was applied + self.was_optimised = False + # status flag indicating that tuner step was applied` + self.was_tuned = False + self.context = context + self.init_cache() + + def init_cache(self): + cache_plan = build_cache_init_plan( + use_operations_cache=self.params.get('use_operations_cache'), + use_preprocessing_cache=self.params.get('use_preprocessing_cache'), + use_predictions_cache=self.params.get('use_predictions_cache'), + use_input_preprocessing=self.params.get('use_input_preprocessing'), + cache_dir=self.params.get('cache_dir'), + use_stats=self.params.get('use_stats'), + ) + + if cache_plan.use_operations_cache: + self.operations_cache = OperationsCache(cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) + self.operations_cache.reset() + if cache_plan.use_preprocessing_cache: + self.preprocessing_cache = PreprocessingCache( + cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) + self.preprocessing_cache.reset() + if cache_plan.use_predictions_cache: + self.predictions_cache = PredictionsCache(cache_dir=cache_plan.cache_dir, use_stats=cache_plan.use_stats) + self.predictions_cache.reset() + + def obtain_model(self, train_data: InputData) -> Tuple[Pipeline, Sequence[Pipeline], OptHistory]: + """ Function for composing FEDOT pipeline model """ + + with fedot_composer_timer.launch_composing(): + timeout: float = self.params.timeout + with_tuning = self.params.get('with_tuning') + + self.timer = ApiTime(time_for_automl=timeout, with_tuning=with_tuning) + + initial_assumption, fitted_assumption = self.propose_and_fit_initial_assumption(train_data) + + multi_objective = len(self.metrics) > 1 + self.params.init_params_for_composing(self.timer.timedelta_composing, multi_objective) + + self.log.message(f"AutoML configured." + f" Parameters tuning: {with_tuning}." + f" Time limit: {timeout} min." + f" Set of candidate models: {self.params.get('available_operations')}.") + + best_pipeline, best_pipeline_candidates, gp_composer = self.compose_pipeline( + train_data, + initial_assumption, + fitted_assumption + ) + + timeout_for_tuning = abs(self.timer.determine_resources_for_tuning()) / 60 + execution_plan = build_composer_execution_plan( + with_tuning=with_tuning, + have_time_for_composing=self.was_optimised, + have_time_for_tuning=self.timer.have_time_for_tuning(), + tuning_timeout_minutes=timeout_for_tuning, + ) + + if execution_plan.should_tune: + with fedot_composer_timer.launch_tuning('composing'): + best_pipeline = self.tune_final_pipeline(train_data, best_pipeline, execution_plan) + elif with_tuning: + self.log.message( + f'Time for pipeline composing was {str(self.timer.composing_spend_time)}.\n' + f'The remaining {max(0, round(execution_plan.tuning_timeout_minutes, 1))} seconds are not enough ' + f'to tune the hyperparameters.') + self.log.message('Composed pipeline returned without tuning.') + self.was_tuned = False + + if gp_composer.history: + adapter = self.params.graph_generation_params.adapter + gp_composer.history.tuning_result = adapter.adapt(best_pipeline) + gc.collect() + + self.log.message('Model generation finished') + return best_pipeline, best_pipeline_candidates, gp_composer.history + + def propose_and_fit_initial_assumption(self, train_data: InputData) -> Tuple[Sequence[Pipeline], Pipeline]: + """ Method for obtaining and fitting initial assumption""" + available_operations = self.params.get('available_operations') + + preset = self.params.get('preset') + + assumption_handler = AssumptionsHandler(train_data) + + initial_assumption = assumption_handler.propose_assumptions(self.params.get('initial_assumption'), + available_operations, + use_input_preprocessing=self.params.get( + 'use_input_preprocessing')) + + with self.timer.launch_assumption_fit(n_folds=self.params.data['cv_folds']): + fitted_assumption = \ + assumption_handler.fit_assumption_and_check_correctness(deepcopy(initial_assumption[0]), + operations_cache=self.operations_cache, + preprocessing_cache=self.preprocessing_cache, + eval_n_jobs=self.params.n_jobs) + + self.log.message( + f'Initial pipeline was fitted in ' + f'{round(self.timer.assumption_fit_spend_time_single_fold.total_seconds(), 1)} sec.') + + self.log.message( + f'Taking into account n_folds={self.params.data["cv_folds"]}, estimated fit time for initial assumption ' + f'is {round(self.timer.assumption_fit_spend_time.total_seconds(), 1)} sec.') + + self.params.update(preset=assumption_handler.propose_preset(preset, self.timer, n_jobs=self.params.n_jobs)) + + return initial_assumption, fitted_assumption + + def compose_pipeline(self, train_data: InputData, initial_assumption: Sequence[Pipeline], + fitted_assumption: Pipeline) -> Tuple[Pipeline, List[Pipeline], GPComposer]: + + gp_composer: GPComposer = (ComposerBuilder(task=self.params.task) + .with_requirements(self.params.composer_requirements) + .with_initial_pipelines(initial_assumption) + .with_optimizer(self.params.get('optimizer')) + .with_optimizer_params(parameters=self.params.optimizer_params) + .with_metrics(self.metrics) + .with_context(self.context) + .with_cache(self.operations_cache, self.preprocessing_cache, self.predictions_cache) + .with_graph_generation_param(self.params.graph_generation_params) + .build()) + + have_time_for_composing = self.timer.have_time_for_composing(self.params.get('pop_size'), self.params.n_jobs) + execution_plan = build_composer_execution_plan( + with_tuning=self.params.get('with_tuning'), + have_time_for_composing=have_time_for_composing, + have_time_for_tuning=False, + tuning_timeout_minutes=0, + ) + + if execution_plan.should_compose: + with self.timer.launch_composing(): + self.log.message('Pipeline composition started.') + self.was_optimised = False + best_pipelines = gp_composer.compose_pipeline(data=train_data) + best_pipeline_candidates = gp_composer.best_models + self.was_optimised = True + else: + self.log.message(f'Timeout is too small for composing and is skipped ' + f'because fit_time is {self.timer.assumption_fit_spend_time.total_seconds()} sec.') + best_pipelines = fitted_assumption + best_pipeline_candidates = [fitted_assumption] + self.was_optimised = False + + for pipeline in best_pipeline_candidates: + pipeline.log = self.log + best_pipeline = best_pipelines[0] if isinstance(best_pipelines, Sequence) else best_pipelines + return best_pipeline, best_pipeline_candidates, gp_composer + + def tune_final_pipeline(self, train_data: InputData, + pipeline_gp_composed: Pipeline, + execution_plan=None) -> Pipeline: + """ Launch tuning procedure for obtained pipeline by composer """ + timeout_for_tuning = execution_plan.tuning_timeout_minutes if execution_plan else abs( + self.timer.determine_resources_for_tuning()) / 60 + tuner_plan = build_tuner_plan( + metrics=self.metrics, + timeout_minutes=timeout_for_tuning, + iterations=DEFAULT_TUNING_ITERATIONS_NUMBER, + ) + tuner = (TunerBuilder(self.params.task) + .with_tuner(SimultaneousTuner) + .with_metric(tuner_plan.metric) + .with_iterations(tuner_plan.iterations) + .with_timeout(datetime.timedelta(minutes=tuner_plan.timeout_minutes)) + .with_eval_time_constraint(self.params.composer_requirements.max_graph_fit_time) + .with_requirements(self.params.composer_requirements) + .with_context(self.context) + .build(train_data)) + + with self.timer.launch_tuning(): + self.was_tuned = False + self.log.message(f'Hyperparameters tuning started with {round(tuner_plan.timeout_minutes)} min. timeout') + tuned_pipeline = tuner.tune(pipeline_gp_composed) + self.log.message('Hyperparameters tuning finished') + self.was_tuned = tuner.was_tuned + return tuned_pipeline diff --git a/fedot/api/api_utils/api_params_repository.py b/fedot/api/api_utils/api_params_repository.py index b091104251..9dda08094a 100644 --- a/fedot/api/api_utils/api_params_repository.py +++ b/fedot/api/api_utils/api_params_repository.py @@ -25,9 +25,10 @@ class ApiParamsRepository: STATIC_INDIVIDUAL_METADATA_KEYS = {'use_input_preprocessing'} - def __init__(self, task_type: TaskTypesEnum): + def __init__(self, task_type: TaskTypesEnum, context: Optional[ExecutionContext] = None): self.task_type = task_type self.default_params = ApiParamsRepository.default_params_for_task(self.task_type) + self.context = context @staticmethod def default_params_for_task(task_type: TaskTypesEnum) -> dict: @@ -75,14 +76,21 @@ def get_params_for_gp_algorithm_params(self, params: dict) -> dict: if params.get('genetic_scheme') == 'steady_state': gp_algorithm_params['genetic_scheme_type'] = GeneticSchemeTypesEnum.steady_state - # gp_algorithm_params['mutation_types'] = ApiParamsRepository._get_default_mutations(self.task_type, params) - gp_algorithm_params['mutation_types'] = context.api_params_repository__get_default_mutations(self.task_type, - params) + gp_algorithm_params['mutation_types'] = ApiParamsRepository._get_default_mutations(self.task_type, params, + self.context) gp_algorithm_params['seed'] = params['seed'] return gp_algorithm_params + + @staticmethod + def _get_default_mutations(task_type: TaskTypesEnum, params, context: Optional[ExecutionContext] = None) -> Sequence[MutationTypesEnum]: + if context: + return context.default_mutations.get_default_mutation(task_type, params) + else: + return _get_default_mutations_core(task_type, params) + @staticmethod - def _get_default_mutations(task_type: TaskTypesEnum, params) -> Sequence[MutationTypesEnum]: + def _get_default_mutations_core(task_type: TaskTypesEnum, params) -> Sequence[MutationTypesEnum]: mutations = [parameter_change_mutation, MutationTypesEnum.single_change, MutationTypesEnum.single_drop, diff --git a/fedot/api/main.py b/fedot/api/main.py index b7b7c955e2..e1055b1ecf 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -51,7 +51,7 @@ from fedot.utilities.define_metric_by_task import MetricByTask from fedot.utilities.memory import MemoryAnalytics from fedot.utilities.project_import_export import export_project_to_zip, import_project_from_zip -from fedot.core.context.context import ExecutionContext +from fedot.core.contex.context import ExecutionContext NOT_FITTED_ERR_MSG = 'Model not fitted yet' @@ -107,11 +107,11 @@ def __init__(self, **composer_tuner_params ): + self.context = ExecutionContext(extension_name=context) + set_random_seed(seed) self.log = self._init_logger(logging_level) - self.context = ExecutionContext(extension_name=context) - # Attributes for dealing with metrics, data sources and hyperparameters self.params = ApiParams(composer_tuner_params, problem, task_params, n_jobs, timeout, seed) @@ -303,7 +303,8 @@ def tune_tensordata(self, .with_n_jobs(common_tune_plan.n_jobs) .with_metric(common_tune_plan.metric) .with_iterations(iterations) - .with_timeout(timeout)) + .with_timeout(timeout) + .with_context(self.context)) pipeline_tuner = getattr(pipeline_tuner, tune_plan.builder_method_name)( tensor_data if tune_plan.use_tensor_runtime else common_tune_plan.input_data ) @@ -377,6 +378,7 @@ def tune(self, .with_metric(tune_plan.metric) .with_iterations(iterations) .with_timeout(timeout) + .with_context(self.context) .build(tune_input_data)) self.current_pipeline = pipeline_tuner.tune(self.current_pipeline, show_progress=show_progress) @@ -682,7 +684,8 @@ def get_metrics(self, data_producer=lambda: (yield self.train_data, self.test_data), validation_blocks=validation_blocks, eval_n_jobs=self.params.n_jobs, - do_unfit=False) + do_unfit=False, + context=self.context) metrics = obj_eval.evaluate(self.current_pipeline).values metrics = {metric_name: round(abs(metric), rounding_order) for (metric_name, metric) in diff --git a/fedot/core/composer/composer_builder.py b/fedot/core/composer/composer_builder.py index 51f868fb5a..1081e41b7d 100644 --- a/fedot/core/composer/composer_builder.py +++ b/fedot/core/composer/composer_builder.py @@ -13,9 +13,9 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache from fedot.core.caching.predictions_cache import PredictionsCache -from fedot.core.context import ExecutionContext from fedot.core.composer.composer import Composer from fedot.core.composer.gp_composer.gp_composer import GPComposer +from fedot.core.context.context import ExecutionContext from fedot.core.optimisers.objective.metrics_objective import MetricsObjective from fedot.core.pipelines.pipeline import Pipeline from fedot.core.pipelines.pipeline_composer_requirements import PipelineComposerRequirements @@ -61,8 +61,9 @@ def __init__(self, task: Task): self.context: Optional[ExecutionContext] = None - def with_context(self, context: ExecutionContext): - self.context = context or ExecutionContext() + def with_context(self, context): + if context: + self.context = context return self def with_composer(self, composer_cls: Optional[Type[Composer]]): @@ -118,8 +119,7 @@ def with_cache(self, @staticmethod def _get_default_composer_params(task: Task) -> PipelineComposerRequirements: # Get all available operations for task - # operations = get_operations_for_task(task=task, mode='all') - operations = self.context.operation_registry.get_operation_for_task(task=task, mode='all') + operations = get_operations_for_task(task=task, mode='all') return PipelineComposerRequirements(primary=operations, secondary=operations) def _get_default_graph_generation_params(self) -> GraphGenerationParams: @@ -169,7 +169,6 @@ def build(self) -> Composer: self.composer_requirements, self.operations_cache, self.preprocessing_cache, - self.predictions_cache, - self.context) + self.predictions_cache) return composer diff --git a/fedot/core/composer/gp_composer/gp_composer.py b/fedot/core/composer/gp_composer/gp_composer.py index ada295b0ed..705110480e 100644 --- a/fedot/core/composer/gp_composer/gp_composer.py +++ b/fedot/core/composer/gp_composer/gp_composer.py @@ -11,10 +11,10 @@ from fedot.core.caching.operations_cache import OperationsCache from fedot.core.caching.predictions_cache import PredictionsCache from fedot.core.caching.preprocessing_cache import PreprocessingCache -from fedot.core.context.context import ExecutionContext from fedot.core.composer.composer import Composer from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData +from fedot.core.context.context import ExecutionContext from fedot.core.optimisers.objective.data_objective_eval import ( PipelineObjectiveEvaluate, ) @@ -25,7 +25,6 @@ ) from fedot.core.utils import default_fedot_data_dir -from functools import partial class GPComposer(Composer): """ @@ -43,7 +42,7 @@ def __init__(self, optimizer: GraphOptimizer, operations_cache: Optional[OperationsCache] = None, preprocessing_cache: Optional[PreprocessingCache] = None, predictions_cache: Optional[PredictionsCache] = None, - context: Optional[ExecutionContext] = None): + context: Optional[str] = None,): super().__init__(optimizer, composer_requirements) self.composer_requirements = composer_requirements self.operations_cache: Optional[OperationsCache] = operations_cache @@ -51,15 +50,15 @@ def __init__(self, optimizer: GraphOptimizer, self.predictions_cache: Optional[PredictionsCache] = predictions_cache self.best_models: Collection[Pipeline] = () - - self.context = context or ExecutionContext() + self.context = ExecutionContext(extension_name=context) def compose_pipeline(self, data: Union[InputData, MultiModalData]) -> Union[Pipeline, Sequence[Pipeline]]: # Define data source data_splitter = DataSourceSplitter(self.composer_requirements.cv_folds, shuffle=True) - - data_producer = self.context.data_source_splitter_build(data_splitter, data) + if self.context: + data_splitter.build = self.context.data_source_splitter.build + data_producer = data_splitter.build(data) parallelization_mode = self.composer_requirements.parallelization_mode if parallelization_mode == 'populational': @@ -77,10 +76,10 @@ def compose_pipeline(self, data: Union[InputData, MultiModalData]) -> Union[Pipe preprocessing_cache=self.preprocessing_cache, predictions_cache=self.predictions_cache, validation_blocks=data_splitter.validation_blocks, - eval_n_jobs=n_jobs_for_evaluation) + eval_n_jobs=n_jobs_for_evaluation, + context=context) - # objective_function = objective_evaluator.evaluate - objective_function = partial(self.context.evaluator_evaluate, objective_evaluator) + objective_function = objective_evaluator.evaluate # Define callback for computing intermediate metrics if needed if self.composer_requirements.collect_intermediate_metric: diff --git a/fedot/core/context/context.py b/fedot/core/context/context.py index 2443f346b0..b77dd3c995 100644 --- a/fedot/core/context/context.py +++ b/fedot/core/context/context.py @@ -1,69 +1,33 @@ from typing import Dict, Any, Optional, Callable from fedot.extensions.registry import get_registered_extension, register_extension -from fedot.core.context.industrial_manifest import FEDOT_INDUSTRIAL_MANIFEST -register_extension(FEDOT_INDUSTRIAL_MANIFEST) class ExecutionContext: - def __init__(self, extension_name: str = "core", extra_params: Optional[Dict[str, Any]] = None): + def __init__(self, extension_name: str = "industrial", extra_params: Optional[Dict[str, Any]] = None): self.extension_name = extension_name self.extra_params = extra_params or {} self._instances: Dict[str, Any] = {} self._overridden: Dict[str, Any] = {} - self._manifest = None - if extension_name != "core": - from fedot.extensions.registry import _REGISTERED_EXTENSIONS - manifest = _REGISTERED_EXTENSIONS.get(extension_name) - if manifest is None: - raise ValueError(f"Extension '{extension_name}' not registered") - self._manifest = manifest - - self._core_implementations = self._get_core_implementations() - - self._protocol_classes = self._core_implementations.copy() - if self._manifest and self._manifest.protocols: - self._protocol_classes.update(self._manifest.protocols) - - def _get_core_implementations(self) -> Dict[str, Callable]: - from fedot.core.context.default_backend import ( - CoreSplitter, CoreDataMerger, CoreImageMerger, - CoreTSMerger, CoreTextMerger, CoreTuner, - CoreDataSourceSplitter, CoreOperationPredict, - CoreLaggedTransformer, CoreTopologicalFeatures, - CoreTsSmoothing, CoreApiComposerTune, CoreReproduction, - CoreSearchSpace, CoreDefaultMutations, CoreEvaluator - ) - return { - "splitter": CoreSplitter, - "data_merger": CoreDataMerger, - "image_merger": CoreImageMerger, - "ts_merger": CoreTSMerger, - "text_merger": CoreTextMerger, - "tuner_class": CoreTuner, - "data_source_splitter": CoreDataSourceSplitter, - "operation_predict": CoreOperationPredict, - "lagged_transformer": CoreLaggedTransformer, - "topological_features": CoreTopologicalFeatures, - "ts_smoothing": CoreTsSmoothing, - "api_composer_tune": CoreApiComposerTune, - "reproduction": CoreReproduction, - "search_space": CoreSearchSpace, - "default_mutations": CoreDefaultMutations, - "evaluator": CoreEvaluator, - } + manifest = get_registered_extension(extension_name) + if manifest is None: + raise ValueError(f"Extension '{extension_name}' not registered") + self._manifest = manifest - def _get_protocol_class(self, protocol_name: str) -> Callable: - if self._manifest and self._manifest.protocols: - if protocol_name in self._manifest.protocols: - return self._manifest.protocols[protocol_name] - - if protocol_name in self._core_implementations: - return self._core_implementations[protocol_name] + self._protocol_classes = self._manifest.protocols or {} + def _get_protocol_class(self, protocol_name: str) -> Callable: + if protocol_name in self._protocol_classes: + return self._protocol_classes[protocol_name] raise ValueError(f"No implementation for protocol '{protocol_name}'") def _get_instance(self, protocol_name: str) -> Any: + if protocol_name in self._overridden: + override = self._overridden[protocol_name] + if isinstance(override, type): + return override(**self.extra_params) + return override + if protocol_name not in self._instances: protocol_class = self._get_protocol_class(protocol_name) self._instances[protocol_name] = protocol_class(**self.extra_params) @@ -135,7 +99,7 @@ def evaluator(self): def __setattr__(self, name: str, value: Any) -> None: if name in ('extra_params', '_instances', '_overridden', '_protocol_classes', - '_manifest', '_core_implementations', 'extension_name'): + '_manifest', 'extension_name'): super().__setattr__(name, value) else: self._overridden[name] = value @@ -144,7 +108,7 @@ def __getattr__(self, name: str): if name in self._overridden: return self._overridden[name] - if name in ('_protocol_classes', '_instances', '_manifest', '_core_implementations'): + if name in ('_protocol_classes', '_instances', '_manifest'): return super().__getattribute__(name) - raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") \ No newline at end of file + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") diff --git a/fedot/core/context/factories.py b/fedot/core/context/factories.py deleted file mode 100644 index 3ed22ca720..0000000000 --- a/fedot/core/context/factories.py +++ /dev/null @@ -1,60 +0,0 @@ -from fedot.core.context.industrial_backend import (IndustrialSplitter, IndustrialDataMerger, IndustrialImageMerger, - IndustrialTSMerger, IndustrialTextMerger, - IndustrialDataSourceSplitterBuilder, IndustrialTunerClass, - IndustrialReproduction, IndustrialEvaluator, IndustrialSearchSpace, - IndustrialDefaultMutations,IndustrialOperationPredict, - IndustrialLaggedTransformer, IndustrialTopologicalFeatures, - IndustrialTsSmoothing, IndustrialApiComposerTune) - - - -def industrial_context_factory(backend: str = "default"): - return IndustrialContext(backend=backend) - -def splitters_factory(): - return IndustrialSplitter() - -def data_merger_factory(): - return IndustrialDataMerger() - -def image_merger_factory(): - return IndustrialImageMerger() - -def ts_merger_factory(): - return IndustrialTSMerger() - -def text_merger_factory(): - return IndustrialTextMerger() - -def data_source_splitter_factory(): - return IndustrialDataSourceSplitterBuilder() - -def tuner_class_factory(backend: str = "default"): - return IndustrialTunerClass(backend) - -def reproduction_factory(): - return IndustrialReproduction() - -def evaluator_factory(): - return IndustrialEvaluator() - -def search_space_factory(): - return IndustrialSearchSpace() - -def mutations_factory(): - return IndustrialDefaultMutations() - -def operation_predict_factory(): - return IndustrialOperationPredict() - -def lagged_transformer_factory(): - return IndustrialLaggedTransformer() - -def topo_features_factory(): - return IndustrialTopologicalFeatures() - -def ts_smoothing_factory(): - return IndustrialTsSmoothing() - -def api_composer_tune_factory(): - return IndustrialApiComposerTune() \ No newline at end of file diff --git a/fedot/core/context/industrial_backend.py b/fedot/core/context/industrial_backend.py index 79737b8620..8ddbe9c01d 100644 --- a/fedot/core/context/industrial_backend.py +++ b/fedot/core/context/industrial_backend.py @@ -62,23 +62,8 @@ def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: from fedot.industrial.core.repository.industrial_implementations.abstract import preprocess_industrial_predicts return preprocess_industrial_predicts(predicts) - def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: - from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts - return merge_industrial_predicts(predicts) - class IndustrialTSMerger(TSMergerProtocol): - def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: - from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts - return merge_industrial_predicts(predicts) - - def merge_targets(self, targets: List[np.ndarray]) -> np.ndarray: - from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_targets - return merge_industrial_targets(targets) - - def preprocess_predicts(self, predicts: List[np.ndarray]) -> List[np.ndarray]: - from fedot.industrial.core.repository.industrial_implementations.abstract import preprocess_industrial_predicts - return preprocess_industrial_predicts(predicts) def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: from fedot.industrial.core.repository.industrial_implementations.abstract import postprocess_industrial_predicts @@ -90,9 +75,6 @@ def merge_predicts(self, predicts: List[np.ndarray]) -> np.ndarray: from fedot.industrial.core.repository.industrial_implementations.abstract import merge_industrial_predicts return merge_industrial_predicts(predicts) - def postprocess_predicts(self, merged: np.ndarray) -> np.ndarray: - return merged - class IndustrialDataSourceSplitterBuilder(DataSourceSplitterProtocol): def build(self, data: Union['InputData', 'MultiModalData']): @@ -104,7 +86,7 @@ class IndustrialTunerClass(TunerClassProtocol): def __init__(self, **kwargs): self.backend = kwargs.get("backend", "default") - def __call__(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): + def optuna_tuner(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): from golem.core.tuning.optuna_tuner import OptunaTuner from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import DaskOptunaTuner @@ -139,7 +121,7 @@ def get_parameters_dict(self): class IndustrialDefaultMutations(DefaultMutationsProtocol): @staticmethod - def __call__(task_type: 'TaskTypesEnum', params): + def get_default_mutations(task_type: 'TaskTypesEnum', params): from fedot.industrial.core.repository.industrial_implementations.optimisation import \ _get_default_industrial_mutations return _get_default_industrial_mutations(task_type, params) @@ -165,7 +147,7 @@ class IndustrialLaggedTransformer(LaggedTransformerProtocol): def _update_column_types(self, output_data: 'OutputData'): from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ update_column_types_industrial - update_column_types_industrial(output_data) + return update_column_types_industrial(output_data) def transform(self, input_data: 'InputData') -> 'OutputData': from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ @@ -180,7 +162,7 @@ def transform_for_fit(self, input_data: 'InputData') -> 'OutputData': def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): from fedot.industrial.core.repository.industrial_implementations.data_transformation import \ _check_and_correct_window_size_industrial - _check_and_correct_window_size_industrial(time_series, forecast_length) + return _check_and_correct_window_size_industrial(time_series, forecast_length) class IndustrialTopologicalFeatures(TopologicalFeaturesProtocol): @@ -202,6 +184,6 @@ def transform(self, input_data: 'InputData') -> 'OutputData': class IndustrialApiComposerTune(ApiComposerTuneProtocol): - def __call__(self, train_data: 'InputData', pipeline: 'Pipeline', execution_plan=None) -> 'Pipeline': + def tune_pipeline(self, train_data: 'InputData', pipeline: 'Pipeline', execution_plan=None) -> 'Pipeline': from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import tune_pipeline_industrial - return tune_pipeline_industrial(train_data, pipeline, execution_plan) \ No newline at end of file + return tune_pipeline_industrial(train_data, pipeline, execution_plan) diff --git a/fedot/core/context/industrial_manifest.py b/fedot/core/context/industrial_manifest.py index 9bf32c5835..36a1439450 100644 --- a/fedot/core/context/industrial_manifest.py +++ b/fedot/core/context/industrial_manifest.py @@ -1,4 +1,5 @@ from fedot.extensions.contracts import ExtensionManifest +from fedot.extensions.registry import get_registered_extension, register_extension from fedot.core.context.industrial_backend import ( IndustrialSplitter, IndustrialDataMerger, @@ -42,3 +43,5 @@ "api_composer_tune": IndustrialApiComposerTune, } ) + +register_extension(FEDOT_INDUSTRIAL_MANIFEST) diff --git a/fedot/core/data/data_split.py b/fedot/core/data/data_split.py index 8b48223982..a18221dfa0 100644 --- a/fedot/core/data/data_split.py +++ b/fedot/core/data/data_split.py @@ -6,9 +6,9 @@ from fedot.core.data.data import InputData from fedot.core.data.multi_modal import MultiModalData -from fedot.core.context import ExecutionContext from fedot.core.repository.dataset_types import DataTypesEnum from fedot.core.repository.tasks import TaskTypesEnum +from fedot.core.context.context import ExecutionContext def _split_input_data_by_indexes(origin_input_data: Union[InputData, MultiModalData], @@ -175,7 +175,8 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], stratify: bool = True, random_seed: int = 42, validation_blocks: Optional[int] = None, - context: Optional[ExecutionContext] = None) -> Tuple[Union[InputData, MultiModalData], Union[InputData, MultiModalData]]: + context: Optional[ExecutionContext] = None) -> Tuple[Union[InputData, MultiModalData], + Union[InputData, MultiModalData]]: """ Function for train and test split for both InputData and MultiModalData :param data: InputData object to split @@ -185,13 +186,10 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], :param stratify: make stratified sample or not :param random_seed: random_seed for shuffle :param validation_blocks: validation blocks are used for test - :param context: FEDOT Core or Fedot.Industrial funcs :return: data for train, data for validation """ - context = context or ExecutionContext() - # for backward compatibility shuffle |= shuffle_flag # check that stratification may be done @@ -207,17 +205,18 @@ def train_test_data_setup(data: Union[InputData, MultiModalData], 'random_seed': random_seed, 'validation_blocks': validation_blocks} if isinstance(data, InputData): - # split_func_dict = {DataTypesEnum.multi_ts: _split_time_series, - # DataTypesEnum.ts: _split_time_series, - # DataTypesEnum.table: _split_any, - # DataTypesEnum.image: _split_any, - # DataTypesEnum.text: _split_any} - - split_func_dict = {DataTypesEnum.multi_ts: context.data_split__split_time_series, - DataTypesEnum.ts: context.data_split__split_time_series, - DataTypesEnum.table: context.data_split__split_any, - DataTypesEnum.image: context.data_split__split_any, - DataTypesEnum.text: context.data_split__split_any} + if context: + split_func_dict = {DataTypesEnum.multi_ts: context.splitter.split_time_series, + DataTypesEnum.ts: context.splitter.split_time_series, + DataTypesEnum.table: context.splitter.split_any, + DataTypesEnum.image: context.splitter.split_any, + DataTypesEnum.text: context.splitter.split_any,} + else: + split_func_dict = {DataTypesEnum.multi_ts: _split_time_series, + DataTypesEnum.ts: _split_time_series, + DataTypesEnum.table: _split_any, + DataTypesEnum.image: _split_any, + DataTypesEnum.text: _split_any} if data.data_type not in split_func_dict: raise TypeError((f'Unknown data type {type(data)}. Supported data types:' diff --git a/fedot/core/data/merge/data_merger.py b/fedot/core/data/merge/data_merger.py index 064075c594..c4cfb1b150 100644 --- a/fedot/core/data/merge/data_merger.py +++ b/fedot/core/data/merge/data_merger.py @@ -8,8 +8,8 @@ from fedot.core.data.array_utilities import find_common_elements, atleast_2d, atleast_4d, flatten_extra_dim from fedot.core.data.data import OutputData, InputData from fedot.core.data.merge.supplementary_data_merger import SupplementaryDataMerger -from fedot.core.context import ExecutionContext from fedot.core.repository.dataset_types import DataTypesEnum +from fedot.core.context.context import ExecutionContext class DataMerger: @@ -24,12 +24,11 @@ class DataMerger: :param outputs: list with OutputData from parent nodes for merging """ - def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None, - context: Optional[ExecutionContext] = None): + def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None, context: Optional[ExecutionContext] = None,): self.log = default_log(self) self.outputs = outputs self.data_type = data_type or DataMerger.get_datatype_for_merge(output.data_type for output in outputs) - self.context = context or ExecutionContext() + self.context = context # Ensure outputs are of equal length, find common index if it is not idx_list = [np.asarray(output.idx) for output in outputs] @@ -38,11 +37,10 @@ def __init__(self, outputs: List['OutputData'], data_type: DataTypesEnum = None, raise ValueError('There are no common indices for outputs') # Find first output with the main target & resulting task - # self.main_output = DataMerger.find_main_output(outputs) - self.main_output = self.context.merger_find_main_output(outputs) + self.main_output = DataMerger.find_main_output(outputs, self.context) @staticmethod - def get(outputs: List['OutputData']) -> 'DataMerger': + def get_core(outputs: List['OutputData']) -> 'DataMerger': """ Construct appropriate data merger for the outputs. """ # Ensure outputs can be merged @@ -75,8 +73,7 @@ def merge(self) -> 'InputData': common_predicts = self.find_common_predicts() mergeable_predicts = self.preprocess_predicts(common_predicts) - # merged_features = self.merge_predicts(mergeable_predicts) - merged_features = self.context.merger_merge_predicts(mergeable_predicts) + merged_features = self.merge_predicts(mergeable_predicts) merged_features = self.postprocess_predicts(merged_features) updated_metadata = SupplementaryDataMerger(self.outputs, self.main_output).merge() @@ -121,7 +118,7 @@ def preprocess_predicts(self, predicts: List[np.array]) -> List[np.array]: """ Pre-process (e.g. equalizes sizes, reshapes) and return list of arrays that can be merged. """ return list(map(atleast_2d, predicts)) - def merge_predicts(self, predicts: List[np.array]) -> np.array: + def merge_predicts_core(self, predicts: List[np.array]) -> np.array: # Finally, merge predictions into features for the next stage return np.concatenate(predicts, axis=-1) @@ -142,7 +139,7 @@ def is_forecast_index(output: 'OutputData'): return len(output.idx) != len(output.predict) @staticmethod - def find_main_output(outputs: List['OutputData']) -> 'OutputData': + def find_main_output_core(outputs: List['OutputData']) -> 'OutputData': """ Returns first output with main target or (if there are no main targets) the output with priority secondary target. """ priority_output = next((output for output in outputs @@ -152,11 +149,35 @@ def find_main_output(outputs: List['OutputData']) -> 'OutputData': i_priority_secondary = np.argmin(flow_lengths) priority_output = outputs[i_priority_secondary] return priority_output + @staticmethod + def find_main_output(outputs: List['OutputData'], context: Optional[ExecutionContext] = None) -> 'OutputData': + if context: + return context.data_merger.find_main_output(outputs) + else: + return find_main_output_core(outputs) + @staticmethod + def get(outputs: List['OutputData'], context: Optional[ExecutionContext] = None) -> 'DataMerger': + if context: + return context.data_merger.get(outputs) + else: + return get_core(outputs) + + def merge_predicts(self, predicts: List[np.array]) -> np.array: + if self.context: + return self.context.data_merger.merge_predicts(predicts) + else: + return self.merge_predicts_core(predicts) class ImageDataMerger(DataMerger): def preprocess_predicts(self, predicts: List[np.array]) -> List[np.array]: + if self.context: + return self.context.image_merger.preprocess_predicts(predicts) + else: + return self.preprocess_predicts_core(predicts) + + def preprocess_predicts_core(self, predicts: List[np.array]) -> List[np.array]: # Reshape predicts to 4d (idx, width, height, channels) reshaped_predicts = list(map(atleast_4d, predicts)) @@ -172,6 +193,12 @@ def preprocess_predicts(self, predicts: List[np.array]) -> List[np.array]: class TSDataMerger(DataMerger): def postprocess_predicts(self, merged_predicts: np.array) -> np.array: + if self.context: + return self.context.ts_merger.postprocess_predicts(merged_predicts) + else: + return self.postprocess_predicts_core(merged_predicts) + + def postprocess_predicts_core(self, merged_predicts: np.array) -> np.array: # Ensure that 1d-column timeseries remains 1d timeseries return flatten_extra_dim(merged_predicts) diff --git a/fedot/core/operations/evaluation/operation_implementations/data_operations/topological/fast_topological_extractor.py b/fedot/core/operations/evaluation/operation_implementations/data_operations/topological/fast_topological_extractor.py index cf734fa8e0..584aaced1b 100644 --- a/fedot/core/operations/evaluation/operation_implementations/data_operations/topological/fast_topological_extractor.py +++ b/fedot/core/operations/evaluation/operation_implementations/data_operations/topological/fast_topological_extractor.py @@ -3,6 +3,7 @@ from typing import Optional import numpy as np +from fedot.core.context.context import ExecutionContext try: from gph import ripser_parallel as ripser @@ -20,7 +21,7 @@ class TopologicalFeaturesImplementation(DataOperationImplementation): - def __init__(self, params: Optional[OperationParameters] = None): + def __init__(self, params: Optional[OperationParameters] = None, context: Optional[ExecutionContext] = None): super().__init__(params) self.window_size_as_share = params.get('window_size_as_share') self.max_homology_dimension = params.get('max_homology_dimension') @@ -30,15 +31,23 @@ def __init__(self, params: Optional[OperationParameters] = None): self.quantiles = (0.1, 0.25, 0.5, 0.75, 0.9) self._shape = len(self.quantiles) self._window_size = None + self.context = context def fit(self, input_data: InputData): + if self.context: + return self.context.topological_feature.fit(input_data) + else: + return self.fit_core(input_data) + + + def fit_core(self, input_data: InputData): self._window_size = int(input_data.features.shape[1] * self.window_size_as_share) self._window_size = max(self._window_size, 2) self._window_size = min(self._window_size, input_data.features.shape[1] - 2) self._window_size = max(self._window_size, 1) return self - def transform(self, input_data: InputData) -> OutputData: + def transform_core(self, input_data: InputData) -> OutputData: features = input_data.features with Parallel(n_jobs=self.n_jobs, prefer='processes') as parallel: topological_features = parallel(delayed(self._extract_features) @@ -52,6 +61,12 @@ def transform(self, input_data: InputData) -> OutputData: np.nan_to_num(result, copy=False, nan=0, posinf=0, neginf=0) return result + def transform(self, input_data: InputData): + if self.context: + return self.context.topological_feature.transform(input_data) + else: + return self.transform_core(input_data) + def _extract_features(self, x): x_sliced = np.array([x[i:self._window_size + i] for i in range(x.shape[0] - self._window_size + 1)]) x_processed = ripser(x_sliced, diff --git a/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py b/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py index bd141cb1df..ea835d43fe 100644 --- a/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py +++ b/fedot/core/operations/evaluation/operation_implementations/data_operations/ts_transformations.py @@ -11,21 +11,18 @@ from sklearn.decomposition import TruncatedSVD from fedot.core.data.data import InputData, OutputData -from fedot.core.context import ExecutionContext from fedot.core.operations.evaluation.operation_implementations.implementation_interfaces import ( DataOperationImplementation ) from fedot.core.operations.operation_parameters import OperationParameters from fedot.core.repository.dataset_types import DataTypesEnum from fedot.preprocessing.data_types import TYPE_TO_ID - +from fedot.core.context.context import ExecutionContext class LaggedImplementation(DataOperationImplementation): def __init__(self, params: Optional[OperationParameters], context: Optional[ExecutionContext] = None): super().__init__(params) - self.context = context or ExecutionContext() - self.window_size_minimum = None self.sparse_transform = False self.use_svd = False @@ -33,6 +30,7 @@ def __init__(self, params: Optional[OperationParameters], context: Optional[Exec # Define logger object self.log = default_log(self) + self.context = context @property def window_size(self) -> Optional[int]: @@ -54,7 +52,13 @@ def fit(self, input_data): pass - def transform(self, input_data: InputData) -> OutputData: + def tranfsorm(self, input_data: InputData) -> OutputData: + if self.context: + return self.context.lagged_transformer.transform(input_data) + else: + return self.transform_core(input_data) + + def transform_core(self, input_data: InputData) -> OutputData: """ Method for transformation of time series to lagged form for predict stage Args: @@ -75,11 +79,10 @@ def transform(self, input_data: InputData) -> OutputData: output_data = self._convert_to_output(new_input_data, self.features_columns, data_type=DataTypesEnum.table) - # self._update_column_types(output_data) - self.context.lagged__update_column_types(self, output_data) + self._update_column_types(output_data) return output_data - def transform_for_fit(self, input_data: InputData) -> OutputData: + def transform_for_fit_core(self, input_data: InputData) -> OutputData: """Method for transformation of time series to lagged form for fit stage Args: @@ -107,11 +110,17 @@ def transform_for_fit(self, input_data: InputData) -> OutputData: output_data = self._convert_to_output(new_input_data, self.features_columns, data_type=DataTypesEnum.table) - # self._update_column_types(output_data) - self.context.lagged__update_column_types(self, output_data) + self._update_column_types(output_data) return output_data - def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): + def transform_for_fit(self, input_data: InputData) -> OutputData: + if self.context: + return self.context.lagged_transformer.transform_for_fit(input_data) + else: + return self.transform_for_fit_core(input_data) + + + def _check_and_correct_window_size_core(self, time_series: np.ndarray, forecast_length: int): """ Method check if the length of the time series is not enough for lagged transformation @@ -147,7 +156,19 @@ def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_lengt f"from {self.params.get('window_size')} to {self.window_size_minimum}")) self.params.update(window_size=self.window_size_minimum) + def _check_and_correct_window_size(self, time_series: np.ndarray, forecast_length: int): + if self.context: + return self.context.lagged_transformer._check_and_correct_window_size(time_series, forecast_length) + else: + return self._check_and_correct_window_size_core(time_series, forecast_length) + def _update_column_types(self, output_data: OutputData): + if self.context: + return self.context.lagged_transformer._update_column_types(output_data) + else: + return self._update_column_types_core(output_data) + + def _update_column_types_core(self, output_data: OutputData): """Update column types after lagged transformation. All features becomes ``float`` """ @@ -376,8 +397,9 @@ def __init__(self, params: Optional[OperationParameters]): class TsSmoothingImplementation(DataOperationImplementation): - def __init__(self, params: Optional[OperationParameters]): + def __init__(self, params: Optional[OperationParameters], context: Optional[ExecutionContext] = None): super().__init__(params) + self.context = context @property def window_size(self) -> int: @@ -392,7 +414,13 @@ def fit(self, input_data: InputData): pass - def transform(self, input_data: InputData) -> OutputData: + def transform(self, input_data: InputData): + if self.context: + self.context.ts_smoothing.transform(input_data) + else: + return self.transform_core(input_data) + + def transform_core(self, input_data: InputData) -> OutputData: """Method for smoothing time series Args: diff --git a/fedot/core/operations/operation.py b/fedot/core/operations/operation.py index faf2fdf341..18e5ecd8ed 100644 --- a/fedot/core/operations/operation.py +++ b/fedot/core/operations/operation.py @@ -27,7 +27,7 @@ class Operation: operation_type: name of the operation """ - def __init__(self, operation_type: str, **kwargs): + def __init__(self, operation_type: str, context: Optional[ExecutionContext] = None, **kwargs): self.operation_type = operation_type self._eval_strategy = None @@ -35,6 +35,7 @@ def __init__(self, operation_type: str, **kwargs): self.fitted_operation = None self.log = default_log(self) + self.context = context def _init(self, task: Task, **kwargs): params = kwargs.get('params') @@ -110,6 +111,48 @@ def predict(self, predictions_cache: Optional[PredictionsCache] = None, fold_id: Optional[int] = None, descriptive_id: Optional[str] = None): + if self.context: + return self.context.operation_predict.predict(fitted_operation, data, params, output_mode, predictions_cache, fold_id, descriptive_id) + else: + return self.predict_core(fitted_operation, data, params, output_mode, predictions_cache, fold_id, descriptive_id) + + def predict_for_fit(self, + fitted_operation, + data: InputData, + params: Optional[OperationParameters] = None, + output_mode: str = 'default', + predictions_cache: Optional[PredictionsCache] = None, + fold_id: Optional[int] = None, + descriptive_id: Optional[str] = None): + if self.context: + return self.context.operation_predict.predict_for_fit(fitted_operation, data, params, output_mode, predictions_cache, fold_id, descriptive_id) + else: + return self.predict_for_fit_core(fitted_operation, data, params, output_mode, predictions_cache, fold_id, descriptive_id) + + def _predict(self, + fitted_operation, + data: InputData, + params: Optional[OperationParameters] = None, + output_mode: str = 'default', + is_fit_stage: bool = False, + predictions_cache: Optional[PredictionsCache] = None, + fold_id: Optional[int] = None, + descriptive_id: Optional[str] = None): + if self.context: + return self.context.operation_predict._predict(fitted_operation, data, params, output_mode, + is_fit_stage, predictions_cache, fold_id, descriptive_id) + else: + return self._predict_core(fitted_operation, data, params, output_mode, is_fit_stage, + predictions_cache, fold_id, descriptive_id) + + def predict_core(self, + fitted_operation, + data: InputData, + params: Optional[Union[OperationParameters, dict]] = None, + output_mode: str = 'default', + predictions_cache: Optional[PredictionsCache] = None, + fold_id: Optional[int] = None, + descriptive_id: Optional[str] = None): """This method is used for defining and running of the evaluation strategy to predict with the data provided @@ -120,10 +163,10 @@ def predict(self, output_mode: string with information about output of operation, for example, is the operation predict probabilities or class labels """ - return self._predict(fitted_operation, data, params, output_mode, is_fit_stage=False, + return self._predict_core(fitted_operation, data, params, output_mode, is_fit_stage=False, predictions_cache=predictions_cache, fold_id=fold_id, descriptive_id=descriptive_id) - def predict_for_fit(self, + def predict_for_fit_core(self, fitted_operation, data: InputData, params: Optional[OperationParameters] = None, @@ -144,7 +187,7 @@ def predict_for_fit(self, return self._predict(fitted_operation, data, params, output_mode, is_fit_stage=True, predictions_cache=predictions_cache, fold_id=fold_id, descriptive_id=descriptive_id) - def _predict(self, + def _predict_core(self, fitted_operation, data: InputData, params: Optional[OperationParameters] = None, diff --git a/fedot/core/optimisers/objective/data_objective_eval.py b/fedot/core/optimisers/objective/data_objective_eval.py index 47fa1cc131..f547127c40 100644 --- a/fedot/core/optimisers/objective/data_objective_eval.py +++ b/fedot/core/optimisers/objective/data_objective_eval.py @@ -15,7 +15,6 @@ from fedot.core.operations.model import Model from fedot.core.pipelines.pipeline import Pipeline from fedot.utilities.debug import is_recording_mode, save_debug_info_for_pipeline - from fedot.core.context.context import ExecutionContext DataSource = Callable[[], Iterable[Tuple[InputData, InputData]]] @@ -47,7 +46,8 @@ def __init__(self, predictions_cache: Optional[PredictionsCache] = None, eval_n_jobs: int = 1, do_unfit: bool = True, - context: Optional[ExecutionContext] = None): + context: Optional[ExecutionContext] = None + ): super().__init__(objective, eval_n_jobs=eval_n_jobs) self._data_producer = data_producer self._time_constraint = time_constraint @@ -55,12 +55,21 @@ def __init__(self, self._operations_cache = operations_cache self._preprocessing_cache = preprocessing_cache self._predictions_cache = predictions_cache + self.context = context self._log = default_log(self) self._do_unfit = do_unfit def evaluate(self, graph: Pipeline) -> Fitness: # Seems like a workaround for situation when logger is lost # when adapting and restoring it to/from OptGraph. + + if self.context: + return self.context.evaluator.evaluate(graph) + + else: + return self.evaluation_core(graph) + + def evaluation_core(self, graph: Pipeline) -> Fitness: graph.log = self._log graph_id = graph.root_node.descriptive_id diff --git a/fedot/core/optimisers/objective/data_source_splitter.py b/fedot/core/optimisers/objective/data_source_splitter.py index c004b86b23..0a31428f6f 100644 --- a/fedot/core/optimisers/objective/data_source_splitter.py +++ b/fedot/core/optimisers/objective/data_source_splitter.py @@ -39,7 +39,8 @@ def __init__(self, split_ratio: Optional[float] = None, shuffle: bool = False, stratify: bool = True, - random_seed: int = 42): + random_seed: int = 42, + context: Optional[ExecutionContext] = None): self.cv_folds = cv_folds self.validation_blocks = validation_blocks self.split_ratio = split_ratio @@ -47,12 +48,19 @@ def __init__(self, self.stratify = stratify self.random_seed = random_seed self.log = default_log(self) + self.context = context def build_tensordata(self, tensor_data) -> DataSource: input_data = tensordata_to_input_data(tensor_data) return self.build(input_data) def build(self, data: Union[InputData, MultiModalData]) -> DataSource: + if self.context: + return self.context.data_source_splitter.build(data) + else: + return self.build_core(data) + + def build_core(self, data: Union[InputData, MultiModalData]) -> DataSource: # define split_ratio self.split_ratio = self.split_ratio or default_data_split_ratio_by_task[data.task.task_type] diff --git a/fedot/core/pipelines/tuning/search_space.py b/fedot/core/pipelines/tuning/search_space.py index 2724175253..70b594ce29 100644 --- a/fedot/core/pipelines/tuning/search_space.py +++ b/fedot/core/pipelines/tuning/search_space.py @@ -6,6 +6,8 @@ from fedot.core.utils import NESTED_PARAMS_LABEL +from fedot.core.context.context import ExecutionContext + class PipelineSearchSpace(SearchSpace): """ @@ -18,13 +20,21 @@ class PipelineSearchSpace(SearchSpace): def __init__(self, custom_search_space: Optional[OperationParametersMapping] = None, - replace_default_search_space: bool = False): + replace_default_search_space: bool = False, + context: Optional[ExecutionContext] = None): self.custom_search_space = custom_search_space self.replace_default_search_space = replace_default_search_space + self.context = context parameters_per_operation = self.get_parameters_dict() super().__init__(parameters_per_operation) def get_parameters_dict(self): + if self.context: + return self.context.search_space.get_parameters_dict() + else: + return self.get_parameters_dict_core() + + def get_parameters_dict_core(self): parameters_per_operation = { 'kmeans': { 'n_clusters': { diff --git a/fedot/core/pipelines/tuning/tuner_builder.py b/fedot/core/pipelines/tuning/tuner_builder.py index 3d9e33b7e9..6ae9c2c953 100644 --- a/fedot/core/pipelines/tuning/tuner_builder.py +++ b/fedot/core/pipelines/tuning/tuner_builder.py @@ -37,6 +37,11 @@ def __init__(self, task: Task): self.eval_time_constraint = None self.additional_params = {} self.adapter = PipelineAdapter() + self.context: Optional[ExecutionContext] = None + + def with_context(self, context: ExecutionContext): # ← добавить метод + self.context = context + return self def with_tuner(self, tuner: Type[BaseTuner]): self.tuner_class = tuner @@ -110,6 +115,7 @@ def _build_tuner(self, data_producer, validation_blocks: int) -> BaseTuner: time_constraint=self.eval_time_constraint, eval_n_jobs=self.n_jobs, # because tuners are not parallelized validation_blocks=validation_blocks, + context=context ) tuner = self.tuner_class(objective_evaluate=objective_evaluate, adapter=self.adapter, @@ -122,11 +128,11 @@ def _build_tuner(self, data_producer, validation_blocks: int) -> BaseTuner: return tuner def build(self, data: InputData) -> BaseTuner: - data_splitter = DataSourceSplitter(self.cv_folds, validation_blocks=self.validation_blocks) + data_splitter = DataSourceSplitter(self.cv_folds, validation_blocks=self.validation_blocks, context=self.context) data_producer = data_splitter.build(data) return self._build_tuner(data_producer, data_splitter.validation_blocks) def build_tensordata(self, tensor_data) -> BaseTuner: - data_splitter = DataSourceSplitter(self.cv_folds, validation_blocks=self.validation_blocks) + data_splitter = DataSourceSplitter(self.cv_folds, validation_blocks=self.validation_blocks, context=self.context) data_producer = data_splitter.build_tensordata(tensor_data) return self._build_tuner(data_producer, data_splitter.validation_blocks) From 751f9e5a0f48abdc50bf9ec56ebb72855998101a Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:46:18 +0300 Subject: [PATCH 10/15] Delete test/unit/context/test_initialization.py --- test/unit/context/test_initialization.py | 90 ------------------------ 1 file changed, 90 deletions(-) delete mode 100644 test/unit/context/test_initialization.py diff --git a/test/unit/context/test_initialization.py b/test/unit/context/test_initialization.py deleted file mode 100644 index d9ce940c82..0000000000 --- a/test/unit/context/test_initialization.py +++ /dev/null @@ -1,90 +0,0 @@ -import pytest -from unittest.mock import Mock -from fedot.core.context.context import ExecutionContext - -@pytest.fixture -def ctx(): - return ExecutionContext() - - -def test_default_initialization(ctx): - assert ctx.extra_params == {} - assert ctx._instances == {} - assert ctx._overridden == {} - assert "splitter" in ctx._protocol_classes - assert "evaluator" in ctx._protocol_classes - - -def test_extra_params_stored(): - extra = {"backend": "dask", "timeout": 30} - ctx = ExecutionContext(extra_params=extra) - assert ctx.extra_params == extra - - -def test_lazy_instantiation(ctx): - assert "splitter" not in ctx._instances - splitter1 = ctx.splitter - splitter2 = ctx.splitter - assert splitter1 is splitter2 - assert "splitter" in ctx._instances - - -def test_core_implementations_are_used_by_default(ctx): - from fedot.core.context.default_backend import CoreSplitter, CoreEvaluator, CoreDataMerger - - assert isinstance(ctx.splitter, CoreSplitter) - assert isinstance(ctx.evaluator, CoreEvaluator) - assert isinstance(ctx.data_merger, CoreDataMerger) - - -def test_method_override(): - ctx_ind = ExecutionContext(extension_name="industrial") - - from fedot.core.context.industrial_backend import IndustrialSplitter - assert isinstance(ctx_ind.splitter, IndustrialSplitter) - -def test_missing(ctx): - with pytest.raises(ValueError, match="No implementation for protocol 'unknown'"): - ctx._get_protocol_class("unknown") - - -def test_attribute_override(ctx): - ctx.custom_attr = 123 - assert ctx._overridden["custom_attr"] == 123 - assert ctx.custom_attr == 123 - -def test_multiple_contexts_independence(): - ctx1 = ExecutionContext(extra_params={"p": 1}) - ctx2 = ExecutionContext(extra_params={"p": 2}) - assert ctx1.extra_params["p"] == 1 - assert ctx2.extra_params["p"] == 2 - - -def test_method_called_with_parameters(ctx): - from unittest.mock import patch - from fedot.core.data.data import InputData - - with patch('fedot.core.context.default_backend.CoreSplitter.split_any') as mock_split: - mock_split.return_value = ("train_data", "test_data") - - mock_data = Mock(spec=InputData) - - splitter = ctx.splitter - result = splitter.split_any( - data=mock_data, - split_ratio=0.75, - shuffle=True, - stratify=True, - random_seed=42, - extra_arg="custom_value" - ) - - mock_split.assert_called_once() - - args, kwargs = mock_split.call_args - assert kwargs['data'] == mock_data - assert kwargs['split_ratio'] == 0.75 - assert kwargs['shuffle'] is True - assert kwargs['stratify'] is True - assert kwargs['random_seed'] == 42 - assert kwargs['extra_arg'] == "custom_value" \ No newline at end of file From 394eabff3b40b43bf965b5a651a169bb82ac3e6c Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:47:21 +0300 Subject: [PATCH 11/15] Delete test/unit/data/test_supplementary_data.py --- test/unit/data/test_supplementary_data.py | 133 ---------------------- 1 file changed, 133 deletions(-) delete mode 100644 test/unit/data/test_supplementary_data.py diff --git a/test/unit/data/test_supplementary_data.py b/test/unit/data/test_supplementary_data.py deleted file mode 100644 index 732a1859ca..0000000000 --- a/test/unit/data/test_supplementary_data.py +++ /dev/null @@ -1,133 +0,0 @@ -import numpy as np -import pytest - -from fedot.core.data.data import OutputData -from fedot.core.data.merge.data_merger import DataMerger -from fedot.core.data.merge.supplementary_data_merger import SupplementaryDataMerger -from fedot.core.data.supplementary_data import SupplementaryData -from fedot.core.context import ExecutionContext -from fedot.core.pipelines.node import PipelineNode -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum -from fedot.preprocessing.data_types import TYPE_TO_ID -from test.unit.data.test_data_merge import unequal_outputs_table # noqa, fixture -from test.unit.tasks.test_regression import get_synthetic_regression_data - - -@pytest.fixture() -def outputs_table_with_different_types(): - """ Create datasets with different types of columns in predictions """ - task = Task(TaskTypesEnum.regression) - idx = [0, 1, 2] - target = [1, 2, 10] - data_info_first = SupplementaryData(col_type_ids={'features': np.array([TYPE_TO_ID[str], TYPE_TO_ID[float]]), - 'target': np.array([TYPE_TO_ID[int]])}) - output_first = OutputData(idx=idx, features=None, - predict=np.array([['a', 1.1], ['b', 2], ['c', 3]], dtype=object), - task=task, target=target, data_type=DataTypesEnum.table, - supplementary_data=data_info_first) - - data_info_second = SupplementaryData(col_type_ids={'features': np.array([TYPE_TO_ID[float]]), - 'target': np.array([TYPE_TO_ID[int]])}) - output_second = OutputData(idx=idx, features=None, - predict=np.array([[2.5], [2.1], [9.3]], dtype=float), - task=task, target=target, data_type=DataTypesEnum.table, - supplementary_data=data_info_second) - - return [output_first, output_second] - - -def generate_straight_pipeline(): - """ Simple linear pipeline """ - node_scaling = PipelineNode('scaling') - node_ridge = PipelineNode('ridge', nodes_from=[node_scaling]) - node_linear = PipelineNode('linear', nodes_from=[node_ridge]) - pipeline = Pipeline(node_linear) - return pipeline - - -def test_parent_mask_correct(unequal_outputs_table, context: Optional[ExecutionContext] = None): # noqa, fixture - """ Test correctness of function for tables mask generation """ - - context = context or ExecutrionContext() - - correct_parent_mask = {'input_ids': [0, 1], 'flow_lens': [1, 0]} - - # Calculate parent mask from outputs - # main_output = DataMerger.find_main_output(unequal_outputs_table) - main_output = context.merger_find_main_output(unequal_outputs_table) - p_mask = SupplementaryDataMerger(unequal_outputs_table, main_output).prepare_parent_mask() - - assert tuple(p_mask['input_ids']) == tuple(correct_parent_mask['input_ids']) - assert tuple(p_mask['flow_lens']) == tuple(correct_parent_mask['flow_lens']) - - -def test_calculate_data_flow_len_correct(): - """ Function checks whether the number of nodes visited by the data block - is calculated correctly """ - - # Pipeline consists of 3 nodes - simple_pipeline = generate_straight_pipeline() - data = get_synthetic_regression_data(n_samples=100, n_features=2) - - simple_pipeline.fit(data) - predict_output = simple_pipeline.predict(data) - - assert predict_output.supplementary_data.data_flow_length == 2 - - -def test_get_compound_mask_correct(): - """ Checking whether the procedure for combining lists with keys is - performed correctly for features_mask """ - - synthetic_mask = {'input_ids': [0, 0, 1, 1], - 'flow_lens': [1, 1, 0, 0]} - output_example = OutputData(idx=[0, 0], features=[0, 0], predict=[0, 0], - task=Task(TaskTypesEnum.regression), - target=[0, 0], data_type=DataTypesEnum.table, - supplementary_data=SupplementaryData(features_mask=synthetic_mask)) - - mask = output_example.supplementary_data.compound_mask - - assert ('01', '01', '10', '10') == tuple(mask) - - -def test_define_parents_with_equal_lengths(): - """ - Check the processing of the case when the decompose operation receives - data whose flow_lens is not different. In this case, the data that came - from the data_operation node is used as the "Data parent". - - Such case is common for time series forecasting pipelines. So we imitate - merged output from ARIMA and lagged operations - """ - sd = SupplementaryData(is_main_target=True, - data_flow_length=1, - features_mask={'input_ids': [0, 0, 0, 1, 1, 1], - 'flow_lens': [0, 0, 0, 0, 0, 0]}, - previous_operations=['arima', 'lagged']) - features_mask = np.array(sd.compound_mask) - unique_features_masks = np.unique(features_mask) - - model_parent, data_parent = sd.define_parents(unique_features_masks, task=TaskTypesEnum.ts_forecasting) - - assert model_parent == '00' - assert data_parent == '10' - - -def test_define_types_after_merging(outputs_table_with_different_types): - """ Check if column types for features table perform correctly """ - outputs = outputs_table_with_different_types - # new_idx, features, target, task, d_type, updated_info = DataMerger(outputs).merge() - merged_data = DataMerger.get(outputs).merge() - updated_info = merged_data.supplementary_data - - feature_type_ids = updated_info.col_type_ids['features'] - target_type_ids = updated_info.col_type_ids['target'] - - # Target type must stay the same - ancestor_target_type = outputs[0].supplementary_data.col_type_ids['target'][0] - assert target_type_ids[0] == ancestor_target_type - assert len(feature_type_ids) == 3 - assert tuple(feature_type_ids) == (TYPE_TO_ID[str], TYPE_TO_ID[float], TYPE_TO_ID[float]) From 87e2a52c568279d2fedd8242ae6a632937cc69ee Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:48:11 +0300 Subject: [PATCH 12/15] Delete test/integration/pipelines/tuning/test_pipeline_tuning.py --- .../pipelines/tuning/test_pipeline_tuning.py | 566 ------------------ 1 file changed, 566 deletions(-) delete mode 100644 test/integration/pipelines/tuning/test_pipeline_tuning.py diff --git a/test/integration/pipelines/tuning/test_pipeline_tuning.py b/test/integration/pipelines/tuning/test_pipeline_tuning.py deleted file mode 100644 index d88098273d..0000000000 --- a/test/integration/pipelines/tuning/test_pipeline_tuning.py +++ /dev/null @@ -1,566 +0,0 @@ -import os -from time import time - -import pytest -from golem.core.tuning.hyperopt_tuner import get_node_parameters_for_hyperopt -from golem.core.tuning.iopt_tuner import IOptTuner -from golem.core.tuning.optuna_tuner import OptunaTuner -from golem.core.tuning.sequential import SequentialTuner -from golem.core.tuning.simultaneous import SimultaneousTuner -from golem.utilities.data_structures import ensure_wrapped_in_sequence -from hyperopt import hp -from hyperopt.pyll.stochastic import sample as hp_sample - -from examples.simple.time_series_forecasting.ts_pipelines import ts_complex_ridge_smoothing_pipeline, \ - ts_polyfit_ridge_pipeline -from fedot.core.data.data import InputData -from fedot.core.data.data_split import train_test_data_setup -from fedot.core.operations.evaluation.operation_implementations.models.ts_implementations.statsmodels import \ - GLMImplementation -from fedot.core.pipelines.node import PipelineNode -from fedot.core.pipelines.pipeline import Pipeline -from fedot.core.pipelines.pipeline_builder import PipelineBuilder -from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace -from fedot.core.pipelines.tuning.tuner_builder import TunerBuilder -from fedot.core.repository.dataset_types import DataTypesEnum -from fedot.core.repository.metrics_repository import RegressionMetricsEnum, ClassificationMetricsEnum -from fedot.core.repository.tasks import Task, TaskTypesEnum -from fedot.core.utils import fedot_project_root, NESTED_PARAMS_LABEL -from test.unit.multimodal.data_generators import get_single_task_multimodal_tabular_data, get_multimodal_pipeline -from test.unit.tasks.test_forecasting import get_ts_data - - -@pytest.fixture(scope='package') -def regression_dataset(): - test_file_path = str(os.path.dirname(__file__)) - file = os.path.join(str(fedot_project_root()), 'test/data/simple_regression_train.csv') - return InputData.from_csv(os.path.join(test_file_path, file), task=Task(TaskTypesEnum.regression)) - - -@pytest.fixture() -def classification_dataset(): - test_file_path = str(os.path.dirname(__file__)) - file = os.path.join(str(fedot_project_root()), 'test/data/simple_classification.csv') - return InputData.from_csv(os.path.join(test_file_path, file), task=Task(TaskTypesEnum.classification)) - - -@pytest.fixture() -def tiny_classification_dataset(): - test_file_path = str(os.path.dirname(__file__)) - file = os.path.join(str(fedot_project_root()), 'test/data/tiny_simple_classification.csv') - return InputData.from_csv(os.path.join(test_file_path, file), task=Task(TaskTypesEnum.classification)) - - -@pytest.fixture() -def multi_classification_dataset(): - test_file_path = str(os.path.dirname(__file__)) - file = os.path.join(str(fedot_project_root()), 'test/data/multiclass_classification.csv') - return InputData.from_csv(os.path.join(test_file_path, file), task=Task(TaskTypesEnum.classification)) - - -@pytest.fixture() -def ts_forecasting_dataset(): - train_data, _ = get_ts_data(n_steps=700, forecast_length=20) - return train_data - - -@pytest.fixture() -def multimodal_dataset(): - data, _ = get_single_task_multimodal_tabular_data() - return data - - -def get_simple_regr_pipeline(operation_type='rfr'): - final = PipelineNode(operation_type=operation_type) - pipeline = Pipeline(final) - - return pipeline - - -def get_complex_regr_pipeline(): - node_scaling = PipelineNode(operation_type='scaling') - node_ridge = PipelineNode('ridge', nodes_from=[node_scaling]) - node_linear = PipelineNode('linear', nodes_from=[node_scaling]) - final = PipelineNode('rfr', nodes_from=[node_ridge, node_linear]) - pipeline = Pipeline(final) - - return pipeline - - -def get_regr_pipelines(): - simple_pipelines = [get_simple_regr_pipeline(operation_type) for operation_type in get_regr_operation_types()] - - return simple_pipelines + [get_complex_regr_pipeline()] - - -def get_simple_class_pipeline(operation_type='logit'): - final = PipelineNode(operation_type=operation_type) - pipeline = Pipeline(final) - - return pipeline - - -def get_complex_class_pipeline(): - first = PipelineNode(operation_type='knn') - second = PipelineNode(operation_type='pca') - final = PipelineNode(operation_type='logit', - nodes_from=[first, second]) - - pipeline = Pipeline(final) - - return pipeline - - -def get_pipeline_with_no_params_to_tune(): - first = PipelineNode(operation_type='scaling') - final = PipelineNode(operation_type='bernb', - nodes_from=[first]) - - pipeline = Pipeline(final) - - return pipeline - - -def get_class_pipelines(): - simple_pipelines = [get_simple_class_pipeline(operation_type) for operation_type in get_class_operation_types()] - - return simple_pipelines + [get_complex_class_pipeline()] - - -def get_ts_forecasting_pipelines(): - pipelines = [ts_polyfit_ridge_pipeline(2), ts_complex_ridge_smoothing_pipeline()] - return pipelines - - -def get_multimodal_pipelines(): - return [get_multimodal_pipeline()] - - -def get_regr_operation_types(): - return ['lgbmreg'] - - -def get_class_operation_types(): - return ['rf'] - - -def get_regr_losses(): - return [RegressionMetricsEnum.RMSE, RegressionMetricsEnum.MAPE] - - -def get_class_losses(): - return [ClassificationMetricsEnum.ROCAUC, ClassificationMetricsEnum.accuracy] - - -def get_not_default_search_space(): - custom_search_space = { - 'logit': { - 'C': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [1e-1, 5.0], - 'type': 'continuous'} - }, - 'ridge': { - 'alpha': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [0.01, 5.0], - 'type': 'continuous'} - }, - 'lgbmreg': { - 'learning_rate': { - 'hyperopt-dist': hp.loguniform, - 'sampling-scope': [0.03, 0.1], - 'type': 'continuous'}, - 'colsample_bytree': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [0.2, 0.8], - 'type': 'continuous'}, - 'subsample': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [0.1, 0.8], - 'type': 'continuous'} - }, - 'dt': { - 'max_depth': { - 'hyperopt-dist': hp.uniformint, - 'sampling-scope': [1, 5], - 'type': 'discrete'}, - 'min_samples_split': { - 'hyperopt-dist': hp.uniformint, - 'sampling-scope': [10, 25], - 'type': 'discrete'} - }, - 'ar': { - 'lag_1': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [2, 100], - 'type': 'continuous'}, - 'lag_2': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [2, 500], - 'type': 'continuous'} - }, - 'pca': { - 'n_components': { - 'hyperopt-dist': hp.uniform, - 'sampling-scope': [0.1, 0.5], - 'type': 'continuous'} - } - } - return PipelineSearchSpace(custom_search_space=custom_search_space) - - -def run_pipeline_tuner(train_data, - pipeline, - loss_function, - tuner=SimultaneousTuner, - search_space=PipelineSearchSpace(), - cv=None, - iterations=5, - early_stopping_rounds=None, **kwargs): - # if data is time series then lagged window should be tuned correctly - # because lagged window raises error if windows size is uncorrect - # and tuner will fall - if train_data.data_type in (DataTypesEnum.ts, DataTypesEnum.multi_ts): - forecast_length = train_data.task.task_params.forecast_length - folds = cv or 1 - validation_blocks = 1 - max_window = int(train_data.features.shape[0] / (folds + 1)) - (forecast_length * validation_blocks) - 1 - ssp = {'window_size': {'hyperopt-dist': hp.uniformint, 'sampling-scope': [2, max_window], 'type': 'discrete'}} - if search_space.custom_search_space is None: - search_space.custom_search_space = {'lagged': ssp} - else: - search_space.custom_search_space['lagged'] = ssp - search_space.replace_default_search_space = True - # search_space.parameters_per_operation = search_space.get_parameters_dict() - search_space.parameters_per_operation = context.search_space_get_parameters_dict(search_space) - - # Pipeline tuning - pipeline_tuner = TunerBuilder(train_data.task) \ - .with_tuner(tuner) \ - .with_metric(loss_function) \ - .with_cv_folds(cv) \ - .with_iterations(iterations) \ - .with_n_jobs(1) \ - .with_early_stopping_rounds(early_stopping_rounds) \ - .with_search_space(search_space) \ - .with_additional_params(**kwargs) \ - .build(train_data) - tuned_pipeline = pipeline_tuner.tune(pipeline, show_progress=False) - return pipeline_tuner, tuned_pipeline - - -def run_node_tuner(train_data, - pipeline, - loss_function, - search_space=PipelineSearchSpace(), - cv=None, - node_index=0, - iterations=3, - early_stopping_rounds=None): - # Pipeline tuning - node_tuner = TunerBuilder(train_data.task) \ - .with_tuner(SequentialTuner) \ - .with_metric(loss_function) \ - .with_cv_folds(cv) \ - .with_iterations(iterations) \ - .with_search_space(search_space) \ - .with_early_stopping_rounds(early_stopping_rounds) \ - .build(train_data) - tuned_pipeline = node_tuner.tune_node(pipeline, node_index) - return node_tuner, tuned_pipeline - - -@pytest.mark.parametrize('data_fixture', ['classification_dataset']) -def test_custom_params_setter(data_fixture, request): - data = request.getfixturevalue(data_fixture) - pipeline = get_complex_class_pipeline() - - custom_params = dict(C=10) - - pipeline.root_node.parameters = custom_params - pipeline.fit(data) - params = pipeline.root_node.fitted_operation.get_params() - - assert params['C'] == 10 - - -@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', - [('regression_dataset', get_regr_pipelines(), get_regr_losses()), - ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), - ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), - ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, OptunaTuner]) -def test_pipeline_tuner_correct(data_fixture, pipelines, loss_functions, request, tuner): - """ Test all tuners for pipeline """ - data = request.getfixturevalue(data_fixture) - cvs = [None, 2] - - for pipeline in pipelines: - for loss_function in loss_functions: - for cv in cvs: - print(pipeline) - pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, - train_data=data, - pipeline=pipeline, - loss_function=loss_function, - cv=cv) - assert pipeline_tuner.obtained_metric is not None - assert tuned_pipeline is not None - assert not tuned_pipeline.is_fitted - - -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) -def test_pipeline_tuner_with_no_parameters_to_tune(classification_dataset, tuner): - pipeline = get_pipeline_with_no_params_to_tune() - pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, - train_data=classification_dataset, - pipeline=pipeline, - loss_function=ClassificationMetricsEnum.ROCAUC, - iterations=20) - assert pipeline_tuner.obtained_metric is not None - assert tuned_pipeline is not None - assert pipeline_tuner.obtained_metric == pipeline_tuner.init_metric - assert not tuned_pipeline.is_fitted - - -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, OptunaTuner]) -def test_pipeline_tuner_with_initial_params(classification_dataset, tuner): - """ Test all tuners for pipeline with initial parameters """ - # a model - node = PipelineNode(content={'name': 'xgboost', 'params': {'max_depth': 3, - 'learning_rate': 0.03, - 'min_child_weight': 2}}) - pipeline = Pipeline(node) - pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, - train_data=classification_dataset, - pipeline=pipeline, - loss_function=ClassificationMetricsEnum.ROCAUC, - iterations=20) - assert pipeline_tuner.obtained_metric is not None - assert tuned_pipeline is not None - assert not tuned_pipeline.is_fitted - - -@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', - [('regression_dataset', get_regr_pipelines(), get_regr_losses()), - ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), - ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), - ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, OptunaTuner]) -def test_pipeline_tuner_with_custom_search_space(data_fixture, pipelines, loss_functions, request, tuner): - """ Test tuners with different search spaces """ - data = request.getfixturevalue(data_fixture) - train_data, test_data = train_test_data_setup(data=data) - search_spaces = [PipelineSearchSpace(), get_not_default_search_space()] - - for search_space in search_spaces: - pipeline_tuner, tuned_pipeline = run_pipeline_tuner(tuner=tuner, - train_data=train_data, - pipeline=pipelines[0], - loss_function=loss_functions[0], - search_space=search_space) - assert pipeline_tuner.obtained_metric is not None - assert tuned_pipeline is not None - - -@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', - [('regression_dataset', get_regr_pipelines(), get_regr_losses()), - ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), - ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), - ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) -def test_certain_node_tuning_correct(data_fixture, pipelines, loss_functions, request): - """ Test SequentialTuner for particular node based on hyperopt library """ - data = request.getfixturevalue(data_fixture) - cvs = [None, 2] - - for pipeline in pipelines: - for loss_function in loss_functions: - for cv in cvs: - node_tuner, tuned_pipeline = run_node_tuner(train_data=data, - pipeline=pipeline, - loss_function=loss_function, - cv=cv) - assert node_tuner.obtained_metric is not None - assert not tuned_pipeline.is_fitted - assert tuned_pipeline is not None - - -@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', - [('regression_dataset', get_regr_pipelines(), get_regr_losses()), - ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), - ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), - ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) -def test_certain_node_tuner_with_custom_search_space(data_fixture, pipelines, loss_functions, request): - """ Test SequentialTuner for particular node with different search spaces """ - data = request.getfixturevalue(data_fixture) - train_data, test_data = train_test_data_setup(data=data) - search_spaces = [PipelineSearchSpace(), get_not_default_search_space()] - - for search_space in search_spaces: - node_tuner, tuned_pipeline = run_node_tuner(train_data=train_data, - pipeline=pipelines[0], - loss_function=loss_functions[0], - search_space=search_space) - assert node_tuner.obtained_metric is not None - assert tuned_pipeline is not None - - -@pytest.mark.parametrize('n_steps', [100, 133, 217, 300]) -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) -def test_ts_pipeline_with_stats_model(n_steps, tuner): - """ Tests tuners for time series forecasting task with AR model """ - train_data, test_data = get_ts_data(n_steps=n_steps, forecast_length=5) - - ar_pipeline = Pipeline(PipelineNode('ar')) - - for search_space in [PipelineSearchSpace(), get_not_default_search_space()]: - # Tune AR model - tuner_ar = TunerBuilder(train_data.task) \ - .with_tuner(tuner) \ - .with_metric(RegressionMetricsEnum.MSE) \ - .with_iterations(3) \ - .with_search_space(search_space).build(train_data) - tuned_pipeline = tuner_ar.tune(ar_pipeline, show_progress=False) - assert tuned_pipeline is not None - assert tuner_ar.obtained_metric is not None - - -@pytest.mark.parametrize('data_fixture', ['tiny_classification_dataset']) -def test_early_stop_in_tuning(data_fixture, request): - data = request.getfixturevalue(data_fixture) - train_data, test_data = train_test_data_setup(data=data) - - start_pipeline_tuner = time() - _ = run_pipeline_tuner(tuner=SimultaneousTuner, - train_data=train_data, - pipeline=get_class_pipelines()[0], - loss_function=ClassificationMetricsEnum.ROCAUC, - iterations=1000, - early_stopping_rounds=1) - assert time() - start_pipeline_tuner < 1.3 - - start_sequential_tuner = time() - _ = run_pipeline_tuner(tuner=SequentialTuner, - train_data=train_data, - pipeline=get_class_pipelines()[0], - loss_function=ClassificationMetricsEnum.ROCAUC, - iterations=1000, - early_stopping_rounds=1) - assert time() - start_sequential_tuner < 1.3 - - start_node_tuner = time() - _ = run_node_tuner(train_data=train_data, - pipeline=get_class_pipelines()[0], - loss_function=ClassificationMetricsEnum.ROCAUC, - iterations=1000, - early_stopping_rounds=1) - assert time() - start_node_tuner < 1.3 - - -def test_search_space_correctness_after_customization(): - default_search_space = PipelineSearchSpace() - - custom_search_space = {'gbr': {'max_depth': { - 'hyperopt-dist': hp.choice, - 'sampling-scope': [[3, 7, 31, 127, 8191, 131071]], - 'type': 'categorical'}}} - custom_search_space_without_replace = PipelineSearchSpace(custom_search_space=custom_search_space, - replace_default_search_space=False) - custom_search_space_with_replace = PipelineSearchSpace(custom_search_space=custom_search_space, - replace_default_search_space=True) - - default_params, _ = get_node_parameters_for_hyperopt(default_search_space, - node_id=0, - node=PipelineNode('gbr')) - custom_without_replace_params, _ = get_node_parameters_for_hyperopt(custom_search_space_without_replace, - node_id=0, - node=PipelineNode('gbr')) - custom_with_replace_params, _ = get_node_parameters_for_hyperopt(custom_search_space_with_replace, - node_id=0, - node=PipelineNode('gbr')) - - assert default_params.keys() == custom_without_replace_params.keys() - assert default_params.keys() != custom_with_replace_params.keys() - assert default_params['0 || gbr | max_depth'] != custom_without_replace_params['0 || gbr | max_depth'] - assert default_params['0 || gbr | max_depth'] != custom_with_replace_params['0 || gbr | max_depth'] - - -def test_search_space_get_operation_parameter_range(): - default_search_space = PipelineSearchSpace() - gbr_operations = ['loss', 'learning_rate', 'max_depth', 'min_samples_split', - 'min_samples_leaf', 'subsample', 'max_features', 'alpha'] - - custom_search_space = {'gbr': {'max_depth': { - 'hyperopt-dist': hp.choice, - 'sampling-scope': [[3, 7, 31, 127, 8191, 131071]], - 'type': 'categorical'}}} - custom_search_space_without_replace = PipelineSearchSpace(custom_search_space=custom_search_space, - replace_default_search_space=False) - custom_search_space_with_replace = PipelineSearchSpace(custom_search_space=custom_search_space, - replace_default_search_space=True) - - default_operations = default_search_space.get_parameters_for_operation('gbr') - custom_without_replace_operations = custom_search_space_without_replace.get_parameters_for_operation('gbr') - custom_with_replace_operations = custom_search_space_with_replace.get_parameters_for_operation('gbr') - - assert default_operations == gbr_operations - assert custom_without_replace_operations == gbr_operations - assert custom_with_replace_operations == ['max_depth'] - - -def test_complex_search_space(): - space = PipelineSearchSpace() - for i in range(20): - operation_parameters = space.parameters_per_operation.get("glm") - new_value = hp_sample(operation_parameters[NESTED_PARAMS_LABEL]) - for params in new_value['sampling-scope'][0]: - assert params['link'] in GLMImplementation.family_distribution[params['family']]['available_links'] - - -@pytest.mark.parametrize('tuner', [SimultaneousTuner, SequentialTuner, IOptTuner, OptunaTuner]) -def test_complex_search_space_tuning_correct(tuner): - """ Tests Tuners for time series forecasting task with GLM model that has a complex glm search space""" - train_data, test_data = get_ts_data(n_steps=700, forecast_length=20) - - # ridge added because IOpt requires at least one continuous parameter - glm_pipeline = PipelineBuilder().add_sequence('glm', 'ridge', branch_idx=0).build() - initial_parameters = glm_pipeline.nodes[0].parameters - tuner = TunerBuilder(train_data.task) \ - .with_tuner(tuner) \ - .with_metric(RegressionMetricsEnum.MSE) \ - .with_iterations(100) \ - .build(train_data) - tuned_glm_pipeline = tuner.tune(glm_pipeline) - found_parameters = tuned_glm_pipeline.nodes[0].parameters - assert initial_parameters != found_parameters - - -@pytest.mark.parametrize('data_fixture, pipelines, loss_functions', - [('regression_dataset', get_regr_pipelines(), get_regr_losses()), - ('classification_dataset', get_class_pipelines(), get_class_losses()), - ('multi_classification_dataset', get_class_pipelines(), get_class_losses()), - ('ts_forecasting_dataset', get_ts_forecasting_pipelines(), get_regr_losses()), - ('multimodal_dataset', get_multimodal_pipelines(), get_class_losses())]) -@pytest.mark.parametrize('tuner', [OptunaTuner]) -def test_multiobj_tuning(data_fixture, pipelines, loss_functions, request, tuner): - """ Test multi objective tuning is correct """ - data = request.getfixturevalue(data_fixture) - cvs = [None, 2] - - for pipeline in pipelines: - for cv in cvs: - pipeline_tuner, tuned_pipelines = run_pipeline_tuner(tuner=tuner, - train_data=data, - pipeline=pipeline, - loss_function=loss_functions, - cv=cv) - assert tuned_pipelines is not None - assert all([tuned_pipeline is not None for tuned_pipeline in ensure_wrapped_in_sequence(tuned_pipelines)]) - for metrics in pipeline_tuner.obtained_metric: - assert len(metrics) == len(loss_functions) - assert all(metric is not None for metric in metrics) From 56e8a2b461da2afb1895c3e114d523e8579351dd Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:49:07 +0300 Subject: [PATCH 13/15] Delete fedot/industrial/industrial_extension.py --- fedot/industrial/industrial_extension.py | 77 ------------------------ 1 file changed, 77 deletions(-) delete mode 100644 fedot/industrial/industrial_extension.py diff --git a/fedot/industrial/industrial_extension.py b/fedot/industrial/industrial_extension.py deleted file mode 100644 index a87d6fd229..0000000000 --- a/fedot/industrial/industrial_extension.py +++ /dev/null @@ -1,77 +0,0 @@ -import fedot.industrial.core.repository.model_repository as MODEL_REPO -from fedot.industrial.core.metrics.pipeline import industrial_evaluate_pipeline -from fedot.industrial.core.repository.constanst_repository import IND_DATA_OPERATION_PATH, IND_MODEL_OPERATION_PATH, DEFAULT_DATA_OPERATION_PATH, DEFAULT_MODEL_OPERATION_PATH -from fedot.industrial.core.repository.industrial_implementations.abstract import ( - preprocess_industrial_predicts, merge_industrial_predicts, merge_industrial_targets, - build_industrial, postprocess_industrial_predicts, split_any_industrial, - split_time_series_industrial, predict_operation_industrial, predict_industrial, - predict_for_fit_industrial, update_column_types_industrial, fit_topo_extractor_industrial, - transform_topo_extractor_industrial, find_main_output_industrial, get_merger_industrial -) -from fedot.industrial.core.repository.industrial_implementations.data_transformation import ( - transform_lagged_industrial, transform_lagged_for_fit_industrial, - _check_and_correct_window_size_industrial, transform_smoothing_industrial -) -from fedot.industrial.core.repository.industrial_implementations.ml_optimisation import ( - DaskOptunaTuner, tune_pipeline_industrial -) -from fedot.industrial.core.repository.industrial_implementations.optimisation import ( - _get_default_industrial_mutations, has_no_lagged_conflicts_in_ts_pipeline, - reproduce_controlled_industrial, reproduce_industrial, - has_no_data_flow_conflicts_in_industrial_pipeline -) -from fedot.industrial.core.tuning.search_space import get_industrial_search_space - -from fedot.core.context import ExecutionContext - - -class IndustrialExtension: - """Overrides ExecutionContext with industrial implementations.""" - def __init__(self, backend: str = "default") -> None: - self.backend = backend - - def apply(self, context: ExecutionContext) -> None: - """Mutates context with industrial implementations.""" - context.optuna_optuna_tuner = DaskOptunaTuner if "dask" in self.backend else OptunaTuner - context.evaluator_evaluate = industrial_evaluate_pipeline - context.search_space_get_parameters_dict = get_industrial_search_space - context.api_params_repository__get_default_mutations = _get_default_industrial_mutations - context.merger_find_main_output = find_main_output_industrial - context.merger_get = get_merger_industrial - context.merger_merge_predicts = merge_industrial_predicts - context.image_merger_preprocess_predicts = preprocess_industrial_predicts - context.image_merger_merge_predicts = merge_industrial_predicts - context.ts_merger_merge_predicts = merge_industrial_predicts - context.ts_merger_merge_targets = merge_industrial_targets - context.ts_merger_postprocess_predicts = postprocess_industrial_predicts - context.ts_merger_preprocess_predicts = preprocess_industrial_predicts - context.data_source_splitter_build = build_industrial - context.data_split__split_any = split_any_industrial - context.data_split__split_time_series = split_time_series_industrial - context.operation__predict = predict_operation_industrial - context.operation_predict = predict_industrial - context.operation_predict_for_fit = predict_for_fit_industrial - context.lagged__update_column_types = update_column_types_industrial - context.lagged_transform = transform_lagged_industrial - context.lagged_transform_for_fit = transform_lagged_for_fit_industrial - context.lagged__check_and_correct_window_size = _check_and_correct_window_size_industrial - context.topo_features_fit = fit_topo_extractor_industrial - context.topo_features_transform = transform_topo_extractor_industrial - context.ts_smoothing_transform = transform_smoothing_industrial - context.api_composer_tune_final_pipeline = tune_pipeline_industrial - context.reproduction_reproduce = reproduce_industrial - context.reproduction_reproduce_uncontrolled = reproduce_controlled_industrial - context.class_rules.append(has_no_data_flow_conflicts_in_industrial_pipeline) - context.ts_rules.append(has_no_lagged_conflicts_in_ts_pipeline) - -class IndustrialContext(ExecutionContext): - """Fedot.Industrial execution context""" - def __init__(self, backend: str = "default") -> None: - super().__init__() - - self.operation_registry = OperationTypesRepository() - self.operation_registry.load_operations(IND_DATA_OPERATION_PATH, 'data_operation') - self.operation_registry.load_operations(IND_MODEL_OPERATION_PATH, 'model') - - self.extension = IndustrialExtension(backend=backend) - self.extension.apply(self) \ No newline at end of file From 5988ce6194f7c3100f93ee1cd20c86f0d9aa93da Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 01:50:40 +0300 Subject: [PATCH 14/15] Delete fedot/core/context/default_backend.py --- fedot/core/context/default_backend.py | 207 -------------------------- 1 file changed, 207 deletions(-) delete mode 100644 fedot/core/context/default_backend.py diff --git a/fedot/core/context/default_backend.py b/fedot/core/context/default_backend.py deleted file mode 100644 index d6796a8e87..0000000000 --- a/fedot/core/context/default_backend.py +++ /dev/null @@ -1,207 +0,0 @@ -from fedot.core.protocols.protocols import ( - SplitterProtocol, - DataMergerProtocol, - ImageMergerProtocol, - TSMergerProtocol, - TextMergerProtocol, - DataSourceSplitterProtocol, - TunerClassProtocol, - ReproductionProtocol, - EvaluatorProtocol, - SearchSpaceProtocol, - DefaultMutationsProtocol, - OperationPredictProtocol, - LaggedTransformerProtocol, - TopologicalFeaturesProtocol, - TsSmoothingProtocol, - ApiComposerTuneProtocol, -) - - -class CoreEvaluator(EvaluatorProtocol): - def evaluate(self, graph): - from fedot.core.optimisers.objective import PipelineObjectiveEvaluate - # from golem.core.optimisers.fitness import Fitness - return PipelineObjectiveEvaluate.evaluate(graph) - - -class CoreSearchSpace(SearchSpaceProtocol): - def get_parameters_dict(self) -> dict: - from fedot.core.pipelines.tuning.search_space import PipelineSearchSpace - return PipelineSearchSpace.get_parameters_dict() - - -class CoreDefaultMutations(DefaultMutationsProtocol): - @staticmethod - def __call__(task_type, params): - from fedot.api.api_utils.api_params_repository import ApiParamsRepository - # from typing import Sequence - return ApiParamsRepository._get_default_mutations(task_type, params) - - -class CoreDataMerger(DataMergerProtocol): - @staticmethod - def get(outputs): - from fedot.core.data.merge.data_merger import DataMerger - return DataMerger.get(outputs) - - def merge_predicts(self, predicts): - from fedot.core.data.merge.data_merger import DataMerger - return DataMerger.merge_predicts(predicts) - - @staticmethod - def find_main_output(outputs): - from fedot.core.data.merge.data_merger import DataMerger - return DataMerger.find_main_output(outputs) - - def preprocess_predicts(self, predicts): - return predicts - - def postprocess_predicts(self, merged): - return merged - - -class CoreImageMerger(ImageMergerProtocol): - def preprocess_predicts(self, predicts): - from fedot.core.data.merge.data_merger import ImageDataMerger - return ImageDataMerger.preprocess_predicts(predicts) - - def merge_predicts(self, predicts): - from fedot.core.data.merge.data_merger import ImageDataMerger - return ImageDataMerger.merge_predicts(predicts) - - -class CoreTSMerger(TSMergerProtocol): - def merge_predicts(self, predicts): - from fedot.core.data.merge.data_merger import TSDataMerger - return TSDataMerger.merge_predicts(predicts) - - def merge_targets(self, targets): - from fedot.core.data.merge.data_merger import TSDataMerger - return TSDataMerger.merge_targets(targets) - - def preprocess_predicts(self, predicts): - from fedot.core.data.merge.data_merger import TSDataMerger - return TSDataMerger.preprocess_predicts(predicts) - - def postprocess_predicts(self, merged): - from fedot.core.data.merge.data_merger import TSDataMerger - return TSDataMerger.postprocess_predicts(merged) - - -class CoreTextMerger(TextMergerProtocol): - def merge_predicts(self, predicts): - from fedot.core.data.merge.data_merger import TextDataMerger - return TextDataMerger.merge_predicts(predicts) - - def postprocess_predicts(self, merged): - return merged - - -class CoreDataSourceSplitter(DataSourceSplitterProtocol): - def build(self, data): - from fedot.core.optimisers.objective.data_source_splitter import DataSourceSplitter - return DataSourceSplitter.build(data) - - -class CoreSplitter(SplitterProtocol): - def split_any(self, data, split_ratio, shuffle, stratify, random_seed, **kwargs): - from fedot.core.data.data_split import _split_any - return _split_any(data, split_ratio, shuffle, stratify, random_seed, **kwargs) - - def split_time_series(self, data, validation_blocks=None, **kwargs): - from fedot.core.data.data_split import _split_time_series - return _split_time_series(data, validation_blocks, **kwargs) - - -class CoreOperationPredict(OperationPredictProtocol): - def predict(self, fitted_operation, data, params=None, output_mode='default'): - from fedot.core.operations.operation import Operation - return Operation.predict(fitted_operation, data, params, output_mode) - - def predict_for_fit(self, fitted_operation, data, params=None, output_mode='default'): - from fedot.core.operations.operation import Operation - return Operation.predict_for_fit(fitted_operation, data, params, output_mode) - - def _predict(self, fitted_operation, data, params=None, output_mode='default', - is_fit_stage=False, predictions_cache=None, fold_id=None, descriptive_id=None): - from fedot.core.operations.operation import Operation - return Operation._predict(fitted_operation, data, params, output_mode, - is_fit_stage, predictions_cache, fold_id, descriptive_id) - - -class CoreLaggedTransformer(LaggedTransformerProtocol): - def _update_column_types(self, output_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation - ) - return LaggedImplementation._update_column_types(output_data) - - def transform(self, input_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation - ) - return LaggedImplementation.transform(input_data) - - def transform_for_fit(self, input_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation - ) - return LaggedImplementation.transform_for_fit(input_data) - - def _check_and_correct_window_size(self, time_series, forecast_length): - from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - LaggedImplementation - ) - return LaggedImplementation._check_and_correct_window_size(time_series, forecast_length) - - -class CoreTopologicalFeatures(TopologicalFeaturesProtocol): - def fit(self, input_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( - TopologicalFeaturesImplementation - ) - return TopologicalFeaturesImplementation.fit(input_data) - - def transform(self, input_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.topological.fast_topological_extractor import ( - TopologicalFeaturesImplementation - ) - return TopologicalFeaturesImplementation.transform(input_data) - - -class CoreTsSmoothing(TsSmoothingProtocol): - def transform(self, input_data): - from fedot.core.operations.evaluation.operation_implementations.data_operations.ts_transformations import ( - TsSmoothingImplementation - ) - return TsSmoothingImplementation.transform(input_data) - - -class CoreTuner(TunerClassProtocol): - def __init__(self, **kwargs): - self.backend = kwargs.get("backend", "default") - - def __call__(self, objective_evaluate, task, iterations, max_lead_time=None, **kwargs): - from golem.core.tuning.optuna_tuner import OptunaTuner, DaskOptunaTuner - # from fedot.core.pipelines.tuning.tuner import BaseTuner - - if "dask" in self.backend: - return DaskOptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) - return OptunaTuner(objective_evaluate, task, iterations, max_lead_time, **kwargs) - - -class CoreApiComposerTune(ApiComposerTuneProtocol): - def __call__(self, train_data, pipeline, execution_plan=None): - from fedot.api.api_utils.api_composer import ApiComposer - return ApiComposer.tune_final_pipeline(train_data, pipeline, execution_plan) - - -class CoreReproduction(ReproductionProtocol): - def reproduce(self, population, evaluator, **kwargs): - from golem.core.optimisers.genetic.operators.reproduction import ReproductionController - return ReproductionController.reproduce(population, evaluator, **kwargs) - - def reproduce_uncontrolled(self, population, **kwargs): - from golem.core.optimisers.genetic.operators.reproduction import ReproductionController - return ReproductionController.reproduce_uncontrolled(population, **kwargs) \ No newline at end of file From f492f8b36b97930c910ac99a05bd3624400d91be Mon Sep 17 00:00:00 2001 From: PavelMarian Date: Fri, 29 May 2026 08:51:04 +0300 Subject: [PATCH 15/15] updated main.py --- fedot/api/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fedot/api/main.py b/fedot/api/main.py index e1055b1ecf..9a4770881a 100644 --- a/fedot/api/main.py +++ b/fedot/api/main.py @@ -107,7 +107,10 @@ def __init__(self, **composer_tuner_params ): - self.context = ExecutionContext(extension_name=context) + if context is None: + self.context = None + elif isinstance(context, str): + self.context = ExecutionContext(extension_name=context) set_random_seed(seed) self.log = self._init_logger(logging_level)