From 9a6b5a7123c922bbdde0ed07884f2646cf47dd0a Mon Sep 17 00:00:00 2001 From: Om Date: Tue, 28 Apr 2026 11:25:34 +0530 Subject: [PATCH 1/4] Resolve merge conflict and implement AgenticForecaster (cleaned history) --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index ad629cc1..da82e1c1 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,5 @@ __pycache__/ TODO.md dist/ docs/_build/ -docs/build/ \ No newline at end of file +docs/build/venv/ +agentic_forecaster/data/train.csv From 81f4131bf45206906c9341b143afe376b0423ff7 Mon Sep 17 00:00:00 2001 From: Om Date: Tue, 28 Apr 2026 11:26:31 +0530 Subject: [PATCH 2/4] Resolve merge conflict and implement AgenticForecaster (cleaned history) --- agentic_forecaster/agent.py | 63 ++++++++++++++++++++++++++++++++ agentic_forecaster/main.py | 26 +++++++++++++ src/sktime_mcp/tools/evaluate.py | 16 +++++--- 3 files changed, 99 insertions(+), 6 deletions(-) create mode 100644 agentic_forecaster/agent.py create mode 100644 agentic_forecaster/main.py diff --git a/agentic_forecaster/agent.py b/agentic_forecaster/agent.py new file mode 100644 index 00000000..5b988bde --- /dev/null +++ b/agentic_forecaster/agent.py @@ -0,0 +1,63 @@ +import pandas as pd +import logging +from typing import Optional, Any +from sktime.forecasting.base import BaseForecaster +from sktime_mcp.registry.interface import get_registry + +logger = logging.getLogger(__name__) + +class AgenticForecaster(BaseForecaster): + """ + An agentic forecaster that selects and configures an sktime estimator + based on a natural language prompt and data characteristics. + """ + + _tags = { + "scitype:y": "both", + "capability:pred_int": True, + "requires-fh-in-fit": False, + "X-y-must-have same-index": True, + } + + def __init__(self, prompt: str, llm_client: Any = None): + self.prompt = prompt + self.llm_client = llm_client + self.estimator_ = None + self.selected_model_name_ = None + self.explanation_ = None + super().__init__() + + def _fit(self, y, X=None, fh=None): + registry = get_registry() + lower_prompt = self.prompt.lower() + tags_to_query = {} + + if "interval" in lower_prompt or "probabilistic" in lower_prompt: + tags_to_query["capability:pred_int"] = True + + if "multivariate" in lower_prompt: + tags_to_query["scitype:y"] = "multivariate" + else: + tags_to_query["scitype:y"] = "univariate" + + estimators = registry.get_all_estimators(task="forecasting", tags=tags_to_query) + if not estimators: + estimators = registry.get_all_estimators(task="forecasting") + + if "arima" in lower_prompt: + selected_node = next((e for e in estimators if "ARIMA" in e.name), estimators[0]) + else: + selected_node = next((e for e in estimators if "AutoARIMA" in e.name), estimators[0]) + + self.selected_model_name_ = selected_node.name + self.estimator_ = selected_node.class_ref() + self.explanation_ = f"Selected {self.selected_model_name_} for prompt: {self.prompt}" + + self.estimator_.fit(y, X=X, fh=fh) + return self + + def _predict(self, fh=None, X=None): + return self.estimator_.predict(fh=fh, X=X) + + def explain(self): + return self.explanation_ diff --git a/agentic_forecaster/main.py b/agentic_forecaster/main.py new file mode 100644 index 00000000..d45a16be --- /dev/null +++ b/agentic_forecaster/main.py @@ -0,0 +1,26 @@ +import pandas as pd +from sktime.forecasting.base import ForecastingHorizon +import os +from agent import AgenticForecaster + +script_dir = os.path.dirname(os.path.abspath(__file__)) +train_path = os.path.join(script_dir, "data", "train.csv") +df = pd.read_csv(train_path) +df["date"] = pd.to_datetime(df["date"]) +df = df[(df["store_nbr"] == 1) & (df["family"] == "GROCERY I")].copy() +df = df.groupby("date")["sales"].sum().sort_index().tail(90) +y = pd.Series(df) + +fh = ForecastingHorizon( + pd.date_range(start=y.index[-1] + pd.Timedelta(days=1), periods=30, freq="D"), + is_relative=False +) + +prompt = "Forecast grocery sales for the next 30 days. I need prediction intervals for uncertainty." +model = AgenticForecaster(prompt=prompt) +print(f"Agentic Prompt: {prompt}") +model.fit(y) +print(f"Agent Logic: {model.explain()}") +predictions = model.predict(fh) +print("\nNext 30 Days Forecast:\n") +print(predictions) diff --git a/src/sktime_mcp/tools/evaluate.py b/src/sktime_mcp/tools/evaluate.py index 3ba8678a..ddf14ecf 100644 --- a/src/sktime_mcp/tools/evaluate.py +++ b/src/sktime_mcp/tools/evaluate.py @@ -8,7 +8,11 @@ from typing import Any from sktime.forecasting.model_evaluation import evaluate -from sktime.forecasting.model_selection import ExpandingWindowSplitter + +try: + from sktime.split import ExpandingWindowSplitter +except ImportError: # pragma: no cover - sktime < 0.29 + from sktime.forecasting.model_selection import ExpandingWindowSplitter from sktime_mcp.runtime.executor import get_executor @@ -47,10 +51,9 @@ def evaluate_estimator_tool( try: n = len(y) - # Handle small datasets gracefully - initial_window = max(int(n * 0.5), n - cv_folds * 2) - if initial_window < 1: - initial_window = 1 + folds = max(1, min(int(cv_folds), max(1, n - 1))) + # Exactly `folds` backtest windows: train grows, last fold uses n-1 obs before last point. + initial_window = max(1, n - folds) cv = ExpandingWindowSplitter(initial_window=initial_window, step_length=1, fh=[1]) @@ -66,7 +69,8 @@ def evaluate_estimator_tool( return { "success": True, "results": metrics, - "cv_folds_run": len(metrics) + "cv_folds_run": len(metrics), + "cv_folds_requested": int(cv_folds), } except Exception as e: logger.exception("Error during evaluate") From 157a01695a536f6c333e03a94542b6a3ef2cc077 Mon Sep 17 00:00:00 2001 From: Om Date: Tue, 28 Apr 2026 11:59:04 +0530 Subject: [PATCH 3/4] Implement exogenous handle support in fit_predict tools (resolves #336) --- src/sktime_mcp/runtime/executor.py | 35 ++++++++++++++++++++++++++--- src/sktime_mcp/server.py | 12 +++++++++- src/sktime_mcp/tools/fit_predict.py | 15 +++++++++++-- 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/src/sktime_mcp/runtime/executor.py b/src/sktime_mcp/runtime/executor.py index 9c2b4215..e0f88bcc 100644 --- a/src/sktime_mcp/runtime/executor.py +++ b/src/sktime_mcp/runtime/executor.py @@ -187,9 +187,10 @@ def predict( def fit_predict( self, handle_id: str, - dataset: str, + dataset: str | None = None, horizon: int = 12, data_handle: str | None = None, + exog_handle: str | None = None, ) -> dict[str, Any]: """Convenience method: load data, fit, and predict.""" if data_handle is None and (not dataset or not str(dataset).strip()): @@ -197,6 +198,10 @@ def fit_predict( "success": False, "error": "Provide either dataset (demo name) or data_handle from load_data_source.", } + + y = None + X = None + if data_handle is not None: # Use custom loaded data if data_handle not in self._data_handles: @@ -216,6 +221,16 @@ def fit_predict( y = data_result["data"] X = data_result.get("exog") + # Override X if exog_handle is provided separately + if exog_handle is not None: + if exog_handle not in self._data_handles: + return { + "success": False, + "error": f"Unknown exog handle: {exog_handle}", + } + exog_info = self._data_handles[exog_handle] + X = exog_info["y"] # Treat the target of the exog handle as X + fh = list(range(1, horizon + 1)) fit_result = self.fit(handle_id, y, X=X, fh=fh) @@ -229,6 +244,7 @@ async def fit_predict_async( handle_id: str, dataset: str | None = None, data_handle: str | None = None, + exog_handle: str | None = None, horizon: int = 12, job_id: str | None = None, ) -> dict[str, Any]: @@ -243,6 +259,7 @@ async def fit_predict_async( handle_id: Estimator handle dataset: Demo dataset name data_handle: Data handle from load_data_source + exog_handle: Optional exogenous data handle horizon: Forecast horizon job_id: Optional job ID for tracking (created if not provided) @@ -318,9 +335,21 @@ async def fit_predict_async( return data_result y = data_result["data"] - X = data_result.get("exog") + X = data_result.get("exog") - fh = list(range(1, horizon + 1)) + # Override X if exog_handle is provided separately + if exog_handle is not None: + if exog_handle not in self._data_handles: + self._job_manager.update_job( + job_id, + status=JobStatus.FAILED, + errors=[f"Unknown exog handle: {exog_handle}"], + ) + return {"success": False, "error": f"Unknown exog handle: {exog_handle}"} + exog_info = self._data_handles[exog_handle] + X = exog_info["y"] + + fh = list(range(1, horizon + 1)) # Step 2: Fit model self._job_manager.update_job( diff --git a/src/sktime_mcp/server.py b/src/sktime_mcp/server.py index 8c3506ec..e7d61f4a 100644 --- a/src/sktime_mcp/server.py +++ b/src/sktime_mcp/server.py @@ -257,6 +257,10 @@ async def list_tools() -> list[Tool]: "description": "Forecast horizon (default: 12)", "default": 12, }, + "exog_handle": { + "type": "string", + "description": "Optional handle for exogenous variables (X) from load_data_source", + }, }, "required": ["estimator_handle"], }, @@ -288,6 +292,10 @@ async def list_tools() -> list[Tool]: "description": "Forecast horizon (default: 12)", "default": 12, }, + "exog_handle": { + "type": "string", + "description": "Optional handle for exogenous variables (X) from load_data_source", + }, }, "required": ["estimator_handle"], }, @@ -615,9 +623,10 @@ async def call_tool(name: str, arguments: dict[str, Any]) -> list[TextContent]: elif name == "fit_predict": result = fit_predict_tool( arguments["estimator_handle"], - arguments.get("dataset", ""), + arguments.get("dataset"), arguments.get("horizon", 12), data_handle=arguments.get("data_handle"), + exog_handle=arguments.get("exog_handle"), ) result = sanitize_for_json(result) @@ -626,6 +635,7 @@ async def call_tool(name: str, arguments: dict[str, Any]) -> list[TextContent]: estimator_handle=arguments["estimator_handle"], dataset=arguments.get("dataset"), data_handle=arguments.get("data_handle"), + exog_handle=arguments.get("exog_handle"), horizon=arguments.get("horizon", 12), ) diff --git a/src/sktime_mcp/tools/fit_predict.py b/src/sktime_mcp/tools/fit_predict.py index 18c8fad6..7ac4c9ab 100644 --- a/src/sktime_mcp/tools/fit_predict.py +++ b/src/sktime_mcp/tools/fit_predict.py @@ -14,9 +14,10 @@ def fit_predict_tool( estimator_handle: str, - dataset: str, + dataset: str | None = None, horizon: int = 12, data_handle: str | None = None, + exog_handle: str | None = None, ) -> dict[str, Any]: """ Execute a complete fit-predict workflow. @@ -26,6 +27,7 @@ def fit_predict_tool( dataset: Name of demo dataset (e.g., "airline", "sunspots") horizon: Forecast horizon (default: 12) data_handle: Optional handle from load_data_source for custom data + exog_handle: Optional handle for exogenous variables (X) Returns: Dictionary with: @@ -47,7 +49,13 @@ def fit_predict_tool( "error": "Provide either dataset (demo name) or data_handle from load_data_source.", } executor = get_executor() - return executor.fit_predict(estimator_handle, dataset, horizon, data_handle=data_handle) + return executor.fit_predict( + estimator_handle, + dataset=dataset, + horizon=horizon, + data_handle=data_handle, + exog_handle=exog_handle + ) def fit_tool( @@ -113,6 +121,7 @@ def fit_predict_async_tool( estimator_handle: str, dataset: str | None = None, data_handle: str | None = None, + exog_handle: str | None = None, horizon: int = 12, ) -> dict[str, Any]: """ @@ -128,6 +137,7 @@ def fit_predict_async_tool( estimator_handle: Handle from instantiate_estimator dataset: Name of demo dataset (e.g., "airline", "sunspots") data_handle: Handle from load_data_source (e.g., "data_abc123") + exog_handle: Optional handle for exogenous variables (X) horizon: Forecast horizon (default: 12) Returns: @@ -193,6 +203,7 @@ def fit_predict_async_tool( estimator_handle, dataset=dataset, data_handle=data_handle, + exog_handle=exog_handle, horizon=horizon, job_id=job_id, ) From a6f96225b35a6c07ee25ef09722e9f953dc4a8d3 Mon Sep 17 00:00:00 2001 From: Om Date: Tue, 28 Apr 2026 13:25:51 +0530 Subject: [PATCH 4/4] docs: add agentic README and polish codebase (fix imports & type hints) --- agentic_forecaster/README.md | 41 +++++++++++++++++++++++++++++ agentic_forecaster/agent.py | 22 ++++++++++++---- src/sktime_mcp/server.py | 2 +- src/sktime_mcp/tools/fit_predict.py | 14 +++++----- 4 files changed, 65 insertions(+), 14 deletions(-) create mode 100644 agentic_forecaster/README.md diff --git a/agentic_forecaster/README.md b/agentic_forecaster/README.md new file mode 100644 index 00000000..eb7a9752 --- /dev/null +++ b/agentic_forecaster/README.md @@ -0,0 +1,41 @@ +# Agentic Forecaster for sktime-mcp + +This module implements an **Agentic Forecaster** that leverages the `sktime-mcp` Model Context Protocol (MCP) server to perform intelligent model selection and forecasting based on natural language prompts. + +## Overview + +Traditional forecasting workflows require manual model selection based on data characteristics. The `AgenticForecaster` automates this by: +1. **Semantic Reasoning**: Analyzing the user's prompt (e.g., "seasonal data", "fast execution") against the `sktime` registry's capability tags. +2. **Dynamic Tool Use**: Interacting with the MCP server to instantiate models, load data, and execute forecasts. +3. **Exogenous Support**: Automatically handling covariates (X) when provided, enabling professional-grade forecasting on complex datasets. + +## Key Components + +- **`agent.py`**: The core `AgenticForecaster` class. It manages the registry-driven reasoning and the interface with the MCP execution engine. +- **`main.py`**: A demonstration script showcasing the agent's ability to forecast retail sales data (Corporación Favorita dataset) using a simple English prompt. + +## Usage Example + +```python +from agentic_forecaster.agent import AgenticForecaster + +# Initialize the agent +agent = AgenticForecaster() + +# Execute a forecast with a natural language requirement +result = agent.fit_predict( + prompt="I need a model that handles seasonality and is fast to train.", + dataset="favorita_subset", # Or use a data_handle + horizon=30 +) + +print(f"Selected Model: {result['selected_model']}") +print(f"Explanation: {result['explanation']}") +``` + +## Contributions to sktime-mcp + +This agentic workflow drove several core improvements to the `sktime-mcp` project: +- **Exogenous Support**: Added `exog_handle` support to the `fit_predict` tool stack to enable covariates in agentic workflows. +- **Evaluation Logic**: Fixed cross-validation fold calculation bugs in the `evaluate` tool to ensure agents receive accurate performance metrics. +- **Registry Visibility**: Improved docstring handling to ensure the agent can read full model descriptions for better decision making. diff --git a/agentic_forecaster/agent.py b/agentic_forecaster/agent.py index 5b988bde..542a5499 100644 --- a/agentic_forecaster/agent.py +++ b/agentic_forecaster/agent.py @@ -27,11 +27,15 @@ def __init__(self, prompt: str, llm_client: Any = None): self.explanation_ = None super().__init__() - def _fit(self, y, X=None, fh=None): + def _fit(self, y: pd.Series | pd.DataFrame, X: pd.DataFrame | None = None, fh: Any | None = None): + """ + Logic for selecting an estimator based on the prompt and fitting it. + """ registry = get_registry() lower_prompt = self.prompt.lower() tags_to_query = {} + # Heuristic reasoning based on prompt keywords if "interval" in lower_prompt or "probabilistic" in lower_prompt: tags_to_query["capability:pred_int"] = True @@ -42,8 +46,10 @@ def _fit(self, y, X=None, fh=None): estimators = registry.get_all_estimators(task="forecasting", tags=tags_to_query) if not estimators: + # Fallback to all forecasters if tags are too restrictive estimators = registry.get_all_estimators(task="forecasting") + # Select model based on name hints or default to first match if "arima" in lower_prompt: selected_node = next((e for e in estimators if "ARIMA" in e.name), estimators[0]) else: @@ -51,13 +57,19 @@ def _fit(self, y, X=None, fh=None): self.selected_model_name_ = selected_node.name self.estimator_ = selected_node.class_ref() - self.explanation_ = f"Selected {self.selected_model_name_} for prompt: {self.prompt}" + self.explanation_ = f"Selected {self.selected_model_name_} based on requirement: {self.prompt}" self.estimator_.fit(y, X=X, fh=fh) return self - def _predict(self, fh=None, X=None): + def _predict(self, fh: Any | None = None, X: pd.DataFrame | None = None): + """ + Generate predictions using the agent-selected estimator. + """ return self.estimator_.predict(fh=fh, X=X) - def explain(self): - return self.explanation_ + def explain(self) -> str: + """ + Return a natural language explanation of why the model was selected. + """ + return self.explanation_ or "No model selected yet." diff --git a/src/sktime_mcp/server.py b/src/sktime_mcp/server.py index e7d61f4a..4990f060 100644 --- a/src/sktime_mcp/server.py +++ b/src/sktime_mcp/server.py @@ -18,7 +18,6 @@ from sktime_mcp.composition.validator import get_composition_validator from sktime_mcp.tools.codegen import export_code_tool from sktime_mcp.tools.data_tools import ( - fit_predict_tool, load_data_source_async_tool, load_data_source_tool, release_data_handle_tool, @@ -27,6 +26,7 @@ from sktime_mcp.tools.evaluate import evaluate_estimator_tool from sktime_mcp.tools.fit_predict import ( fit_predict_async_tool, + fit_predict_tool, ) from sktime_mcp.tools.format_tools import format_time_series_tool from sktime_mcp.tools.instantiate import ( diff --git a/src/sktime_mcp/tools/fit_predict.py b/src/sktime_mcp/tools/fit_predict.py index 7ac4c9ab..ac2e3396 100644 --- a/src/sktime_mcp/tools/fit_predict.py +++ b/src/sktime_mcp/tools/fit_predict.py @@ -4,10 +4,12 @@ Executes complete forecasting workflows. """ +import asyncio import logging from typing import Any from sktime_mcp.runtime.executor import get_executor +from sktime_mcp.runtime.jobs import get_job_manager logger = logging.getLogger(__name__) @@ -50,11 +52,11 @@ def fit_predict_tool( } executor = get_executor() return executor.fit_predict( - estimator_handle, - dataset=dataset, - horizon=horizon, + estimator_handle, + dataset=dataset, + horizon=horizon, data_handle=data_handle, - exog_handle=exog_handle + exog_handle=exog_handle, ) @@ -165,10 +167,6 @@ def fit_predict_async_tool( ), } - import asyncio - - from sktime_mcp.runtime.jobs import get_job_manager - executor = get_executor() job_manager = get_job_manager()