ROB-858 holmes telemetry (#330)

Author: moshemorad

holmes/clients/robusta_client.py

@@ -0,0 +1,24 @@
+import logging
+from typing import Optional
+import requests
+from pydantic import BaseModel, ConfigDict
+from holmes.common.env_vars import ROBUSTA_API_ENDPOINT
+
+HOLMES_GET_INFO_URL = f"{ROBUSTA_API_ENDPOINT}/api/holmes/get_info"
+TIMEOUT = 2
+
+
+class HolmesInfo(BaseModel):
+    model_config = ConfigDict(extra="ignore")
+    latest_version: Optional[str] = None
+
+
+def fetch_holmes_info() -> Optional[HolmesInfo]:
+    try:
+        response = requests.get(HOLMES_GET_INFO_URL, timeout=TIMEOUT)
+        response.raise_for_status()
+        result = response.json()
+        return HolmesInfo(**result)
+    except Exception:
+        logging.info("Failed to fetch holmes info")
+        return None

holmes/config.py

@@ -4,6 +4,8 @@
 import os.path
 from typing import Any, Dict, List, Optional, Union
 
+from holmes import get_version
+from holmes.clients.robusta_client import HolmesInfo, fetch_holmes_info
 from holmes.core.llm import LLM, DefaultLLM
 from pydantic import FilePath, SecretStr
 from pydash.arrays import concat
@@ -211,6 +213,28 @@ class Config(RobustaBaseConfig):
 
     _server_tool_executor: Optional[ToolExecutor] = None
 
+    _version: Optional[str] = None
+    _holmes_info: Optional[HolmesInfo] = None
+
+    @property
+    def is_latest_version(self) -> bool:
+        if self._holmes_info and self._holmes_info.latest_version:
+            return self._version.startswith(self._holmes_info.latest_version)
+
+        # We couldn't resolve version, assume we are running the latest version
+        return True
+
+    def model_post_init(self, __context: Any) -> None:
+        self._version = get_version()
+        self._holmes_info = fetch_holmes_info()
+
+        if not self.is_latest_version:
+            logging.warning(
+                "You are running version %s of holmes, but the latest version is %s. Please update to the latest version.",
+                self._version,
+                self._holmes_info.latest_version,
+            )
+
     @classmethod
     def load_from_env(cls):
         kwargs = {}

holmes/main.py

@@ -287,10 +287,14 @@ def ask(
         slack_token=slack_token,
         slack_channel=slack_channel,
     )
-    system_prompt = load_and_render_prompt(system_prompt)
+
     ai = config.create_console_toolcalling_llm(
         allowed_toolsets=allowed_toolsets, dal=None
     )
+    template_context = {
+        "enabled_toolsets": ai.tool_executor.enabled_toolsets_names,
+    }
+    system_prompt = load_and_render_prompt(system_prompt, template_context)
     if echo_request:
         console.print("[bold yellow]User:[/bold yellow] " + prompt)
     for path in include_file:

poetry.lock

@@ -52,6 +52,7 @@ ruff = "^0.7.3"
 braintrust = "^0.0.168"
 autoevals = "^0.0.103"
 pre-commit = "^4.0.1"
+responses = "^0.23.1"
 
 [build-system]
 requires = ["poetry-core"]

pyproject.toml

@@ -0,0 +1,52 @@
+from http import HTTPStatus
+from unittest import mock
+from holmes.config import Config
+import responses
+
+
+@responses.activate
+def test_config_load_version_matches_latest():
+    with mock.patch("holmes.config.get_version", return_value="1.0.0"):
+        responses.add(
+            responses.GET,
+            "https://api.robusta.dev/api/holmes/get_info",
+            json={"latest_version": "1.0.0"},
+        )
+        config = Config.load_from_env()
+        assert config.is_latest_version
+
+
+@responses.activate
+def test_config_load_version_matches_latest_on_branch():
+    with mock.patch("holmes.config.get_version", return_value="1.0.0-dev"):
+        responses.add(
+            responses.GET,
+            "https://api.robusta.dev/api/holmes/get_info",
+            json={"latest_version": "1.0.0"},
+        )
+        config = Config.load_from_env()
+        assert config.is_latest_version
+
+
+@responses.activate
+def test_config_load_version_old():
+    with mock.patch("holmes.config.get_version", return_value="0.9.0"):
+        responses.add(
+            responses.GET,
+            "https://api.robusta.dev/api/holmes/get_info",
+            json={"latest_version": "1.0.0"},
+        )
+        config = Config.load_from_env()
+        assert not config.is_latest_version
+
+
+@responses.activate
+def test_config_load_failed_fetch_version():
+    responses.add(
+        responses.GET,
+        "https://api.robusta.dev/api/holmes/get_info",
+        status=HTTPStatus.INTERNAL_SERVER_ERROR,
+    )
+    config = Config.load_from_env()
+    assert config._holmes_info is None
+    assert config.is_latest_version