Merge pull request #36 from fanqiNO1/fix_runtime_bugs

[Fix] Fix runtime bugs

Author: wangxingjun778

pyproject.toml

@@ -35,6 +35,8 @@ dependencies = [
     "pyreadline3>=3.5; sys_platform == 'win32'",
     "rich>=13.0",
     "prompt_toolkit>=3.0.40",
+    "pyobjc-framework-Quartz>=12.2; sys_platform == 'darwin'",
+    "pynput>=1.8.0"
 ]
 
 [project.optional-dependencies]

src/leapflow/cli/banner.py

@@ -47,13 +47,20 @@
 def _categorize_tools(
     tool_defs: Sequence[Dict[str, Any]],
 ) -> Dict[str, List[str]]:
-    """Group tool names by category for display."""
+    """Group tool names by category for display.
+
+    The static display map wins for known names; tools injected at runtime
+    (semantic desktop schemas) fall back to their declared x_leapflow
+    category instead of collapsing into "other".
+    """
     groups: Dict[str, List[str]] = {}
     for td in tool_defs:
         name = td.get("function", {}).get("name", "")
         if not name:
             continue
-        cat = _TOOL_CATEGORIES.get(name, "other")
+        cat = _TOOL_CATEGORIES.get(name)
+        if not cat:
+            cat = str((td.get("x_leapflow") or {}).get("category") or "") or "other"
         groups.setdefault(cat, []).append(name)
     return dict(sorted(groups.items()))
 

src/leapflow/cli/commands/host.py

@@ -79,6 +79,8 @@ def _cua_driver_version() -> Optional[str]:
             [_CUA_DRIVER_CMD, "--version"],
             capture_output=True,
             text=True,
+            encoding="utf-8",
+            errors="replace",
             timeout=5.0,
         )
         if result.returncode == 0 and result.stdout.strip():
@@ -377,11 +379,13 @@ async def _cmd_doctor() -> int:
         client.start()
         _ok("MCP session established")
 
-        # Step 3: Ping test (list_apps as health probe)
+        # Step 3: Ping test (get_screen_size as health probe — a real driver
+        # round-trip that responds instantly; list_apps enumerates the whole
+        # UI tree and can take 20s+ on Windows, making it a poor probe)
         print()
         print(f"  {_BOLD}3. Ping test{_RESET}")
-        _info("Sending probe (list_apps)...")
-        result = client._session.call_tool_sync("list_apps", {}, timeout=5.0)
+        _info("Sending probe (get_screen_size)...")
+        result = client._session.call_tool_sync("get_screen_size", {}, timeout=10.0)
         if result.get("isError"):
             _warn("Probe returned error (non-fatal)")
         else:

src/leapflow/cli/commands/slash_handlers.py

@@ -23,15 +23,18 @@
 def build_tool_payload(ctx: "Context") -> dict[str, Any]:
     """Build a serializable tool summary for local or daemon rendering."""
     from leapflow.cli.banner import _categorize_tools
-    from leapflow.tools.registry_bootstrap import TOOL_DEFINITIONS
+    from leapflow.tools.registry_bootstrap import _capability_catalog
 
-    tool_groups = _categorize_tools(TOOL_DEFINITIONS)
+    # Live catalog: static registry plus semantic desktop tools while
+    # perception is online (falls back to the static list otherwise).
+    tool_groups = _categorize_tools(_capability_catalog())
     groups = {category: sorted(names) for category, names in tool_groups.items()}
     mcp_count = 0
     if hasattr(ctx.rpc, "connected") and ctx.rpc.connected:
         mcp_count = len(getattr(ctx, "platform_tools", []))
     return {
         "ok": True,
+        "view": "tools",
         "groups": groups,
         "total": sum(len(names) for names in groups.values()),
         "mcp_count": mcp_count,
@@ -1886,6 +1889,9 @@ def render_command_payload(console: "LeapConsole", payload: dict[str, Any]) -> N
     if view == "status":
         _render_status_view(console, payload)
         return
+    if view == "tools":
+        render_tool_payload(console, payload)
+        return
     if view == "model":
         render_model_payload(console, payload)
         return

src/leapflow/cli/context.py

@@ -5,7 +5,6 @@
 import asyncio
 import logging
 import os
-import re
 import sys
 import time
 from concurrent.futures import ThreadPoolExecutor
@@ -263,13 +262,6 @@ def _promote(frag: MemoryFragment) -> None:
     return _promote
 
 
-def sanitize_skill_name(title: str) -> str:
-    """Convert a skill title to a registry-safe name."""
-    name = re.sub(r"[^\w\s-]", "", title.lower())
-    name = re.sub(r"[\s]+", "-", name.strip())
-    return name or "unnamed-skill"
-
-
 def _make_stored_skill_fn(stored: "StoredSkill", llm: Any):
     """Create an LLM-backed execution function from a StoredSkill."""
     steps_text = "\n".join(f"  {i+1}. {step}" for i, step in enumerate(stored.steps))
@@ -310,14 +302,17 @@ def _register_stored_skill_fallbacks(
     llm: Any,
 ) -> int:
     """Register StoredSkills that lack a parameterized or doc counterpart."""
+    from leapflow.learning.document import title_to_kebab
     from leapflow.skills.registry import Skill, SkillMetadata
 
     registered_names = set(registry.names()) if hasattr(registry, 'names') else {s.name for s in registry.list_all()}
     stored = skill_lib.load_all_active()
     count = 0
 
     for s in stored:
-        name = sanitize_skill_name(s.title)
+        # Same naming function as the SKILL.md write paths, otherwise the
+        # dedup below misses doc-backed skills and registers a duplicate.
+        name = title_to_kebab(s.title)
         if name in registered_names:
             continue
         if not s.trigger_phrases:
@@ -1512,6 +1507,12 @@ async def _summarize_via_llm(prompt: str) -> str:
             logger.debug("Shell approval gate: action orchestrator mode")
         except Exception:
             logger.debug("Shell approval gate setup skipped", exc_info=True)
+        try:
+            from leapflow.tools.registry_bootstrap import set_desktop_gate
+            set_desktop_gate(self._approval_orchestrator)
+            logger.debug("Desktop approval gate: action orchestrator mode")
+        except Exception:
+            logger.debug("Desktop approval gate setup skipped", exc_info=True)
 
         self._critical_tool_bridge = tool_bridge
 

src/leapflow/config.py

@@ -484,6 +484,7 @@ class Settings:
     # ── Cua Driver ──
     use_cua_driver: bool = True
     cua_driver_cmd: str = "cua-driver"
+    desktop_tools_enabled: bool = True
 
     # ── Workflow Copilot ──
     copilot_enabled: bool = True
@@ -989,6 +990,7 @@ def _tuple_env(key: str, default: tuple) -> tuple:
     # Cua Driver
     use_cua_driver = _bool("LEAPFLOW_USE_CUA_DRIVER", "true")
     cua_driver_cmd = os.getenv("LEAPFLOW_CUA_DRIVER_CMD", "cua-driver").strip()
+    desktop_tools_enabled = _bool("LEAPFLOW_DESKTOP_TOOLS_ENABLED", "true")
 
     # Workflow Copilot
     copilot_enabled = _bool("LEAPFLOW_COPILOT_ENABLED", "true")
@@ -1312,6 +1314,7 @@ def _tuple_env(key: str, default: tuple) -> tuple:
         # Cua Driver
         use_cua_driver=use_cua_driver,
         cua_driver_cmd=cua_driver_cmd,
+        desktop_tools_enabled=desktop_tools_enabled,
         # Workflow Copilot
         copilot_enabled=copilot_enabled,
         copilot_min_idle_ms=copilot_min_idle_ms,

src/leapflow/copilot/context.py

@@ -181,7 +181,7 @@ def _warmup_pipeline(self, event: "SystemEvent", state: "ContextState") -> None:
             loop = asyncio.get_running_loop()
             ctx_snapshot = self._encoder.snapshot()
             loop.create_task(on_observed(
-                action_id=f"{event.event_type}:{event.source}",
+                # action_id=f"{event.event_type}:{event.source}",
                 context=ctx_snapshot,
             ))
         except RuntimeError:

src/leapflow/daemon/approval_coordinator.py

@@ -31,7 +31,11 @@ def install_gate(self, ctx: Any, service: Any) -> None:
             from leapflow.security.orchestrator import ApprovalOrchestrator
             from leapflow.tools.config_tools import set_config_approval_gate
             from leapflow.tools.gateway_tool import set_gateway_approval_gate
-            from leapflow.tools.registry_bootstrap import set_file_read_gate, set_file_write_gate
+            from leapflow.tools.registry_bootstrap import (
+                set_desktop_gate,
+                set_file_read_gate,
+                set_file_write_gate,
+            )
             from leapflow.tools.shell_tools import set_approval_gate
             from leapflow.tools.web_fetch import set_web_approval_gate
 
@@ -51,6 +55,9 @@ def install_gate(self, ctx: Any, service: Any) -> None:
             set_config_approval_gate(orchestrator)
             # Same for outbound fetches that resolve to internal addresses.
             set_web_approval_gate(orchestrator)
+            # Mutating semantic desktop tools (click, type_text, ...) share the
+            # same approval path.
+            set_desktop_gate(orchestrator)
 
             class _FileReadGate:
                 def __init__(self) -> None: