diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 904a0945..d3377b6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -350,61 +350,12 @@ jobs: name: oxo-call-${{ matrix.target }} path: dist/*.zip - # ─── Build WebAssembly binary (always, release artifacts only on tags) ────── - build-wasm: - name: Build WebAssembly (wasm32-wasip1) - runs-on: ubuntu-latest - needs: [test] - # always() breaks the skip-cascade from sync-version through test. - if: always() && needs.test.result == 'success' - - steps: - - uses: actions/checkout@v6 - with: - ref: ${{ (github.event_name == 'workflow_dispatch' && inputs.tag != '') && format('refs/tags/{0}', inputs.tag) || github.ref }} - - - name: Install Rust (stable) with wasm32-wasip1 - uses: dtolnay/rust-toolchain@stable - with: - targets: wasm32-wasip1 - - - name: Cache cargo - uses: actions/cache@v5 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-wasm32-wasip1-${{ hashFiles('**/Cargo.lock') }} - - - name: Build release (wasm32-wasip1) - run: cargo build --release --target wasm32-wasip1 - - - name: Package binary - if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.tag != '') - shell: bash - run: | - set -euxo pipefail - BIN_NAME="oxo-call" - TARGET_DIR="target/wasm32-wasip1/release" - mkdir -p dist - tar -C "${TARGET_DIR}" -czvf \ - "dist/${BIN_NAME}-${RELEASE_TAG}-wasm32-wasip1.tar.gz" \ - "${BIN_NAME}.wasm" - - - name: Upload artifact - if: startsWith(github.ref, 'refs/tags/v') || (github.event_name == 'workflow_dispatch' && inputs.tag != '') - uses: actions/upload-artifact@v7 - with: - name: oxo-call-wasm32-wasip1 - path: dist/*.tar.gz - # ─── GitHub Release (attach all binary artifacts) ───────────────────────── release: permissions: contents: write name: GitHub Release - needs: [sync-version, build-linux, build-macos, build-windows, build-wasm] + needs: [sync-version, build-linux, build-macos, build-windows] runs-on: ubuntu-latest if: >- always() && @@ -412,8 +363,7 @@ jobs: (needs.sync-version.result == 'success' || needs.sync-version.result == 'skipped') && needs.build-linux.result == 'success' && needs.build-macos.result == 'success' && - needs.build-windows.result == 'success' && - needs.build-wasm.result == 'success' + needs.build-windows.result == 'success' steps: - uses: actions/checkout@v6 diff --git a/Cargo.toml b/Cargo.toml index 805ca1f7..4c225472 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,9 +34,6 @@ tracing = "0.1" regex = "1.12.3" lru = "0.12" hex = "0.4" - -# ── Native-only dependencies (not available on wasm32) ──────────────────────── -[target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio = { version = "1", features = ["full"] } reqwest = { version = "0.13", features = ["json", "rustls"], default-features = false } directories = "6.0" @@ -44,10 +41,6 @@ which = "8.0" indicatif = "0.18" termimad = "0.34" -# ── WebAssembly dependencies ────────────────────────────────────────────────── -[target.'cfg(target_arch = "wasm32")'.dependencies] -tokio = { version = "1", features = ["rt", "macros"] } - [dev-dependencies] tempfile = "3" wiremock = "0.6.5" diff --git a/docs/guide/src/reference/architecture.md b/docs/guide/src/reference/architecture.md index 76d0c0ea..d2e4f088 100644 --- a/docs/guide/src/reference/architecture.md +++ b/docs/guide/src/reference/architecture.md @@ -1,7 +1,5 @@ # System Architecture - - ## Overview oxo-call is a Rust workspace with three crates: @@ -12,42 +10,177 @@ oxo-call is a Rust workspace with three crates: | `crates/license-issuer` | Maintainer-only license signing tool | No | | `crates/oxo-bench` | Benchmarking and evaluation suite | No | -The architecture is designed to make command generation usable in production science and engineering workflows, not just impressive in a demo. The key idea is that `oxo-call` reduces ambiguity before the model answers, then records enough provenance afterward for users to trust and reproduce the result. +The architecture is designed around a layered system that makes command generation usable in production science and engineering workflows. The core idea: **Describe your task in plain language — oxo-call fetches the tool's documentation, asks your LLM backend to generate the exact flags you need.** -## Module Structure +## Layered Architecture + +```text +┌─────────────────────────────────────────────────────────────────────────┐ +│ User Interface Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ +│ │ CLI Client │ │ Chat Mode │ │ Web API │ │ SDK/API │ │ +│ │ (cli.rs) │ │ (chat.rs) │ │ (server.rs) │ │ (lib.rs) │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Language Processing Layer │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ Universal Task Translator (Any Language → Optimized English) │ │ +│ │ • task_normalizer.rs • task_complexity.rs • sanitize.rs │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ AI Orchestration Layer │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ Runner Pipeline (runner/) │ │ +│ │ • core.rs (orchestration) • batch.rs (parallel execution) │ │ +│ │ • retry.rs (error recovery) • utils.rs (tool detection) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ LLM Integration (llm/) │ │ +│ │ • provider.rs (multi-provider support) • types.rs (traits) │ │ +│ │ • Copilot / OpenAI / Anthropic / Ollama / DeepSeek / etc. │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ Command Generation (generator.rs) │ │ +│ │ • LLM-based • Rule-based • Composite strategies │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Knowledge Enhancement Layer │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ Documentation System │ │ +│ │ • docs.rs (resolver + caching) • doc_processor.rs (extraction)│ │ +│ │ • doc_summarizer.rs (compression) • index.rs (search index) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ +│ │ Skill │ │ MCP Skill │ │ Mini Skill │ │ Context │ │ +│ │ Manager │ │ Provider │ │ Cache │ │ Builder │ │ +│ │ (skill.rs) │ │ (mcp.rs) │ │ │ │ (context.rs│ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Execution & Monitoring Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ +│ │ Workflow │ │ DAG │ │ History │ │ Job │ │ +│ │ Templates │ │ Engine │ │ Tracker │ │ Manager │ │ +│ │ (workflow.rs) │ │ (engine.rs) │ │ (history.rs) │ │ (job.rs) │ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └────────────┘ │ +│ │ +│ ┌──────────────────────────────────────────────────────────────────┐ │ +│ │ Workflow Graph Visualization (workflow_graph.rs) │ │ +│ └──────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ + │ + ▼ +┌─────────────────────────────────────────────────────────────────────────┐ +│ Infrastructure Layer │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ ┌────────────┐ │ +│ │ LLM Backend │ │ Cache Layer │ │ Config │ │ Remote │ │ +│ │ (Multiple │ │ (cache.rs) │ │ Management │ │ Execution │ │ +│ │ Providers) │ │ │ │ (config.rs) │ │ (server.rs)│ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ └────────────┘ │ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ License │ │ Error │ │ Copilot │ │ +│ │ Verifier │ │ Handling │ │ Auth │ │ +│ │ (license.rs) │ │ (error.rs) │ │(copilot_auth)│ │ +│ └──────────────┘ └──────────────┘ └──────────────┘ │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### Layer Descriptions + +**User Interface Layer** — Multiple entry points for interacting with oxo-call: +- **CLI Client** (`cli.rs`, `main.rs`): Primary command-line interface with Clap-based argument parsing +- **Chat Mode** (`chat.rs`): Interactive conversational AI for bioinformatics tool guidance +- **Web API** (`server.rs`): Remote server management for SSH/HPC execution +- **SDK/API** (`lib.rs`): Programmatic Rust API for downstream crates and integrations + +**Language Processing Layer** — Normalizes and analyzes user input before LLM processing: +- **Task Normalizer** (`task_normalizer.rs`): Translates natural-language tasks into optimized prompts +- **Task Complexity** (`task_complexity.rs`): Estimates task complexity for adaptive prompt tier selection +- **Sanitizer** (`sanitize.rs`): Anonymizes sensitive data before sending to LLM + +**AI Orchestration Layer** — Core intelligence pipeline: +- **Runner Pipeline** (`runner/`): Orchestrates the full docs→skill→LLM→execute flow +- **LLM Integration** (`llm/`): Multi-provider abstraction (GitHub Copilot, OpenAI, Anthropic, Ollama, DeepSeek, and more) +- **Command Generator** (`generator.rs`): Extensible generation strategies via the `CommandGenerator` trait + +**Knowledge Enhancement Layer** — Grounds LLM calls in real documentation and domain expertise: +- **Documentation System** (`docs.rs`, `doc_processor.rs`, `doc_summarizer.rs`): Fetches, parses, and caches tool documentation +- **Skill System** (`skill.rs`): Domain-specific knowledge injection (user → community → MCP → built-in) +- **MCP Provider** (`mcp.rs`): Model Context Protocol for external skill servers +- **Context Builder** (`context.rs`): Assembles enriched context for LLM prompts + +**Execution & Monitoring Layer** — Runs commands and tracks results: +- **Workflow Engine** (`engine.rs`): DAG-based parallel workflow execution with tokio +- **Workflow Templates** (`workflow.rs`): Pre-built bioinformatics pipelines (RNA-seq, WGS, etc.) +- **History Tracker** (`history.rs`): JSONL command history with full provenance (UUID, model, exit code) +- **Job Manager** (`job.rs`): Background job tracking and management + +**Infrastructure Layer** — Platform services and configuration: +- **LLM Backend**: Multi-provider support with adaptive prompt tiers +- **Cache Layer** (`cache.rs`): Semantic hash-based response caching to reduce API costs +- **Config Management** (`config.rs`): TOML-based configuration with environment variable overrides +- **License Verifier** (`license.rs`): Ed25519 offline license verification -The main CLI crate contains the following modules with clear separation of concerns: +## Module Structure ```text main.rs — Command dispatcher & license gate ├─→ cli.rs — Command definitions (Clap) ├─→ handlers.rs — Extracted command-handler helpers (formatting, suggestions) ├─→ license.rs — Ed25519 offline verification - ├─→ runner.rs — Core orchestration pipeline + provenance tracking - │ ├─→ docs.rs — Documentation resolver - │ ├─→ doc_processor.rs — Structured doc extraction (flag catalog, examples, quality) - │ ├─→ skill.rs — Skill loading system + depth validation - │ │ └─→ mcp.rs — MCP skill provider (JSON-RPC / HTTP) - │ ├─→ llm.rs — LLM client, prompt builder & provider trait - │ ├─→ llm_workflow.rs — Fast/Quality workflow executor - │ ├─→ cache.rs — LLM response cache with semantic hash - │ ├─→ generator.rs — CommandGenerator trait (extensible strategies) - │ └─→ history.rs — Command history tracker with provenance - ├─→ chat.rs — Interactive chat with AI about bioinformatics tools - ├─→ sanitize.rs — Data anonymization for LLM contexts - ├─→ server.rs — Remote server management (SSH / HPC) - ├─→ workflow.rs — Templates & registry - │ └─→ engine.rs — DAG execution engine - ├─→ config.rs — Configuration management - ├─→ index.rs — Documentation index - └─→ error.rs — Error type definitions + ├─→ runner/ — Core orchestration pipeline + provenance tracking + │ ├─→ core.rs — Main runner logic + │ ├─→ batch.rs — Batch/parallel execution + │ ├─→ retry.rs — Auto-retry with error recovery + │ └─→ utils.rs — Tool detection & spinner utilities + ├─→ docs.rs — Documentation resolver + ├─→ doc_processor.rs — Structured doc extraction (flag catalog, examples) + ├─→ doc_summarizer.rs — Documentation compression + ├─→ skill.rs — Skill loading system + depth validation + │ └─→ mcp.rs — MCP skill provider (JSON-RPC / HTTP) + ├─→ llm/ — LLM integration + │ ├─→ provider.rs — Multi-provider client + │ └─→ types.rs — LlmProvider trait & types + ├─→ llm_workflow.rs — Fast/Quality workflow executor + ├─→ generator.rs — CommandGenerator trait (extensible strategies) + ├─→ cache.rs — LLM response cache with semantic hash + ├─→ history.rs — Command history tracker with provenance + ├─→ chat.rs — Interactive AI chat mode + ├─→ sanitize.rs — Data anonymization for LLM contexts + ├─→ server.rs — Remote server management (SSH / HPC) + ├─→ workflow.rs — Templates & registry + │ └─→ engine.rs — DAG execution engine + ├─→ workflow_graph.rs — DAG visualization + ├─→ task_normalizer.rs — Task normalization + ├─→ task_complexity.rs — Complexity estimation + ├─→ context.rs — Context assembly + ├─→ config.rs — Configuration management + ├─→ index.rs — Documentation index + ├─→ job.rs — Job management + ├─→ format.rs — Output formatting + ├─→ mini_skill_cache.rs — Lightweight skill caching + ├─→ copilot_auth.rs — GitHub Copilot authentication + └─→ error.rs — Error type definitions lib.rs — Programmatic API surface (re-exports all modules) ``` ## Execution Flow - - ### Command Generation (run/dry-run) ```text @@ -84,7 +217,7 @@ lib.rs — Programmatic API surface (re-exports all modules) 2. **Docs-first grounding**: Documentation fetched before LLM call to prevent hallucination 3. **Offline-first**: Cached docs, no license server, optional remote fetching 4. **Skill-augmented prompting**: Domain knowledge injected without code changes -5. **Platform independence**: WASM conditional compilation, cross-platform config dirs +5. **Native performance**: Direct native compilation for all major platforms (Linux, macOS, Windows) 6. **Strict LLM contract**: ARGS:/EXPLANATION: format with retry on invalid response 7. **Adaptive prompt compression**: Three prompt tiers (Full/Medium/Compact) auto-selected by model size and context window, ensuring reliable output from 0.5B to 200B+ parameter models 8. **Extensible generation**: CommandGenerator trait enables multiple generation strategies (LLM, rule-based, composite) with chain-of-responsibility pattern diff --git a/docs/guide/src/tutorials/installation.md b/docs/guide/src/tutorials/installation.md index 82eeffaf..6a4ef371 100644 --- a/docs/guide/src/tutorials/installation.md +++ b/docs/guide/src/tutorials/installation.md @@ -18,7 +18,6 @@ Pre-built binaries are the easiest way to get started. Download from the [Releas | macOS | aarch64 (Apple Silicon) | `oxo-call-vX.Y.Z-aarch64-apple-darwin.tar.gz` | | Windows | x86_64 | `oxo-call-vX.Y.Z-x86_64-pc-windows-msvc.zip` | | Windows | aarch64 | `oxo-call-vX.Y.Z-aarch64-pc-windows-msvc.zip` | -| WebAssembly | wasm32-wasip1 | `oxo-call-vX.Y.Z-wasm32-wasip1.tar.gz` (advanced) | 2. Extract and move to your PATH: diff --git a/docs/index.html b/docs/index.html index 67e79ff5..e58cda1d 100644 --- a/docs/index.html +++ b/docs/index.html @@ -603,19 +603,9 @@ .community-links { grid-template-columns: 1fr; } } - /* ── Wasm note ──────────────────────────────────────────────────────── */ + /* ── Wasm note (removed) ──────────────────────────────────────────── */ .wasm-badge { - display: inline-flex; - align-items: center; - gap: 0.4rem; - padding: 0.2rem 0.7rem; - background: rgba(88,166,255,0.1); - border: 1px solid rgba(88,166,255,0.3); - border-radius: 20px; - font-family: 'JetBrains Mono', monospace; - font-size: 0.72rem; - color: var(--accent); - margin-bottom: 1rem; + display: none; } .pulse { width: 6px; height: 6px; @@ -902,7 +892,6 @@ 🦀 Rust 2024 🧬 Bioinformatics 🤖 LLM-powered - ⚡ WebAssembly
@@ -1413,7 +1402,7 @@
# Download the .wasm binary from the GitHub Releases page, then: -wasmtime oxo-call.wasm -- dry-run samtools "sort input.bam by coordinate"-