Feature: mimic the qualifire integration in python sdk

.github/workflows/rogue.yml

@@ -1,7 +1,7 @@
 name: Rogue
 
 on:
-  pull_request_target:
+  pull_request:
   push:
     branches:
       - main
@@ -11,17 +11,11 @@ jobs:
   rogue_sanity:
     runs-on: ubuntu-latest
     timeout-minutes: 15
-    environment: rogue-sanity-ci-secrets
+    # Only run for PRs from within the same repository, not from forks
+    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
-        with:
-          # 1. Checkout the actual PR commit, not just the base branch
-          ref: ${{ github.event.pull_request.head.sha }}
-          # 2. DO NOT expose the GITHUB_TOKEN write-permissions to the untrusted code
-          # This is essential to prevent untrusted code from exfiltrating secrets
-          # by manipulating the repository itself.
-          persist-credentials: false
 
       - name: Install uv
         uses: astral-sh/setup-uv@v5

.gitignore

@@ -90,3 +90,6 @@ dist/
 .adk/
 **/.rogue
 bin/
+
+# Go TUI
+packages/tui/cmd/rogue/__debug*

.vscode/launch.json

@@ -63,6 +63,14 @@
             ],
             "envFile": "${workspaceFolder}/.env"
         },
+        {
+            "name": "Rogue TUI",
+            "type": "go",
+            "request": "launch",
+            "mode": "auto",
+            "program": "${workspaceFolder}/packages/tui/cmd/rogue/main.go",
+            "console": "integratedTerminal"
+        },
         {
             "name": "Rogue AIO",
             "type": "debugpy",

examples/mcp/tshirt_store_langgraph_mcp/__main__.py

@@ -15,7 +15,7 @@
     "--transport",
     "transport",
     default="streamable-http",
-    choices=["streamable-http", "sse"],
+    type=click.Choice(["streamable-http", "sse"]),
     help="Transport to use for the mcp server",
 )
 def main(host: str, port: int, transport: Literal["streamable-http", "sse"]) -> None:

packages/sdk/src/qualifire.ts

@@ -0,0 +1,92 @@
+import { EvaluationResult, ReportSummaryRequest, StructuredSummary } from "./types";
+
+interface QualifireReportPayload {
+  job_id: string;
+  evaluations: EvaluationResult;
+  structured: StructuredSummary | null;
+  deep_test: boolean;
+  start_time: string;
+  judge_model: string | null;
+}
+
+export interface QualifireClientOptions {
+  logger?: (message: string) => void;
+}
+
+export class QualifireClient {
+  private static convertWithStructuredSummary(
+    evaluationResults: EvaluationResult,
+    request: ReportSummaryRequest
+  ): QualifireReportPayload {
+    return {
+      job_id: request.job_id,
+      evaluations: evaluationResults,
+      structured: request.structuredSummary || null,
+      deep_test: request.deepTest ?? false,
+      start_time: request.startTime ?? new Date().toISOString(),
+      judge_model: request.judgeModel || null,
+    };
+  }
+
+  /**
+   * Reports evaluation summary to Qualifire.
+   * 
+   * @param evaluationResults - The evaluation results to report
+   * @param request - Configuration including Qualifire URL, API key, and metadata
+   * @throws {Error} If the API request fails or returns a non-2xx status
+   * @returns A promise that resolves when the report is successfully submitted
+   */
+  public static async reportSummaryToQualifire(
+    evaluationResults: EvaluationResult,
+    request: ReportSummaryRequest,
+    options?: QualifireClientOptions
+  ): Promise<void> {
+    options?.logger?.("Reporting summary to Qualifire");
+
+    const apiKey = request.qualifireApiKey;
+    const baseUrl = request.qualifireUrl ?? "https://api.qualifire.com";
+    const endpoint = `${baseUrl}/api/v1/evaluations`;
+
+    if (!apiKey) {
+      throw new Error("qualifireApiKey is required but was undefined");
+    }
+
+    if (!baseUrl || baseUrl === "undefined") {
+      throw new Error("Invalid qualifireUrl provided");
+    }
+
+    const apiEvaluationResult = this.convertWithStructuredSummary(
+      evaluationResults,
+      request
+    );
+
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 30000);
+
+    try {
+      const response = await fetch(endpoint, {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "X-Qualifire-API-Key": apiKey,
+        },
+        body: JSON.stringify(apiEvaluationResult),
+        signal: controller.signal
+      });
+
+      if (!response.ok) {
+        const errText = await response.text();
+        throw new Error(
+          `Qualifire report failed: ${response.status} ${response.statusText} - ${errText}`
+        );
+      }
+      clearTimeout(timeoutId);
+    } catch (error) {
+      clearTimeout(timeoutId);
+      if (error instanceof Error && error.name === 'AbortError') {
+        throw new Error('Qualifire report timed out after 30 seconds');
+      }
+      throw error;
+    }
+  }
+}

packages/sdk/src/types.ts

@@ -118,6 +118,23 @@ export interface RogueClientConfig {
   retries?: number;
 }
 
+export interface StructuredSummary {
+  overall_summary: string,
+  key_findings: string[],
+  recommendations: string[],
+  detailed_breakdown: object[]
+}
+
+export interface ReportSummaryRequest {
+  job_id: string,
+  qualifireUrl?: string;
+  qualifireApiKey?: string;
+  structuredSummary?: StructuredSummary;
+  deepTest?: boolean;
+  startTime?: string;
+  judgeModel?: string;
+}
+
 // Event Types for WebSocket
 export type WebSocketEventType =
   | "job_update"

packages/tui/internal/tui/common_controller.go

@@ -45,19 +45,17 @@ func (m Model) handlePasteMsg(msg tea.PasteMsg) (Model, tea.Cmd) {
 		}
 
 		// Only paste into text fields (Agent URL and Judge Model)
-		if m.evalState.currentField <= 1 {
-			switch m.evalState.currentField {
-			case 0: // Agent URL
-				// Insert at cursor position
-				runes := []rune(m.evalState.AgentURL)
-				m.evalState.AgentURL = string(runes[:m.evalState.cursorPos]) + cleanText + string(runes[m.evalState.cursorPos:])
-				m.evalState.cursorPos += len([]rune(cleanText))
-			case 1: // Judge Model
-				// Insert at cursor position
-				runes := []rune(m.evalState.JudgeModel)
-				m.evalState.JudgeModel = string(runes[:m.evalState.cursorPos]) + cleanText + string(runes[m.evalState.cursorPos:])
-				m.evalState.cursorPos += len([]rune(cleanText))
-			}
+		switch m.evalState.currentField {
+		case 0: // Agent URL
+			// Insert at cursor position
+			runes := []rune(m.evalState.AgentURL)
+			m.evalState.AgentURL = string(runes[:m.evalState.cursorPos]) + cleanText + string(runes[m.evalState.cursorPos:])
+			m.evalState.cursorPos += len([]rune(cleanText))
+		case 3: // Judge Model
+			// Insert at cursor position
+			runes := []rune(m.evalState.JudgeModel)
+			m.evalState.JudgeModel = string(runes[:m.evalState.cursorPos]) + cleanText + string(runes[m.evalState.cursorPos:])
+			m.evalState.cursorPos += len([]rune(cleanText))
 		}
 		return m, nil
 	}
@@ -202,14 +200,17 @@ func (m Model) handleCommandSelectedMsg(msg components.CommandSelectedMsg) (Mode
 			// Use the configured model in provider/model format
 			judgeModel = m.config.SelectedProvider + "/" + m.config.SelectedModel
 		}
+		// TODO read agent url and protocol .rogue/user_config.json
 		m.evalState = &EvaluationViewState{
-			ServerURL:    m.config.ServerURL,
-			AgentURL:     "http://localhost:10001",
-			JudgeModel:   judgeModel,
-			ParallelRuns: 1,
-			DeepTest:     false,
-			Scenarios:    loadScenariosFromWorkdir(),
-			cursorPos:    len([]rune("http://localhost:10001")), // Set cursor to end of Agent URL
+			ServerURL:      m.config.ServerURL,
+			AgentURL:       "http://localhost:10001",
+			AgentProtocol:  ProtocolA2A,
+			AgentTransport: TransportHTTP,
+			JudgeModel:     judgeModel,
+			ParallelRuns:   1,
+			DeepTest:       false,
+			Scenarios:      loadScenariosFromWorkdir(),
+			cursorPos:      len([]rune("http://localhost:10001")), // Set cursor to end of Agent URL
 		}
 	case "configure_models":
 		// Open LLM configuration dialog

packages/tui/internal/tui/eval_form.go

@@ -37,8 +37,8 @@ func (m Model) renderNewEvaluation() string {
 
 	title := titleStyle.Render("🧪 New Evaluation")
 
-	// Helper function to render a field with inline label and value
-	renderField := func(fieldIndex int, label, value string) string {
+	// Helper function to render a text field with inline label and value
+	renderTextField := func(fieldIndex int, label, value string) string {
 		active := m.evalState.currentField == fieldIndex
 
 		labelStyle := lipgloss.NewStyle().
@@ -83,8 +83,86 @@ func (m Model) renderNewEvaluation() string {
 		return fieldContainer.Render(fieldContent)
 	}
 
+	// Helper function to render a dropdown field with indicators
+	renderDropdownField := func(fieldIndex int, label, value string) string {
+		active := m.evalState.currentField == fieldIndex
+
+		labelStyle := lipgloss.NewStyle().
+			Foreground(t.TextMuted()).
+			Background(t.Background()).
+			Width(20).
+			Align(lipgloss.Right)
+
+		valueStyle := lipgloss.NewStyle().
+			Foreground(t.Text()).
+			Background(t.Background()).
+			Padding(0, 1)
+
+		if active {
+			labelStyle = labelStyle.Foreground(t.Primary()).Bold(true)
+			valueStyle = valueStyle.
+				Foreground(t.Primary()).
+				Background(t.Background()).
+				Bold(true)
+			// Add dropdown indicators
+			value = "◀ " + value + " ▶"
+		}
+
+		// Create a full-width container for the field
+		fieldContainer := lipgloss.NewStyle().
+			Width(m.width-4).
+			Background(t.Background()).
+			Padding(0, 2)
+
+		fieldContent := lipgloss.JoinHorizontal(lipgloss.Left,
+			labelStyle.Render(label),
+			valueStyle.Render(value),
+		)
+
+		return fieldContainer.Render(fieldContent)
+	}
+
+	// Helper function to render a toggle field
+	renderToggleField := func(fieldIndex int, label, value string) string {
+		active := m.evalState.currentField == fieldIndex
+
+		labelStyle := lipgloss.NewStyle().
+			Foreground(t.TextMuted()).
+			Background(t.Background()).
+			Width(20).
+			Align(lipgloss.Right)
+
+		valueStyle := lipgloss.NewStyle().
+			Foreground(t.Text()).
+			Background(t.Background()).
+			Padding(0, 1)
+
+		if active {
+			labelStyle = labelStyle.Foreground(t.Primary()).Bold(true)
+			valueStyle = valueStyle.
+				Foreground(t.Primary()).
+				Background(t.Background()).
+				Bold(true)
+		}
+
+		// Create a full-width container for the field
+		fieldContainer := lipgloss.NewStyle().
+			Width(m.width-4).
+			Background(t.Background()).
+			Padding(0, 2)
+
+		fieldContent := lipgloss.JoinHorizontal(lipgloss.Left,
+			labelStyle.Render(label),
+			valueStyle.Render(value),
+		)
+
+		return fieldContainer.Render(fieldContent)
+	}
+
 	// Prepare field values
 	agent := m.evalState.AgentURL
+	protocol := string(m.evalState.AgentProtocol)
+	transport := string(m.evalState.AgentTransport)
 	judge := m.evalState.JudgeModel
 	deep := "❌"
 	if m.evalState.DeepTest {
@@ -93,7 +171,7 @@ func (m Model) renderNewEvaluation() string {
 
 	// Helper function to render the start button
 	renderStartButton := func() string {
-		active := m.evalState.currentField == 3
+		active := m.evalState.currentField == 5
 		var buttonText string
 
 		if m.evalSpinner.IsActive() {
@@ -151,9 +229,11 @@ func (m Model) renderNewEvaluation() string {
 
 	// Build the content sections
 	formSection := lipgloss.JoinVertical(lipgloss.Left,
-		renderField(0, "Agent URL:", agent),
-		renderField(1, "Judge LLM:", judge),
-		renderField(2, "Deep Test:", deep),
+		renderTextField(0, "Agent URL:", agent),
+		renderDropdownField(1, "Protocol:", protocol),
+		renderDropdownField(2, "Transport:", transport),
+		renderTextField(3, "Judge LLM:", judge),
+		renderToggleField(4, "Deep Test:", deep),
 	)
 
 	var infoLines []string
@@ -168,7 +248,7 @@ func (m Model) renderNewEvaluation() string {
 
 	buttonSection := renderStartButton()
 
-	helpText := helpStyle.Render("t Test Server   ↑/↓ switch fields   ←/→ move cursor    Space toggle   Enter activate   Esc Back")
+	helpText := helpStyle.Render("t Test Server   ↑/↓ switch fields   ←/→ move cursor/cycle dropdown    Space toggle   Enter activate   Esc Back")
 
 	// Calculate content area height (excluding title and help)
 	contentHeight := m.height - 6 // title(3) + help(1) + margins(2)