feat: separate model/judge API keys, fix f-string syntax, rename jobs

- Add judge_api_key input (defaults to llm_api_key when not set)
- Fix Python SyntaxError: backslash in f-string (use constants instead)
- Rename workflow job to "VerifyWise LLM Evaluation (model_name)"
- Update README to document two-key architecture and when each is needed
- Update reusable workflow header with cross-provider example

Made-with: Cursor

Author: EfeAcar6431

.github/workflows/e2e-test.yml

@@ -1,11 +1,11 @@
-name: E2E Evaluation Test
+name: VerifyWise LLM Evaluation
 
 on:
   workflow_dispatch:
 
 jobs:
-  eval-chatbot:
-    name: Chatbot Eval (answer_relevancy + bias)
+  evaluate:
+    name: VerifyWise LLM Evaluation (gpt-4o-mini)
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -14,7 +14,7 @@ jobs:
         id: eval
         uses: ./
         with:
-          api_url: https://a5d433af817b90.lhr.life
+          api_url: https://29ae92ae4bd295.lhr.life
           project_id: project_20260403_144244_140917
           dataset_id: "52"
           metrics: answer_relevancy,bias
@@ -27,6 +27,7 @@ jobs:
           fail_on_threshold: "false"
           vw_api_token: ${{ secrets.VW_API_TOKEN }}
           llm_api_key: ${{ secrets.LLM_API_KEY }}
+          # judge_api_key not set — defaults to llm_api_key
 
       - name: Show results
         if: always()
@@ -37,7 +38,3 @@ jobs:
             echo "--- Results ---"
             cat "${{ steps.eval.outputs.results_path }}"
           fi
-          if [ -f "${{ steps.eval.outputs.summary_path }}" ]; then
-            echo "--- Summary ---"
-            cat "${{ steps.eval.outputs.summary_path }}"
-          fi

README.md

@@ -128,10 +128,13 @@ jobs:
 
 **Required secrets** — add these in your repo's Settings > Secrets and variables > Actions:
 
-| Secret | Where to get it |
-|--------|----------------|
-| `VW_API_TOKEN` | VerifyWise dashboard > Settings > API Tokens |
-| `LLM_API_KEY` | Your LLM provider (OpenAI, Anthropic, etc.) |
+| Secret | Required | Where to get it |
+|--------|----------|----------------|
+| `VW_API_TOKEN` | yes | VerifyWise dashboard > Settings > API Tokens |
+| `LLM_API_KEY` | yes | API key for the model being evaluated (e.g. OpenAI, Anthropic) |
+| `JUDGE_API_KEY` | no | API key for the judge LLM. Defaults to `LLM_API_KEY` if not set. Only needed when the model and judge use different providers. |
+
+> **How it works:** The evaluation uses two LLMs — the **model** generates responses to your prompts, and the **judge** scores those responses against the selected metrics. If both use the same provider (e.g. both OpenAI), a single `LLM_API_KEY` is enough. If they use different providers (e.g. evaluating a Claude model with GPT-4o as judge), set `JUDGE_API_KEY` separately.
 
 ---
 
@@ -146,7 +149,8 @@ jobs:
 | `model_name` | **yes** | — | Model to evaluate (e.g. `gpt-4o-mini`, `claude-3-5-sonnet`) |
 | `model_provider` | **yes** | — | `openai`, `anthropic`, `google`, `mistral`, `xai`, or `self-hosted` |
 | `vw_api_token` | **yes** | — | VerifyWise API token (store as a repository secret) |
-| `llm_api_key` | **yes** | — | API key for the LLM provider (store as a repository secret) |
+| `llm_api_key` | **yes** | — | API key for the model being evaluated |
+| `judge_api_key` | no | *(same as llm_api_key)* | API key for the judge LLM (only needed when model and judge use different providers) |
 | `judge_model` | no | `gpt-4o` | LLM used to judge responses |
 | `judge_provider` | no | `openai` | Provider for the judge LLM |
 | `threshold` | no | `0.7` | Pass/fail threshold (0.0–1.0) |

action.yml

@@ -73,8 +73,12 @@ inputs:
     description: 'VerifyWise API token'
     required: true
   llm_api_key:
-    description: 'LLM provider API key'
+    description: 'API key for the model being evaluated'
     required: true
+  judge_api_key:
+    description: 'API key for the judge LLM (defaults to llm_api_key if not set)'
+    required: false
+    default: ''
 
 outputs:
   passed:
@@ -120,6 +124,7 @@ runs:
         VW_POLL_INTERVAL: ${{ inputs.poll_interval_seconds }}
         VW_EXPERIMENT_NAME: ${{ inputs.experiment_name }}
         LLM_API_KEY: ${{ inputs.llm_api_key }}
+        JUDGE_API_KEY: ${{ inputs.judge_api_key }}
       run: |
         RESULTS="${{ runner.temp }}/vw-results.json"
         SUMMARY="${{ runner.temp }}/vw-summary.md"
@@ -176,12 +181,10 @@ runs:
 
         passed = data.get("passed", False)
         metrics = data.get("metrics", [])
-        samples = data.get("samples", [])
         name = data.get("name", "Evaluation")
         model = data.get("model", "unknown")
 
         failing = [m for m in metrics if not m.get("passed")]
-        passing = [m for m in metrics if m.get("passed")]
 
         with open(os.environ["GITHUB_OUTPUT"], "a") as out:
             out.write(f"passed={'true' if passed else 'false'}\n")

ci_eval_runner.py

@@ -27,6 +27,9 @@
     sys.exit(2)
 
 INVERTED_KEYWORDS = ("bias", "toxicity", "hallucination", "conversationsafety")
+EM_DASH = "\u2014"
+CHECK = "\u2705"
+CROSS = "\u274C"
 
 
 def parse_args() -> argparse.Namespace:
@@ -84,7 +87,8 @@ def create_experiment(
     name: str,
 ) -> Dict[str, Any]:
     url = f"{base_url}/api/deepeval/experiments"
-    llm_api_key = os.getenv("LLM_API_KEY", "")
+    model_api_key = os.getenv("LLM_API_KEY", "")
+    judge_api_key = os.getenv("JUDGE_API_KEY", "") or model_api_key
 
     metric_configs = []
     for m in metrics:
@@ -110,7 +114,7 @@ def create_experiment(
                 "name": model_name,
                 "model_name": model_name,
                 "provider": model_provider,
-                "apiKey": llm_api_key if model_provider != "self-hosted" else "",
+                "apiKey": model_api_key if model_provider != "self-hosted" else "",
             },
             "dataset": {
                 "id": dataset_id,
@@ -122,7 +126,7 @@ def create_experiment(
             "judgeLlm": {
                 "provider": judge_provider,
                 "model": judge_model,
-                "apiKey": llm_api_key,
+                "apiKey": judge_api_key,
             },
         },
     }
@@ -235,7 +239,6 @@ def parse_results(
         })
 
     # Build samples from detailed_results, enriched with log data
-    # Logs are ordered newest-first; reverse to match sample order
     sorted_logs = list(reversed(logs))
 
     samples = []
@@ -269,7 +272,6 @@ def parse_results(
                 }
         samples.append(sample_entry)
 
-    # If we have logs but no detailed_results, build samples from logs alone
     if not detailed_results and sorted_logs:
         for i, log in enumerate(sorted_logs):
             samples.append({
@@ -294,38 +296,40 @@ def parse_results(
 
 
 def _escape_md(text: str) -> str:
-    """Prepare text for display in markdown without truncation."""
+    """Prepare text for display in markdown."""
     if not text:
         return ""
     return text.replace("\n", " ").replace("|", "\\|").strip()
 
 
 def generate_markdown(results: Dict[str, Any]) -> str:
+    model = results["model"]
+    overall_icon = CHECK if results["passed"] else CROSS
+    overall_word = "PASS" if results["passed"] else "FAIL"
+
     lines = [
         "## VerifyWise LLM Evaluation Results",
         "",
         f"**Experiment:** {results['name']}  ",
-        f"**Model:** {results['model']}  ",
+        f"**Model:** {model}  ",
         f"**Status:** {results['status']}  ",
         f"**Samples:** {results['total_prompts']}  ",
     ]
 
     if results.get("duration_ms"):
         lines.append(f"**Duration:** {results['duration_ms'] / 1000:.1f}s  ")
 
-    icon = "\u2705" if results["passed"] else "\u274C"
-    overall = "PASS" if results["passed"] else "FAIL"
     lines.extend([
         "",
-        f"### Overall: {icon} {overall}",
+        f"### Overall: {overall_icon} {overall_word}",
         "",
         "| Metric | Score | Threshold | Result |",
         "|--------|------:|----------:|--------|",
     ])
 
     for m in results["metrics"]:
         inv = " *(inverted)*" if m["inverted"] else ""
-        mi = "\u2705" if m["passed"] else "\u274C"
+        mi = CHECK if m["passed"] else CROSS
         lines.append(
             f"| {m['name']}{inv} | {m['score']*100:.1f}% | {m['threshold']*100:.0f}% | {mi} |"
         )
@@ -342,23 +346,20 @@ def generate_markdown(results: Dict[str, Any]) -> str:
 
         for sample in samples:
             sample_scores = sample.get("metric_scores", {})
-
-            # Show all samples that have metric scores (not just failing ones)
-            # so the user can compare passing vs failing on the same prompt
             if not sample_scores:
                 continue
 
-            input_text = _escape_md(sample["input"])
-            output_text = _escape_md(sample["output"])
+            input_text = _escape_md(sample["input"]) or "*(empty)*"
+            output_text = _escape_md(sample["output"]) or "*(not captured)*"
             expected_text = _escape_md(sample.get("expected", ""))
 
             lines.extend([
                 "",
                 f"#### Sample {sample['index']}",
                 "",
-                f"**Input:** {input_text}" if input_text else "**Input:** *(empty)*",
+                f"**Input:** {input_text}",
                 "",
-                f"**Response:** {output_text}" if output_text else "**Response:** *(not captured)*",
+                f"**Response:** {output_text}",
             ])
 
             if expected_text:
@@ -380,13 +381,14 @@ def generate_markdown(results: Dict[str, Any]) -> str:
                     score_str = "N/A"
 
                 if passed is True:
-                    result_str = "\u2705"
+                    result_str = CHECK
                 elif passed is False:
-                    result_str = "\u274C"
+                    result_str = CROSS
                 else:
-                    result_str = "\u2014"
+                    result_str = EM_DASH
 
-                lines.append(f"| {metric_name} | {score_str} | {result_str} | {explanation or '\u2014'} |")
+                explanation_str = explanation if explanation else EM_DASH
+                lines.append(f"| {metric_name} | {score_str} | {result_str} | {explanation_str} |")
 
     lines.extend([
         "",
@@ -397,15 +399,6 @@ def generate_markdown(results: Dict[str, Any]) -> str:
     return "\n".join(lines)
 
 
-def _metric_name_matches(name: str, targets: set) -> bool:
-    """Check if a metric name matches any target, case-insensitively."""
-    lower = name.lower()
-    for t in targets:
-        if t.lower() == lower or t.lower().replace("_", "") == lower.replace("_", ""):
-            return True
-    return False
-
-
 def main():
     args = parse_args()