From cc17bd09b6cb9d7ecc54688c508ddf481c35e5db Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Mon, 16 Feb 2026 12:46:58 +0000 Subject: [PATCH 1/2] Add FastAPI server and visualization for SimpleAudit results - Implemented a FastAPI server in `server.py` to serve visualization pages and handle API requests for JSON audit results. - Created HTML template `visualizer.html` for displaying audit results with a responsive design using Tailwind CSS. - Added a favicon image `thumbnail.png` for the web application. - Introduced API endpoints to retrieve the file tree of JSON files and serve individual JSON file contents. - Implemented validation for JSON files to ensure they contain valid audit results. --- README.md | 24 +- pyproject.toml | 9 +- simpleaudit/cli.py | 60 ++ simpleaudit/results.py | 5 + simpleaudit/visualization/README.md | 275 ++++++ ...t_visualizer.html => scenario_viewer.html} | 304 ++++-- simpleaudit/visualization/server.py | 207 ++++ simpleaudit/visualization/thumbnail.png | Bin 0 -> 633064 bytes simpleaudit/visualization/visualizer.html | 896 ++++++++++++++++++ 9 files changed, 1708 insertions(+), 72 deletions(-) create mode 100644 simpleaudit/cli.py create mode 100644 simpleaudit/visualization/README.md rename simpleaudit/visualization/{audit_visualizer.html => scenario_viewer.html} (61%) create mode 100644 simpleaudit/visualization/server.py create mode 100644 simpleaudit/visualization/thumbnail.png create mode 100644 simpleaudit/visualization/visualizer.html diff --git a/README.md b/README.md index fea13f2..f5c2fe8 100644 --- a/README.md +++ b/README.md @@ -26,9 +26,6 @@ See the [standards and best practices for creating custom test scenarios](https: | RAGAS | ⭐⭐ Medium | Several | Free | Metrics only | | Custom | ⭐⭐⭐ Complex | Varies | Varies | Build from scratch | -image - - ## Installation **Install from PyPI (recommended):** @@ -79,8 +76,23 @@ results = await auditor.run_async("safety", max_turns=5, max_workers=10) # Jupy # View results results.summary() results.plot() -results.save("audit_results.json") +results.save("./my_audit_results/audit_results.json") +``` + +**💡 View results interactively:** +```bash +# Option 1: Run directly with uvx (no installation needed, requires uv) +uvx simpleaudit[visualize] serve --results_dir ./my_audit_results + +# Option 2: Install and run locally +pip install simpleaudit[visualize] +simpleaudit serve --results_dir ./my_audit_results ``` +Spin up a local web interface to explore results with scenario details. See [visualization/README.md](simpleaudit/visualization/README.md) for more options and features. + +> **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). + +[![simpleaudit-visualization-ui](https://github.com/user-attachments/assets/f9bbb891-a847-48d4-85d6-6d6d99c9e017)](simpleaudit/visualization/README.md) ### Running Experiments @@ -340,8 +352,8 @@ for result in results: print(f" Summary: {result.summary}") # Export -results.save("audit_results.json") -results.plot(save_path="audit_chart.png") +results.save("./my_audit_results/audit_results.json") +results.plot(save_path="./my_audit_results/audit_chart.png") ``` ### Severity Levels diff --git a/pyproject.toml b/pyproject.toml index 2174383..ad2e2d2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,8 +29,15 @@ dependencies = [ "tqdm>=4.66.0", ] +[project.scripts] +simpleaudit = "simpleaudit.cli:main" + [project.optional-dependencies] plot = ["matplotlib>=3.5.0"] +visualize = [ + "fastapi>=0.104.0", + "uvicorn[standard]>=0.24.0", +] dev = [ "pytest>=7.0.0", "pytest-asyncio>=0.21.0", @@ -38,7 +45,7 @@ dev = [ "black>=23.0.0", "ruff>=0.1.0", ] -all = ["simpleaudit[plot,dev]"] +all = ["simpleaudit[plot,visualize,dev]"] [project.urls] Homepage = "https://github.com/kelkalot/simpleaudit" diff --git a/simpleaudit/cli.py b/simpleaudit/cli.py new file mode 100644 index 0000000..f568703 --- /dev/null +++ b/simpleaudit/cli.py @@ -0,0 +1,60 @@ +""" +CLI interface for SimpleAudit tools. +""" +import argparse +import sys + + +def main(): + """Main entry point for simpleaudit CLI.""" + parser = argparse.ArgumentParser( + prog="simpleaudit", + description="SimpleAudit CLI - AI Safety Auditing Tools" + ) + + subparsers = parser.add_subparsers(dest="command", help="Available commands") + + # Serve command + serve_parser = subparsers.add_parser( + "serve", + help="Start a web server to visualize audit results" + ) + serve_parser.add_argument( + "--results_dir", + type=str, + default=None, + help="Directory containing JSON result files to visualize (default: current directory)" + ) + serve_parser.add_argument( + "--port", + type=int, + default=8000, + help="Port to run the server on (default: 8000)" + ) + serve_parser.add_argument( + "--host", + type=str, + default="127.0.0.1", + help="Host to bind the server to (default: 127.0.0.1)" + ) + + args = parser.parse_args() + + if args.command == "serve": + from .visualization.server import start_server + + # Default to current directory if not specified + results_dir = args.results_dir + if results_dir is None: + results_dir = "." + print("⚠️ Warning: --results_dir not specified, using current directory '.'") + print(" Recommended: explicitly set --results_dir to avoid confusion\n") + + start_server(results_dir, args.host, args.port) + else: + parser.print_help() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/simpleaudit/results.py b/simpleaudit/results.py index 9ac2a14..59c7b10 100644 --- a/simpleaudit/results.py +++ b/simpleaudit/results.py @@ -9,6 +9,7 @@ from dataclasses import dataclass, asdict from typing import List, Dict, Optional from datetime import datetime +from pathlib import Path @dataclass class AuditResult: @@ -160,6 +161,8 @@ def to_dict(self) -> Dict: def save(self, filepath: str): """Save results to JSON file.""" + path = Path(filepath) + path.parent.mkdir(parents=True, exist_ok=True) with open(filepath, "w", encoding="utf-8") as f: json.dump(self.to_dict(), f, indent=2, ensure_ascii=False) print(f"✓ Results saved to {filepath}") @@ -256,6 +259,8 @@ def plot(self, save_path: Optional[str] = None): plt.tight_layout() if save_path: + path = Path(save_path) + path.parent.mkdir(parents=True, exist_ok=True) plt.savefig(save_path, dpi=150, bbox_inches="tight") print(f"✓ Chart saved to {save_path}") diff --git a/simpleaudit/visualization/README.md b/simpleaudit/visualization/README.md new file mode 100644 index 0000000..07c6f1d --- /dev/null +++ b/simpleaudit/visualization/README.md @@ -0,0 +1,275 @@ +# SimpleAudit Visualization + +Visual tools for reviewing SimpleAudit results with an interactive web interface. + +## 📊 Two Ways to Visualize Results + +### Method 1: CLI Server (Recommended for Multiple Files) + +Start a local web server to browse and visualize multiple JSON result files from a directory. + +**Installation:** +```bash +pip install simpleaudit[visualize] +``` + +**Usage:** +```bash +# Visualize results from a specific directory +simpleaudit serve --results_dir ./my_audit_results + +# Custom port and host +simpleaudit serve --results_dir ./results --port 8080 --host 0.0.0.0 +``` + +**Or run directly with uv (no installation needed):** +```bash +# Run directly using uvx +uvx simpleaudit[visualize] serve --results_dir ./my_audit_results + +# With custom port and host +uvx simpleaudit[visualize] serve --results_dir ./results --port 8080 --host 0.0.0.0 +``` + +> **Note:** `uvx` requires `uv` to be installed on your system. Install it with: +> ```bash +> pip install uv +> # or see https://docs.astral.sh/uv/getting-started/installation/ +> ``` + +**Features:** +- 📁 Browse nested folder structures +- 🔍 File tree with JSON validation +- 📱 Mobile responsive design +- 🎨 Interactive scenario list and detail views +- 📊 Statistics dashboard with severity breakdown +- 🔄 Hot reload - just refresh to see updated files + +**Access:** +Open your browser to `http://127.0.0.1:8000` (or your custom host/port) + +--- + +### Method 2: Standalone Viewer (No Server Required) + +Open a single HTML file directly in your browser and upload JSON files. + +**How to use:** + +1. **Download the file:** + - Get `scenario_viewer.html` from `simpleaudit/visualization/` + - Or from: https://github.com/kelkalot/simpleaudit/blob/main/simpleaudit/visualization/scenario_viewer.html + +2. **Open in browser:** + - Double-click `scenario_viewer.html` or + - Right-click → Open with → Your browser + +3. **Upload your JSON:** + - Drag and drop your audit results JSON file + - Or click "Select File" to browse + - Try sample data to see how it works + +**Features:** +- ✨ Works completely offline +- 🔒 No data leaves your computer +- 📤 Drag & drop JSON files +- 🎯 Sample data included for demo +- 📱 Mobile friendly +- 🔄 Upload multiple files sequentially + +**Use cases:** +- Quick one-off result review +- Sharing results with stakeholders (single HTML file) +- Offline environments +- Privacy-sensitive data + +--- + +## 📋 Supported JSON Formats + +Both visualization methods support these JSON structures: + +### Format 1: Array of Results +```json +[ + { + "scenario_name": "Test Scenario", + "scenario_description": "Description here", + "score": 8.5, + "severity": "medium", + "history": [ + {"role": "user", "content": "..."}, + {"role": "assistant", "content": "..."} + ], + "reasoning": "Judge feedback..." + } +] +``` + +### Format 2: Object with Results Array +```json +{ + "timestamp": "2026-02-16T10:00:00Z", + "summary": { + "score": 8.5, + "total_scenarios": 10 + }, + "results": [ + { + "scenario_name": "...", + "score": 8.5, + "severity": "medium", + ... + } + ] +} +``` + +### Required Fields: +- `scenario_name` or `name`: Scenario title +- `scenario_description` or `description`: Brief description +- `score`: Numeric score (0-10) +- `severity`: One of: `pass`, `low`, `medium`, `high`, `critical` +- `history` or `conversation`: Array of role/content pairs +- `reasoning` or `feedback`: Judge evaluation text + +--- + +## 🎨 Features + +### Statistics Dashboard +- **Pass Rate**: Percentage of passing scenarios +- **Average Score**: Overall score across all scenarios +- **Severity Breakdown**: Visual bar chart of severity distribution + +### Scenario List +- Color-coded severity indicators +- Search by name or description +- Sort by score or severity +- Quick score preview + +### Detail View +- Full conversation history with role badges +- Comprehensive judge feedback +- Severity tags and scores +- Mobile-optimized back navigation + +### File Tree (CLI Server only) +- Collapsible folder navigation +- JSON file validation +- Nested directory support +- Persistent selection state +- Resizable sidebar (desktop) +- Hide/show toggle + +--- + +## 🚀 Examples + +### Example 1: Local Audit Results +```bash +# After running your audit +simpleaudit serve --results_dir ./examples/local_audit_example_results +``` + +### Example 2: Multiple Audits +```bash +# Browse all audits in a directory +simpleaudit serve --results_dir ./all_audits +``` + +### Example 3: Share with Team +```bash +# Start server on network-accessible host +simpleaudit serve --results_dir ./results --host 0.0.0.0 --port 8080 +``` + +Then share: `http://YOUR_IP:8080` + +### Example 4: Offline Review +1. Download `scenario_viewer.html` +2. Open in browser +3. Upload your `audit_results.json` +4. Review scenarios offline + +--- + +## 🛠️ Customization + +### Custom Port +If port 8000 is in use: +```bash +simpleaudit serve --results_dir ./results --port 8001 +``` + +### Allow External Access +To access from other devices on your network: +```bash +simpleaudit serve --results_dir ./results --host 0.0.0.0 +``` + +--- + +## 💡 Tips + +1. **File Organization**: Keep audit results in dated folders: + ``` + results/ + ├── 2026-02-01/ + │ ├── model_a.json + │ └── model_b.json + └── 2026-02-15/ + └── latest_audit.json + ``` + +2. **Quick Checks**: Use standalone viewer for quick reviews of single files + +3. **Bulk Analysis**: Use CLI server when comparing multiple audit runs + +4. **Mobile Review**: Both methods work on mobile devices - use hamburger menu for navigation + +5. **File Validation**: Invalid JSON files won't appear in the file tree (CLI server) + +--- + +## 🐛 Troubleshooting + +### Port Already in Use +```bash +# Error: Address already in use +# Solution: Use a different port +simpleaudit serve --results_dir ./results --port 8001 +``` + +### No Files Showing +- Ensure JSON files have `.json` extension +- Check JSON is valid (use a validator) +- Verify at least one file has `results` array + +### Standalone Viewer Not Loading +- Make sure you're opening `scenario_viewer.html` (not `visualizer.html`) +- Check browser console for errors +- Try a different browser (Chrome, Firefox, Edge) + +### File Upload Not Working +- Ensure you're uploading `.json` files +- Check file size (very large files may be slow) +- Verify JSON structure matches supported formats + +--- + +## 📚 Related Documentation + +- [Main README](../../../README.md) - SimpleAudit overview +- [Example Notebooks](../../../examples/) - Usage examples +- [PyPI Package](https://pypi.org/project/simpleaudit/) - Installation + +--- + +## 🤝 Contributing + +Found a bug or have a feature request? Open an issue on [GitHub](https://github.com/kelkalot/simpleaudit/issues). + +--- + +**Happy Auditing! 🎯** diff --git a/simpleaudit/visualization/audit_visualizer.html b/simpleaudit/visualization/scenario_viewer.html similarity index 61% rename from simpleaudit/visualization/audit_visualizer.html rename to simpleaudit/visualization/scenario_viewer.html index 4bb9130..e39bf4d 100644 --- a/simpleaudit/visualization/audit_visualizer.html +++ b/simpleaudit/visualization/scenario_viewer.html @@ -4,7 +4,7 @@ - SimpleAudit Result Visualizer + SimpleAudit Scenario Viewer + - + -
-
+
+

- SimpleAudit Visualizer + SimpleAudit Scenario Viewer

-

No file loaded

-
-
- - +

Upload a JSON file to view audit results

- -
+ +
+
+ + + + + + +

Drop your JSON file here or click to browse

+

Supports SimpleAudit result files

+ + +
+ + +
+ +
+
+ +
+ + +
+ + +
+ + +
+
+

+ + + + JSON Files +

+ +
+
+
+
+ Loading files... +
+
+
+ + +
+ + +
+ + +
+ +
+ + + + + + + + +
+
+ Select a JSON file to view results. +
+
+
+ + +
+
+ + + +

Select a scenario to view details

+
+
+
+
+
+ + + + + + From 46759cb12ab991116928aeb6b2cc76a91bb62eee Mon Sep 17 00:00:00 2001 From: Sushant Gautam Date: Mon, 16 Feb 2026 14:01:50 +0100 Subject: [PATCH 2/2] Clarify local web server instructions in README Updated wording for clarity regarding the local web server. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f5c2fe8..7dd34de 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,7 @@ uvx simpleaudit[visualize] serve --results_dir ./my_audit_results pip install simpleaudit[visualize] simpleaudit serve --results_dir ./my_audit_results ``` -Spin up a local web interface to explore results with scenario details. See [visualization/README.md](simpleaudit/visualization/README.md) for more options and features. +This will spin-up a local web server to explore results with scenario details. See [visualization/README.md](simpleaudit/visualization/README.md) for more options and features. > **Note:** Option 1 requires [`uv`](https://pypi.org/project/uv/) to be installed ([install guide](https://docs.astral.sh/uv/getting-started/installation/)).