drop dummy dataset files

Author: viraatc

examples/03_BenchmarkComparison/README.md

@@ -4,15 +4,11 @@ Compare `inference-endpoint` with vLLM's benchmarking tool using identical promp
 
 ## Prerequisites
 
-1. **Running inference server** (OpenAI-compatible):
+**Running inference server** (OpenAI-compatible):
 
-   ```bash
-   vllm serve Qwen/Qwen2.5-0.5B-Instruct --port 8000
-   ```
-
-2. **Pre-generated dataset files** (included in this directory):
-   - `dummy_prompts_vllm.jsonl` - vLLM format: `{"prompt": "..."}`
-   - `dummy_prompts_ie.jsonl` - inference-endpoint format: `{"text_input": "..."}`
+```bash
+vllm serve Qwen/Qwen2.5-0.5B-Instruct --port 8000
+```
 
 ## Usage
 

examples/03_BenchmarkComparison/compare_with_vllm.py

@@ -14,29 +14,44 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Compare inference-endpoint and vLLM benchmarking tools using JSONL datasets.
+"""Compare inference-endpoint and vLLM benchmarking tools.
 
-Prerequisites:
-    - Pre-generated dataset files in the same directory:
-      - dummy_prompts_vllm.jsonl (vLLM custom format)
-      - dummy_prompts_ie.jsonl (inference-endpoint format)
-    - Contain the same text prompts.
+Generates dummy prompts on the fly from a base set of 10 phrases.
+Prompts are cycled if num-prompts exceeds the base set size.
 
 Usage:
     python compare_with_vllm.py --model "Qwen/Qwen2.5-0.5B-Instruct" --num-prompts 100
 """
 
 import argparse
+import json
 import logging
 import re
 import subprocess
 import sys
+import tempfile
 import time
+from collections.abc import Generator
+from contextlib import contextmanager
 from pathlib import Path
 from typing import Any
 
 import requests
 
+# Base prompts for generating benchmark datasets (cycled as needed)
+BASE_PROMPTS = [
+    "Explain quantum entanglement in simple terms.",
+    "What is machine learning and how does it work?",
+    "Describe the process of photosynthesis.",
+    "How does blockchain technology work?",
+    "What are the health benefits of regular exercise?",
+    "Explain the theory of relativity in simple terms.",
+    "How do neural networks learn from data?",
+    "What causes climate change and its effects?",
+    "Describe how the internet works at a high level.",
+    "What is the difference between AI and machine learning?",
+]
+
 # Constants
 DEFAULT_ENDPOINT = "http://localhost:8000"
 DEFAULT_NUM_PROMPTS = 100
@@ -50,6 +65,38 @@
 logger = logging.getLogger(__name__)
 
 
+@contextmanager
+def temp_dataset_files(num_prompts: int) -> Generator[tuple[Path, Path], None, None]:
+    """Context manager that creates temporary dataset files for benchmarking.
+
+    Generates prompts by cycling through BASE_PROMPTS and creates two JSONL files:
+    - vLLM format: {"prompt": "..."}
+    - inference-endpoint format: {"text_input": "..."}
+
+    Args:
+        num_prompts: Number of prompts to generate
+
+    Yields:
+        Tuple of (vllm_dataset_path, ie_dataset_path)
+    """
+    with tempfile.TemporaryDirectory() as temp_dir:
+        temp_path = Path(temp_dir)
+        prompts = [BASE_PROMPTS[i % len(BASE_PROMPTS)] for i in range(num_prompts)]
+
+        vllm_path = temp_path / "prompts_vllm.jsonl"
+        ie_path = temp_path / "prompts_ie.jsonl"
+
+        with open(vllm_path, "w") as f:
+            for prompt in prompts:
+                f.write(json.dumps({"prompt": prompt}) + "\n")
+
+        with open(ie_path, "w") as f:
+            for prompt in prompts:
+                f.write(json.dumps({"text_input": prompt}) + "\n")
+
+        yield vllm_path, ie_path
+
+
 def parse_args() -> argparse.Namespace:
     """Parse command line arguments."""
     parser = argparse.ArgumentParser(
@@ -457,27 +504,10 @@ def warmup_server(url: str, model: str, timeout: int = 120):
 def main():
     args = parse_args()
 
-    # Dataset files are expected to exist in the script directory
-    script_dir = Path(__file__).parent
-    vllm_dataset = script_dir / "dummy_prompts_vllm.jsonl"
-    ie_dataset = script_dir / "dummy_prompts_ie.jsonl"
-
-    # Validate dataset files exist
-    if not vllm_dataset.exists():
-        logger.error(f"vLLM dataset not found: {vllm_dataset}")
-        logger.error("Run: python convert_dataset.py dummy_prompts.json")
-        sys.exit(1)
-    if not ie_dataset.exists():
-        logger.error(f"inference-endpoint dataset not found: {ie_dataset}")
-        logger.error("Run: python convert_dataset.py dummy_prompts.json")
-        sys.exit(1)
-
     logger.info(f"Model: {args.model}")
     logger.info(f"Endpoint: {args.endpoint}")
     logger.info(f"Number of prompts: {args.num_prompts}")
     logger.info(f"Max output tokens: {args.max_output_tokens}")
-    logger.info(f"vLLM dataset: {vllm_dataset}")
-    logger.info(f"IE dataset: {ie_dataset}")
     if args.dry:
         logger.info("DRY RUN MODE: Commands will be printed but not executed")
 
@@ -492,26 +522,29 @@ def main():
     else:
         logger.info("Skipping warmup in dry run mode")
 
-    # Run benchmarks
-    vllm_res = run_vllm_benchmark(
-        args.model,
-        vllm_dataset,
-        args.endpoint,
-        args.num_prompts,
-        args.max_output_tokens,
-        dry_run=args.dry,
-    )
-
-    ie_res = run_inference_endpoint(
-        args.model,
-        ie_dataset,
-        args.endpoint,
-        args.num_prompts,
-        args.max_output_tokens,
-        args.timeout,
-        args.workers,
-        dry_run=args.dry,
-    )
+    # Generate temp dataset files and run benchmarks
+    with temp_dataset_files(args.num_prompts) as (vllm_dataset, ie_dataset):
+        logger.info(f"Generated {args.num_prompts} prompts in temp directory")
+
+        vllm_res = run_vllm_benchmark(
+            args.model,
+            vllm_dataset,
+            args.endpoint,
+            args.num_prompts,
+            args.max_output_tokens,
+            dry_run=args.dry,
+        )
+
+        ie_res = run_inference_endpoint(
+            args.model,
+            ie_dataset,
+            args.endpoint,
+            args.num_prompts,
+            args.max_output_tokens,
+            args.timeout,
+            args.workers,
+            dry_run=args.dry,
+        )
 
     if not args.dry:
         print_comparison(ie_res, vllm_res)