From 206e1aa58e0450ea9bf3b8bbd9bf4dcd800e876c Mon Sep 17 00:00:00 2001 From: Gloire Rubambiza Date: Fri, 29 May 2026 15:56:58 -0400 Subject: [PATCH 1/4] feat: Add FMA actuation path classification, hit rates, and per-path timing Phase 1: Replace pod-name heuristic with timestamp-based classification. Launcher creationTimestamp vs requester creationTimestamp determines warm (pre-existing launcher) vs cold-with-launcher (DPC created new). Add Hot_hit_rate, Warm_hit_rate, cold_launcher_rate per iteration. Phase 2: Compute upper-bound per-path timing using Kube timestamps: T_wake (hot), T_instance_create (warm), T_cold_launcher (cold). Rename T_LUKE_WARM to T_COLD_LAUNCHER across harness and analysis. Closes llm-d/llm-d-benchmark#1422 Assisted-By: Claude Opus 4.6 Signed-off-by: Gloire Rubambiza --- .../analysis/scripts/nop-analyze_results.py | 8 +- workload/harnesses/fma_functions.py | 83 +++++++++++++++++-- 2 files changed, 77 insertions(+), 14 deletions(-) diff --git a/llmdbenchmark/analysis/scripts/nop-analyze_results.py b/llmdbenchmark/analysis/scripts/nop-analyze_results.py index 34419a4e2..b4e7a450a 100755 --- a/llmdbenchmark/analysis/scripts/nop-analyze_results.py +++ b/llmdbenchmark/analysis/scripts/nop-analyze_results.py @@ -293,11 +293,9 @@ def write_fma_metrics( # pylint: disable=too-many-locals,too-many-statements file.write("\n\n") file.write("Actuation Conditions:\n") - file.write( - " T_luke_warm: when new launcher created by Dual Pod Controller + new vLLM\n" - ) - file.write(" T_warm: when existing launcher creates new vLLM\n") - file.write(" T_hot: when waking up sleeping vLLM\n\n") + file.write(" T_cold_launcher: DPC creates new launcher + new vLLM instance\n") + file.write(" T_warm: existing launcher creates new vLLM instance\n") + file.write(" T_hot: waking sleeping vLLM instance\n\n") file.write("T_actuation: Time for the Requester Pod to be ready\n") file.write("TTRD: Time for the Requester Pod to have dual label set\n") file.write("T_first_token: Time for vLLM server to return first token\n") diff --git a/workload/harnesses/fma_functions.py b/workload/harnesses/fma_functions.py index e663cd455..0ff4b88f4 100755 --- a/workload/harnesses/fma_functions.py +++ b/workload/harnesses/fma_functions.py @@ -78,6 +78,10 @@ class FMALauncherInfo: # pylint: disable=too-many-instance-attributes vllm_endpoint: str = "" ttft: float = 0.0 actuation_condition: FMAActuationCondition | None = None + launcher_creation_timestamp: float = 0.0 + t_wake: float | None = None + t_instance_create: float | None = None + t_cold_launcher: float | None = None def dump(self) -> dict[str, Any]: """Convert FMALauncherInfo to dict. @@ -102,9 +106,9 @@ def dump(self) -> dict[str, Any]: class FMAActuationCondition(StrEnum): """Type of actuation""" - T_LUKE_WARM = "T_luke_warm" # when new launcher created by DPC + new vllm - T_WARM = "T_warm" # when existing launcher creates new vllm - T_HOT = "T_hot" # when waking up sleeping vllm + T_COLD_LAUNCHER = "T_cold_launcher" # DPC creates new launcher + new vllm + T_WARM = "T_warm" # existing launcher creates new vllm + T_HOT = "T_hot" # waking sleeping vllm def dump(self) -> str: """Convert FMAActuationCondition to str. @@ -121,6 +125,9 @@ class FMAMetricsIteration: iteration: int launcher_infos: list[FMALauncherInfo] + hot_hit_rate: float = 0.0 + warm_hit_rate: float = 0.0 + cold_launcher_rate: float = 0.0 def dump(self) -> dict[str, Any]: """Convert FMAMetricsIteration to dict. @@ -268,6 +275,11 @@ def get_fma_launcher_infos( # pylint: disable=too-many-locals,too-many-argument launcher_info.container_name = container.name launcher_info.name = engine.name launcher_info.requester_info = requester_info + launcher_info.launcher_creation_timestamp = ( + launcher_pod.metadata.creation_timestamp.astimezone( + timezone.utc + ).timestamp() + ) launcher_info.launcher_endpoint = ( f"http://{launcher_pod_ip}:{fma_launcher_port}" ) @@ -832,12 +844,45 @@ def benchmark_fma( # pylint: disable=too-many-arguments,too-many-positional-arg FMAActuationCondition.T_HOT ) - # TODO: Improve the warm/luke_warm check instead of pod name if launcher_info.actuation_condition is None: - launcher_info.actuation_condition = ( - FMAActuationCondition.T_WARM - if launcher_info.name.startswith("launcher-fma-") - else FMAActuationCondition.T_LUKE_WARM + if ( + launcher_info.launcher_creation_timestamp > 0.0 + and launcher_info.requester_info.creation_timestamp + > 0.0 + and launcher_info.launcher_creation_timestamp + < launcher_info.requester_info.creation_timestamp + ): + launcher_info.actuation_condition = ( + FMAActuationCondition.T_WARM + ) + else: + launcher_info.actuation_condition = ( + FMAActuationCondition.T_COLD_LAUNCHER + ) + + # Compute per-path timing (upper bound via Kube timestamps) + ready_ts = launcher_info.requester_info.ready_timestamp + creation_ts = launcher_info.requester_info.creation_timestamp + if ( + launcher_info.actuation_condition + == FMAActuationCondition.T_HOT + and ready_ts > 0.0 + ): + launcher_info.t_wake = ready_ts - creation_ts + elif ( + launcher_info.actuation_condition + == FMAActuationCondition.T_WARM + and ready_ts > 0.0 + ): + launcher_info.t_instance_create = ready_ts - creation_ts + elif ( + launcher_info.actuation_condition + == FMAActuationCondition.T_COLD_LAUNCHER + and ready_ts > 0.0 + and launcher_info.launcher_creation_timestamp > 0.0 + ): + launcher_info.t_cold_launcher = ( + ready_ts - launcher_info.launcher_creation_timestamp ) except Exception as e: @@ -845,7 +890,27 @@ def benchmark_fma( # pylint: disable=too-many-arguments,too-many-positional-arg f"error on benchmark FMA '{launcher_info.name}' launcher" ) from e - fma_metrics_iteration = FMAMetricsIteration(iteration, launcher_infos) + # Compute hit rates for this iteration + total = len(launcher_infos) + hot_count = sum( + li.actuation_condition == FMAActuationCondition.T_HOT + for li in launcher_infos + ) + warm_count = sum( + li.actuation_condition == FMAActuationCondition.T_WARM + for li in launcher_infos + ) + cold_count = sum( + li.actuation_condition == FMAActuationCondition.T_COLD_LAUNCHER + for li in launcher_infos + ) + fma_metrics_iteration = FMAMetricsIteration( + iteration, + launcher_infos, + hot_hit_rate=hot_count / total if total > 0 else 0.0, + warm_hit_rate=warm_count / total if total > 0 else 0.0, + cold_launcher_rate=cold_count / total if total > 0 else 0.0, + ) fma_metrics.iterations.append(fma_metrics_iteration) finally: logger.info("Benchmark FMA iteration '%d' end.", iteration) From d28086c10ecaf55051a625d5db6c6fe2dd5fc7a2 Mon Sep 17 00:00:00 2001 From: Gloire Rubambiza Date: Fri, 5 Jun 2026 14:53:28 -0400 Subject: [PATCH 2/4] feat: Address PR review feedback on FMA metrics - Rename cold_launcher_rate to cold_launcher_hit_rate - Add per-path timing (T_hot, T_warm, T_cold) to analysis summary table - Add launcher node ID to results and analysis output - Display all hit rates (hot, warm, cold_launcher) instead of just hot - Add sleeper_limit to scenario metadata and analysis output - Handle None values in per-path timing columns (show "--") - Extend benchmark_report conversion for new FMA fields - Fix LLMDBENCH_FMA_SLEEPER_LIMIT not being read from env Assisted-By: Claude Opus 4.6 Signed-off-by: Gloire Rubambiza --- config/templates/jinja/20_harness_pod.yaml.j2 | 2 + .../benchmark_report/native_to_br0_1.py | 187 +++++++++++++----- .../analysis/scripts/nop-analyze_results.py | 72 +++++-- workload/harnesses/fma_functions.py | 6 +- workload/harnesses/nop-llm-d-benchmark.py | 10 +- workload/harnesses/nop_functions.py | 1 + 6 files changed, 210 insertions(+), 68 deletions(-) diff --git a/config/templates/jinja/20_harness_pod.yaml.j2 b/config/templates/jinja/20_harness_pod.yaml.j2 index ec8e739b8..8b0fb1a5b 100644 --- a/config/templates/jinja/20_harness_pod.yaml.j2 +++ b/config/templates/jinja/20_harness_pod.yaml.j2 @@ -113,6 +113,8 @@ spec: value: "{{ fma.launcherConfigurator.port }}" - name: LLMDBENCH_FMA_ITERATIONS value: "{{ fma.iterations }}" + - name: LLMDBENCH_FMA_SLEEPER_LIMIT + value: "{{ fma.dualPod.sleeperLimit }}" {% endif %} {% if harness.extraEnvVars is defined and harness.extraEnvVars %} {% for env_entry in harness.extraEnvVars %} diff --git a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py index 37a0b2871..ed88eacec 100644 --- a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py +++ b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py @@ -52,13 +52,19 @@ def _get_llmd_benchmark_envars() -> dict: * int(os.environ.get("LLMDBENCH_VLLM_COMMON_REPLICAS", "-1")), "accelerator": [ { - "model": os.environ.get("LLMDBENCH_VLLM_COMMON_AFFINITY", "").split( - ":", 1 - )[-1], + "model": os.environ.get( + "LLMDBENCH_VLLM_COMMON_AFFINITY", "" + ).split(":", 1)[-1], "count": int( - os.environ.get("LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM", "-1") + os.environ.get( + "LLMDBENCH_VLLM_COMMON_TENSOR_PARALLELISM", "-1" + ) ) - * int(os.environ.get("LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM", "-1")), + * int( + os.environ.get( + "LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM", "-1" + ) + ), "parallelism": { "tp": int( os.environ.get( @@ -66,7 +72,9 @@ def _get_llmd_benchmark_envars() -> dict: ) ), "dp": int( - os.environ.get("LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM", "-1") + os.environ.get( + "LLMDBENCH_VLLM_COMMON_DATA_PARALLELISM", "-1" + ) ), }, } @@ -97,14 +105,18 @@ def _get_llmd_benchmark_envars() -> dict: }, }, "metadata": { - "load_format": os.environ.get("LLMDBENCH_VLLM_COMMON_VLLM_LOAD_FORMAT", ""), + "load_format": os.environ.get( + "LLMDBENCH_VLLM_COMMON_VLLM_LOAD_FORMAT", "" + ), "logging_level": os.environ.get( "LLMDBENCH_VLLM_COMMON_VLLM_LOGGING_LEVEL", "" ), "vllm_server_dev_mode": os.environ.get( "LLMDBENCH_VLLM_COMMON_VLLM_SERVER_DEV_MODE", "" ), - "preprocess": os.environ.get("LLMDBENCH_VLLM_STANDALONE_PREPROCESS", ""), + "preprocess": os.environ.get( + "LLMDBENCH_VLLM_STANDALONE_PREPROCESS", "" + ), }, }, "metadata": { @@ -150,89 +162,117 @@ def _get_llmd_benchmark_envars() -> dict: "model": {"name": os.environ.get("LLMDBENCH_DEPLOY_CURRENT_MODEL", "")}, "host": { "type": ["prefill"] - * int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1")) + * int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1" + ) + ) + ["decode"] - * int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1")), + * int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1" + ) + ), "accelerator": [ { - "model": os.environ.get("LLMDBENCH_VLLM_COMMON_AFFINITY", "").split( - ":", 1 - )[-1], + "model": os.environ.get( + "LLMDBENCH_VLLM_COMMON_AFFINITY", "" + ).split(":", 1)[-1], "count": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_TENSOR_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_TENSOR_PARALLELISM", + "-1", ) ) * int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_LOCAL_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_LOCAL_PARALLELISM", + "-1", ) ), "parallelism": { "tp": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_TENSOR_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_TENSOR_PARALLELISM", + "-1", ) ), "dp": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_PARALLELISM", + "-1", ) ), "dpLocal": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_LOCAL_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_DATA_LOCAL_PARALLELISM", + "-1", ) ), "workers": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_NUM_WORKERS_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_NUM_WORKERS_PARALLELISM", + "-1", ) ), }, } ] - * int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1")) + * int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1" + ) + ) + [ { - "model": os.environ.get("LLMDBENCH_VLLM_COMMON_AFFINITY", "").split( - ":", 1 - )[-1], + "model": os.environ.get( + "LLMDBENCH_VLLM_COMMON_AFFINITY", "" + ).split(":", 1)[-1], "count": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM", + "-1", ) ) * int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_LOCAL_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_LOCAL_PARALLELISM", + "-1", ) ), "parallelism": { "tp": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_TENSOR_PARALLELISM", + "-1", ) ), "dp": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_PARALLELISM", + "-1", ) ), "dpLocal": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_LOCAL_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_DATA_LOCAL_PARALLELISM", + "-1", ) ), "workers": int( os.environ.get( - "LLMDBENCH_VLLM_MODELSERVICE_DECODE_NUM_WORKERS_PARALLELISM", "-1" + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_NUM_WORKERS_PARALLELISM", + "-1", ) ), }, } ] - * int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1")), + * int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1" + ) + ), }, "platform": { "metadata": { @@ -257,8 +297,16 @@ def _get_llmd_benchmark_envars() -> dict: } ] * ( - int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1")) - + int(os.environ.get("LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1")) + int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_PREFILL_REPLICAS", "-1" + ) + ) + + int( + os.environ.get( + "LLMDBENCH_VLLM_MODELSERVICE_DECODE_REPLICAS", "-1" + ) + ) ), }, }, @@ -1393,7 +1441,9 @@ def _stats(raw: dict | None, units: Units) -> dict | None: "sessions_per_second": results.get("sessions_per_second"), "session_duration": _stats(results.get("session_duration_sec"), Units.S), "num_events": _stats(results.get("num_events"), Units.COUNT), - "num_events_cancelled": _stats(results.get("num_events_cancelled"), Units.COUNT), + "num_events_cancelled": _stats( + results.get("num_events_cancelled"), Units.COUNT + ), "total_input_tokens": _stats(results.get("total_input_tokens"), Units.COUNT), "total_output_tokens": _stats(results.get("total_output_tokens"), Units.COUNT), } @@ -1418,7 +1468,8 @@ def _stats(raw: dict | None, units: Units) -> dict | None: }, "requests": { "total": results.get("total_events", 0), - "failures": results.get("total_events", 0) - results.get("total_events_completed", 0), + "failures": results.get("total_events", 0) + - results.get("total_events_completed", 0), "input_length": { "units": Units.COUNT, "mean": get_nested(results, ["total_input_tokens", "mean"], 0), @@ -2134,7 +2185,7 @@ def _import_categories(cat_list: list[dict[str, Any]]) -> list[dict[str, Any]]: "args": engine["args"], "metadata": { "image": engine["image"], - } + }, } engines.append(e) @@ -2154,6 +2205,7 @@ def _import_categories(cat_list: list[dict[str, Any]]) -> list[dict[str, Any]]: "deploy_methods": results["scenario"]["deploy_methods"], "load_format": results["scenario"]["load_format"], "sleep_mode": results["scenario"]["sleep_mode"], + "sleeper_limit": results["scenario"].get("sleeper_limit", 0), "gpus": results["scenario"]["gpus"], }, }, @@ -2341,42 +2393,87 @@ def _import_categories(cat_list: list[dict[str, Any]]) -> list[dict[str, Any]]: } vllm_metadatas.append(metadata_dict) - results_dict["metrics"]["metadata"].append({"name": metrics_name, "value": vllm_metadatas}) + results_dict["metrics"]["metadata"].append( + {"name": metrics_name, "value": vllm_metadatas} + ) metrics_name = "extra_metrics" fma_metadatas = [] - for extra_metric in results.get(metrics_name, []): + for extra_metric in results.get(metrics_name, []): if extra_metric["name"] != "fma": continue metadata_dict = {"name": extra_metric["name"]} iterations = [] for iteration in extra_metric.get("iterations", []): - it = { "iteration": { "units": Units.COUNT, "value": iteration["iteration"] } } + it = {"iteration": {"units": Units.COUNT, "value": iteration["iteration"]}} launcher_infos = [] for launcher_info in iteration.get("launcher_infos", []): - info = { "name": launcher_info["name"] } + info = {"name": launcher_info["name"]} requester_info = launcher_info["requester_info"] - ri = { "name": requester_info["name"] } - ri["creation_timestamp"] = { "units": Units.S, "value": requester_info["creation_timestamp"]} - ri["ready_timestamp"] = { "units": Units.S, "value": requester_info["ready_timestamp"]} - ri["dual_label_timestamp"] = { "units": Units.S, "value": requester_info["dual_label_timestamp"]} + ri = {"name": requester_info["name"]} + ri["creation_timestamp"] = { + "units": Units.S, + "value": requester_info["creation_timestamp"], + } + ri["ready_timestamp"] = { + "units": Units.S, + "value": requester_info["ready_timestamp"], + } + ri["dual_label_timestamp"] = { + "units": Units.S, + "value": requester_info["dual_label_timestamp"], + } info["requester_info"] = ri info["actuation_condition"] = launcher_info["actuation_condition"] info["launcher_endpoint"] = launcher_info["launcher_endpoint"] info["vllm_endpoint"] = launcher_info["vllm_endpoint"] - info["ttft"] = { "units": Units.S, "value": launcher_info["ttft"]} + info["ttft"] = {"units": Units.S, "value": launcher_info["ttft"]} + info["launcher_creation_timestamp"] = { + "units": Units.S, + "value": launcher_info.get("launcher_creation_timestamp", 0.0), + } + info["launcher_node"] = launcher_info.get("launcher_node", "") + if launcher_info.get("t_wake") is not None: + info["t_wake"] = { + "units": Units.S, + "value": launcher_info["t_wake"], + } + if launcher_info.get("t_instance_create") is not None: + info["t_instance_create"] = { + "units": Units.S, + "value": launcher_info["t_instance_create"], + } + if launcher_info.get("t_cold_launcher") is not None: + info["t_cold_launcher"] = { + "units": Units.S, + "value": launcher_info["t_cold_launcher"], + } launcher_infos.append(info) it["launcher_infos"] = launcher_infos + it["hot_hit_rate"] = { + "units": Units.COUNT, + "value": iteration.get("hot_hit_rate", 0.0), + } + it["warm_hit_rate"] = { + "units": Units.COUNT, + "value": iteration.get("warm_hit_rate", 0.0), + } + it["cold_launcher_hit_rate"] = { + "units": Units.COUNT, + "value": iteration.get("cold_launcher_hit_rate", 0.0), + } iterations.append(it) metadata_dict["iterations"] = iterations fma_metadatas.append(metadata_dict) - results_dict["metrics"]["metadata"].append({"name": metrics_name, "value": fma_metadatas}) + results_dict["metrics"]["metadata"].append( + {"name": metrics_name, "value": fma_metadatas} + ) update_dict(br_dict, results_dict) diff --git a/llmdbenchmark/analysis/scripts/nop-analyze_results.py b/llmdbenchmark/analysis/scripts/nop-analyze_results.py index b4e7a450a..fb905cff0 100755 --- a/llmdbenchmark/analysis/scripts/nop-analyze_results.py +++ b/llmdbenchmark/analysis/scripts/nop-analyze_results.py @@ -121,6 +121,9 @@ def write_benchmark_scenario(file: io.TextIOWrapper, scenario: Scenario): file.write(f" Harness : {scenario.load.name}\n") file.write(f" Load Format : {scenario.metadata['load_format']}\n") file.write(f" Sleep Mode On : {scenario.metadata['sleep_mode']}\n") + file.write( + f" Sleeper Limit : {scenario.metadata.get('sleeper_limit', 'N/A')}\n" + ) file.write(f" Model : {scenario.model.name}\n") for engine in scenario.platform.engine: file.write(" Engine\n") @@ -297,54 +300,78 @@ def write_fma_metrics( # pylint: disable=too-many-locals,too-many-statements file.write(" T_warm: existing launcher creates new vLLM instance\n") file.write(" T_hot: waking sleeping vLLM instance\n\n") file.write("T_actuation: Time for the Requester Pod to be ready\n") - file.write("TTRD: Time for the Requester Pod to have dual label set\n") + file.write("T_hot: Hot-start timing (upper bound)\n") + file.write("T_warm: Warm-start timing (upper bound)\n") + file.write("T_cold_launcher: Cold-start-with-launcher timing (upper bound)\n") file.write("T_first_token: Time for vLLM server to return first token\n") - file.write("TTRD + T_first_token == T_e2e\n") file.write("Each iteration scales ReplicaSet from 0 to 1 and then from 1 to 0\n") - file.write("Hit_rate (count(hot starts) / total iterations)\n") hot_starts = 0 + warm_starts = 0 + cold_starts = 0 total_iterations = len(iterations) pandas_datas = [] for iteration in iterations: for launcher_info in iteration["launcher_infos"]: ct = float(launcher_info["requester_info"]["creation_timestamp"]["value"]) rt = float(launcher_info["requester_info"]["ready_timestamp"]["value"]) - dt = float(launcher_info["requester_info"]["dual_label_timestamp"]["value"]) ttrr = rt - ct if rt > 0.0 else 0.0 - ttrd = dt - ct if dt > 0.0 else 0.0 ttft = float(launcher_info["ttft"]["value"]) actuation_condition = launcher_info["actuation_condition"] if actuation_condition == "T_hot": hot_starts += 1 + elif actuation_condition == "T_warm": + warm_starts += 1 + elif actuation_condition == "T_cold_launcher": + cold_starts += 1 + + t_hot = launcher_info.get("t_wake", {}) + t_hot_val = float(t_hot["value"]) if isinstance(t_hot, dict) else None + t_warm = launcher_info.get("t_instance_create", {}) + t_warm_val = float(t_warm["value"]) if isinstance(t_warm, dict) else None + t_cold = launcher_info.get("t_cold_launcher", {}) + t_cold_val = float(t_cold["value"]) if isinstance(t_cold, dict) else None + node = launcher_info.get("launcher_node", "") pandas_datas.append( { "Iteration": iteration["iteration"]["value"], - "vLLM Name": launcher_info["name"], + "Node": node, "Actuation Condition": actuation_condition, - "T_actuation(secs)": ttrr, - "TTRD(secs)": ttrd, - "T_first_token(secs)": ttft, - "T_e2e": ttrd + ttft, + "T_actuation(s)": ttrr, + "T_hot(s)": t_hot_val, + "T_warm(s)": t_warm_val, + "T_cold(s)": t_cold_val, + "T_first_token(s)": ttft, } ) - hit_rate = hot_starts / total_iterations if total_iterations > 0 else 0.0 + hot_hit_rate = hot_starts / total_iterations if total_iterations > 0 else 0.0 + warm_hit_rate = warm_starts / total_iterations if total_iterations > 0 else 0.0 + cold_hit_rate = cold_starts / total_iterations if total_iterations > 0 else 0.0 df = pd.DataFrame(pandas_datas) file.write("\n") # Float formatting - float_columns = ["T_actuation(secs)", "TTRD(secs)", "T_first_token(secs)", "T_e2e"] - - # Compute column widths dynamically + float_columns = [ + "T_actuation(s)", + "T_hot(s)", + "T_warm(s)", + "T_cold(s)", + "T_first_token(s)", + ] + + # Compute column widths dynamically. + # FMA per-path columns (T_hot, T_warm, T_cold) contain None for + # inapplicable paths, so we handle NaN when computing widths. col_widths = {} for col in df.columns: if col in float_columns: - # max width between header and max formatted float - max_float_width = max(df[col].apply(lambda x: len(f"{x:.4f}"))) + max_float_width = max( + df[col].apply(lambda x: len(f"{x:.4f}") if pd.notna(x) else 2) + ) col_widths[col] = max(len(col), max_float_width) else: col_widths[col] = max(len(col), df[col].astype(str).apply(len).max()) @@ -363,20 +390,25 @@ def write_fma_metrics( # pylint: disable=too-many-locals,too-many-statements ) file.write(f"{' ' * left_padding}{separator}\n") - # Rows + # Rows -- per-path timing columns show "--" when not applicable for _, r in df.iterrows(): row = [] for col in df.columns: val = r[col] if col in float_columns: - row.append(f"{val:>{col_widths[col]}.4f}") # right-align numbers + if pd.notna(val): + row.append(f"{val:>{col_widths[col]}.4f}") # right-align numbers + else: + row.append(f"{'--':>{col_widths[col]}}") # N/A for this path elif isinstance(val, int): row.append(f"{val:>{col_widths[col]}}") else: - row.append(f"{val:<{col_widths[col]}}") # left-align strings + row.append(f"{val:<{col_widths[col]}}") file.write(f"{' ' * left_padding}{(' ' * space_between_cols).join(row)}\n") - file.write(f"\nHit_rate: {hit_rate:8.2f}\n") + file.write(f"\n Hot_hit_rate: {hot_hit_rate:.2f}\n") + file.write(f" Warm_hit_rate: {warm_hit_rate:.2f}\n") + file.write(f" Cold_launcher_hit_rate: {cold_hit_rate:.2f}\n") file.write("\n") diff --git a/workload/harnesses/fma_functions.py b/workload/harnesses/fma_functions.py index 0ff4b88f4..eaaa8db44 100755 --- a/workload/harnesses/fma_functions.py +++ b/workload/harnesses/fma_functions.py @@ -79,6 +79,7 @@ class FMALauncherInfo: # pylint: disable=too-many-instance-attributes ttft: float = 0.0 actuation_condition: FMAActuationCondition | None = None launcher_creation_timestamp: float = 0.0 + launcher_node: str = "" t_wake: float | None = None t_instance_create: float | None = None t_cold_launcher: float | None = None @@ -127,7 +128,7 @@ class FMAMetricsIteration: launcher_infos: list[FMALauncherInfo] hot_hit_rate: float = 0.0 warm_hit_rate: float = 0.0 - cold_launcher_rate: float = 0.0 + cold_launcher_hit_rate: float = 0.0 def dump(self) -> dict[str, Any]: """Convert FMAMetricsIteration to dict. @@ -280,6 +281,7 @@ def get_fma_launcher_infos( # pylint: disable=too-many-locals,too-many-argument timezone.utc ).timestamp() ) + launcher_info.launcher_node = launcher_pod.spec.node_name or "" launcher_info.launcher_endpoint = ( f"http://{launcher_pod_ip}:{fma_launcher_port}" ) @@ -909,7 +911,7 @@ def benchmark_fma( # pylint: disable=too-many-arguments,too-many-positional-arg launcher_infos, hot_hit_rate=hot_count / total if total > 0 else 0.0, warm_hit_rate=warm_count / total if total > 0 else 0.0, - cold_launcher_rate=cold_count / total if total > 0 else 0.0, + cold_launcher_hit_rate=cold_count / total if total > 0 else 0.0, ) fma_metrics.iterations.append(fma_metrics_iteration) finally: diff --git a/workload/harnesses/nop-llm-d-benchmark.py b/workload/harnesses/nop-llm-d-benchmark.py index 2a97b5563..1c3115c43 100755 --- a/workload/harnesses/nop-llm-d-benchmark.py +++ b/workload/harnesses/nop-llm-d-benchmark.py @@ -82,7 +82,13 @@ def main(): ] ) if fma_enabled: - keys.extend(["LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "LLMDBENCH_FMA_ITERATIONS"]) + keys.extend( + [ + "LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", + "LLMDBENCH_FMA_ITERATIONS", + "LLMDBENCH_FMA_SLEEPER_LIMIT", + ] + ) envs.update(get_env_variables(keys)) logger.info("Environment variables:") @@ -106,6 +112,7 @@ def main(): fma_launcher_port = envs.get("LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "0") fma_iterations = int(envs.get("LLMDBENCH_FMA_ITERATIONS", "0")) + fma_sleeper_limit = int(envs.get("LLMDBENCH_FMA_SLEEPER_LIMIT", "0")) deploy_methods = [] if standalone_enabled: @@ -121,6 +128,7 @@ def main(): benchmark_result = BenchmarkResult() benchmark_result.scenario.deploy_methods = ",".join(deploy_methods) + benchmark_result.scenario.sleeper_limit = fma_sleeper_limit if fma_enabled: try: logger.info("Benchmark FMA launcher start...") diff --git a/workload/harnesses/nop_functions.py b/workload/harnesses/nop_functions.py index 3b42f58eb..1c6bf6726 100755 --- a/workload/harnesses/nop_functions.py +++ b/workload/harnesses/nop_functions.py @@ -545,6 +545,7 @@ class BenchmarkScenario: deploy_methods: str = "" load_format: LoadFormat = LoadFormat.UNKNOWN sleep_mode: bool = False + sleeper_limit: int = 0 model: ModelScenario = field(default_factory=ModelScenario) platform: PlatformScenario = field(default_factory=PlatformScenario) gpus: list[GPUScenario] = field(default_factory=list[GPUScenario]) From 23beecc6233dd9b371b1143077b6895054a1b472 Mon Sep 17 00:00:00 2001 From: Gloire Rubambiza Date: Tue, 9 Jun 2026 15:27:21 -0400 Subject: [PATCH 3/4] feat: Replace maxSleepingInstances with maxInstances, add to metrics - Change LauncherConfig template from maxSleepingInstances to maxInstances - Add maxInstances default (4) in defaults.yaml under fma.launcher - Add LLMDBENCH_FMA_MAX_INSTANCES env var to harness pod - Display Max Instances in analysis output (replaces Sleeper Limit display) - Propagate max_instances through scenario metadata and benchmark_report Assisted-By: Claude Opus 4.6 Signed-off-by: Gloire Rubambiza --- config/templates/jinja/20_harness_pod.yaml.j2 | 2 ++ config/templates/jinja/24_fma-deployment.yaml.j2 | 2 +- config/templates/values/defaults.yaml | 1 + llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py | 1 + llmdbenchmark/analysis/scripts/nop-analyze_results.py | 2 +- workload/harnesses/nop-llm-d-benchmark.py | 3 +++ workload/harnesses/nop_functions.py | 1 + 7 files changed, 10 insertions(+), 2 deletions(-) diff --git a/config/templates/jinja/20_harness_pod.yaml.j2 b/config/templates/jinja/20_harness_pod.yaml.j2 index e5db1f7ce..be72099dc 100644 --- a/config/templates/jinja/20_harness_pod.yaml.j2 +++ b/config/templates/jinja/20_harness_pod.yaml.j2 @@ -126,6 +126,8 @@ spec: value: "{{ fma.iterations }}" - name: LLMDBENCH_FMA_SLEEPER_LIMIT value: "{{ fma.dualPod.sleeperLimit }}" + - name: LLMDBENCH_FMA_MAX_INSTANCES + value: "{{ fma.launcher.maxInstances }}" {% endif %} {% if harness.extraEnvVars is defined and harness.extraEnvVars %} {% for env_entry in harness.extraEnvVars %} diff --git a/config/templates/jinja/24_fma-deployment.yaml.j2 b/config/templates/jinja/24_fma-deployment.yaml.j2 index 21a33f87d..b61bb2757 100644 --- a/config/templates/jinja/24_fma-deployment.yaml.j2 +++ b/config/templates/jinja/24_fma-deployment.yaml.j2 @@ -44,7 +44,7 @@ metadata: {% endif %} namespace: {{ namespace.name }} spec: - maxSleepingInstances: 3 + maxInstances: {{ fma.launcher.maxInstances }} podTemplate: metadata: {% if fma.launcher.podTemplate.metadata.labels is defined %} diff --git a/config/templates/values/defaults.yaml b/config/templates/values/defaults.yaml index 21cfa1843..4d76fb95e 100644 --- a/config/templates/values/defaults.yaml +++ b/config/templates/values/defaults.yaml @@ -1305,6 +1305,7 @@ fma: limitsMemory: 250Mi launcher: + maxInstances: 4 image: repository: ghcr.io/llm-d-incubation/llm-d-fast-model-actuation/launcher tag: v0.6.0-alpha.13 diff --git a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py index 4cb38ec1c..9b9494bfa 100644 --- a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py +++ b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py @@ -2324,6 +2324,7 @@ def _import_categories(cat_list: list[dict[str, Any]]) -> list[dict[str, Any]]: "load_format": results["scenario"]["load_format"], "sleep_mode": results["scenario"]["sleep_mode"], "sleeper_limit": results["scenario"].get("sleeper_limit", 0), + "max_instances": results["scenario"].get("max_instances", 0), "gpus": results["scenario"]["gpus"], }, }, diff --git a/llmdbenchmark/analysis/scripts/nop-analyze_results.py b/llmdbenchmark/analysis/scripts/nop-analyze_results.py index fb905cff0..38feee1f2 100755 --- a/llmdbenchmark/analysis/scripts/nop-analyze_results.py +++ b/llmdbenchmark/analysis/scripts/nop-analyze_results.py @@ -122,7 +122,7 @@ def write_benchmark_scenario(file: io.TextIOWrapper, scenario: Scenario): file.write(f" Load Format : {scenario.metadata['load_format']}\n") file.write(f" Sleep Mode On : {scenario.metadata['sleep_mode']}\n") file.write( - f" Sleeper Limit : {scenario.metadata.get('sleeper_limit', 'N/A')}\n" + f" Max Instances : {scenario.metadata.get('max_instances', 'N/A')}\n" ) file.write(f" Model : {scenario.model.name}\n") for engine in scenario.platform.engine: diff --git a/workload/harnesses/nop-llm-d-benchmark.py b/workload/harnesses/nop-llm-d-benchmark.py index d049869af..fbca884b6 100755 --- a/workload/harnesses/nop-llm-d-benchmark.py +++ b/workload/harnesses/nop-llm-d-benchmark.py @@ -87,6 +87,7 @@ def main(): "LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "LLMDBENCH_FMA_ITERATIONS", "LLMDBENCH_FMA_SLEEPER_LIMIT", + "LLMDBENCH_FMA_MAX_INSTANCES", ] ) @@ -113,6 +114,7 @@ def main(): fma_launcher_port = envs.get("LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "0") fma_iterations = int(envs.get("LLMDBENCH_FMA_ITERATIONS", "0")) fma_sleeper_limit = int(envs.get("LLMDBENCH_FMA_SLEEPER_LIMIT", "0")) + fma_max_instances = int(envs.get("LLMDBENCH_FMA_MAX_INSTANCES", "0")) deploy_methods = [] if standalone_enabled: @@ -135,6 +137,7 @@ def main(): benchmark_result = BenchmarkResult() benchmark_result.scenario.deploy_methods = ",".join(deploy_methods) benchmark_result.scenario.sleeper_limit = fma_sleeper_limit + benchmark_result.scenario.max_instances = fma_max_instances if fma_enabled: try: logger.info("Benchmark FMA launcher start...") diff --git a/workload/harnesses/nop_functions.py b/workload/harnesses/nop_functions.py index 1c6bf6726..27b8a05ed 100755 --- a/workload/harnesses/nop_functions.py +++ b/workload/harnesses/nop_functions.py @@ -546,6 +546,7 @@ class BenchmarkScenario: load_format: LoadFormat = LoadFormat.UNKNOWN sleep_mode: bool = False sleeper_limit: int = 0 + max_instances: int = 0 model: ModelScenario = field(default_factory=ModelScenario) platform: PlatformScenario = field(default_factory=PlatformScenario) gpus: list[GPUScenario] = field(default_factory=list[GPUScenario]) From c56262e271682f61ead16bf40a1698b1eecc7d8d Mon Sep 17 00:00:00 2001 From: Gloire Rubambiza Date: Wed, 10 Jun 2026 07:21:44 -0400 Subject: [PATCH 4/4] refactor: Remove sleeperLimit from harness reporting pipeline sleeperLimit is an M2-only DPC config that doesn't affect M3 (launcher-based) actuation paths. Remove it from the harness env vars, scenario metadata, and benchmark report conversion. The Helm chart still passes it to the DPC for M2 compatibility. Assisted-By: Claude Opus 4.6 Signed-off-by: Gloire Rubambiza --- config/templates/jinja/20_harness_pod.yaml.j2 | 2 -- llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py | 1 - workload/harnesses/nop-llm-d-benchmark.py | 3 --- workload/harnesses/nop_functions.py | 1 - 4 files changed, 7 deletions(-) diff --git a/config/templates/jinja/20_harness_pod.yaml.j2 b/config/templates/jinja/20_harness_pod.yaml.j2 index be72099dc..60275ad31 100644 --- a/config/templates/jinja/20_harness_pod.yaml.j2 +++ b/config/templates/jinja/20_harness_pod.yaml.j2 @@ -124,8 +124,6 @@ spec: value: "{{ fma.launcherConfigurator.port }}" - name: LLMDBENCH_FMA_ITERATIONS value: "{{ fma.iterations }}" - - name: LLMDBENCH_FMA_SLEEPER_LIMIT - value: "{{ fma.dualPod.sleeperLimit }}" - name: LLMDBENCH_FMA_MAX_INSTANCES value: "{{ fma.launcher.maxInstances }}" {% endif %} diff --git a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py index 9b9494bfa..f0152c2bf 100644 --- a/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py +++ b/llmdbenchmark/analysis/benchmark_report/native_to_br0_1.py @@ -2323,7 +2323,6 @@ def _import_categories(cat_list: list[dict[str, Any]]) -> list[dict[str, Any]]: "deploy_methods": results["scenario"]["deploy_methods"], "load_format": results["scenario"]["load_format"], "sleep_mode": results["scenario"]["sleep_mode"], - "sleeper_limit": results["scenario"].get("sleeper_limit", 0), "max_instances": results["scenario"].get("max_instances", 0), "gpus": results["scenario"]["gpus"], }, diff --git a/workload/harnesses/nop-llm-d-benchmark.py b/workload/harnesses/nop-llm-d-benchmark.py index fbca884b6..56c995ee2 100755 --- a/workload/harnesses/nop-llm-d-benchmark.py +++ b/workload/harnesses/nop-llm-d-benchmark.py @@ -86,7 +86,6 @@ def main(): [ "LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "LLMDBENCH_FMA_ITERATIONS", - "LLMDBENCH_FMA_SLEEPER_LIMIT", "LLMDBENCH_FMA_MAX_INSTANCES", ] ) @@ -113,7 +112,6 @@ def main(): fma_launcher_port = envs.get("LLMDBENCH_FMA_LAUNCHER_CONFIG_PORT", "0") fma_iterations = int(envs.get("LLMDBENCH_FMA_ITERATIONS", "0")) - fma_sleeper_limit = int(envs.get("LLMDBENCH_FMA_SLEEPER_LIMIT", "0")) fma_max_instances = int(envs.get("LLMDBENCH_FMA_MAX_INSTANCES", "0")) deploy_methods = [] @@ -136,7 +134,6 @@ def main(): benchmark_result = BenchmarkResult() benchmark_result.scenario.deploy_methods = ",".join(deploy_methods) - benchmark_result.scenario.sleeper_limit = fma_sleeper_limit benchmark_result.scenario.max_instances = fma_max_instances if fma_enabled: try: diff --git a/workload/harnesses/nop_functions.py b/workload/harnesses/nop_functions.py index 27b8a05ed..c629e9edd 100755 --- a/workload/harnesses/nop_functions.py +++ b/workload/harnesses/nop_functions.py @@ -545,7 +545,6 @@ class BenchmarkScenario: deploy_methods: str = "" load_format: LoadFormat = LoadFormat.UNKNOWN sleep_mode: bool = False - sleeper_limit: int = 0 max_instances: int = 0 model: ModelScenario = field(default_factory=ModelScenario) platform: PlatformScenario = field(default_factory=PlatformScenario)