From 9d7350a444202f0eb46c0ed9669cb1a339b13021 Mon Sep 17 00:00:00 2001 From: ahnsv Date: Wed, 18 Jun 2025 18:14:49 -0400 Subject: [PATCH 1/3] test(genai-perf): add tests for plots to cover more on plots code --- genai-perf/tests/test_plots/test_base_plot.py | 42 ++++++++++++ genai-perf/tests/test_plots/test_box_plot.py | 44 +++++++++++++ genai-perf/tests/test_plots/test_heat_map.py | 55 ++++++++++++++++ .../tests/test_plots/test_plot_manager.py | 65 +++++++++++++++++++ .../tests/test_plots/test_scatter_plot.py | 47 ++++++++++++++ 5 files changed, 253 insertions(+) create mode 100644 genai-perf/tests/test_plots/test_base_plot.py create mode 100644 genai-perf/tests/test_plots/test_box_plot.py create mode 100644 genai-perf/tests/test_plots/test_heat_map.py create mode 100644 genai-perf/tests/test_plots/test_plot_manager.py create mode 100644 genai-perf/tests/test_plots/test_scatter_plot.py diff --git a/genai-perf/tests/test_plots/test_base_plot.py b/genai-perf/tests/test_plots/test_base_plot.py new file mode 100644 index 00000000..ea47c048 --- /dev/null +++ b/genai-perf/tests/test_plots/test_base_plot.py @@ -0,0 +1,42 @@ +import pytest +from genai_perf.plots.base_plot import BasePlot +from genai_perf.plots.plot_config import PlotConfig, PlotType + + +class TestBasePlot: + @pytest.fixture + def sample_data(self): + return {"x": [1, 2, 3, 4, 5], "y": [10, 20, 30, 40, 50]} + + @pytest.fixture + def plot_config(self, tmp_path): + return PlotConfig( + title="Test Plot", + x_label="X Axis", + y_label="Y Axis", + height=1000, + width=1000, + output=tmp_path / "output", + type=PlotType.BOX, + ) + + def test_base_plot_initialization(self, sample_data, plot_config): + plot = BasePlot(sample_data, plot_config) + assert plot.data == sample_data + assert plot.config == plot_config + + def test_base_plot_validation(self): + with pytest.raises(ValueError): + BasePlot(None) + + def test_base_plot_empty_data(self): + with pytest.raises(ValueError): + BasePlot({}) + + def test_base_plot_missing_required_data(self): + with pytest.raises(ValueError): + BasePlot({"x": [1, 2, 3]}) # Missing y data + + def test_base_plot_data_length_mismatch(self): + with pytest.raises(ValueError): + BasePlot({"x": [1, 2, 3], "y": [1, 2]}) diff --git a/genai-perf/tests/test_plots/test_box_plot.py b/genai-perf/tests/test_plots/test_box_plot.py new file mode 100644 index 00000000..1f09b243 --- /dev/null +++ b/genai-perf/tests/test_plots/test_box_plot.py @@ -0,0 +1,44 @@ +import pytest +from genai_perf.plots.box_plot import BoxPlot +from genai_perf.plots.plot_config import PlotConfig + + +class TestBoxPlot: + @pytest.fixture + def box_plot_data(self): + return { + "data": [[1, 2, 3, 4, 5], [2, 3, 4, 5, 6], [3, 4, 5, 6, 7]], + "labels": ["Group A", "Group B", "Group C"], + } + + @pytest.fixture + def box_plot_config(self): + return PlotConfig( + title="Box Plot Test", + x_label="Groups", + y_label="Values", + show_grid=True, + show_outliers=True, + ) + + def test_box_plot_creation(self, box_plot_data, box_plot_config): + plot = BoxPlot(box_plot_data, box_plot_config) + assert plot.data == box_plot_data + assert plot.config == box_plot_config + + def test_box_plot_data_validation(self): + with pytest.raises(ValueError): + BoxPlot({"data": [], "labels": []}) + + def test_box_plot_labels_mismatch(self): + with pytest.raises(ValueError): + BoxPlot( + { + "data": [[1, 2, 3], [4, 5, 6]], + "labels": ["Group A"], # Missing label + } + ) + + def test_box_plot_empty_data_groups(self): + with pytest.raises(ValueError): + BoxPlot({"data": [[], [1, 2, 3]], "labels": ["Group A", "Group B"]}) diff --git a/genai-perf/tests/test_plots/test_heat_map.py b/genai-perf/tests/test_plots/test_heat_map.py new file mode 100644 index 00000000..8085b4bc --- /dev/null +++ b/genai-perf/tests/test_plots/test_heat_map.py @@ -0,0 +1,55 @@ +import pytest +import numpy as np +from genai_perf.plots.heat_map import HeatMap +from genai_perf.plots.plot_config import PlotConfig + + +class TestHeatMap: + @pytest.fixture + def heat_map_data(self): + return { + "data": np.random.rand(5, 5), + "x_labels": ["A", "B", "C", "D", "E"], + "y_labels": ["1", "2", "3", "4", "5"], + } + + @pytest.fixture + def heat_map_config(self): + return PlotConfig( + title="Heat Map Test", + x_label="X Axis", + y_label="Y Axis", + show_grid=True, + color_map="viridis", + ) + + def test_heat_map_creation(self, heat_map_data, heat_map_config): + plot = HeatMap(heat_map_data, heat_map_config) + assert plot.data == heat_map_data + assert plot.config == heat_map_config + + def test_heat_map_data_validation(self): + with pytest.raises(ValueError): + HeatMap({"data": np.array([])}) + + def test_heat_map_labels_mismatch(self): + with pytest.raises(ValueError): + HeatMap( + { + "data": np.random.rand(3, 3), + "x_labels": ["A", "B"], # Missing label + "y_labels": ["1", "2", "3"], + } + ) + + def test_heat_map_invalid_color_map(self): + with pytest.raises(ValueError): + config = PlotConfig(color_map="invalid_map") + HeatMap( + { + "data": np.random.rand(3, 3), + "x_labels": ["A", "B", "C"], + "y_labels": ["1", "2", "3"], + }, + config, + ) diff --git a/genai-perf/tests/test_plots/test_plot_manager.py b/genai-perf/tests/test_plots/test_plot_manager.py new file mode 100644 index 00000000..bacfb732 --- /dev/null +++ b/genai-perf/tests/test_plots/test_plot_manager.py @@ -0,0 +1,65 @@ +import pytest +from genai_perf.plots.plot_manager import PlotManager +from genai_perf.plots.box_plot import BoxPlot +from genai_perf.plots.scatter_plot import ScatterPlot +from genai_perf.plots.plot_config import PlotConfig, PlotType + + +class TestPlotManager: + @pytest.fixture + def plot_config(self, tmp_path): + return PlotConfig( + title="Test Plot", + data=[{"x": [1, 2, 3], "y": [1, 2, 3]}], + x_label="X Axis", + y_label="Y Axis", + height=1000, + width=1000, + output=tmp_path / "output", + type=PlotType.BOX, + ) + + @pytest.fixture + def plot_manager(self, plot_config): + return PlotManager(plot_config) + + @pytest.fixture + def sample_plots(self, plot_config): + return { + "box": BoxPlot( + {"data": [[1, 2, 3], [4, 5, 6]], "labels": ["A", "B"]}, plot_config + ), + "scatter": ScatterPlot({"x": [1, 2, 3], "y": [1, 2, 3]}, plot_config), + } + + def test_plot_manager_initialization(self, plot_manager): + assert plot_manager.plots == {} + + def test_add_plot(self, plot_manager, sample_plots): + for name, plot in sample_plots.items(): + plot_manager.add_plot(name, plot) + assert len(plot_manager.plots) == len(sample_plots) + + def test_get_plot(self, plot_manager, sample_plots): + plot_manager.add_plot("test", sample_plots["box"]) + assert plot_manager.get_plot("test") == sample_plots["box"] + + def test_get_nonexistent_plot(self, plot_manager): + with pytest.raises(KeyError): + plot_manager.get_plot("nonexistent") + + def test_remove_plot(self, plot_manager, sample_plots): + plot_manager.add_plot("test", sample_plots["box"]) + plot_manager.remove_plot("test") + assert "test" not in plot_manager.plots + + def test_clear_plots(self, plot_manager, sample_plots): + for name, plot in sample_plots.items(): + plot_manager.add_plot(name, plot) + plot_manager.clear_plots() + assert len(plot_manager.plots) == 0 + + def test_plot_names(self, plot_manager, sample_plots): + for name, plot in sample_plots.items(): + plot_manager.add_plot(name, plot) + assert set(plot_manager.plot_names()) == set(sample_plots.keys()) diff --git a/genai-perf/tests/test_plots/test_scatter_plot.py b/genai-perf/tests/test_plots/test_scatter_plot.py new file mode 100644 index 00000000..f39102a3 --- /dev/null +++ b/genai-perf/tests/test_plots/test_scatter_plot.py @@ -0,0 +1,47 @@ +import pytest +from genai_perf.plots.scatter_plot import ScatterPlot +from genai_perf.plots.plot_config import PlotConfig + + +class TestScatterPlot: + @pytest.fixture + def scatter_plot_data(self): + return { + "x": [1, 2, 3, 4, 5], + "y": [10, 20, 30, 40, 50], + "sizes": [100, 200, 300, 400, 500], + "colors": ["red", "blue", "green", "yellow", "purple"], + } + + @pytest.fixture + def scatter_plot_config(self): + return PlotConfig( + title="Scatter Plot Test", + x_label="X Axis", + y_label="Y Axis", + show_grid=True, + show_legend=True, + ) + + def test_scatter_plot_creation(self, scatter_plot_data, scatter_plot_config): + plot = ScatterPlot(scatter_plot_data, scatter_plot_config) + assert plot.data == scatter_plot_data + assert plot.config == scatter_plot_config + + def test_scatter_plot_minimal_data(self): + data = {"x": [1, 2, 3], "y": [1, 2, 3]} + plot = ScatterPlot(data, PlotConfig()) + assert plot.data == data + + def test_scatter_plot_data_validation(self): + with pytest.raises(ValueError): + ScatterPlot({"x": [1, 2, 3]}) # Missing y data + + def test_scatter_plot_optional_data_validation(self): + data = { + "x": [1, 2, 3], + "y": [1, 2, 3], + "sizes": [100, 200], # Mismatched length + } + with pytest.raises(ValueError): + ScatterPlot(data) From ad8ac14df318ef0cce0af3db9b932f130db97cc8 Mon Sep 17 00:00:00 2001 From: ahnsv Date: Thu, 19 Jun 2025 12:35:12 -0400 Subject: [PATCH 2/3] test(plots): add tests --- genai-perf/genai_perf/plots/base_plot.py | 3 + genai-perf/genai_perf/plots/exceptions.py | 43 +++++++ genai-perf/tests/test_plots/test_base_plot.py | 42 ------- genai-perf/tests/test_plots/test_box_plot.py | 107 ++++++++++------ genai-perf/tests/test_plots/test_heat_map.py | 117 ++++++++++-------- .../tests/test_plots/test_plot_manager.py | 87 ++++++------- .../tests/test_plots/test_scatter_plot.py | 107 +++++++++------- 7 files changed, 279 insertions(+), 227 deletions(-) create mode 100644 genai-perf/genai_perf/plots/exceptions.py delete mode 100644 genai-perf/tests/test_plots/test_base_plot.py diff --git a/genai-perf/genai_perf/plots/base_plot.py b/genai-perf/genai_perf/plots/base_plot.py index 470e0b94..3785b849 100755 --- a/genai-perf/genai_perf/plots/base_plot.py +++ b/genai-perf/genai_perf/plots/base_plot.py @@ -30,6 +30,7 @@ import pandas as pd from genai_perf.exceptions import GenAIPerfException +from genai_perf.plots.exceptions import EmptyDataError from genai_perf.plots.plot_config import ProfileRunData from plotly.graph_objects import Figure @@ -40,6 +41,8 @@ class BasePlot: """ def __init__(self, data: List[ProfileRunData]) -> None: + if not data: + raise EmptyDataError("Data is empty") self._profile_data = data def create_plot( diff --git a/genai-perf/genai_perf/plots/exceptions.py b/genai-perf/genai_perf/plots/exceptions.py new file mode 100644 index 00000000..7d7e3ba4 --- /dev/null +++ b/genai-perf/genai_perf/plots/exceptions.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from genai_perf.exceptions import GenAIPerfException + + +class GenAIPerfPlotException(GenAIPerfException): + """ + Exception raised when there is an error with a plot + """ + + pass + + +class EmptyDataError(GenAIPerfPlotException): + """ + Exception raised when data is empty + """ + + pass diff --git a/genai-perf/tests/test_plots/test_base_plot.py b/genai-perf/tests/test_plots/test_base_plot.py deleted file mode 100644 index ea47c048..00000000 --- a/genai-perf/tests/test_plots/test_base_plot.py +++ /dev/null @@ -1,42 +0,0 @@ -import pytest -from genai_perf.plots.base_plot import BasePlot -from genai_perf.plots.plot_config import PlotConfig, PlotType - - -class TestBasePlot: - @pytest.fixture - def sample_data(self): - return {"x": [1, 2, 3, 4, 5], "y": [10, 20, 30, 40, 50]} - - @pytest.fixture - def plot_config(self, tmp_path): - return PlotConfig( - title="Test Plot", - x_label="X Axis", - y_label="Y Axis", - height=1000, - width=1000, - output=tmp_path / "output", - type=PlotType.BOX, - ) - - def test_base_plot_initialization(self, sample_data, plot_config): - plot = BasePlot(sample_data, plot_config) - assert plot.data == sample_data - assert plot.config == plot_config - - def test_base_plot_validation(self): - with pytest.raises(ValueError): - BasePlot(None) - - def test_base_plot_empty_data(self): - with pytest.raises(ValueError): - BasePlot({}) - - def test_base_plot_missing_required_data(self): - with pytest.raises(ValueError): - BasePlot({"x": [1, 2, 3]}) # Missing y data - - def test_base_plot_data_length_mismatch(self): - with pytest.raises(ValueError): - BasePlot({"x": [1, 2, 3], "y": [1, 2]}) diff --git a/genai-perf/tests/test_plots/test_box_plot.py b/genai-perf/tests/test_plots/test_box_plot.py index 1f09b243..44e9b49d 100644 --- a/genai-perf/tests/test_plots/test_box_plot.py +++ b/genai-perf/tests/test_plots/test_box_plot.py @@ -1,44 +1,69 @@ +from unittest.mock import patch + import pytest + from genai_perf.plots.box_plot import BoxPlot -from genai_perf.plots.plot_config import PlotConfig - - -class TestBoxPlot: - @pytest.fixture - def box_plot_data(self): - return { - "data": [[1, 2, 3, 4, 5], [2, 3, 4, 5, 6], [3, 4, 5, 6, 7]], - "labels": ["Group A", "Group B", "Group C"], - } - - @pytest.fixture - def box_plot_config(self): - return PlotConfig( - title="Box Plot Test", - x_label="Groups", - y_label="Values", - show_grid=True, - show_outliers=True, - ) +from genai_perf.plots.exceptions import EmptyDataError +from genai_perf.plots.plot_config import ProfileRunData + + +@pytest.fixture +def profile_run_data_list(): + return [ + ProfileRunData(name="run1", x_metric=[1, 2, 3], y_metric=[4, 5, 6]), + ProfileRunData(name="run2", x_metric=[1, 2, 3], y_metric=[7, 8, 9]), + ] + + +def test_box_plot_init(profile_run_data_list): + plot = BoxPlot(profile_run_data_list) + assert plot._profile_data == profile_run_data_list - def test_box_plot_creation(self, box_plot_data, box_plot_config): - plot = BoxPlot(box_plot_data, box_plot_config) - assert plot.data == box_plot_data - assert plot.config == box_plot_config - - def test_box_plot_data_validation(self): - with pytest.raises(ValueError): - BoxPlot({"data": [], "labels": []}) - - def test_box_plot_labels_mismatch(self): - with pytest.raises(ValueError): - BoxPlot( - { - "data": [[1, 2, 3], [4, 5, 6]], - "labels": ["Group A"], # Missing label - } - ) - - def test_box_plot_empty_data_groups(self): - with pytest.raises(ValueError): - BoxPlot({"data": [[], [1, 2, 3]], "labels": ["Group A", "Group B"]}) + +@patch("genai_perf.plots.base_plot.BasePlot._generate_parquet") +@patch("genai_perf.plots.base_plot.BasePlot._generate_graph_file") +def test_box_plot_create_plot( + mock_gen_graph, mock_gen_parquet, profile_run_data_list, tmp_path +): + plot = BoxPlot(profile_run_data_list) + plot.create_plot( + graph_title="Test Title", + x_label="X", + y_label="Y", + width=800, + height=600, + filename_root="testfile", + output_dir=tmp_path, + ) + # Should call parquet and graph file generation twice (html, jpeg) + assert mock_gen_parquet.called + assert mock_gen_graph.call_count == 2 + html_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".html")] + jpeg_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".jpeg")] + assert html_call and jpeg_call + + +def test_box_plot_create_dataframe(profile_run_data_list): + plot = BoxPlot(profile_run_data_list) + df = plot._create_dataframe("X", "Y") + assert list(df.columns) == ["X", "Y", "Run Name"] + assert len(df) == 2 + assert df["Run Name"].tolist() == ["run1", "run2"] + assert df["Y"].tolist() == [[4, 5, 6], [7, 8, 9]] + + +@patch("genai_perf.plots.base_plot.BasePlot._generate_parquet") +@patch("genai_perf.plots.base_plot.BasePlot._generate_graph_file") +def test_box_plot_create_plot_empty_data(mock_gen_graph, mock_gen_parquet, tmp_path): + with pytest.raises(EmptyDataError) as exc: + plot = BoxPlot([]) + plot.create_plot( + graph_title="Empty", + x_label="X", + y_label="Y", + width=700, + height=450, + filename_root="emptyfile", + output_dir=tmp_path, + ) + assert "Data is empty" in str(exc.value) diff --git a/genai-perf/tests/test_plots/test_heat_map.py b/genai-perf/tests/test_plots/test_heat_map.py index 8085b4bc..c26f156a 100644 --- a/genai-perf/tests/test_plots/test_heat_map.py +++ b/genai-perf/tests/test_plots/test_heat_map.py @@ -1,55 +1,68 @@ +from unittest.mock import patch + import pytest -import numpy as np + +from genai_perf.plots.exceptions import EmptyDataError from genai_perf.plots.heat_map import HeatMap -from genai_perf.plots.plot_config import PlotConfig - - -class TestHeatMap: - @pytest.fixture - def heat_map_data(self): - return { - "data": np.random.rand(5, 5), - "x_labels": ["A", "B", "C", "D", "E"], - "y_labels": ["1", "2", "3", "4", "5"], - } - - @pytest.fixture - def heat_map_config(self): - return PlotConfig( - title="Heat Map Test", - x_label="X Axis", - y_label="Y Axis", - show_grid=True, - color_map="viridis", - ) +from genai_perf.plots.plot_config import ProfileRunData + + +@pytest.fixture +def profile_run_data_list(): + return [ + ProfileRunData(name="run1", x_metric=[1, 2, 3], y_metric=[4, 5, 6]), + ProfileRunData(name="run2", x_metric=[1, 2, 3], y_metric=[7, 8, 9]), + ] + + +def test_heat_map_init(profile_run_data_list): + plot = HeatMap(profile_run_data_list) + assert plot._profile_data == profile_run_data_list - def test_heat_map_creation(self, heat_map_data, heat_map_config): - plot = HeatMap(heat_map_data, heat_map_config) - assert plot.data == heat_map_data - assert plot.config == heat_map_config - - def test_heat_map_data_validation(self): - with pytest.raises(ValueError): - HeatMap({"data": np.array([])}) - - def test_heat_map_labels_mismatch(self): - with pytest.raises(ValueError): - HeatMap( - { - "data": np.random.rand(3, 3), - "x_labels": ["A", "B"], # Missing label - "y_labels": ["1", "2", "3"], - } - ) - - def test_heat_map_invalid_color_map(self): - with pytest.raises(ValueError): - config = PlotConfig(color_map="invalid_map") - HeatMap( - { - "data": np.random.rand(3, 3), - "x_labels": ["A", "B", "C"], - "y_labels": ["1", "2", "3"], - }, - config, - ) + +@patch("genai_perf.plots.base_plot.BasePlot._generate_parquet") +@patch("genai_perf.plots.base_plot.BasePlot._generate_graph_file") +def test_heat_map_create_plot( + mock_gen_graph, mock_gen_parquet, profile_run_data_list, tmp_path +): + plot = HeatMap(profile_run_data_list) + plot.create_plot( + graph_title="Test Title", + x_label="X", + y_label="Y", + width=800, + height=600, + filename_root="testfile", + output_dir=tmp_path, + ) + assert mock_gen_parquet.called + assert mock_gen_graph.call_count == 2 + html_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".html")] + jpeg_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".jpeg")] + assert html_call and jpeg_call + + +def test_heat_map_create_dataframe(profile_run_data_list): + plot = HeatMap(profile_run_data_list) + df = plot._create_dataframe("X", "Y") + assert list(df.columns) == ["X", "Y", "Run Name"] + assert len(df) == 2 + assert df["Run Name"].tolist() == ["run1", "run2"] + assert df["Y"].tolist() == [[4, 5, 6], [7, 8, 9]] + + +@patch("genai_perf.plots.base_plot.BasePlot._generate_parquet") +@patch("genai_perf.plots.base_plot.BasePlot._generate_graph_file") +def test_heat_map_create_plot_empty_data(mock_gen_graph, mock_gen_parquet, tmp_path): + with pytest.raises(EmptyDataError) as exc: + plot = HeatMap([]) + plot.create_plot( + graph_title="Empty", + x_label="X", + y_label="Y", + width=700, + height=450, + filename_root="emptyfile", + output_dir=tmp_path, + ) + assert "Data is empty" in str(exc.value) diff --git a/genai-perf/tests/test_plots/test_plot_manager.py b/genai-perf/tests/test_plots/test_plot_manager.py index bacfb732..dc13777f 100644 --- a/genai-perf/tests/test_plots/test_plot_manager.py +++ b/genai-perf/tests/test_plots/test_plot_manager.py @@ -1,65 +1,56 @@ +from pathlib import Path + import pytest + +from genai_perf.plots.plot_config import PlotConfig, PlotType, ProfileRunData from genai_perf.plots.plot_manager import PlotManager -from genai_perf.plots.box_plot import BoxPlot -from genai_perf.plots.scatter_plot import ScatterPlot -from genai_perf.plots.plot_config import PlotConfig, PlotType class TestPlotManager: @pytest.fixture - def plot_config(self, tmp_path): + def box_plot_config(self, tmp_path: Path): + output_path = tmp_path / "output" + output_path.mkdir(parents=True, exist_ok=True) return PlotConfig( - title="Test Plot", - data=[{"x": [1, 2, 3], "y": [1, 2, 3]}], + title="Test Box Plot", + data=[ProfileRunData(name="test", x_metric=[1, 2, 3], y_metric=[1, 2, 3])], x_label="X Axis", y_label="Y Axis", height=1000, width=1000, - output=tmp_path / "output", + output=output_path, type=PlotType.BOX, ) @pytest.fixture - def plot_manager(self, plot_config): - return PlotManager(plot_config) + def scatter_plot_config(self, tmp_path: Path): + output_path = tmp_path / "output" + output_path.mkdir(parents=True, exist_ok=True) + return PlotConfig( + title="Test Scatter Plot", + data=[ProfileRunData(name="test", x_metric=[1, 2, 3], y_metric=[1, 2, 3])], + x_label="X Axis", + y_label="Y Axis", + height=1000, + width=1000, + output=output_path, + type=PlotType.SCATTER, + ) @pytest.fixture - def sample_plots(self, plot_config): - return { - "box": BoxPlot( - {"data": [[1, 2, 3], [4, 5, 6]], "labels": ["A", "B"]}, plot_config - ), - "scatter": ScatterPlot({"x": [1, 2, 3], "y": [1, 2, 3]}, plot_config), - } - - def test_plot_manager_initialization(self, plot_manager): - assert plot_manager.plots == {} - - def test_add_plot(self, plot_manager, sample_plots): - for name, plot in sample_plots.items(): - plot_manager.add_plot(name, plot) - assert len(plot_manager.plots) == len(sample_plots) - - def test_get_plot(self, plot_manager, sample_plots): - plot_manager.add_plot("test", sample_plots["box"]) - assert plot_manager.get_plot("test") == sample_plots["box"] - - def test_get_nonexistent_plot(self, plot_manager): - with pytest.raises(KeyError): - plot_manager.get_plot("nonexistent") - - def test_remove_plot(self, plot_manager, sample_plots): - plot_manager.add_plot("test", sample_plots["box"]) - plot_manager.remove_plot("test") - assert "test" not in plot_manager.plots - - def test_clear_plots(self, plot_manager, sample_plots): - for name, plot in sample_plots.items(): - plot_manager.add_plot(name, plot) - plot_manager.clear_plots() - assert len(plot_manager.plots) == 0 - - def test_plot_names(self, plot_manager, sample_plots): - for name, plot in sample_plots.items(): - plot_manager.add_plot(name, plot) - assert set(plot_manager.plot_names()) == set(sample_plots.keys()) + def plot_manager( + self, box_plot_config: PlotConfig, scatter_plot_config: PlotConfig + ): + return PlotManager([box_plot_config, scatter_plot_config]) + + def test_plot_manager_generated_plots_are_all_present( + self, + box_plot_config: PlotConfig, + scatter_plot_config: PlotConfig, + plot_manager: PlotManager, + ): + plot_manager.generate_plots() + assert Path(box_plot_config.output / "test_box_plot.html").exists() + assert Path(box_plot_config.output / "test_box_plot.jpeg").exists() + assert Path(scatter_plot_config.output / "test_scatter_plot.html").exists() + assert Path(scatter_plot_config.output / "test_scatter_plot.jpeg").exists() diff --git a/genai-perf/tests/test_plots/test_scatter_plot.py b/genai-perf/tests/test_plots/test_scatter_plot.py index f39102a3..245d126e 100644 --- a/genai-perf/tests/test_plots/test_scatter_plot.py +++ b/genai-perf/tests/test_plots/test_scatter_plot.py @@ -1,47 +1,66 @@ +from unittest.mock import patch + import pytest + +from genai_perf.plots.exceptions import EmptyDataError +from genai_perf.plots.plot_config import ProfileRunData from genai_perf.plots.scatter_plot import ScatterPlot -from genai_perf.plots.plot_config import PlotConfig - - -class TestScatterPlot: - @pytest.fixture - def scatter_plot_data(self): - return { - "x": [1, 2, 3, 4, 5], - "y": [10, 20, 30, 40, 50], - "sizes": [100, 200, 300, 400, 500], - "colors": ["red", "blue", "green", "yellow", "purple"], - } - - @pytest.fixture - def scatter_plot_config(self): - return PlotConfig( - title="Scatter Plot Test", - x_label="X Axis", - y_label="Y Axis", - show_grid=True, - show_legend=True, - ) - def test_scatter_plot_creation(self, scatter_plot_data, scatter_plot_config): - plot = ScatterPlot(scatter_plot_data, scatter_plot_config) - assert plot.data == scatter_plot_data - assert plot.config == scatter_plot_config - - def test_scatter_plot_minimal_data(self): - data = {"x": [1, 2, 3], "y": [1, 2, 3]} - plot = ScatterPlot(data, PlotConfig()) - assert plot.data == data - - def test_scatter_plot_data_validation(self): - with pytest.raises(ValueError): - ScatterPlot({"x": [1, 2, 3]}) # Missing y data - - def test_scatter_plot_optional_data_validation(self): - data = { - "x": [1, 2, 3], - "y": [1, 2, 3], - "sizes": [100, 200], # Mismatched length - } - with pytest.raises(ValueError): - ScatterPlot(data) + +@pytest.fixture +def profile_run_data_list(): + return [ + ProfileRunData(name="run1", x_metric=[1, 2, 3], y_metric=[4, 5, 6]), + ProfileRunData(name="run2", x_metric=[1, 2, 3], y_metric=[7, 8, 9]), + ] + + +def test_scatter_plot_init(profile_run_data_list): + plot = ScatterPlot(profile_run_data_list) + assert plot._profile_data == profile_run_data_list + + +@patch("genai_perf.plots.base_plot.BasePlot._generate_parquet") +@patch("genai_perf.plots.base_plot.BasePlot._generate_graph_file") +def test_scatter_plot_create_plot( + mock_gen_graph, mock_gen_parquet, profile_run_data_list, tmp_path +): + plot = ScatterPlot(profile_run_data_list) + plot.create_plot( + graph_title="Test Title", + x_label="X", + y_label="Y", + width=800, + height=600, + filename_root="testfile", + output_dir=tmp_path, + ) + assert mock_gen_parquet.called + assert mock_gen_graph.call_count == 2 + html_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".html")] + jpeg_call = [c for c in mock_gen_graph.call_args_list if c[0][2].endswith(".jpeg")] + assert html_call and jpeg_call + + +def test_scatter_plot_create_dataframe(profile_run_data_list): + plot = ScatterPlot(profile_run_data_list) + df = plot._create_dataframe("X", "Y") + assert list(df.columns) == ["X", "Y", "Run Name"] + assert len(df) == 2 + assert df["Run Name"].tolist() == ["run1", "run2"] + assert df["Y"].tolist() == [[4, 5, 6], [7, 8, 9]] + + +def test_scatter_plot_create_plot_empty_data(tmp_path): + with pytest.raises(EmptyDataError) as exc: + plot = ScatterPlot([]) + plot.create_plot( + graph_title="Empty", + x_label="X", + y_label="Y", + width=700, + height=450, + filename_root="emptyfile", + output_dir=tmp_path, + ) + assert "Data is empty" in str(exc.value) From fe1766eb4b9e065febd546f971110a08743f4ba4 Mon Sep 17 00:00:00 2001 From: ahnsv Date: Thu, 19 Jun 2025 12:40:17 -0400 Subject: [PATCH 3/3] refactor: adding copyright comment on the top --- genai-perf/tests/test_plot_configs.py | 114 ------- genai-perf/tests/test_plots/test_box_plot.py | 25 ++ genai-perf/tests/test_plots/test_heat_map.py | 25 ++ .../tests/test_plots/test_plot_configs.py | 300 ++++++++++++++++++ .../tests/test_plots/test_plot_manager.py | 25 ++ .../tests/test_plots/test_scatter_plot.py | 25 ++ 6 files changed, 400 insertions(+), 114 deletions(-) delete mode 100644 genai-perf/tests/test_plot_configs.py create mode 100644 genai-perf/tests/test_plots/test_plot_configs.py diff --git a/genai-perf/tests/test_plot_configs.py b/genai-perf/tests/test_plot_configs.py deleted file mode 100644 index 1e0ef250..00000000 --- a/genai-perf/tests/test_plot_configs.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -from pathlib import Path - -# Skip type checking to avoid mypy error -# Issue: https://github.com/python/mypy/issues/10632 -import yaml # type: ignore -from genai_perf.config.input.config_command import ConfigCommand -from genai_perf.plots.plot_config import PlotType -from genai_perf.plots.plot_config_parser import PlotConfigParser - - -class TestPlotConfigParser: - yaml_config = """ - plot1: - title: TTFT vs ITL - x_metric: time_to_first_tokens - y_metric: inter_token_latencies - x_label: TTFT (ms) - y_label: ITL (ms) - width: 1000 - height: 3000 - type: box - paths: - - run1/concurrency32.json - - run2/concurrency32.json - - run3/concurrency32.json - output: test_output_1 - - plot2: - title: Input Sequence Length vs Output Sequence Length - x_metric: input_sequence_lengths - y_metric: output_sequence_lengths - x_label: Input Sequence Length - y_label: Output Sequence Length - width: 1234 - height: 5678 - type: scatter - paths: - - run4/concurrency1.json - output: test_output_2 - """ - - def test_generate_configs(self, monkeypatch) -> None: - monkeypatch.setattr( - "genai_perf.plots.plot_config_parser.load_yaml", - lambda _: yaml.safe_load(self.yaml_config), - ) - monkeypatch.setattr(PlotConfigParser, "_get_statistics", lambda *_: {}) - monkeypatch.setattr(PlotConfigParser, "_get_metric", lambda *_: [1, 2, 3]) - - config_parser = PlotConfigParser(Path("test_config.yaml")) - config = ConfigCommand({"model_name": "test_model"}) - plot_configs = config_parser.generate_configs(config) - - assert len(plot_configs) == 2 - pc1, pc2 = plot_configs - - # plot config 1 - assert pc1.title == "TTFT vs ITL" - assert pc1.x_label == "TTFT (ms)" - assert pc1.y_label == "ITL (ms)" - assert pc1.width == 1000 - assert pc1.height == 3000 - assert pc1.type == PlotType.BOX - assert pc1.output == Path("test_output_1") - - assert len(pc1.data) == 3 # profile run data - prd1, prd2, prd3 = pc1.data - assert prd1.name == "run1/concurrency32" - assert prd2.name == "run2/concurrency32" - assert prd3.name == "run3/concurrency32" - for prd in pc1.data: - assert prd.x_metric == [1, 2, 3] - assert prd.y_metric == [1, 2, 3] - - # plot config 2 - assert pc2.title == "Input Sequence Length vs Output Sequence Length" - assert pc2.x_label == "Input Sequence Length" - assert pc2.y_label == "Output Sequence Length" - assert pc2.width == 1234 - assert pc2.height == 5678 - assert pc2.type == PlotType.SCATTER - assert pc2.output == Path("test_output_2") - - assert len(pc2.data) == 1 # profile run data - prd = pc2.data[0] - assert prd.name == "run4/concurrency1" - assert prd.x_metric == [1, 2, 3] - assert prd.y_metric == [1, 2, 3] diff --git a/genai-perf/tests/test_plots/test_box_plot.py b/genai-perf/tests/test_plots/test_box_plot.py index 44e9b49d..4ee0f19b 100644 --- a/genai-perf/tests/test_plots/test_box_plot.py +++ b/genai-perf/tests/test_plots/test_box_plot.py @@ -1,3 +1,28 @@ +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from unittest.mock import patch import pytest diff --git a/genai-perf/tests/test_plots/test_heat_map.py b/genai-perf/tests/test_plots/test_heat_map.py index c26f156a..861fb0ba 100644 --- a/genai-perf/tests/test_plots/test_heat_map.py +++ b/genai-perf/tests/test_plots/test_heat_map.py @@ -1,3 +1,28 @@ +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from unittest.mock import patch import pytest diff --git a/genai-perf/tests/test_plots/test_plot_configs.py b/genai-perf/tests/test_plots/test_plot_configs.py new file mode 100644 index 00000000..fbbfcbdf --- /dev/null +++ b/genai-perf/tests/test_plots/test_plot_configs.py @@ -0,0 +1,300 @@ +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from pathlib import Path + +# Skip type checking to avoid mypy error +# Issue: https://github.com/python/mypy/issues/10632 +import yaml # type: ignore +from genai_perf.config.input.config_command import ConfigCommand +from genai_perf.plots.plot_config import PlotType +from genai_perf.plots.plot_config_parser import PlotConfigParser +import io +import tempfile +import shutil +import os +import pytest +from unittest import mock + + +class TestPlotConfigParser: + yaml_config = """ + plot1: + title: TTFT vs ITL + x_metric: time_to_first_tokens + y_metric: inter_token_latencies + x_label: TTFT (ms) + y_label: ITL (ms) + width: 1000 + height: 3000 + type: box + paths: + - run1/concurrency32.json + - run2/concurrency32.json + - run3/concurrency32.json + output: test_output_1 + + plot2: + title: Input Sequence Length vs Output Sequence Length + x_metric: input_sequence_lengths + y_metric: output_sequence_lengths + x_label: Input Sequence Length + y_label: Output Sequence Length + width: 1234 + height: 5678 + type: scatter + paths: + - run4/concurrency1.json + output: test_output_2 + """ + + def test_generate_configs(self, monkeypatch) -> None: + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.load_yaml", + lambda _: yaml.safe_load(self.yaml_config), + ) + monkeypatch.setattr(PlotConfigParser, "_get_statistics", lambda *_: {}) + monkeypatch.setattr(PlotConfigParser, "_get_metric", lambda *_: [1, 2, 3]) + + config_parser = PlotConfigParser(Path("test_config.yaml")) + config = ConfigCommand({"model_name": "test_model"}) + plot_configs = config_parser.generate_configs(config) + + assert len(plot_configs) == 2 + pc1, pc2 = plot_configs + + # plot config 1 + assert pc1.title == "TTFT vs ITL" + assert pc1.x_label == "TTFT (ms)" + assert pc1.y_label == "ITL (ms)" + assert pc1.width == 1000 + assert pc1.height == 3000 + assert pc1.type == PlotType.BOX + assert pc1.output == Path("test_output_1") + + assert len(pc1.data) == 3 # profile run data + prd1, prd2, prd3 = pc1.data + assert prd1.name == "run1/concurrency32" + assert prd2.name == "run2/concurrency32" + assert prd3.name == "run3/concurrency32" + for prd in pc1.data: + assert prd.x_metric == [1, 2, 3] + assert prd.y_metric == [1, 2, 3] + + # plot config 2 + assert pc2.title == "Input Sequence Length vs Output Sequence Length" + assert pc2.x_label == "Input Sequence Length" + assert pc2.y_label == "Output Sequence Length" + assert pc2.width == 1234 + assert pc2.height == 5678 + assert pc2.type == PlotType.SCATTER + assert pc2.output == Path("test_output_2") + + assert len(pc2.data) == 1 # profile run data + prd = pc2.data[0] + assert prd.name == "run4/concurrency1" + assert prd.x_metric == [1, 2, 3] + assert prd.y_metric == [1, 2, 3] + + +class DummyStats: + def __init__(self, data=None, chunked=None): + self.metrics = mock.Mock() + self.metrics.data = data or {} + if chunked is not None: + setattr(self.metrics, "_chunked_inter_token_latencies", chunked) + + +class DummyConfig: + pass + + +def test_get_run_name_with_parent(): + parser = PlotConfigParser(Path("/foo/bar/baz.json")) + assert parser._get_run_name(Path("/foo/bar/baz.json")) == "bar/baz" + + +def test_get_run_name_without_parent(): + parser = PlotConfigParser(Path("baz.json")) + assert parser._get_run_name(Path("baz.json")) == "baz" + + +def test_get_metric_empty(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({}) + assert parser._get_metric(stats, "") == [] + + +def test_get_metric_inter_token_latencies(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"inter_token_latencies": [1000000, 2000000]}) + # Should scale from ns to ms + result = parser._get_metric(stats, "inter_token_latencies") + assert result == [1.0, 2.0] + + +def test_get_metric_token_positions(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"token_positions": []}, chunked=[[1, 2, 3], [1, 2]]) + result = parser._get_metric(stats, "token_positions") + assert result == [1, 2, 3, 1, 2] + + +def test_get_metric_time_to_first_tokens(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"time_to_first_tokens": [1000000, 2000000]}) + result = parser._get_metric(stats, "time_to_first_tokens") + assert result == [1.0, 2.0] + + +def test_get_metric_time_to_second_tokens(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"time_to_second_tokens": [1000000, 3000000]}) + result = parser._get_metric(stats, "time_to_second_tokens") + assert result == [1.0, 3.0] + + +def test_get_metric_request_latencies(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"request_latencies": [1000000, 4000000]}) + result = parser._get_metric(stats, "request_latencies") + assert result == [1.0, 4.0] + + +def test_get_metric_fallback(): + parser = PlotConfigParser(Path("dummy")) + stats = DummyStats({"foo": [42, 43]}) + result = parser._get_metric(stats, "foo") + assert result == [42, 43] + + +def test_get_plot_type_valid(): + parser = PlotConfigParser(Path("dummy")) + assert parser._get_plot_type("scatter") == PlotType.SCATTER + assert parser._get_plot_type("box") == PlotType.BOX + assert parser._get_plot_type("heatmap") == PlotType.HEATMAP + + +def test_get_plot_type_invalid(): + parser = PlotConfigParser(Path("dummy")) + with pytest.raises(ValueError): + parser._get_plot_type("invalid_type") + + +def test_get_statistics_calls_parser(monkeypatch): + parser = PlotConfigParser(Path("dummy")) + dummy_stats = object() + dummy_parser = mock.Mock() + dummy_parser.get_profile_load_info.return_value = [("mode", "level")] + dummy_parser.get_statistics.return_value = dummy_stats + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.LLMProfileDataParser", + lambda **kwargs: dummy_parser, + ) + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.get_tokenizer", lambda config: "tok" + ) + result = parser._get_statistics("foo.json", DummyConfig()) + assert result is dummy_stats + dummy_parser.get_profile_load_info.assert_called_once() + dummy_parser.get_statistics.assert_called_once_with("mode", "level") + + +def test_get_statistics_assert(monkeypatch): + parser = PlotConfigParser(Path("dummy")) + dummy_parser = mock.Mock() + dummy_parser.get_profile_load_info.return_value = [] + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.LLMProfileDataParser", + lambda **kwargs: dummy_parser, + ) + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.get_tokenizer", lambda config: "tok" + ) + with pytest.raises(AssertionError): + parser._get_statistics("foo.json", DummyConfig()) + + +def test_create_init_yaml_config(tmp_path): + files = [tmp_path / "a.json", tmp_path / "b.json"] + for f in files: + f.write_text("{}") + output_dir = tmp_path + PlotConfigParser.create_init_yaml_config(files, output_dir) + config_path = output_dir / "config.yaml" + assert config_path.exists() + with open(config_path) as f: + config = yaml.safe_load(f) + # Check structure and some keys + assert "plot1" in config + assert config["plot1"]["title"] == "Time to First Token" + assert config["plot1"]["paths"] == [str(f) for f in files] + assert config["plot1"]["output"] == str(output_dir) + assert "plot5" in config + assert config["plot5"]["type"] == "scatter" + + +def test_generate_configs_integration(monkeypatch, tmp_path): + # Create a minimal YAML config file + yaml_config = { + "plot1": { + "title": "Test Plot", + "x_metric": "foo", + "y_metric": "bar", + "x_label": "X", + "y_label": "Y", + "width": 100, + "height": 200, + "type": "scatter", + "paths": [str(tmp_path / "run1.json")], + "output": str(tmp_path), + } + } + config_path = tmp_path / "test.yaml" + with open(config_path, "w") as f: + yaml.dump(yaml_config, f) + # Patch dependencies + monkeypatch.setattr( + "genai_perf.plots.plot_config_parser.load_yaml", lambda _: yaml_config + ) + monkeypatch.setattr(PlotConfigParser, "_get_statistics", lambda *_: None) + monkeypatch.setattr(PlotConfigParser, "_get_metric", lambda *_: [1, 2]) + parser = PlotConfigParser(config_path) + config = ConfigCommand({"model_name": "test_model"}) + plot_configs = parser.generate_configs(config) + assert len(plot_configs) == 1 + pc = plot_configs[0] + assert pc.title == "Test Plot" + assert pc.x_label == "X" + assert pc.y_label == "Y" + assert pc.width == 100 + assert pc.height == 200 + assert pc.type == PlotType.SCATTER + assert pc.output == tmp_path + assert len(pc.data) == 1 + prd = pc.data[0] + assert prd.x_metric == [1, 2] + assert prd.y_metric == [1, 2] diff --git a/genai-perf/tests/test_plots/test_plot_manager.py b/genai-perf/tests/test_plots/test_plot_manager.py index dc13777f..3201a697 100644 --- a/genai-perf/tests/test_plots/test_plot_manager.py +++ b/genai-perf/tests/test_plots/test_plot_manager.py @@ -1,3 +1,28 @@ +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from pathlib import Path import pytest diff --git a/genai-perf/tests/test_plots/test_scatter_plot.py b/genai-perf/tests/test_plots/test_scatter_plot.py index 245d126e..6f81c6b4 100644 --- a/genai-perf/tests/test_plots/test_scatter_plot.py +++ b/genai-perf/tests/test_plots/test_scatter_plot.py @@ -1,3 +1,28 @@ +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. from unittest.mock import patch import pytest