+ {% for plot_type in figure.plot_types %}
+ {% set plot_type_i = loop.index %}
+
+ {% endfor %}
+
+ {% endif %}
+ {% for subfigure in figure.subfigures %}
+ {% set subfigure_i = loop.index %}
+
+
+ {% for export in subfigure.exports %}
+
+ {% endfor %}
+
-++++
-
-++++
\ No newline at end of file
+{% endfor %}
\ No newline at end of file
diff --git a/src/feelpp/benchmarking/report/templates/js/figureHelpers.js b/src/feelpp/benchmarking/report/templates/js/figureHelpers.js
new file mode 100644
index 00000000..c2cee0f5
--- /dev/null
+++ b/src/feelpp/benchmarking/report/templates/js/figureHelpers.js
@@ -0,0 +1,26 @@
+
+function downloadString(data, filename, mimeType ) {
+ const blob = new Blob([data], { type: `${mimeType};charset=utf-8` });
+ const url = URL.createObjectURL(blob);
+ const a = document.createElement('a');
+ a.href = url;
+ a.download = filename;
+ document.body.appendChild(a);
+ a.click();
+ document.body.removeChild(a);
+ URL.revokeObjectURL(url);
+}
+
+function switchTab(figureIndex, tabIndex) {
+ let subfigures = document.querySelectorAll(`[id^="subfig_${figureIndex}_"]`);
+ subfigures.forEach(subfig => {
+ subfig.classList.remove('active');
+ subfig.classList.add('inactive');
+ });
+
+ let activeSubfig = document.getElementById(`subfig_${figureIndex}_${tabIndex}`);
+ if(activeSubfig){
+ activeSubfig.classList.remove('inactive');
+ activeSubfig.classList.add('active');
+ }
+}
\ No newline at end of file
diff --git a/tests/configtest/test_benchmark.py b/tests/configtest/test_benchmark.py
index 45fc11ff..f1afc305 100644
--- a/tests/configtest/test_benchmark.py
+++ b/tests/configtest/test_benchmark.py
@@ -7,36 +7,6 @@
-class TestImage:
- """Tests for the Image schema"""
-
- dummy_image_path = "tests/data/configs/mockAppConfig.json"
-
- def test_extractProtocol(self):
- """Tests the correct extraction of the protocol from the image name"""
- image = Image(name=self.dummy_image_path)
- assert image.protocol == "local"
- assert image.name == self.dummy_image_path
-
- with pytest.raises(ValueError,match="Unkown Protocol"):
- image = Image(name="unkown_local://tests/data/configs")
-
- image = Image(name="docker://test_name")
- assert image.protocol == "docker"
- assert image.name == "docker://test_name"
-
-
- def test_checkImage(self):
- """Tests that checking if image exists is done correctly"""
-
- with pytest.raises(FileNotFoundError):
- image = Image(name="nonexistant_image.sif")
-
- #Dry run
- image = Image.model_validate({"name":"nonexistant_image.sif"},context={"dry_run":True})
- assert image.protocol == "local"
-
-
class TestPlatform:
pass
diff --git a/tests/data/configs/mockAppConfig.json b/tests/data/configs/mockAppConfig.json
index 1f5fedab..23da17c0 100644
--- a/tests/data/configs/mockAppConfig.json
+++ b/tests/data/configs/mockAppConfig.json
@@ -9,8 +9,6 @@
}
},
"options": [],
- "outputs": [],
- "additional_files":{},
"scalability": {
"directory":"",
"stages":[]
diff --git a/tests/scalability/test_appSetup.py b/tests/scalability/test_appSetup.py
new file mode 100644
index 00000000..67e52f33
--- /dev/null
+++ b/tests/scalability/test_appSetup.py
@@ -0,0 +1,35 @@
+import pytest
+import tempfile, os
+from feelpp.benchmarking.reframe.config.configMachines import MachineConfig
+from feelpp.benchmarking.reframe.config.configReader import ConfigReader
+
+
+class AdditionalFilesMocker:
+ def __init__(self,description_filepath="",parameterized_descriptions_filepath=""):
+ self.description_filepath=description_filepath
+ self.parameterized_descriptions_filepath = parameterized_descriptions_filepath
+
+class TestAdditionalFilesCopy:
+
+ def test_copyFile(self):
+ """Test the copyDescription method of the AppSetup class.
+ It checks that a file is correctly copied as expected, with its content intact"""
+
+ os.environ["MACHINE_CONFIG_FILEPATH"] = "./tests/data/configs/mockMachineConfig.json"
+ os.environ["APP_CONFIG_FILEPATH"] = "./tests/data/configs/mockAppConfig.json"
+
+ from feelpp.benchmarking.reframe.setup import AppSetup
+
+ with tempfile.NamedTemporaryFile() as file:
+ with open(file.name,"w") as f:
+ f.write("TEST DESCRIPTION FILE")
+
+ app_setup = AppSetup("./tests/data/configs/mockAppConfig.json",ConfigReader("./tests/data/configs/mockMachineConfig.json",MachineConfig).config)
+ app_setup.reader.config.additional_files = True
+
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ app_setup.copyFile(dir_path=tmp_dir,name="test_description",filepath=file.name)
+
+ assert os.path.isfile(os.path.join(tmp_dir,"partials","test_description"))
+ with open(os.path.join(tmp_dir,"partials","test_description"),"r") as f:
+ assert f.read() == "TEST DESCRIPTION FILE"
\ No newline at end of file
diff --git a/tests/scalability/test_outputsHandler.py b/tests/scalability/test_outputsHandler.py
deleted file mode 100644
index c2032222..00000000
--- a/tests/scalability/test_outputsHandler.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import pytest
-import tempfile, os
-from feelpp.benchmarking.reframe.outputs import OutputsHandler
-
-
-class OutputsConfigMocker:
- def __init__(self,filepath="",format=""):
- self.filepath = filepath
- self.format = format
-
-class AdditionalFilesMocker:
- def __init__(self,description_filepath="",parameterized_descriptions_filepath=""):
- self.description_filepath=description_filepath
- self.parameterized_descriptions_filepath = parameterized_descriptions_filepath
-
-class TestOutputsHandler:
-
- @staticmethod
- def buildCsvString(columns,values):
- """Helper function to create the content of a CSV from a list of columns and a list of values"""
- assert len(columns) == len(values)
- return ",".join(columns) + "\n" + ",".join([str(v) for v in values])
-
-
- def test_getOutputsCsv(self):
- """Tests the getOutputs method with csv format"""
- file = tempfile.NamedTemporaryFile()
- columns = ["col1","col2","col3"]
- values = [1,2,3]
- with open(file.name,"w") as f:
- f.write(self.buildCsvString(columns,values))
-
- outputs_handler = OutputsHandler(
- [OutputsConfigMocker(filepath=file.name,format="csv")]
- )
-
- perfvars = outputs_handler.getOutputs()
-
- for i,column in enumerate(columns):
- assert perfvars[column].evaluate() == values[i]
-
- file.close()
-
- def test_copyDescription(self):
- """Test the copyDescription method of the OutputsHandler class.
- It checks that a file is correctly copied as expected, with its content intact"""
- with tempfile.NamedTemporaryFile() as file:
- with open(file.name,"w") as f:
- f.write("TEST DESCRIPTION FILE")
-
- outputs_handler = OutputsHandler(
- outputs_config = [],
- additional_files_config=AdditionalFilesMocker(description_filepath=file.name)
- )
-
- with tempfile.TemporaryDirectory() as tmp_dir:
- outputs_handler.copyDescription(dir_path=tmp_dir,name="test_description")
-
- assert os.path.isfile(os.path.join(tmp_dir,"partials","test_description"))
- with open(os.path.join(tmp_dir,"partials","test_description"),"r") as f:
- assert f.read() == "TEST DESCRIPTION FILE"
-
- #TODO: This should be refactored when the OutputsHandler is reworked
- def test_copyParametrizedDescriptions(self):
- """Test the copyParametrizedDescriptions method of the OutputsHandler class.
- It checks that a file is correctly copied as expected, with its content intact"""
- with tempfile.NamedTemporaryFile() as file:
- with open(file.name,"w") as f:
- f.write("TEST PARAMETRIZED DESCRIPTION FILE")
-
- outputs_handler = OutputsHandler(
- outputs_config = [],
- additional_files_config=AdditionalFilesMocker(parameterized_descriptions_filepath=file.name)
- )
-
- with tempfile.TemporaryDirectory() as tmp_dir:
- outputs_handler.copyParametrizedDescriptions(dir_path=tmp_dir,name="test_parametrized_description")
-
- assert os.path.isfile(os.path.join(tmp_dir,"partials","test_parametrized_description"))
- with open(os.path.join(tmp_dir,"partials","test_parametrized_description"),"r") as f:
- assert f.read() == "TEST PARAMETRIZED DESCRIPTION FILE"
-
diff --git a/tests/scalability/test_scalabilityHandler.py b/tests/scalability/test_scalabilityHandler.py
index 63fd1568..f5d35e03 100644
--- a/tests/scalability/test_scalabilityHandler.py
+++ b/tests/scalability/test_scalabilityHandler.py
@@ -2,16 +2,16 @@
import pytest
import tempfile, json
-from feelpp.benchmarking.reframe.scalability import ScalabilityHandler
-from unittest.mock import patch
+from feelpp.benchmarking.reframe.scalability import ScalabilityHandler, CsvExtractor,TsvExtractor,JsonExtractor,Extractor,ExtractorFactory
import numpy as np
class StageMocker:
- def __init__(self,format="",filepath="",name="",variables_path=[]):
+ def __init__(self,format="",filepath="",name="",variables_path=[],units={"*":"s"}):
self.format = format
self.filepath = filepath
self.name = name
self.variables_path = variables_path
+ self.units = units
class CustomVariableMocker:
def __init__(self, name="",columns=[],op="",unit="s"):
@@ -27,7 +27,38 @@ def __init__(self, directory="",stages=[],custom_variables=[]):
self.custom_variables = custom_variables
-class TestScalabilityHandler:
+class TestExtractors:
+
+ @staticmethod
+ def buildCsvString(columns,values):
+ """Helper function to create the content of a CSV from a list of columns and a list of values.
+ Args:
+ columns(list[str]): List of colum values
+ values(list[list[any]]): List of lists containing the csv values
+ Returns
+ str: The built csv string
+ """
+ assert all(len(columns) == len(v) for v in values)
+ return ",".join(columns) + "\n" + "\n".join([",".join([str(r) for r in row]) for row in values])
+
+
+ @pytest.mark.parametrize(("values"),[
+ ([[1,2,3]]), ([[1,2,3],[4,5,6]])
+ ])
+ def test_extractCsv(self,values):
+ file = tempfile.NamedTemporaryFile()
+ columns = ["col1","col2","col3"]
+ with open(file.name,"w") as f:
+ f.write(self.buildCsvString(columns,values))
+
+ extractor = CsvExtractor(filepath=file.name,stage_name="",units={"*":"s"})
+ perfvars = extractor.extract()
+ for j,column in enumerate(columns):
+ for i in range(len(values)):
+ column_name = column if len(values) == 1 else f"{column}_{i}"
+ assert perfvars[column_name].evaluate() == values[i][j]
+
+ file.close()
@staticmethod
def buildTsvString(index, columns, values):
@@ -35,44 +66,21 @@ def buildTsvString(index, columns, values):
tsv = "# nProc "+ " ".join(columns) + "\n" + f"{index} " + " ".join([str(v) for v in values]) + "\n"
return tsv
- def test_extractCsv(self):
- """ Test performance variable extraction for CSV files"""
- pass
-
def test_extractTsv(self):
""" Test performance variable extraction for special TSV files [WILL BE REMOVED]"""
-
index = 32
- file1 = tempfile.NamedTemporaryFile()
- columns1 = ["col1","col2","col3"]
- values1 = [1,2.5,1e-5]
- with open(file1.name,"w") as f:
- f.write(self.buildTsvString(index,columns1,values=values1))
-
- file2 = tempfile.NamedTemporaryFile()
- columns2 = ["col1","colX"]
- values2 = [4,5.5]
- with open(file2.name,"w") as f:
- f.write(self.buildTsvString(index,columns2,values2))
-
- scalability_handler = ScalabilityHandler(ScalabilityMocker(
- directory="",
- stages = [
- StageMocker(format="tsv",filepath=file1.name,name="file1"),
- StageMocker(format="tsv",filepath=file2.name,name="file2")
- ]
- ))
-
- perf_vars = scalability_handler.getPerformanceVariables(index)
- for i,col1 in enumerate(columns1):
- print(perf_vars["file1_"+col1])
- assert perf_vars[f"file1_{col1}"].evaluate() == values1[i]
+ file = tempfile.NamedTemporaryFile()
+ columns = ["col1","col2","col3"]
+ values = [1,2.5,1e-5]
+ with open(file.name,"w") as f:
+ f.write(self.buildTsvString(index,columns,values=values))
- for j,col2 in enumerate(columns2):
- assert perf_vars[f"file2_{col2}"].evaluate() == values2[j]
+ extractor = TsvExtractor(filepath=file.name,stage_name="file",index=index,units={"*":"s"})
+ perfvars = extractor.extract()
+ for i,col1 in enumerate(columns):
+ assert perfvars[f"file_{col1}"].evaluate() == values[i]
- file1.close()
- file2.close()
+ file.close()
def test_extractJson(self):
@@ -92,25 +100,15 @@ def test_extractJson(self):
json.dump(values,f)
#Test no variables path
- scalability_handler = ScalabilityHandler(ScalabilityMocker(
- directory="",
- stages = [
- StageMocker(format="json",filepath=file.name,name=""),
- ]
- ))
- perf_vars = scalability_handler.getPerformanceVariables(None)
- assert perf_vars == {}
+ extractor = JsonExtractor(file.name,"",units={"*":"s"},variables_path=[])
+ perfvars = extractor.extract()
+ assert perfvars == {}
#Test with *
- scalability_handler = ScalabilityHandler(ScalabilityMocker(
- directory="",
- stages = [
- StageMocker(format="json",filepath=file.name,name="",variables_path=["*"]),
- ]
- ))
- perf_vars = scalability_handler.getPerformanceVariables(None)
- for k,v in perf_vars.items():
+ extractor = JsonExtractor(file.name,"",units={"*":"s"},variables_path=["*"])
+ perfvars = extractor.extract()
+ for k,v in perfvars.items():
path = k.split(".")
dic = values
for p in path:
@@ -119,20 +117,79 @@ def test_extractJson(self):
assert val == v.evaluate()
#Test with specific paths
- scalability_handler = ScalabilityHandler(ScalabilityMocker(
- directory="",
- stages = [
- StageMocker(format="json",filepath=file.name,name="",variables_path=["field2.field2_2.*","field1"]),
- ]
- ))
- perf_vars = scalability_handler.getPerformanceVariables(None)
- assert len(perf_vars.keys()) == 3
- assert perf_vars["field1"].evaluate() == values["field1"]
- assert perf_vars["field2_2_1"].evaluate() == values["field2"]["field2_2"]["field2_2_1"]
- assert perf_vars["field2_2_2"].evaluate() == values["field2"]["field2_2"]["field2_2_2"]
+ extractor = JsonExtractor(file.name,"",units={"*":"s"},variables_path=["field2.field2_2.*","field1"])
+ perfvars = extractor.extract()
+ assert len(perfvars.keys()) == 3
+ assert perfvars["field1"].evaluate() == values["field1"]
+ assert perfvars["field2_2_1"].evaluate() == values["field2"]["field2_2"]["field2_2_1"]
+ assert perfvars["field2_2_2"].evaluate() == values["field2"]["field2_2"]["field2_2_2"]
file.close()
+ #Test with multiple wildcards
+ file = tempfile.NamedTemporaryFile()
+ values = {
+ "hardware": {
+ "gaya3": {
+ "mem": {
+ "available": {
+ "host": "527759648",
+ "physical": "442275",
+ "virtual": "51059"
+ },
+ "total": {
+ "host": "527759648",
+ "physical": "515390",
+ "virtual": "51199"
+ }
+ }
+ },
+ "gaya2":{
+ "mem": {
+ "available": {
+ "host": "101010101",
+ "physical": "442275",
+ },
+ "total": {
+ "host": "202020202",
+ "physical": "515390",
+ "virtual": "51199"
+ }
+ }
+ }
+ }
+ }
+ with open(file.name,"w") as f:
+ json.dump(values,f)
+
+
+ extractor = JsonExtractor(file.name,"",variables_path=["hardware.*.mem.*.host"],units={"*":"s"})
+ perfvars = extractor.extract()
+ assert perfvars["gaya2.available"] == values["hardware"]["gaya2"]["mem"]["available"]["host"]
+ assert perfvars["gaya2.total"] == values["hardware"]["gaya2"]["mem"]["total"]["host"]
+ assert perfvars["gaya3.available"] == values["hardware"]["gaya3"]["mem"]["available"]["host"]
+ assert perfvars["gaya3.total"] == values["hardware"]["gaya3"]["mem"]["total"]["host"]
+
+
+
+ extractor = JsonExtractor(file.name,"",variables_path=["hardware.*.mem.*"],units={"*":"s"})
+ perfvars = extractor.extract()
+ assert perfvars["gaya2.available.host"] == values["hardware"]["gaya2"]["mem"]["available"]["host"]
+ assert perfvars["gaya2.available.physical"] == values["hardware"]["gaya2"]["mem"]["available"]["physical"]
+ assert perfvars["gaya2.total.host"] == values["hardware"]["gaya2"]["mem"]["total"]["host"]
+ assert perfvars["gaya2.total.physical"] == values["hardware"]["gaya2"]["mem"]["total"]["physical"]
+
+ assert perfvars["gaya3.available.host"] == values["hardware"]["gaya3"]["mem"]["available"]["host"]
+ assert perfvars["gaya3.available.physical"] == values["hardware"]["gaya3"]["mem"]["available"]["physical"]
+ assert perfvars["gaya3.available.virtual"] == values["hardware"]["gaya3"]["mem"]["available"]["virtual"]
+ assert perfvars["gaya3.total.host"] == values["hardware"]["gaya3"]["mem"]["total"]["host"]
+ assert perfvars["gaya3.total.physical"] == values["hardware"]["gaya3"]["mem"]["total"]["physical"]
+ assert perfvars["gaya3.total.virtual"] == values["hardware"]["gaya3"]["mem"]["total"]["virtual"]
+
+ file.close()
+
+
+class TestScalabilityHandler:
@pytest.mark.parametrize(("op","fct"),[
@@ -161,7 +218,7 @@ def test_evaluateCustomVariables(self):
columns = ["col1","col2","col3"]
values = [1,2,5.5]
with open(file.name,"w") as f:
- f.write(self.buildTsvString(index,columns,values=values))
+ f.write(TestExtractors.buildTsvString(index,columns,values=values))
scalability_handler = ScalabilityHandler(ScalabilityMocker(
directory="",