Merge branch 'new-benchmark' of https://github.com/feelpp/benchmarkin…

…g.toolboxes into new-benchmark
feelpp · Jan 23, 2025 · c691185 · c691185
2 parents 4c3176b + 4992d9e
commit c691185
Show file tree

Hide file tree

Showing 41 changed files with 1,217 additions and 492 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -215,7 +215,7 @@ jobs:
             -   name: Render docs
                 run: |
                     source .venv/bin/activate
-                    render-benchmarks --config_file=./tmp/website_config.json
+                    render-benchmarks --config-file=./tmp/website_config.json
             -   name: Compile docs
                 run: |
                     source .venv/bin/activate

diff --git a/config/tests_parallelSum/parallelSum.json b/config/tests_parallelSum/parallelSum.json
@@ -20,6 +20,11 @@
             }
         ]
     },
+    // Resources for the test
+    "resources":{
+        "tasks":"{{parameters.resources.tasks.value}}",
+        "exclusive_access":"{{parameters.resources.exclusive_access.value}}"
+    },
     // Files containing app outputs
     "outputs": [
         {
@@ -35,7 +40,7 @@
     // Test parameters
     "parameters": [
         {
-            "name": "nb_tasks",
+            "name": "resources",
             "sequence": [
                 {"tasks" : 1, "exclusive_access":true},
                 {"tasks" : 2, "exclusive_access":true},

diff --git a/config/tests_parallelSum/plots.json b/config/tests_parallelSum/plots.json
@@ -7,7 +7,7 @@
             "variables": [ "computation_time" ],
             "names": ["Time"],
             "xaxis":{
-                "parameter":"nb_tasks.tasks",
+                "parameter":"resources.tasks",
                 "label":"Number of tasks"
             },
             "yaxis":{"label":"Execution time (s)"},
@@ -28,7 +28,7 @@
             },
             "yaxis":{"label":"Execution time (s)"},
             "secondary_axis":{
-                "parameter":"nb_tasks.tasks",
+                "parameter":"resources.tasks",
                 "label":"Number of tasks"
             }
         },
@@ -39,7 +39,7 @@
             "variables": [ "computation_time" ],
             "names": ["Time"],
             "xaxis": {
-                "parameter": "nb_tasks.tasks",
+                "parameter": "resources.tasks",
                 "label": "Number of tasks"
             },
             "yaxis": {

diff --git a/config/toolbox_heat/heat_plots.json b/config/toolbox_heat/heat_plots.json
@@ -13,7 +13,7 @@
             ],
             "names": [ ],
             "xaxis": {
-                "parameter": "meshes",
+                "parameter": "mesh",
                 "label": "mesh levels"
             },
             "yaxis": {
@@ -28,7 +28,7 @@
                 "label": "Measures"
             },
             "aggregations":[
-                {"column":"nb_tasks.tasks","agg":"filter:128"}
+                {"column":"resources.tasks","agg":"filter:640"}
             ]
         },
         {
@@ -42,11 +42,11 @@
             ],
             "names": [ ],
             "xaxis": {
-                "parameter": "nb_tasks.tasks",
+                "parameter": "resources.tasks",
                 "label": "Number of tasks"
             },
             "secondary_axis": {
-                "parameter": "meshes",
+                "parameter": "mesh",
                 "label": "Mesh level"
             },
             "yaxis": {
@@ -71,11 +71,11 @@
             ],
             "names": [ ],
             "xaxis": {
-                "parameter": "nb_tasks.tasks",
+                "parameter": "resources.tasks",
                 "label": "Number of tasks"
             },
             "secondary_axis": {
-                "parameter": "meshes",
+                "parameter": "mesh",
                 "label": "Mesh level"
             },
             "yaxis": {
@@ -100,11 +100,11 @@
             ],
             "names": [ ],
             "xaxis": {
-                "parameter": "nb_tasks.tasks",
+                "parameter": "resources.tasks",
                 "label": "Number of tasks"
             },
             "secondary_axis": {
-                "parameter": "meshes",
+                "parameter": "mesh",
                 "label": "Mesh level"
             },
             "yaxis": {
@@ -129,11 +129,11 @@
             ],
             "names": [ ],
             "xaxis": {
-                "parameter": "nb_tasks.tasks",
+                "parameter": "resources.tasks",
                 "label": "Number of tasks"
             },
             "secondary_axis": {
-                "parameter": "meshes",
+                "parameter": "mesh",
                 "label": "Mesh level"
             },
             "yaxis": {
@@ -146,6 +146,64 @@
             "aggregations":[
                 {"column":"discretization","agg":"filter:P2"}
             ]
+        },
+        {
+            "title": "Performance (P3)",
+            "plot_types": [ "stacked_bar", "grouped_bar" ],
+            "transformation": "performance",
+            "variables": [
+                "Constructor_init",
+                "Solve_algebraic-assembly",
+                "PostProcessing_exportResults"
+            ],
+            "names": [ ],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "secondary_axis": {
+                "parameter": "mesh",
+                "label": "Mesh level"
+            },
+            "yaxis": {
+                "label": "execution time (s)"
+            },
+            "color_axis":{
+                "parameter":"performance_variable",
+                "label":"Performance variable"
+            },
+            "aggregations":[
+                {"column":"discretization","agg":"filter:P3"}
+            ]
+        },
+        {
+            "title": "Speedup (P3)",
+            "plot_types": [ "scatter" ],
+            "transformation": "speedup",
+            "variables": [
+                "Constructor_init",
+                "Solve_algebraic-assembly",
+                "PostProcessing_exportResults"
+            ],
+            "names": [ ],
+            "xaxis": {
+                "parameter": "resources.tasks",
+                "label": "Number of tasks"
+            },
+            "secondary_axis": {
+                "parameter": "mesh",
+                "label": "Mesh level"
+            },
+            "yaxis": {
+                "label": "Speedup"
+            },
+            "color_axis":{
+                "parameter":"performance_variable",
+                "label":"Performance variable"
+            },
+            "aggregations":[
+                {"column":"discretization","agg":"filter:P3"}
+            ]
         }
     ]
 }
diff --git a/config/toolbox_heat/thermal_bridges_case_3.json b/config/toolbox_heat/thermal_bridges_case_3.json
@@ -1,15 +1,19 @@
 {
     "executable": "feelpp_toolbox_heat",
-    "output_directory": "{{machine.output_app_dir}}/toolboxes/heat/ThermalBridgesENISO10211/Case3",
+    "output_directory": "{{machine.output_app_dir}}/javier_test/toolboxes/heat/ThermalBridgesENISO10211/Case3",
     "use_case_name": "ThermalBridgesENISO10211",
     "timeout":"0-00:10:00",
-    "memory":"250G",
+    "resources":{
+        "tasks":"{{parameters.resources.tasks.value}}",
+        "exclusive_access":"{{parameters.resources.exclusive_access.value}}",
+        "memory":"{{parameters.memory.value}}"
+    },
     "platforms": {
         "apptainer":{
             "image": {
                 "name":"{{machine.containers.apptainer.image_base_dir}}/feelpp-noble.sif"
             },
-            "input_dir":"/input_data/",
+            "input_dir":"/input_data",
             "options": [
                 "--home {{machine.output_app_dir}}",
                 "--bind {{machine.input_dataset_base_dir}}/{{use_case_name}}/:{{platforms.apptainer.input_dir}}",
@@ -29,11 +33,8 @@
         "--heat.scalability-save=1",
         "--repository.append.np 0",
         "--case.discretization {{parameters.discretization.value}}",
-        "--heat.json.patch='{\"op\": \"replace\",\"path\": \"/Meshes/heat/Import/filename\",\"value\": \"{{platforms.{{machine.platform}}.input_dir}}/partitioning/case3/{{parameters.meshes.value}}/case3_p{{parameters.nb_tasks.tasks.value}}.json\" }'"
+        "--heat.json.patch='{\"op\": \"replace\",\"path\": \"/Meshes/heat/Import/filename\",\"value\": \"{{platforms.{{machine.platform}}.input_dir}}/partitioning/case3/{{parameters.mesh.value}}/case3_p{{parameters.resources.tasks.value}}.json\" }'"
     ],
-    "env_variables":{
-        "OMP_NUM_THREADS":"1"
-    },
     "outputs": [
         {
             "filepath": "{{output_directory}}/{{instance}}/{{use_case_name}}/heat.measures/values.csv",
@@ -69,18 +70,34 @@
     },
     "parameters": [
         {
-            "name": "nb_tasks",
-            "sequence": [
-                {"tasks":128,"tasks_per_node":128,"exclusive_access":true}
+            "name": "resources",
+            "zip":[
+                {
+                    "name":"tasks",
+                    "sequence":[32,64,128,256]
+                },
+                {
+                    "name":"exclusive_access",
+                    "repeat":{"value":true,"count":4}
+                }
             ]
         },
         {
-            "name": "meshes",
-            "sequence": ["M1"]
+            "name":"mesh",
+            "sequence":["M1","M2","M3"]
         },
         {
-            "name": "discretization",
-            "sequence": ["P1"]
+            "name":"memory",
+            "sequence":[200, 600]
+        },
+        {
+            "name":"discretization",
+            "sequence":["P1","P2","P3"],
+            "conditions":{
+                "P1":{ "memory":[200] },
+                "P2":{ "memory":[200] },
+                "P3":{ "memory":[600], "resources.tasks":[128,256] }
+            }
         },
         {
             "name": "solver",

diff --git a/docs/modules/ROOT/pages/index.adoc b/docs/modules/ROOT/pages/index.adoc
@@ -4,4 +4,4 @@ ifdef::env-github,env-browser,env-vscode[:imagesprefix: ../images/]
 = Benchmarking
 :page-layout: toolboxes
 :page-tags: catalog, catalog-index
-:docdatetime: 2025-01-14T15:32:33
+:docdatetime: 2025-01-23T13:03:51
diff --git a/docs/modules/tutorial/pages/configurationfiles/plots.adoc b/docs/modules/tutorial/pages/configurationfiles/plots.adoc
@@ -113,7 +113,7 @@ For the given example, it produces the following dataframe
 
 [%dynamic%open%hide_code,python]
 ----
-from feelpp.benchmarking.report.strategies import StrategyFactory
+from feelpp.benchmarking.report.transformationFactory import TransformationStrategyFactory
 from feelpp.benchmarking.reframe.config.configPlots import Plot
 plot_config = Plot(**{
     "title": "Absolute performance",
@@ -131,7 +131,7 @@ plot_config = Plot(**{
         "label":"N"
     }
 })
-strategy = StrategyFactory.create(plot_config)
+strategy = TransformationStrategyFactory.create(plot_config)
 strategy.calculate(model.master_df)
 ----
 
@@ -141,7 +141,7 @@ The relative performance strategy computes the proportion of the time that a a `
 
 [%dynamic%open%hide_code,python]
 ----
-from feelpp.benchmarking.report.strategies import StrategyFactory
+from feelpp.benchmarking.report.transformationFactory import TransformationStrategyFactory
 from feelpp.benchmarking.reframe.config.configPlots import Plot
 plot_config = Plot(**{
     "title": "Absolute performance",
@@ -159,7 +159,7 @@ plot_config = Plot(**{
         "label":"N"
     }
 })
-strategy = StrategyFactory.create(plot_config)
+strategy = TransformationStrategyFactory.create(plot_config)
 strategy.calculate(model.master_df)
 ----
 
@@ -172,7 +172,7 @@ For the example, this strategy will produce the following.
 
 [%dynamic%open%hide_code,python]
 ----
-from feelpp.benchmarking.report.strategies import StrategyFactory
+from feelpp.benchmarking.report.transformationFactory import TransformationStrategyFactory
 from feelpp.benchmarking.reframe.config.configPlots import Plot
 plot_config = Plot(**{
     "title": "Absolute performance",
@@ -190,7 +190,7 @@ plot_config = Plot(**{
         "label":"N"
     }
 })
-strategy = StrategyFactory.create(plot_config)
+strategy = TransformationStrategyFactory.create(plot_config)
 strategy.calculate(model.master_df)
 ----
 

diff --git a/docs/modules/tutorial/pages/gettingstarted.adoc b/docs/modules/tutorial/pages/gettingstarted.adoc
@@ -3,28 +3,28 @@
 == Installation
 
 1. Clone the Repository
-[source,cmd]
+[source,bash]
 ----
 git clone https://github.com/feelpp/benchmarking.git
 ----
 
 2. Use a python virtual environment [Optional]
-[source,cmd]
+[source,bash]
 ----
 python3 -m venv .venv
 source .venv/bin/activate
 ----
 
 3. Build the project
-[source,cmd]
+[source,bash]
 ----
 pip3 wheel --no-deps --wheel-dir dist .
 ----
 
 4. Install requirements
 
 This will install necessary dependencies as well as the built project from the previous step.
-[source,cmd]
+[source,bash]
 ----
 python3 -m pip install -r requirements.txt
 ----
@@ -38,7 +38,7 @@ This Feel++ Benchmarking "Hello World" application will compute the sum of an ar
 Additionally, the app will measure the time taken to perform the partial sum, and will save it under a _scalability.json_ file.
 
 The executable is already provided as _tests/data/parallelSum_. You can update it and recompile it for a specific config as needed.
-[source,cmd]
+[source,bash]
 ----
 mpic++ -std=c++17 -o test/data/parallelSum test/data/parallelSum.cpp
 ----
@@ -51,7 +51,7 @@ mpic++ -std=c++17 -o test/data/parallelSum test/data/parallelSum.cpp
 Configuration files might require some changes for specific configurations depending on the system you are running the framework.
 
 Finally, to benchmark the test application, generate the reports and plot the figures, run
-[source,cmd]
+[source,bash]
 ----
 execute-benchmark --machine-config config/machines/local.json \
                     --benchmark-config config/test_parallelSum/parallelSum.json \

diff --git a/requirements.txt b/requirements.txt
@@ -15,4 +15,5 @@ traitlets
 tabulate
 typing-extensions>=4.12.2
 python-dotenv
-.
+tikzplotly
+.