Refactored scripts for benchmark suits (#102)

Author: Egor-Krivov

.github/workflows/execute-test-script.yml

@@ -33,7 +33,7 @@ on:
             test_script:
                 description: Test script to run
                 required: false
-                default: "./mlp.sh"
+                default: "mlp.sh"
                 type: string
         secrets:
             DB_URL:
@@ -122,11 +122,13 @@ jobs:
                       URL="--url ${{ secrets.DB_URL }}"
                   fi
 
-                  export DL_BENCH_ARGS="--host ${{ inputs.runner_type }} --compiler ${{ inputs.compiler }} --device ${{ inputs.device }} --tag ${{ inputs.tag }} ${URL}"
+                  export COMPILER="${{ inputs.compiler }}"
+                  export DEVICE="${{ inputs.device }}"
+                  export OTHER_ARGS="--host ${{ inputs.runner_type }} --tag ${{ inputs.tag }} ${URL}"
 
                   # We mainly want to verify our own backend
                   if [[ ${{ inputs.compiler }} != *torch_mlir* ]]; then
-                    DL_BENCH_ARGS="${DL_BENCH_ARGS} --skip_verification"
+                    OTHER_ARGS="${OTHER_ARGS} --skip_verification"
                   fi
 
                   # HOST CONFIG
@@ -136,11 +138,11 @@ jobs:
                   echo "CPU cores configured: $OMP_NUM_THREADS"
                   if [[ ${LABELS} = *glados* ]]; then
                       export HF_HOME="/cache/torchmlir/huggingface_cache"
-                      
-                      numactl -m 0 --physcpubind=0-31 ${{ inputs.test_script }}
+
+                      numactl -m 0 --physcpubind=0-31 ./bench_suit/${{ inputs.test_script }}
                   else
                       export HF_HOME="/data/torchmlir/huggingface_cache"
-                      source ${{ inputs.test_script}}
+                      source ./bench_suit/${{ inputs.test_script}}
                   fi
 
 

.github/workflows/test-single-config.yml

@@ -55,7 +55,7 @@ on:
             test_scripts:
                 description: Test scripts to execute
                 required: false
-                default: '["./mlp.sh", "./cnn.sh", "./llm.sh"]'
+                default: '["mlp.sh", "cnn.sh", "llm.sh"]'
                 type: string
 
 jobs:

.github/workflows/test.yml

@@ -19,7 +19,7 @@ on:
             test_scripts:
                 description: Test scripts to execute
                 required: false
-                default: '["./mlp.sh", "./cnn.sh", "./llm.sh"]'
+                default: '["mlp.sh", "cnn.sh", "llm.sh"]'
                 type: string
     schedule:
         # Runs at 12pm UTC (6am CST) on every Saturday
@@ -44,13 +44,13 @@ jobs:
                     - ipex_onednn_graph
                     - torch_mlir
                     - torch_mlir_xsmm
-                test_script: ${{ github.event_name == 'workflow_dispatch' && fromJson(inputs.test_scripts) || fromJson('["./mlp.sh", "./cnn.sh", "./llm.sh"]') }}
+                test_script: ${{ github.event_name == 'workflow_dispatch' && fromJson(inputs.test_scripts) || fromJson('["mlp.sh", "cnn.sh", "llm.sh"]') }}
                 exclude:
-                    - test_script: "./llm.sh"
+                    - test_script: "llm.sh"
                       compiler: torchscript
-                    - test_script: "./llm.sh"
+                    - test_script: "llm.sh"
                       compiler: torchscript_onednn
-                    - test_script: "./llm.sh"
+                    - test_script: "llm.sh"
                       compiler: ipex_onednn_graph
             fail-fast: false
         uses: ./.github/workflows/execute-test-script.yml

.github/workflows/test_nvidia.yml

@@ -14,7 +14,7 @@ on:
             test_scripts:
                 description: Test scripts to execute
                 required: false
-                default: '["./mlp.sh", "./cnn.sh", "./llm.sh"]'
+                default: '["mlp.sh", "cnn.sh", "llm.sh"]'
                 type: string
 
 jobs:

bench_suit/cnn.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+. "$(dirname "$0")/common.sh"
+
+DTYPEs="float32 bfloat16"
+if [ "$COMPILER" = 'ipex_onednn_graph' ]; then
+    DTYPEs="$DTYPEs int8"
+fi
+
+run_benchmark_suit cnn "$DTYPEs" "1 32 128" "vgg16 resnet18 resnet50 resnext50 resnext101 densenet121"
+print_report

bench_suit/common.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -x
+
+if [ -z "${COMPILER}" ]; then
+  echo "Please, provide COMPILER environment variable"
+  exit 1
+fi
+
+if [ -z "${DEVICE}" ]; then
+  echo "Please, provide DEVICE environment variable"
+  exit 1
+fi
+
+if [ -z "${OTHER_ARGS}" ]; then
+  echo "Please, provide OTHER_ARGS environment variable"
+  exit 1
+fi
+
+# List of all benchmark runs that failed to report
+failed_commands=()
+
+run_benchmark_suit() {
+    BENCHMARK="$1"
+    DTYPEs="$2"
+    BSs="$3"
+    NAMEs="$4"
+
+    echo "Running suit for benchmark=\"$BENCHMARK\" DTYPEs=\"$DTYPEs\" BSs=\"$BSs\" NAMEs=\"$NAMEs\""
+
+    for DTYPE in $DTYPEs
+    do
+        for BS in $BSs
+        do
+            for NAME in $NAMEs
+            do
+                echo "Benchmark $NAME with BS=$BS and DTYPE=$DTYPE"
+                BS_TXT=$(printf "%04d" $BS)
+
+                CMD="benchmark-run -c ${COMPILER} -d ${DEVICE} -b ${BENCHMARK} -p name='${NAME}' --dtype ${DTYPE} -bs $BS --benchmark_desc ${NAME}_bs${BS_TXT} ${OTHER_ARGS}"
+                $CMD || failed_commands+=("$CMD")
+            done
+        done
+    done
+}
+
+
+print_report() {
+    if [ ${#failed_commands[@]} -gt 0 ]; then
+        echo "Some benchmarks failed, here's the list:"
+        for CMD in "${failed_commands[@]}"; do
+            echo "$CMD"
+        done
+            exit 1
+        fi
+}

bench_suit/llm.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+. "$(dirname "$0")/common.sh"
+
+run_benchmark_suit llm "bfloat16" "1 4 8" "llama2-7b llama2-13b gptj"
+print_report

bench_suit/mlp.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. "$(dirname "$0")/common.sh"
+
+DTYPEs="float32 bfloat16"
+if [ "$COMPILER" = 'ipex_onednn_graph' ]; then
+    DTYPEs="$DTYPEs int8"
+fi
+
+run_benchmark_suit mlp "$DTYPEs" "1024" "size2 size3 size4 size5 size5_sigm size5_tanh size5_gelu size5_linear size5_inplace size5_bn size5_bn_gelu size5_drop_gelu 100@512 25@1024 4@16384 2@16384"
+run_benchmark_suit mlp "$DTYPEs" "1 16 256 2048 8196" "size5"
+print_report