Regression testing suite

.github/workflows/regression-test-pro.yaml

@@ -0,0 +1,40 @@
+name: Regression Test Pro
+
+on:
+  push:
+    branches:
+      - regression-testing
+
+jobs:
+  regression-pro:
+    name: Regression Test Pro Test
+    runs-on: self-hosted
+
+    steps:
+      # - name: Checkout code
+      #   uses: actions/checkout@v4
+      #   with:
+      #     ref: regression-testing
+
+      # - name: Set sysroot
+      #   run: |
+      #     cp /opt/sysroot.tgz ./sysroot.tgz
+      #     tar -xzf ./sysroot.tgz -C ./ariane
+      #     rm -f ./sysroot.tgz
+      #   working-directory: ./soft
+
+      # - name: Install submodule
+      #   run: ./utils/scripts/submodule_init.sh
+
+      - name: Grant permissions to actions-pipeline
+        run: chmod +x *.sh
+        working-directory: ./utils/scripts/actions-pipeline
+
+      - name: Grant permissions to helper
+        run: chmod +x *.sh
+        working-directory: ./utils/scripts/actions-pipeline/helper
+
+      - name: Run workflow script
+        run: |
+          ./run_workflow.sh
+        working-directory: ./utils/scripts/actions-pipeline

socs/defconfig/esp_xilinx-vc707-xc7vx485t_testing

@@ -0,0 +1,41 @@
+CPU_ARCH = ariane
+NCPU_TILE = 1
+CONFIG_HAS_SG = y
+CONFIG_NOC_ROWS = 3
+CONFIG_NOC_COLS = 2
+CONFIG_COH_NOC_WIDTH = 64
+CONFIG_DMA_NOC_WIDTH = 64
+CONFIG_MULTICAST_NOC_EN = y
+CONFIG_MAX_MCAST_DESTS = 4
+CONFIG_QUEUE_SIZE = 4
+CONFIG_CACHE_EN = y
+CONFIG_CACHE_RTL = y
+#CONFIG_CACHE_SPANDEX is not set
+CONFIG_CPU_CACHES = 512 4 1024 16
+CONFIG_ACC_CACHES = 512 4
+CONFIG_CACHE_LINE_SIZE = 128
+CONFIG_SLM_KBYTES = 256
+#CONFIG_JTAG_EN is not set
+CONFIG_ETH_EN = y
+#CONFIG_IOLINK_EN is not set
+CONFIG_IOLINK_WIDTH = 16
+CONFIG_MEM_LINK_WIDTH = 64
+#CONFIG_SVGA_EN is not set
+CONGIG_DSU_IP = C0A80107
+CONGIG_DSU_ETH = A6A7A0F8043D
+CONFIG_CLK_STR = 0
+CONFIG_MON_DDR = y
+CONFIG_MON_MEM = y
+CONFIG_MON_INJ = y
+CONFIG_MON_ROUTERS = y
+CONFIG_MON_ACCELERATORS = y
+CONFIG_MON_L2 = y
+CONFIG_MON_LLC = y
+#CONFIG_MON_DVFS is not set
+TILE_0_0 = 0 mem mem
+TILE_0_1 = 1 cpu cpu
+TILE_1_0 = 2 acc FFT_STRATUS basic_fx32_dma64 1 0 sld
+TILE_1_1 = 3 misc IO
+TILE_2_0 = 4 acc FFT_STRATUS basic_fx32_dma64 0 0 sld
+TILE_2_1 = 5 acc FFT_STRATUS basic_fx32_dma64 0 0 sld
+#CONFIG_HAS_DVFS is not set

socs/xilinx-vc707-xc7vx485t/Makefile

@@ -44,26 +44,26 @@ TOP_VLOG_SIM_SRCS +=
 
 
 ### Xilinx Vivado hw_server ###
-FPGA_HOST ?= localhost
-XIL_HW_SERVER_PORT ?= 3121
+FPGA_HOST ?= espdev.cs.columbia.edu
+XIL_HW_SERVER_PORT ?= 3122
 
 
 ### Network configuration ###
 
 # IP address or host name of the host connected to the FPGA
-UART_IP ?=
-UART_PORT ?=
+UART_IP ?= espdev.cs.columbia.edu
+UART_PORT ?= 4322
 
 # SSH IP address or host name of the ESP Linux instance or gateway
-SSH_IP ?=
-SSH_PORT ?= 22
+SSH_IP ?= 128.59.22.75
+SSH_PORT ?= 5502
 
 # ESPLink IP address or gateway (DO NOT USE HOST NAME)
-ESPLINK_IP ?=
-ESPLINK_PORT ?= 46392
+ESPLINK_IP ?= 128.59.22.75
+ESPLINK_PORT ?= 46307
 
 # MAC address for Linux if using IP address reservation (e.g. 00aabb33cc77)
-# LINUX_MAC ?=
+LINUX_MAC ?= 000A3502CB80
 
 
 ### Include global Makefile ###

soft/common/apps/examples/multifft/multifft.c

@@ -89,8 +89,14 @@ int main(int argc, char **argv)
 
     float *gold[3];
     token_t *buf[3];
+    int *fft_error = (int*)malloc(NACC * sizeof(int));
+    int fft_pass_count = 0;
+    int pass_test_count = 0, total_test_count = 0;
+    char pass_log[256];
+    char fail_log[256];
 
-    const float ERROR_COUNT_TH = 0.01;
+    const float ERROR_COUNT_TH = 0.05;
+    const int TESTING_COUNT_TH = 10;
 
     int k;
 
@@ -102,6 +108,8 @@ int main(int argc, char **argv)
     }
 
     init_buffer(buf[0], gold[0], false);
+    memset(pass_log, 0, sizeof(pass_log));
+    memset(fail_log, 0, sizeof(fail_log));
 
     printf("\n====== Non coherent DMA ======\n\n");
     printf("  .len = %d\n", len);
@@ -117,10 +125,16 @@ int main(int argc, char **argv)
 
     errors = validate_buffer(&buf[0][out_offset], gold[0]);
 
-    if (((float)errors / (float)len) > ERROR_COUNT_TH)
-        printf("  + TEST FAIL: exceeding error count threshold\n");
-    else
-        printf("  + TEST PASS: not exceeding error count threshold\n");
+    if (errors > TESTING_COUNT_TH) {
+        printf("  + OVERALL TEST RESULT: Non coherent DMA: FAIL (%d/%d)\n", errors, 2 * len);
+        strcpy(fail_log, "Non coherent DMA, ");
+    }
+    else {
+        printf("  + OVERALL TEST RESULT: Non coherent DMA: PASS (%d/%d)\n", errors, 2 * len);
+        pass_test_count += 1;
+        strcpy(pass_log, "Non coherent DMA, ");
+    }
+    total_test_count += 1;
 
     printf("\n============\n\n");
 
@@ -141,10 +155,16 @@ int main(int argc, char **argv)
 
     errors = validate_buffer(&buf[0][out_offset], gold[0]);
 
-    if (((float)errors / (float)len) > ERROR_COUNT_TH)
-        printf("  + TEST FAIL: exceeding error count threshold\n");
-    else
-        printf("  + TEST PASS: not exceeding error count threshold\n");
+    if (errors > TESTING_COUNT_TH) {
+        printf("  + OVERALL TEST RESULT: LLC-coherent DMA: FAIL (%d/%d)\n", errors, 2 * len);
+        strcat(fail_log, "LLC-coherent DMA, ");
+    }
+    else {
+        printf("  + OVERALL TEST RESULT: LLC-coherent DMA: PASS (%d/%d)\n", errors, 2 * len);
+        pass_test_count += 1;
+        strcat(pass_log, "LLC-coherent DMA, ");
+    }
+    total_test_count += 1;
 
     printf("\n============\n\n");
 
@@ -165,10 +185,16 @@ int main(int argc, char **argv)
 
     errors = validate_buffer(&buf[0][out_offset], gold[0]);
 
-    if (((float)errors / (float)len) > ERROR_COUNT_TH)
-        printf("  + TEST FAIL: exceeding error count threshold\n");
-    else
-        printf("  + TEST PASS: not exceeding error count threshold\n");
+    if (errors > TESTING_COUNT_TH) {
+        printf("  + OVERALL TEST RESULT: Fully-coherent DMA: FAIL (%d/%d)\n", errors, 2 * len);
+        strcat(fail_log, "Fully-coherent DMA, ");
+    }
+    else {
+        pass_test_count += 1;
+        printf("  + OVERALL TEST RESULT: Fully-coherent DMA: PASS (%d/%d)\n", errors, 2 * len);
+        strcat(pass_log, "Fully-coherent DMA, ");
+    }
+    total_test_count += 1;
 
     printf("\n============\n\n");
 
@@ -196,12 +222,34 @@ int main(int argc, char **argv)
 
     for (k = 0; k < NACC; k++) {
         errors = validate_buffer(&buf[k][out_offset], gold[k]);
+        fft_error[k] = errors;
 
-        if (((float)errors / (float)(len * NACC)) > ERROR_COUNT_TH)
+        if (errors > TESTING_COUNT_TH)
             printf("  + TEST FAIL fft.%d: exceeding error count threshold\n", k);
-        else
+        else {
             printf("  + TEST PASS fft.%d: not exceeding error count threshold\n", k);
+            fft_pass_count ++;
+        }
+    }
+
+    if (fft_pass_count < NACC) {
+        printf("  + OVERALL TEST RESULT: Concurrent execution: FAIL (%d/%d).", NACC - fft_pass_count, NACC);
+        strcat(fail_log, "Concurrent execution, ");
+    }
+    else {
+        printf("  + OVERALL TEST RESULT: Concurrent execution: PASS.");
+        pass_test_count += 1;
+        strcat(pass_log, "Concurrent execution, ");
     }
+    for (k = 0; k < NACC; k++) {
+        errors = fft_error[k];
+        if (errors > TESTING_COUNT_TH)
+            printf(" fft.%d: FAIL (%d/%d).", k, errors, 2 * len);
+        else
+            printf(" fft.%d: PASS (%d/%d).", k, errors, 2 * len);
+    }
+    total_test_count += 1;
+    printf("\n");
 
     printf("\n============\n\n");
 
@@ -225,17 +273,41 @@ int main(int argc, char **argv)
 
     errors = validate_buffer(&buf[0][out_offset], gold[0]);
 
-    if (((float)errors / (float)(len * NACC)) > ERROR_COUNT_TH)
-        printf("  + TEST FAIL: exceeding error count threshold\n");
-    else
-        printf("  + TEST PASS: not exceeding error count threshold\n");
+    if (errors > TESTING_COUNT_TH) {
+        // printf("  + TEST FAIL: exceeding error count threshold of %d\n", TESTING_COUNT_TH);
+        printf("  + OVERALL TEST RESULT: Point-to-point: FAIL (%d/%d)\n", errors, 2 * len);
+        strcat(fail_log, "Point-to-point Test, ");
+    }
+    else {
+        // printf("  + TEST PASS: not exceeding error count threshold of %d\n", TESTING_COUNT_TH);
+        printf("  + OVERALL TEST RESULT: Point-to-point: PASS (%d/%d)\n", errors, 2 * len);
+        pass_test_count += 1;
+        strcat(pass_log, "Point-to-point Test, ");
+    }
+    total_test_count += 1;
 
     printf("\n============\n\n");
 
+    /* Overall result */
+    strcat(pass_log, "\0");
+    strcat(fail_log, "\0");
+
+    if (pass_test_count == total_test_count)
+        printf("[PASS] FFT OVERALL TEST RESULT: PASS ALL! \n");
+    else {
+        printf("[FAIL] FFT OVERALL TEST RESULT: FAIL! ");
+        // detailed result
+        printf("PASS %d: ", pass_test_count);
+        printf("%s", pass_log);
+        printf("FAIL %d: ", total_test_count - pass_test_count);
+        printf("%s\n", fail_log);
+    }
+
     for (k = 0; k < NACC; k++) {
         free(gold[k]);
         esp_free(buf[k]);
     }
+    free(fft_error);
 
     return errors;
 }