Fix data paths in TL3 short tests (#5845)

- unifies data paths in TL3 short tests with other tests
- improves the way the remote file system is detected in the tests

Signed-off-by: Janusz Lisiecki <[email protected]>

Author: JanuszL

qa/TL1_separate_executor/test_nofw.sh

@@ -9,17 +9,11 @@ do_once() {
 }
 
 test_body() {
-    start=`date +%s`
-    (sleep 10 && pkill -HUP ls && true) &
-    (ls /data/imagenet/train-jpeg > /dev/null && pkill -HUP sleep) &
-    wait
-    end=`date +%s`
-    runtime=$((end-start))
-    echo "Data access time: $runtime seconds"
-    if [ $runtime -gt 3 ]; then
-        echo "Data access time is greater than 3 seconds, skipping the test"
+    if [ $(stat /data/imagenet/train-jpeg --format="%T" -f) != "ext2/ext3" ]; then
+        echo "Not available locally, skipping the test"
         return 0
     fi
+
     python test_RN50_data_pipeline.py --gpus ${NUM_GPUS} -b 256 --workers 3 --separate_queue \
         --cpu_size 2 --gpu_size 2 --fp16 --nhwc
     python test_RN50_data_pipeline.py --gpus ${NUM_GPUS} -b 256 --workers 3 --separate_queue \

qa/TL2_FW_iterators_perf/test_pytorch.sh

@@ -13,17 +13,11 @@ test_body() {
         python test_RN50_data_fw_iterators.py --framework ${fw} --gpus ${NUM_GPUS} -b 13 \
             --workers 3 --prefetch 2 --epochs 3
     done
-    start=`date +%s`
-    (sleep 10 && pkill -HUP ls && true) &
-    (ls /data/imagenet/train-jpeg > /dev/null && pkill -HUP sleep) &
-    wait
-    end=`date +%s`
-    runtime=$((end-start))
-    echo "Data access time: $runtime seconds"
-    if [ $runtime -gt 3 ]; then
-        echo "Data access time is greater than 3 seconds, skipping the test"
+    if [ $(stat /data/imagenet/train-jpeg --format="%T" -f) != "ext2/ext3" ]; then
+        echo "Not available locally, skipping the test"
         return 0
     fi
+
     torchrun --nproc_per_node=${NUM_GPUS} ./test_RN50_external_source_parallel_train_ddp.py /data/imagenet/train-jpeg/ --workers 6 --py_workers 6 --epochs 3 --batch_size 256 --reader_queue_depth 2 --worker_init fork --benchmark_iters 500 --test_pipes parallel
     torchrun --nproc_per_node=${NUM_GPUS} ./test_RN50_external_source_parallel_train_ddp.py /data/imagenet/train-jpeg/ --workers 6 --py_workers 6 --epochs 3 --batch_size 256 --reader_queue_depth 2 --worker_init spawn --benchmark_iters 500 --test_pipes parallel
     torchrun --nproc_per_node=${NUM_GPUS} ./test_RN50_external_source_parallel_train_ddp.py /data/imagenet/train-jpeg/ --workers 6 --py_workers 6 --epochs 3 --batch_size 256 --reader_queue_depth 2 --benchmark_iters 500 --test_pipes file_reader

qa/TL2_RN50_data_perf/test.sh

@@ -8,18 +8,11 @@ do_once() {
 }
 
 test_body() {
-    start=`date +%s`
-    (sleep 10 && pkill -HUP ls && true) &
-    (ls /data/imagenet/train-jpeg > /dev/null && pkill -HUP sleep) &
-    wait
-    end=`date +%s`
-    runtime=$((end-start))
-    echo "Data access time: $runtime seconds"
-    if [ $runtime -gt 3 ]; then
-        echo "Data access time is greater than 3 seconds, skipping the test"
+    if [ $(stat /data/imagenet/train-jpeg --format="%T" -f) != "ext2/ext3" ]; then
+        echo "Not available locally, skipping the test"
         return 0
     fi
-    # test code
+
     python test_RN50_data_pipeline.py --gpus ${NUM_GPUS} -b 256 --workers 3 --prefetch 2 --decoder_type "legacy"
     python test_RN50_data_pipeline.py --gpus ${NUM_GPUS} -b 256 --workers 3 --prefetch 2 --decoder_type "experimental"
     python test_RN50_data_pipeline.py --gpus ${NUM_GPUS} -b 16 --workers 3 --prefetch 11 --decoder_type "legacy"

qa/TL3_RN50_short/test_pytorch.sh

@@ -13,10 +13,10 @@ cd /opt/dali/docs/examples/use_cases/pytorch/resnet50
 NUM_GPUS=$(nvidia-smi -L | wc -l)
 
 if [ ! -d "val" ]; then
-   ln -sf /data_raid/imagenet/val-jpeg/ val
+   ln -sf /data/imagenet/val-jpeg/ val
 fi
 if [ ! -d "train" ]; then
-   ln -sf /data_raid/imagenet/train-jpeg/ train
+   ln -sf /data/imagenet/train-jpeg/ train
 fi
 
 LOG=dali.log
@@ -26,7 +26,7 @@ SECONDS=0
 # turn off SHARP to avoid NCCL errors
 export NCCL_NVLS_ENABLE=0
 
-torchrun --nproc_per_node=${NUM_GPUS} main.py -a resnet50 --b 256 --loss-scale 128.0 --workers 8 --lr=0.4 --fp16-mode --epochs 5 ./ 2>&1 | tee $LOG
+torchrun --nproc_per_node=${NUM_GPUS} main.py -a resnet50 --b 256 --loss-scale 128.0 --workers 8 --lr=0.4 --fp16-mode --epochs 2 ./ 2>&1 | tee $LOG
 
 RET=${PIPESTATUS[0]}
 echo "Training ran in $SECONDS seconds"
@@ -57,7 +57,12 @@ printf "TOP-1 Accuracy: %.2f%% (expect at least %f%%) %s\n" $TOP1 $MIN_TOP1 $TOP
 printf "TOP-5 Accuracy: %.2f%% (expect at least %f%%) %s\n" $TOP5 $MIN_TOP5 $TOP5_RESULT
 printf "Average perf: %.2f (expect at least %f) samples/sec %s\n" $PERF $MIN_PERF $PERF_RESULT
 
-if [[ "$TOP1_RESULT" == "OK" && "$TOP5_RESULT" == "OK" && "$PERF_RESULT" == "OK" ]]; then
+# check perf only if data is locally available
+if [ $(stat /data/imagenet/val-jpeg --format="%T" -f) == "ext2/ext3" ] && [ "$PERF_RESULT" != "OK" ]; then
+    CAN_AND_EXIT 4
+fi
+
+if [[ "$TOP1_RESULT" == "OK" && "$TOP5_RESULT" == "OK" ]]; then
     CLEAN_AND_EXIT 0
 fi
 

qa/TL3_RN50_short/test_tensorflow.sh

@@ -6,7 +6,7 @@ mkdir -p idx-files/
 
 NUM_GPUS=$(nvidia-smi -L | wc -l)
 
-DATA_SET_DIR=/data_raid/imagenet/train-val-tfrecord
+DATA_SET_DIR=/data/imagenet/train-val-tfrecord
 for file in $(ls $DATA_SET_DIR/*-of-*);
 do
     file=$(basename ${file})
@@ -69,7 +69,12 @@ printf "TOP-1 Accuracy: %.2f%% (expect at least %f%%) %s\n" $TOP1 $MIN_TOP1 $TOP
 printf "TOP-5 Accuracy: %.2f%% (expect at least %f%%) %s\n" $TOP5 $MIN_TOP5 $TOP5_RESULT
 printf "mean speed %.2f (expect at least %f) samples/sec %s\n" $PERF $MIN_PERF $PERF_RESULT
 
-if [[ "$TOP1_RESULT" == "OK" && "$TOP5_RESULT" == "OK" && "$PERF_RESULT" == "OK" ]]; then
+# check perf only if data is locally available
+if [ $(stat /data/imagenet/train-val-tfrecord --format="%T" -f) == "ext2/ext3" ] && [ "$PERF_RESULT" != "OK" ]; then
+    CAN_AND_EXIT 4
+fi
+
+if [[ "$TOP1_RESULT" == "OK" && "$TOP5_RESULT" == "OK" ]]; then
     CLEAN_AND_EXIT 0
 fi