parallel tests

Dockerfile

@@ -7,8 +7,12 @@ ENV PYTHONPATH=$PYTHONPATH:/dislib
 ENV LC_ALL=C.UTF-8
 RUN python3 -m pip install --upgrade -r /dislib/requirements.txt
 
+# RUN apt-get update && apt-get install -y htop
+
 ENV COMPSS_LOAD_SOURCE false
 
+# RUN sed -i "s/>4</>5</g" /opt/COMPSs//Runtime/configuration/xml/resources/default_resources.xml
+
 # Expose SSH port and run SSHD
 EXPOSE 22
 CMD ["/usr/sbin/sshd","-D"]

Jenkinsfile

@@ -53,7 +53,6 @@ pipeline {
             sh 'docker exec dislib /dislib/bin/print_tests_logs.sh'
             sh 'docker images'
             sh 'docker rm -f dislib &> /dev/null || true'
-            sh 'docker rmi -f bscwdc/dislib &> /dev/null || true'
         }
         success {
             setGithubCommitStatus('success', 'Build Successful')

run_tests.sh

@@ -1,20 +1,56 @@
 #!/bin/bash -e
 
 # Default process per worker
-export ComputingUnits=4
+export ComputingUnits=2
 
-# Run the tests/__main__.py file which calls all the tests named test_*.py
-runcompss \
-    --pythonpath=$(pwd) \
-    --python_interpreter=python3 \
-    ./tests/__main__.py &> >(tee output.log)
+declare -a tests_group=("test_lasso" 
+                "test_array test_pca test_daura"
+                "test_gm test_preproc test_decision_tree"
+                "test_qr test_kmeans test_knn"
+                "test_gridsearch test_tsqr test_linear_regression"
+                "test_dbscan test_matmul test_als"
+                "test_rf_classifier test_randomizedsearch test_data_utils test_kfold"
+                "test_csvm test_rf_regressor test_utils test_rf_dataset"
+                )
+
+declare -a pids
+
+port=43000
+workerid=1
+
+for t in "${tests_group[@]}"
+do
+
+    nextport=$((port + 1))
+
+    sed "s/<MinPort>43001<\/MinPort>/<MinPort>$port<\/MinPort>/g" /opt/COMPSs/Runtime/configuration/xml/resources/default_resources.xml > /tmp/resources-$port.xml
+    sed -i "s/<MaxPort>43002<\/MaxPort>/<MaxPort>$nextport<\/MaxPort>/g" /tmp/resources-$port.xml
+
+    runcompss \
+        --log_level=debug \
+        --pythonpath=$(pwd) \
+        --python_interpreter=python3 \
+        --resources=/tmp/resources-$port.xml \
+        --master_port=$port \
+        ./tests/__main__.py $t -id $workerid &> >(tee output.log) &
+
+    pids+=($!)
+
+    workerid=$((workerid + 1))
+
+    port=$((port + 2))
+    sleep 10
+done
+
+wait ${pids[@]}
 
 # Check the unittest output because PyCOMPSs exits with code 0 even if there
 # are failed tests (the execution itself is successful)
 result=$(cat output.log | egrep "OK|FAILED")
 
 echo "Tests result: ${result}"
 
+
 # If word Failed is in the results, exit 1 so the pull request fails
 if [[ $result =~ FAILED ]]; then 
         exit 1

tests/__init__.py

@@ -1,12 +1,16 @@
-from time import time
+import datetime
 import unittest
 
 
 class BaseTimedTestCase(unittest.TestCase):
     def setUp(self):
-        self.start_time = time()
+        # self.start_time = time()
+        print(f'Test {self.id()} START AT',
+              datetime.datetime.now(), flush=True)
 
     def tearDown(self):
-        self.end_time = time()
-        print("Test %s took: %.3f seconds" %
-              (self.id(), self.end_time - self.start_time))
+        print(f'Test {self.id()} END AT',
+              datetime.datetime.now(), flush=True)
+        # self.end_time = time()
+        # print("Test %s took: %.3f seconds" %
+        #       (self.id(), self.end_time - self.start_time))

tests/__main__.py

@@ -1,9 +1,31 @@
 import unittest
+import argparse
+import datetime
 
+if __name__ == '__main__':
+    suite = list(unittest.loader.defaultTestLoader.discover('./tests/'))
+    parser = argparse.ArgumentParser(description='Process some integers.')
+    parser.add_argument('tests', metavar='T', type=str, nargs='+',
+                        help='an integer for the accumulator')
+    parser.add_argument('-id', type=int)
 
-def load_tests(loader, tests, pattern):
-    return loader.discover('./tests/')
+    args = parser.parse_args()
 
+    print(f'WORKER {args.id} FIND TESTS',
+          datetime.datetime.now(), flush=True)
 
-if __name__ == '__main__':
-    unittest.main(verbosity=2)
+    tests_to_run = []
+    for t in args.tests:
+        for test_case in suite:
+            if t.lower() in str(test_case).lower():
+                tests_to_run.append(test_case)
+
+    print(f'WORKER {args.id} START TEST AT',
+          datetime.datetime.now(), flush=True)
+
+    test_suite = unittest.TestSuite()
+    test_suite.addTests(tests_to_run)
+    unittest.TextTestRunner(verbosity=2).run(test_suite)
+
+    print(f'WORKER {args.id} END TEST AT',
+          datetime.datetime.now(), flush=True)