Use test_cases.txt file to determine the tests to run

Author: felsenhower

conftest.py

@@ -43,7 +43,7 @@ def reference_output_data():
 
 def pytest_generate_tests(metafunc):
     if "test_id" in metafunc.fixturenames:
-        test_ids = list(util.get_reference_output_data_map().keys())
+        test_ids = [" ".join(line) for line in util.get_test_cases()]
         max_num_tests = metafunc.config.getoption("max_num_tests")
         if max_num_tests:
             test_ids = test_ids[:max_num_tests]

make_reference_output.sh

@@ -16,6 +16,7 @@ fi
 
 TIMEOUT_DURATION=2
 OUTPUT_DIRECTORY='./reference_output'
+TEST_CASES_FILE='./test_cases.txt'
 
 PARAM_NUM_RANGE=(1)
 PARAM_METHOD_RANGE=(1 2)
@@ -39,6 +40,7 @@ function run_partdiff {
         output_filename="$(printf '%s/partdiff_%s_%s_%s_%s_%s_%s.txt\n' "$OUTPUT_DIRECTORY" "$num" "$method" "$lines" "$func" "$term" "$preciter")"
         echo "$output" > "$output_filename"
         echo '  (OK)'
+        echo "$num" "$method" "$lines" "$func" "$term" "$preciter" >> "$TEST_CASES_FILE"
         return
     fi
     timed_out="$((exit_code == 124))"
@@ -51,6 +53,7 @@ function run_partdiff {
 }
 
 mkdir -p "$OUTPUT_DIRECTORY"
+true > "$TEST_CASES_FILE"
 
 (
     param_term_range=(2)

test_cases.txt

@@ -0,0 +1,148 @@
+1 1 0 1 2 1
+1 1 0 1 2 10
+1 1 0 1 2 100
+1 1 0 1 2 1000
+1 1 0 1 2 10000
+1 1 0 1 2 100000
+1 1 0 2 2 1
+1 1 0 2 2 10
+1 1 0 2 2 100
+1 1 0 2 2 1000
+1 1 0 2 2 10000
+1 1 0 2 2 100000
+1 1 1 1 2 1
+1 1 1 1 2 10
+1 1 1 1 2 100
+1 1 1 1 2 1000
+1 1 1 1 2 10000
+1 1 1 1 2 100000
+1 1 1 2 2 1
+1 1 1 2 2 10
+1 1 1 2 2 100
+1 1 1 2 2 1000
+1 1 1 2 2 10000
+1 1 1 2 2 100000
+1 1 10 1 2 1
+1 1 10 1 2 10
+1 1 10 1 2 100
+1 1 10 1 2 1000
+1 1 10 1 2 10000
+1 1 10 2 2 1
+1 1 10 2 2 10
+1 1 10 2 2 100
+1 1 10 2 2 1000
+1 1 10 2 2 10000
+1 1 100 1 2 1
+1 1 100 1 2 10
+1 1 100 1 2 100
+1 1 100 2 2 1
+1 1 100 2 2 10
+1 1 100 2 2 100
+1 1 1000 1 2 1
+1 1 1000 2 2 1
+1 2 0 1 2 1
+1 2 0 1 2 10
+1 2 0 1 2 100
+1 2 0 1 2 1000
+1 2 0 1 2 10000
+1 2 0 1 2 100000
+1 2 0 2 2 1
+1 2 0 2 2 10
+1 2 0 2 2 100
+1 2 0 2 2 1000
+1 2 0 2 2 10000
+1 2 0 2 2 100000
+1 2 1 1 2 1
+1 2 1 1 2 10
+1 2 1 1 2 100
+1 2 1 1 2 1000
+1 2 1 1 2 10000
+1 2 1 1 2 100000
+1 2 1 2 2 1
+1 2 1 2 2 10
+1 2 1 2 2 100
+1 2 1 2 2 1000
+1 2 1 2 2 10000
+1 2 1 2 2 100000
+1 2 10 1 2 1
+1 2 10 1 2 10
+1 2 10 1 2 100
+1 2 10 1 2 1000
+1 2 10 1 2 10000
+1 2 10 1 2 100000
+1 2 10 2 2 1
+1 2 10 2 2 10
+1 2 10 2 2 100
+1 2 10 2 2 1000
+1 2 10 2 2 10000
+1 2 100 1 2 1
+1 2 100 1 2 10
+1 2 100 1 2 100
+1 2 100 1 2 1000
+1 2 100 2 2 1
+1 2 100 2 2 10
+1 2 100 2 2 100
+1 2 1000 1 2 1
+1 2 1000 1 2 10
+1 2 1000 2 2 1
+1 1 0 1 1 1e-4
+1 1 0 1 1 1e-8
+1 1 0 1 1 1e-12
+1 1 0 1 1 1e-16
+1 1 0 1 1 1e-20
+1 1 0 2 1 1e-4
+1 1 0 2 1 1e-8
+1 1 0 2 1 1e-12
+1 1 0 2 1 1e-16
+1 1 0 2 1 1e-20
+1 1 1 1 1 1e-4
+1 1 1 1 1 1e-8
+1 1 1 1 1 1e-12
+1 1 1 1 1 1e-16
+1 1 1 1 1 1e-20
+1 1 1 2 1 1e-4
+1 1 1 2 1 1e-8
+1 1 1 2 1 1e-12
+1 1 1 2 1 1e-16
+1 1 1 2 1 1e-20
+1 1 10 1 1 1e-4
+1 1 10 1 1 1e-8
+1 1 10 1 1 1e-12
+1 1 10 1 1 1e-16
+1 1 10 1 1 1e-20
+1 1 10 2 1 1e-4
+1 1 10 2 1 1e-8
+1 1 10 2 1 1e-12
+1 1 10 2 1 1e-16
+1 1 10 2 1 1e-20
+1 1 100 2 1 1e-4
+1 1 1000 2 1 1e-4
+1 2 0 1 1 1e-4
+1 2 0 1 1 1e-8
+1 2 0 1 1 1e-12
+1 2 0 1 1 1e-16
+1 2 0 1 1 1e-20
+1 2 0 2 1 1e-4
+1 2 0 2 1 1e-8
+1 2 0 2 1 1e-12
+1 2 0 2 1 1e-16
+1 2 0 2 1 1e-20
+1 2 1 1 1 1e-4
+1 2 1 1 1 1e-8
+1 2 1 1 1 1e-12
+1 2 1 1 1 1e-16
+1 2 1 1 1 1e-20
+1 2 1 2 1 1e-4
+1 2 1 2 1 1e-8
+1 2 1 2 1 1e-12
+1 2 1 2 1 1e-16
+1 2 1 2 1 1e-20
+1 2 10 1 1 1e-4
+1 2 10 1 1 1e-8
+1 2 10 1 1 1e-12
+1 2 10 1 1 1e-16
+1 2 10 1 1 1e-20
+1 2 10 2 1 1e-4
+1 2 10 2 1 1e-8
+1 2 100 2 1 1e-4
+1 2 1000 2 1 1e-4

util.py

@@ -128,6 +128,7 @@
 ]
 
 REFERENCE_OUTPUT_PATH = Path.cwd() / "reference_output"
+TEST_CASES_FILE_PATH = Path.cwd() / "test_cases.txt"
 
 RE_REF_OUTPUT_FILE = re.compile(
     r"""
@@ -157,6 +158,22 @@ def iter_reference_output_data():
         yield (partdiff_params, reference_output)
 
 
+def iter_test_cases():
+    with TEST_CASES_FILE_PATH.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            fields = line.split()
+            assert len(fields) == 6
+            yield tuple(fields)
+
+
+@cache
+def get_test_cases():
+    return list(iter_test_cases())
+
+
 @cache
 def get_reference_output_data_map():
     return {