Merge pull request #1479 from byuccl/parse_check_golden_issue

Vtr_reg_strong Some Regression tests results are not being checked

Author: vaughnbetz

.travis.yml

@@ -80,7 +80,7 @@ jobs:
       name: "Code Formatting - C/C++"
       script:
         - ./dev/check-format.sh
-    - state: Test
+    - stage: Test
       name: "Code Formatting - Python"
       script:
         - ./dev/check-format-py.sh

vtr_flow/scripts/parse_vtr_task.pl

@@ -130,9 +130,15 @@
 exit $num_golden_failures;
 
 sub parse_single_task {
+	# This routine can operate in a few different ways":
+		# Parse a result file and compare it to a golen (prior) result. This is the default functionality
+		# Parse two result files and check equivelence if a second file is provided in the config file. This option disables the comparison to the golden results.
+		# Optionally, in conjunction with either of the two above options, a qor_parse_file can be provided in the config file and also be compared to the first parsed result file
+
+
 	my $task_name = shift;
 	(my $task_path = $task_name) =~ s/\s+$//;
-
+	
     # first see if task_name is the task path
 	if (! -e "$task_path/config/config.txt") {
 	    ($task_path = "$vtr_flow_path/tasks/$task_name") =~ s/\s+$//;
@@ -150,6 +156,10 @@ sub parse_single_task {
 	my $qor_parse_file;
         my $second_parse_file;
         my $counter = 0;
+
+	#determine if this test individual test is compared to the golden values or not. 
+	my $single_check_golden = $check_golden;
+
 	foreach my $line (@config_data) {
 
 		# Ignore comments
@@ -173,8 +183,9 @@ sub parse_single_task {
                                     #second time parse file
                                     $second_parse_file = expand_user_path($value);
                                     $counter = 0;
-                                    #don't need to check golden, only compare between two files
-                                    $check_golden = 0;
+                                    #don't need to check golden for this test, only compare between two files
+									#If we used the check_golden vairable here, any test following this test would also not compare to golden values. 
+                                    $single_check_golden = 0;
                         }
                         else{
                             $parse_file = expand_user_path($value);
@@ -281,11 +292,10 @@ sub parse_single_task {
         return check_two_files ( $task_name, $task_path, $run_path, "$run_path/parse_results.txt", "$run_path/parse_results_2.txt", 0);
 
     }
-
-	if ($check_golden) {
+	if ($single_check_golden) {
         #Returns 1 if failed
         if ($second_parse_file eq ""){
-            return check_two_files( $task_name, $task_path, $run_path, "$run_path/parse_results.txt", "$task_path/config/golden_results.txt", $check_golden);
+            return check_two_files( $task_name, $task_path, $run_path, "$run_path/parse_results.txt", "$task_path/config/golden_results.txt", $single_check_golden);
         }
 	}
 

vtr_flow/scripts/python_libs/vtr/flow.py

@@ -31,24 +31,26 @@ def __ge__(self, other):
 
 # pylint: disable=too-many-arguments, too-many-locals, too-many-branches, too-many-statements
 def run(
-    architecture_file,
-    circuit_file,
-    power_tech_file=None,
-    start_stage=VtrStage.odin,
-    end_stage=VtrStage.vpr,
-    command_runner=vtr.CommandRunner(),
-    temp_dir=Path("./temp"),
-    odin_args=None,
-    abc_args=None,
-    vpr_args=None,
-    keep_intermediate_files=True,
-    keep_result_files=True,
-    min_hard_mult_size=3,
-    min_hard_adder_size=1,
-    check_equivalent=False,
-    check_incremental_sta_consistency=False,
-    use_old_abc_script=False,
-    relax_w_factor=1.3,
+        architecture_file,
+        circuit_file,
+        power_tech_file=None,
+        start_stage=VtrStage.odin,
+        end_stage=VtrStage.vpr,
+        command_runner=vtr.CommandRunner(),
+        temp_dir=Path("./temp"),
+        odin_args=None,
+        abc_args=None,
+        vpr_args=None,
+        keep_intermediate_files=True,
+        keep_result_files=True,
+        min_hard_mult_size=3,
+        min_hard_adder_size=1,
+        check_equivalent=False,
+        check_incremental_sta_consistency=False,
+        use_old_abc_script=False,
+        relax_w_factor=1.3,
+        check_route = False,
+        check_place = False,
 ):
     """
     Runs the VTR CAD flow to map the specified circuit_file onto the target architecture_file
@@ -118,6 +120,11 @@ def run(
         check_incremental_sta_consistency :
             Do a second-run of the incremental analysis to compare the result files
 
+        check_route:
+            Check first placement and routing by enabling VPR analysis
+
+        check_place:
+            Route existing placement by enabling VPR routing.
     """
 
     #
@@ -236,7 +243,7 @@ def run(
         # Copy the input netlist for input to vpr
         shutil.copyfile(str(next_stage_netlist), str(pre_vpr_netlist))
         route_fixed_w = "route_chan_width" in vpr_args
-        if ("route" in vpr_args or "place" in vpr_args) and not route_fixed_w:
+        if (check_route or check_place) and not route_fixed_w:
             vpr_args["route_chan_width"] = 300
             route_fixed_w = True
 
@@ -245,16 +252,8 @@ def run(
             do_second_run = False
             second_run_args = vpr_args
 
-            if "write_rr_graph" in vpr_args:
-                do_second_run = True
-
-            if "analysis" in vpr_args:
-                do_second_run = True
-                del vpr_args["analysis"]
-
-            if "route" in vpr_args:
+            if "write_rr_graph" in vpr_args or  "analysis" in vpr_args or "route" in vpr_args:
                 do_second_run = True
-                del vpr_args["route"]
 
             vtr.vpr.run(
                 architecture_copy,
@@ -272,6 +271,10 @@ def run(
                     if "write_rr_graph" in second_run_args
                     else ".xml"
                 )
+                if check_place:
+                    second_run_args["route"] = True
+                if check_route:
+                    second_run_args["analysis"] = True
                 vtr.vpr.run_second_time(
                     architecture_copy,
                     pre_vpr_netlist,

vtr_flow/scripts/python_libs/vtr/util.py

@@ -12,8 +12,8 @@
 import argparse
 import csv
 from collections import OrderedDict
-
-from vtr.error import VtrError, InspectError, CommandError
+import vtr.error
+from vtr.error import VtrError, CommandError
 
 VERBOSITY_CHOICES = range(5)
 
@@ -462,11 +462,13 @@ def load_config_lines(filepath, allow_includes=True):
                             include_file_abs, allow_includes=allow_includes
                         )
                     else:
-                        raise InspectError("@include not allowed in this file", filepath)
+                        raise vtr.error.InspectError(
+                            "@include not allowed in this file", filepath
+                        )
                 else:
                     config_lines.append(line)
     except IOError as error:
-        raise InspectError("Error opening config file ({})".format(error)) from error
+        raise vtr.error.InspectError("Error opening config file ({})".format(error))
 
     return config_lines
 

vtr_flow/scripts/python_libs/vtr/vpr/vpr.py

@@ -54,7 +54,7 @@ def run_relax_w(
             Path to the VPR executable
 
         logfile_base:
-            Base name for log files (e.g. "vpr" produces vpr.min_w.out, vpr.relaxed_w.out)
+            Base name for log files (e.g. "vpr" produces vpr.out, vpr.crit_path.out)
 
         vpr_args:
             Extra arguments for VPR
@@ -79,12 +79,6 @@ def run_relax_w(
     if "write_rr_graph" in vpr_args:
         del vpr_args["write_rr_graph"]
 
-    if "analysis" in vpr_args:
-        del vpr_args["analysis"]
-
-    if "route" in vpr_args:
-        del vpr_args["route"]
-
     if vpr_exec is None:
         vpr_exec = find_vtr_file("vpr", is_executable=True)
 
@@ -98,8 +92,8 @@ def run_relax_w(
         vpr_exec=vpr_exec,
         vpr_args=vpr_args,
     )
-
-    if ("pack" in vpr_args or "place" in vpr_args) and "route" not in vpr_args:
+    explicit = "pack" in vpr_args or "place" in vpr_args or "analysis" in vpr_args
+    if explicit and "route" not in vpr_args:
         # Don't look for min W if routing was not run
         return
     if max_router_iterations:

vtr_flow/scripts/run_vtr_flow.py

@@ -431,6 +431,8 @@ def vtr_command_main(arg_list, prog=None):
             check_incremental_sta_consistency=args.check_incremental_sta_consistency,
             use_old_abc_script=args.use_old_abc_script,
             relax_w_factor=args.relax_w_factor,
+            check_route = args.check_route,
+            check_place = args.check_place,
         )
         error_status = "OK"
     except vtr.VtrError as error:
@@ -572,10 +574,6 @@ def process_vpr_args(args, prog, temp_dir, vpr_args):
     if args.verify_rr_graph:
         rr_graph_out_file = "rr_graph" + args.rr_graph_ext
         vpr_args["write_rr_graph"] = rr_graph_out_file
-    if args.check_place:
-        vpr_args["route"] = True
-    if args.check_route:
-        vpr_args["analysis"] = True
 
     return vpr_args
 
@@ -629,7 +627,7 @@ def except_vtr_error(error, expect_fail, verbose):
             print("\tfull command: ", " ".join(error.cmd))
             print("\treturncode  : ", error.returncode)
             print("\tlog file    : ", error.log)
-        exit_status = 1
+        exit_status = error.returncode
     elif isinstance(error, vtr.InspectError):
         # Something went wrong gathering information
         print("\tfile        : ", error.filename)

vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_clock_aliases_set_delay/config/golden_results.txt

@@ -0,0 +1,2 @@
+arch                  	circuit                     	script_params                                           	vtr_flow_elapsed_time	error	odin_synth_time	max_odin_mem	abc_depth	abc_synth_time	abc_cec_time	abc_sec_time	max_abc_mem	ace_time	max_ace_mem	num_clb	num_io	num_memories	num_mult	vpr_status	vpr_revision                	vpr_build_info                	vpr_compiler                                	vpr_compiled       	hostname    	rundir                                                                                                                                                                                                                                       	max_vpr_mem	num_primary_inputs	num_primary_outputs	num_pre_packed_nets	num_pre_packed_blocks	num_netlist_clocks	num_post_packed_nets	num_post_packed_blocks	device_width	device_height	device_grid_tiles	device_limiting_resources	device_name	pack_time	placed_wirelength_est	place_time	place_quench_time	placed_CPD_est	placed_setup_TNS_est	placed_setup_WNS_est	placed_geomean_nonvirtual_intradomain_critical_path_delay_est	place_quench_timing_analysis_time	place_quench_sta_time	place_total_timing_analysis_time	place_total_sta_time	min_chan_width	routed_wirelength	min_chan_width_route_success_iteration	logic_block_area_total	logic_block_area_used	min_chan_width_routing_area_total	min_chan_width_routing_area_per_tile	min_chan_width_route_time	min_chan_width_total_timing_analysis_time	min_chan_width_total_sta_time	crit_path_routed_wirelength	crit_path_route_success_iteration	crit_path_total_nets_routed	crit_path_total_connections_routed	crit_path_total_heap_pushes	crit_path_total_heap_pops	critical_path_delay	geomean_nonvirtual_intradomain_critical_path_delay	setup_TNS	setup_WNS	hold_TNS	hold_WNS	crit_path_routing_area_total	crit_path_routing_area_per_tile	crit_path_route_time	crit_path_total_timing_analysis_time	crit_path_total_sta_time
+timing/k6_N10_40nm.xml	clock_set_delay_aliases.blif	common_-sdc_file_sdc/samples/clock_aliases/set_delay.sdc	0.13                 	     	-1             	-1          	-1       	-1            	-1          	-1          	-1         	-1      	-1         	2      	2     	-1          	-1      	success   	v8.0.0-2368-g3c56542c2-dirty	release IPO VTR_ASSERT_LEVEL=2	GNU 7.5.0 on Linux-4.15.0-111-generic x86_64	2020-08-07T14:08:33	goeders-ssh0	/home/shadtorrie/git/vtr-verilog-to-routing/vtr_flow/tasks/regression_tests/vtr_reg_strong/strong_clock_aliases_set_delay/run116/timing/k6_N10_40nm.xml/clock_set_delay_aliases.blif/common_-sdc_file_sdc/samples/clock_aliases/set_delay.sdc	8512       	2                 	2                  	22                 	24                   	2                 	4                   	6                     	4           	4            	16               	clb                      	auto       	0.01     	4                    	0.01      	0.00             	1.293         	0                   	0                   	1.293                                                        	1.181e-05                        	8.064e-06            	0.000109995                     	9.1012e-05          	6             	20               	2                                     	215576                	107788               	3924.73                          	245.296                             	0.01                     	0.000804665                              	0.000648399                  	7                          	2                                	3                          	3                                 	135                        	78                       	1.293              	1.293                                             	0        	0        	0       	0       	5503.53                     	343.971                        	0.00                	0.000288477                         	0.000259906