fix duplication issues

Author: maxmatical

bigcode_eval/evaluator.py

@@ -59,14 +59,14 @@ def generate_text(self, task_name, intermediate_generations=None):
                 solutions = [[ref] for ref in references]
             return solutions, references
 
-        generations = []  # list[list[str | None] | None]
+        curr_generations = []  # list[list[str | None] | None]
         if intermediate_generations:
-            generations = [gen for gen in intermediate_generations if gen]
-            n_tasks -= len(generations)
+            curr_generations = [gen for gen in intermediate_generations if gen]
+            n_tasks -= len(curr_generations)
         intermediate_save_generations_path = f"{os.path.splitext(self.args.save_generations_path)[0]}_{task_name}_intermediate.json"
-        curr_sample_idx = len(generations)
+        curr_sample_idx = len(curr_generations)
 
-        new_generations = parallel_generations(
+        generations = parallel_generations(
             task,
             dataset,
             self.accelerator,
@@ -76,10 +76,9 @@ def generate_text(self, task_name, intermediate_generations=None):
             args=self.args,
             curr_sample_idx=curr_sample_idx,  # curr_sample_idx will added to limit_start to fix indexing
             save_every_k_tasks=self.args.save_every_k_tasks,
-            intermediate_generations=generations,
+            intermediate_generations=curr_generations,
             intermediate_save_generations_path=intermediate_save_generations_path,
         )
-        generations.extend(new_generations)
 
         if len(generations[0]) > self.args.n_samples:
             generations = [l[: self.args.n_samples] for l in generations]

bigcode_eval/utils.py

@@ -3,6 +3,7 @@
 import re
 import warnings
 from collections import defaultdict
+from copy import deepcopy
 from typing import List, Optional
 
 import torch
@@ -334,8 +335,9 @@ def complete_code(
                     gen_token_dict,
                 )
                 with open(intermediate_save_generations_path, "w") as fp:
-                    intermediate_generations.extend(code_gens)
-                    json.dump(intermediate_generations, fp)
+                    intermediate_save_generations = deepcopy(intermediate_generations)
+                    intermediate_save_generations.extend(code_gens)
+                    json.dump(intermediate_save_generations, fp)
                     print(
                         f"intermediate generations were saved at {intermediate_save_generations_path}"
                     )
@@ -353,7 +355,7 @@ def complete_code(
         gen_token_dict,
     )
 
-    return code_gens
+    return intermediate_generations.extend(code_gens)
 
 
 def update_code_gens(