Dynamically assert model temperature value in argparser

Author: dberardi99

Diff for: llm_toolkit/models.py

@@ -46,7 +46,7 @@
 # Model hyper-parameters.
 MAX_TOKENS: int = 2000
 NUM_SAMPLES: int = 1
-TEMPERATURE: float = 0.4
+TEMPERATURE: float = 1.0
 
 
 class LLM:

Diff for: run_all_experiments.py

@@ -279,6 +279,9 @@ def parse_args() -> argparse.Namespace:
 
   if args.temperature:
     assert 2 >= args.temperature >= 0, '--temperature must be within 0 and 2.'
+    
+  if args.temperature == TEMPERATURE and args.model in models.LLM.all_llm_names():
+    args.temperature = run_one_experiment.get_model_temperature(args)
 
   benchmark_yaml = args.benchmark_yaml
   if benchmark_yaml:

Diff for: run_one_experiment.py

@@ -54,7 +54,7 @@
 NUM_SAMPLES = 2
 MAX_TOKENS: int = 4096
 RUN_TIMEOUT: int = 30
-TEMPERATURE: float = 0.4
+TEMPERATURE: float = 1.0
 
 RESULTS_DIR = './results'
 
@@ -311,3 +311,30 @@ def run(benchmark: Benchmark, model: models.LLM, args: argparse.Namespace,
 
   return AggregatedResult.from_benchmark_result(
       _fuzzing_pipelines(benchmark, model, args, work_dirs))
+
+
+def get_model_temperature(args: argparse.Namespace) -> float:
+  """Retrieves model temperature default value."""
+  default_temperatures = {models.VertexAICodeBisonModel.name: 0.2,
+                          models.VertexAICodeBison32KModel.name: 0.2,
+                          models.GeminiPro.name: 0.9,
+                          models.GeminiUltra.name: 0.2,
+                          models.GeminiExperimental.name: 1.0,
+                          models.GeminiV1D5.name: 1.0,
+                          models.GeminiV2Flash.name: 1.0,
+                          models.GeminiV2.name: 1.0,
+                          models.GeminiV2Think.name: 0.7,
+                          models.ClaudeHaikuV3.name: 0.5,
+                          models.ClaudeOpusV3.name: 0.5,
+                          models.ClaudeSonnetV3D5.name: 0.5,
+                          models.GPT.name: 1.0,
+                          models.GPT4.name: 1.0,
+                          models.GPT4o.name: 1.0,
+                          models.GPT4oMini.name: 1.0,
+                          models.GPT4Turbo.name: 1.0}
+  if args.model.endswith('-chat') or args.model.endswith('-azure'):
+    model_name = '-'.join(args.model.split('-')[:-1])
+  else:
+    model_name = args.model
+  
+  return default_temperatures[model_name]