Some more multimutant runs.

Author: Jacoberts

backend/src/folde/data.py

@@ -173,8 +173,11 @@ def get_proteingym_dataset(
     incomplete_naturalness_df["seq_id"] = incomplete_naturalness_df["seq_id"].apply(
         lambda x: maybe_modify_seq_id(dms_id, x)
     )
-    if any(incomplete_naturalness_df["wt_marginal"] <= 0):
-        raise ValueError("wt_marginal is negative or zero")
+    if any(incomplete_naturalness_df["wt_marginal"] < 0):
+        raise ValueError(f"wt_marginal for {dms_id} is negative, eg: {incomplete_naturalness_df[incomplete_naturalness_df['wt_marginal'] <= 0]['wt_marginal'].tolist()[:5]}")
+    
+    def safe_log_for_wt_naturalness(x):
+        return np.log(max(x, 1e-20))
     if (
         "model" in incomplete_naturalness_df.columns
         and incomplete_naturalness_df.model.unique().size > 1
@@ -184,8 +187,8 @@ def get_proteingym_dataset(
         # Note that we deliberately lose the wt_marginal column - we want this to crash
         # later on, unless the user specifies the proper column names.
         incomplete_naturalness_df["log_wt_marginal"] = incomplete_naturalness_df[
-            "wt_marginal"
-        ].apply(np.log)
+            "wt_marginal"   
+        ].apply(safe_log_for_wt_naturalness)
         incomplete_naturalness_df["model"] = incomplete_naturalness_df["model"].apply(
             lambda x: f"log_wt_marginal_{x}"
         )
@@ -195,7 +198,7 @@ def get_proteingym_dataset(
     else:
         # Otherwise, under normal circumstances, just set up seq_id as index.
         incomplete_naturalness_df = incomplete_naturalness_df.set_index("seq_id", drop=False)
-        incomplete_naturalness_df['log_wt_marginal'] = incomplete_naturalness_df['wt_marginal'].apply(np.log)
+        incomplete_naturalness_df['log_wt_marginal'] = incomplete_naturalness_df['wt_marginal'].apply(safe_log_for_wt_naturalness)
         incomplete_naturalness_df.drop(columns=['wt_marginal'], inplace=True)
 
     seq_ids_with_naturalness = set(incomplete_naturalness_df.index)

backend/src/notebooks/jacob/250905_multimutant_benchmark.py

@@ -177,6 +177,30 @@
                 "zero_shot_model_params.lie_noise_stddev_multiplier": 4.0,
                 "few_shot_model_params.decision_mode": "mean",
             }),
+            ModelDiff(name="with-esmv1-nocl-nowarmstart", diffs={
+                "naturalness_model_id": "1v",
+                "naturalness_columns": [
+                    "log_wt_marginal_1",
+                    "log_wt_marginal_2",
+                    "log_wt_marginal_3",
+                    "log_wt_marginal_4",
+                    "log_wt_marginal_5",
+                ],
+                "few_shot_model_params.decision_mode": "mean",
+            }),
+            ModelDiff(name="with-esmv1-zsclS4-r1only-nowarmstart", diffs={
+                "naturalness_model_id": "1v",
+                "naturalness_columns": [
+                    "log_wt_marginal_1",
+                    "log_wt_marginal_2",
+                    "log_wt_marginal_3",
+                    "log_wt_marginal_4",
+                    "log_wt_marginal_5",
+                ],
+                "zero_shot_model_params.decision_mode": "constantliar",
+                "zero_shot_model_params.lie_noise_stddev_multiplier": 4.0,
+                "few_shot_model_params.decision_mode": "mean",
+            }),
         ],
     )
 )

backend/src/notebooks/jacob/250905_test_benchmark.py

@@ -191,6 +191,30 @@
                 "zero_shot_model_params.lie_noise_stddev_multiplier": 4.0,
                 "few_shot_model_params.decision_mode": "mean",
             }),
+            ModelDiff(name="with-esmv1-nocl-nowarmstart", diffs={
+                "naturalness_model_id": "1v",
+                "naturalness_columns": [
+                    "log_wt_marginal_1",
+                    "log_wt_marginal_2",
+                    "log_wt_marginal_3",
+                    "log_wt_marginal_4",
+                    "log_wt_marginal_5",
+                ],
+                "few_shot_model_params.decision_mode": "mean",
+            }),
+            ModelDiff(name="with-esmv1-zsclS4-r1only-nowarmstart", diffs={
+                "naturalness_model_id": "1v",
+                "naturalness_columns": [
+                    "log_wt_marginal_1",
+                    "log_wt_marginal_2",
+                    "log_wt_marginal_3",
+                    "log_wt_marginal_4",
+                    "log_wt_marginal_5",
+                ],
+                "zero_shot_model_params.decision_mode": "constantliar",
+                "zero_shot_model_params.lie_noise_stddev_multiplier": 4.0,
+                "few_shot_model_params.decision_mode": "mean",
+            }),
         ],
     )
 )