diff --git a/.gitignore b/.gitignore index d858b90..db07c95 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ __pycache__ .env *.jpg testood/ -finetune/ \ No newline at end of file +finetune*/ \ No newline at end of file diff --git a/lamstare/experiments/plt_lcurve.py b/lamstare/experiments/plt_lcurve.py index 108e8df..3ecb7e1 100644 --- a/lamstare/experiments/plt_lcurve.py +++ b/lamstare/experiments/plt_lcurve.py @@ -1,4 +1,5 @@ from pathlib import Path +from lamstare.experiments.plt_test import COLOR from lamstare.utils.plot import fetch_lcurve, sendimg from lamstare.utils.dptest import get_head_weights import matplotlib.pyplot as plt @@ -14,69 +15,75 @@ -def main(exp_path:str, roll:int=50): - run_id=exp_path.split("/")[-1] # Get basename as id +def main(exp_paths:list, roll:int=10): try: - weights = get_head_weights(exp_path) - heads = list(get_head_weights(exp_path).keys()) + weights = get_head_weights(exp_paths[0]) + heads = list(get_head_weights(exp_paths[0]).keys()) except KeyError: heads = [""] # single task n_heads = len(heads) - mult_hist = fetch_lcurve(exp_path) fig, ax = plt.subplots(n_heads, 3, figsize=(12,2*n_heads+1),sharex=True) - if n_heads == 1: - ax[0].loglog(mult_hist["step"], mult_hist[f"rmse_e_trn"].rolling(roll).mean(), linestyle='-',color="blue") - ax[0].loglog(mult_hist["step"], mult_hist[f"rmse_e_val"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[0].set_ylabel(f"rmse_e") + for i, exp_path in enumerate(exp_paths): + color = COLOR[i] + mult_hist = fetch_lcurve(exp_path) + run_id=exp_path.split("/")[-1] # Get basename as id + if n_heads == 1: + ax[0].loglog(mult_hist["step"], mult_hist[f"rmse_e_trn"].rolling(roll).mean(), linestyle='-',color=color) + ax[0].loglog(mult_hist["step"], mult_hist[f"rmse_e_val"].rolling(roll).mean(), linestyle='-.',color=color) + ax[0].set_ylabel(f"rmse_e") - ax[1].loglog(mult_hist["step"], mult_hist[f"rmse_f_trn"].rolling(roll).mean(), linestyle='-',color="blue") - ax[1].loglog(mult_hist["step"], mult_hist[f"rmse_f_val"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[1].set_ylabel(f"rmse_f") + ax[1].loglog(mult_hist["step"], mult_hist[f"rmse_f_trn"].rolling(roll).mean(), linestyle='-',color=color) + ax[1].loglog(mult_hist["step"], mult_hist[f"rmse_f_val"].rolling(roll).mean(), linestyle='-.',color=color) + ax[1].set_ylabel(f"rmse_f") - ax[2].loglog(mult_hist["step"], mult_hist[f"rmse_v_trn"].rolling(roll).mean(), linestyle='-',color="blue") - ax[2].loglog(mult_hist["step"], mult_hist[f"rmse_v_val"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[2].set_ylabel(f"rmse_v") - fig.suptitle(run_id) - else: - for i, head in enumerate(heads): - ax[i][0].loglog(mult_hist["step"], mult_hist[f"rmse_e_trn_{head}"].rolling(roll).mean(), linestyle='-',color="blue") - ax[i][0].loglog(mult_hist["step"], mult_hist[f"rmse_e_val_{head}"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[i][0].set_ylabel(f"rmse_e_{head}") + ax[2].loglog(mult_hist["step"], mult_hist[f"rmse_v_trn"].rolling(roll).mean(), linestyle='-',color=color) + ax[2].loglog(mult_hist["step"], mult_hist[f"rmse_v_val"].rolling(roll).mean(), linestyle='-.',color=color) + ax[2].set_ylabel(f"rmse_v") + fig.suptitle(run_id) + else: + for i, head in enumerate(heads): + ax[i][0].loglog(mult_hist["step"], mult_hist[f"rmse_e_trn_{head}"].rolling(roll).mean(), linestyle='-',color=color) + ax[i][0].loglog(mult_hist["step"], mult_hist[f"rmse_e_val_{head}"].rolling(roll).mean(), linestyle='-.',color=color) + ax[i][0].set_ylabel(f"rmse_e_{head}") - ax[i][1].loglog(mult_hist["step"], mult_hist[f"rmse_f_trn_{head}"].rolling(roll).mean(), linestyle='-',color="blue") - ax[i][1].loglog(mult_hist["step"], mult_hist[f"rmse_f_val_{head}"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[i][1].set_ylabel(f"rmse_f_{head}") + ax[i][1].loglog(mult_hist["step"], mult_hist[f"rmse_f_trn_{head}"].rolling(roll).mean(), linestyle='-',color=color) + ax[i][1].loglog(mult_hist["step"], mult_hist[f"rmse_f_val_{head}"].rolling(roll).mean(), linestyle='-.',color=color) + ax[i][1].set_ylabel(f"rmse_f_{head}") - ax[i][2].loglog(mult_hist["step"], mult_hist[f"rmse_v_trn_{head}"].rolling(roll).mean(), linestyle='-',color="blue") - ax[i][2].loglog(mult_hist["step"], mult_hist[f"rmse_v_val_{head}"].rolling(roll).mean(), linestyle='-.',color="blue") - ax[i][2].set_ylabel(f"rmse_v_{head}") + ax[i][2].loglog(mult_hist["step"], mult_hist[f"rmse_v_trn_{head}"].rolling(roll).mean(), linestyle='-',color=color) + ax[i][2].loglog(mult_hist["step"], mult_hist[f"rmse_v_val_{head}"].rolling(roll).mean(), linestyle='-.',color=color) + ax[i][2].set_ylabel(f"rmse_v_{head}") - if head in BASELINE_MAP: - baseline_hist = fetch_lcurve(BASELINE_MAP[head]) - STEP_NORMAL_PREF = sum(weights.values())/weights[head]*128/120 # need to adjust this value + if head in BASELINE_MAP: + baseline_hist = fetch_lcurve(BASELINE_MAP[head]) + STEP_NORMAL_PREF = sum(weights.values())/weights[head]*128/120 # need to adjust this value - ax[i][0].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_e_trn"].rolling(1000).mean(), linestyle='-',color="red") - ax[i][0].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_e_val"].rolling(1000).mean(), linestyle='-.',color="red") - ax[i][1].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_f_trn"].rolling(1000).mean(), linestyle='-',color="red") - ax[i][1].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_f_val"].rolling(1000).mean(), linestyle='-.',color="red") - if "rmse_v_val" in baseline_hist: - ax[i][2].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_v_trn"].rolling(1000).mean(), linestyle='-',color="red") - ax[i][2].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_v_val"].rolling(1000).mean(), linestyle='-.',color="red") - if head in PREVIOUS_BASELINE: - ax[i][0].axhline(PREVIOUS_BASELINE[head]["rmse_e"],color="green", linestyle="-.") - ax[i][0].axhline(PREVIOUS_BASELINE[head]["e_std"],color="purple", linestyle="-.") - ax[i][1].axhline(PREVIOUS_BASELINE[head]["rmse_f"],color="green", linestyle="-.") - ax[i][1].axhline(PREVIOUS_BASELINE[head]["f_std"],color="purple", linestyle="-.") - ax[i][2].axhline(PREVIOUS_BASELINE[head]["rmse_v"],color="green", linestyle="-.") - ax[i][2].axhline(PREVIOUS_BASELINE[head]["v_std"],color="purple", linestyle="-.") + ax[i][0].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_e_trn"].rolling(1000).mean(), linestyle='-',color="red") + ax[i][0].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_e_val"].rolling(1000).mean(), linestyle='-.',color="red") + ax[i][1].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_f_trn"].rolling(1000).mean(), linestyle='-',color="red") + ax[i][1].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_f_val"].rolling(1000).mean(), linestyle='-.',color="red") + if "rmse_v_val" in baseline_hist: + ax[i][2].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_v_trn"].rolling(1000).mean(), linestyle='-',color="red") + ax[i][2].loglog([s * STEP_NORMAL_PREF for s in baseline_hist["step"]], baseline_hist[f"rmse_v_val"].rolling(1000).mean(), linestyle='-.',color="red") + if head in PREVIOUS_BASELINE: + ax[i][0].axhline(PREVIOUS_BASELINE[head]["rmse_e"],color="green", linestyle="-.") + ax[i][0].axhline(PREVIOUS_BASELINE[head]["e_std"],color="purple", linestyle="-.") + ax[i][1].axhline(PREVIOUS_BASELINE[head]["rmse_f"],color="green", linestyle="-.") + ax[i][1].axhline(PREVIOUS_BASELINE[head]["f_std"],color="purple", linestyle="-.") + ax[i][2].axhline(PREVIOUS_BASELINE[head]["rmse_v"],color="green", linestyle="-.") + ax[i][2].axhline(PREVIOUS_BASELINE[head]["v_std"],color="purple", linestyle="-.") plt.tight_layout() fig.savefig("lcurve.jpg") sendimg(["lcurve.jpg"], run_id) if __name__ == "__main__": - for exp_path in ["/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24"]: - main(exp_path) + exp_path = [ + # "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24", + # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_384_96_24", + "/mnt/data_nas/public/multitask/training_exps/1220_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_64" + ] + main(exp_path) # main("/mnt/data_nas/public/multitask/training_exps/1018_b4_medium_l6_atton_37head_linear_fitting_tanh") \ No newline at end of file diff --git a/lamstare/experiments/plt_ood.py b/lamstare/experiments/plt_ood.py index 000c61d..2edc013 100644 --- a/lamstare/experiments/plt_ood.py +++ b/lamstare/experiments/plt_ood.py @@ -1,6 +1,6 @@ from functools import lru_cache import os - +from typing import Optional import matplotlib.pyplot as plt import numpy as np import pandas @@ -14,7 +14,7 @@ from lamstare.utils.plot import sendimg -with open(os.path.dirname(__file__) + "/../release/OOD_DATASET.yml", "r") as f: +with open(os.path.dirname(__file__) + "/../release/ood_test/OOD_DATASET_v2.yml", "r") as f: OOD_DATASET = yaml.load(f, Loader=yaml.FullLoader) OOD_DATASET = ( DataFrame(OOD_DATASET["OOD_TO_HEAD_MAP"]).T.rename_axis("Dataset").infer_objects() @@ -27,7 +27,7 @@ print(OOD_DATASET) OOD_DATASET_STD = pandas.read_csv( - "/mnt/workspace/cc/LAMstare_new/lamstare/release/ood_data_std.csv" + "/mnt/workspace/cc/LAMstare_new/lamstare/release/ood_test/ood_data_std.csv" ).infer_objects() OOD_DATASET_STD.set_index("Dataset", inplace=True) print(OOD_DATASET_STD) @@ -45,14 +45,14 @@ def get_weighted_result(exp_path: str) -> DataFrame: weighted_avg = all_records_df.groupby( "Training Steps" - ).mean() # provide a baseline with same shape + ).mean().map(lambda x: np.nan) # provide a df with same shape + # mask.inplace and update() won't work; need to assign to a new variable for efv in ["energy", "force", "virial"]: data = all_records_df.loc[ :, [key for key in all_records_df.keys() if efv in key] ] weights = OOD_DATASET[efv + "_weight"] - # data.mask(weights == 0, inplace=True) weighted_avg_efv = ( data.apply(np.log) .mul(weights, axis="index") @@ -60,6 +60,8 @@ def get_weighted_result(exp_path: str) -> DataFrame: .mean() .apply(np.exp) ) + # mask out the results where NAN exists in the original data + weighted_avg_efv.mask(all_records_df.isna().any(axis=1).groupby("Training Steps").any(), inplace=True) weighted_avg.update(weighted_avg_efv) weighted_avg["Dataset"] = "Weighted" @@ -77,6 +79,7 @@ def plotting( all_records_df: DataFrame, color: str, legend_handles: list[Line2D], + metric_key: str="rmse" ): for dataset, records in all_records_df.groupby("Dataset"): assert dataset in dataset_to_subplot.keys(), f"Dataset {dataset} not presented" @@ -95,7 +98,7 @@ def plotting( subsubplot.axhline(std, color="purple", linestyle="-.") # note: this will draw duplicated lines - metric_name = efv + "_rmse" + suffix + metric_name = efv + f"_{metric_key}" + suffix line = subsubplot.loglog( records.index, # step records[metric_name], @@ -107,7 +110,7 @@ def plotting( legend_handles.extend(line) # type: ignore -def main(exps: list[str]): +def main(exps: list[str], metric_key: str="rmse"): # Get dataset list from yaml file to preserve the order datasets: list[str] = OOD_DATASET.index.tolist() datasets.append("Weighted") @@ -126,11 +129,19 @@ def main(exps: list[str]): for exp_path, color in zip(exps, COLOR): all_records_df = get_weighted_result(exp_path) - plotting(dataset_to_subplot, all_records_df, color, legend_handles) - + plotting(dataset_to_subplot, all_records_df, color, legend_handles, metric_key) + + ## to set finer tick + from matplotlib.ticker import FixedLocator + ax[-1][0].yaxis.set_major_locator(FixedLocator(np.arange(0.02, 0.04, 0.002))) + ax[-1][1].yaxis.set_major_locator(FixedLocator(np.arange(0.2, 0.5, 0.04))) + ## to handle hpt explosion + # for ax in dataset_to_subplot["HPt_NC_2022"]: + # ax.set_ylim(0.05,0.2) + fig.tight_layout() fig.subplots_adjust(top=0.975) - title = "Compare OOD" + title = f"Compare OOD-{metric_key}" # fig.suptitle(title) # Poor placement fig.legend( handles=legend_handles, @@ -146,8 +157,17 @@ def main(exps: list[str]): if __name__ == "__main__": exps = [ - - "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_384_96_24", - "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24" + # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_384_96_24", + "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24", + # "/mnt/data_nas/public/multitask/training_exps/1223_prod_shareft_40GPU_finetune_pref0210_10010", + # "/mnt/data_nas/public/multitask/training_exps/1226_prod_shareft_40GPU_finetune_pref0210_10010_lr1e-5", + "/mnt/data_nas/public/multitask/training_exps/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", + "/mnt/data_nas/public/multitask/training_exps/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3", + # "/mnt/workspace/public/multitask/training_exps/N0130_dpa3a_shareft_128_64_32_scp1_e1a_cdsilu10_rc6_120_arc_4_30_l6_64GPU_240by3_float32", + "/mnt/workspace/public/multitask/training_exps/0202_dpa3a_shareft_256_128_32_scp1_e1a_csilu10_rc6_120_arc_4_30_l9_104GPU_240by3" + # "/mnt/data_nas/public/multitask/training_exps/0115_dpa3a_shareft_128_64_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_64GPU_240by3_float32" + + ] main(exps) + main(exps, "mae") diff --git a/lamstare/experiments/plt_test.py b/lamstare/experiments/plt_test.py index 1746e52..7efbc21 100644 --- a/lamstare/experiments/plt_test.py +++ b/lamstare/experiments/plt_test.py @@ -1,5 +1,6 @@ from collections import defaultdict from functools import lru_cache +from pathlib import Path from typing import Union import matplotlib.pyplot as plt @@ -92,7 +93,7 @@ def parse_record_dict_to_df(data: dict) -> DataFrame: # Set multi-index df.set_index(["Dataset", "Training Steps"], inplace=True) # sort by Dataset - df.sort_index(level=0, inplace=True) + # df.sort_index(level=0, inplace=True) # Don't sort, preserve the order in weights definition return df @@ -115,13 +116,14 @@ def get_weighted_result(exp_path: str) -> DataFrame: weighted_avg = all_records_df.groupby( "Training Steps" - ).mean() # provide a baseline with same shape + ).mean().map(lambda x: np.nan) # provide a df with same shape + # mask.inplace and update() won't work; need to assign to a new variable for efv in ["energy", "force", "virial"]: data = all_records_df.loc[ :, [key for key in all_records_df.keys() if efv in key] ] - data.mask(weights == 0, inplace=True) + # data.mask(weights == 0, inplace=True) # Should NOT mask here weighted_avg_efv = ( data.apply(np.log) .mul(weights, axis="index") @@ -129,11 +131,15 @@ def get_weighted_result(exp_path: str) -> DataFrame: .mean() .apply(np.exp) ) + # mask out the results where NAN exists in the original data + # not working: if a result is not generated, it won't have an entry in the dataframe + # weighted_avg_efv.mask(all_records_df.isna().any(axis=1).groupby("Training Steps").any(), inplace=True) weighted_avg.update(weighted_avg_efv) weighted_avg["Dataset"] = "Weighted" weighted_avg.reset_index(inplace=True) weighted_avg.set_index(["Dataset", "Training Steps"], inplace=True) + # weighted_avg.reindex(weights.index,level=0) all_records_df = pandas.concat( [all_records_df_raw, weighted_avg] ) # Preserve masked values @@ -148,7 +154,9 @@ def plotting( legend_handles: list[Line2D], ): for dataset, records in all_records_df.groupby("Dataset"): - assert dataset in dataset_to_subplot.keys(), f"Dataset {dataset} not presented" + # assert dataset in dataset_to_subplot.keys(), f"Dataset {dataset} not presented" + if dataset not in dataset_to_subplot.keys(): + continue subplot = dataset_to_subplot[dataset] # type: ignore # print(dataset) records = records.droplevel("Dataset") @@ -174,17 +182,10 @@ def plotting( def main(exps: list[str]): # Get all datasets - datasets: list[str] = sorted( - set( - dataset - for exp in exps - for dataset in get_weighted_result(exp) - .index.get_level_values("Dataset") - .unique() - .tolist() - ) - ) - datasets.remove("Weighted") # Assuming it exists + datasets: list[str] = DataFrame.from_dict( + data=get_head_weights(exps[0]), orient="index", columns=["weight"] + ).rename_axis("Dataset").index.tolist() + # use -1 for Alex3D->OMat compatibility datasets.append("Weighted") # Move to the end print(datasets) @@ -193,6 +194,10 @@ def main(exps: list[str]): ) ax: list[list[Axes]] legend_handles: list[Line2D] = [] + + # Reverse the order of subplot rows + ax = ax[::-1] + # get axis by dataset name to prevent plotting on wrong axis dataset_to_subplot = dict(zip(datasets, ax)) # add energy/force/virial to the beginning of plots @@ -203,6 +208,8 @@ def main(exps: list[str]): all_records_df = get_weighted_result(exp_path) plotting(dataset_to_subplot, all_records_df, color, legend_handles) + + fig.tight_layout() fig.subplots_adjust(top=0.975) title = "Compare IND" @@ -221,23 +228,14 @@ def main(exps: list[str]): if __name__ == "__main__": exps = [ - # "/mnt/data_nas/public/multitask/training_exps/1107_shareft_pref0021_1000100_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_shareft_240by6_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_shareft_pref0021_1000100_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_sharft_lr1e-3_1e-5_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1113_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1113_shareft_lr1e-3_1e-5_pref0220_10020_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1116_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_8GPU", - # "/mnt/data_nas/public/multitask/training_exps/1116_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_120GPU", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_large_descp", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by6", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_480by3", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_960by3_baseline" - # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_192_48_32", - # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_192_48_12", - "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_384_96_24", - "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24" - + "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24", + # "/mnt/data_nas/public/multitask/training_exps/1223_prod_shareft_40GPU_finetune_pref0210_10010", + # "/mnt/data_nas/public/multitask/training_exps/1226_prod_shareft_40GPU_finetune_pref0210_10010_lr1e-5", + "/mnt/data_nas/public/multitask/training_exps/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", + "/mnt/data_nas/public/multitask/training_exps/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3", + # "/mnt/workspace/public/multitask/training_exps/N0130_dpa3a_shareft_128_64_32_scp1_e1a_cdsilu10_rc6_120_arc_4_30_l6_64GPU_240by3_float32", + "/mnt/workspace/public/multitask/training_exps/0202_dpa3a_shareft_256_128_32_scp1_e1a_csilu10_rc6_120_arc_4_30_l9_104GPU_240by3" + # "/mnt/data_nas/public/multitask/training_exps/0115_dpa3a_shareft_128_64_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_64GPU_240by3_float32" + ] main(exps) diff --git a/lamstare/infra/ood_database.py b/lamstare/infra/ood_database.py index e1710aa..99ca36f 100644 --- a/lamstare/infra/ood_database.py +++ b/lamstare/infra/ood_database.py @@ -1,5 +1,5 @@ import os -from typing import List +from typing import List, Optional from sqlalchemy import ( Column, @@ -28,7 +28,7 @@ class OODRecord(Base): - __tablename__ = "ood_demo" + __tablename__ = "release_ood" id = Column(Integer, primary_key=True) run_id = Column(String(256), index=True) @@ -93,9 +93,12 @@ def query_latest_step(cls, run_id:str) -> int: return cls.query_by_run(run_id)[-1].step @classmethod - def query_best_by_run(cls, run_id: str) -> List["OODRecord"]: + def query_best_by_run(cls, run_id: str, step: Optional[int]=None) -> List["OODRecord"]: records = cls.query_by_run(run_id) - latest_step = cls.query_latest_step(run_id) + if step: + latest_step = step + else: + latest_step = cls.query_latest_step(run_id) ood_datasets = set([record.ood_dataset for record in records]) best_records = [] for ood_dataset in ood_datasets: diff --git a/lamstare/release/2d_test.yml b/lamstare/release/2d_test.yml new file mode 100644 index 0000000..a2012bc --- /dev/null +++ b/lamstare/release/2d_test.yml @@ -0,0 +1,66 @@ +IS_MULTI_TASK: true +OOD_TO_HEAD_MAP: + BN: + head: Alex2D # no path + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/BN + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + BN_mono: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/BN_mono + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + Gr_BN: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/Gr_BN + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + Gr_mono: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/Gr_mono + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + graphene: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/graphene + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + MoS2: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/MoS2 + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + MoS2_mono: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/MoS2_mono + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + WSe2: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/WSe2 + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + WSe2_mono: + head: Alex2D + filepath: + - /mnt/data_nas/public/OOD_test_data/2D_materials/WSe2_mono + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + \ No newline at end of file diff --git a/lamstare/release/PROPERTY_TEST.yml b/lamstare/release/PROPERTY_TEST.yml deleted file mode 100644 index 409e874..0000000 --- a/lamstare/release/PROPERTY_TEST.yml +++ /dev/null @@ -1,65 +0,0 @@ -TASK_TO_HEAD_MAP: - dielectric: - intensive: true - mean: 2.45697701 - property_dim: 1 - std: 2.28836481 - test_path: /mnt/data_nas/public/public_training_data/dielectric/valid - train_path: /mnt/data_nas/public/public_training_data/dielectric/train - train_steps: 200000 - jdft2d: - intensive: true - mean: 117.0396 - property_dim: 1 - std: 145.525 - test_path: /mnt/data_nas/public/public_training_data/jdft2d/valid - train_path: /mnt/data_nas/public/public_training_data/jdft2d/train - train_steps: 100000 - log_gvrh: - intensive: true - mean: 1.55292897 - property_dim: 1 - std: 0.37077067 - test_path: /mnt/data_nas/public/public_training_data/log_gvrh/valid - train_path: /mnt/data_nas/public/public_training_data/log_gvrh/train - train_steps: 100000 - log_kvrh: - intensive: true - mean: 1.8800296 - property_dim: 1 - std: 0.36726444 - test_path: /mnt/data_nas/public/public_training_data/log_kvrh/valid - train_path: /mnt/data_nas/public/public_training_data/log_kvrh/train - train_steps: 100000 - mp_e_form: - intensive: true - mean: -1.4071425 - property_dim: 1 - std: 1.16326513 - test_path: /mnt/data_nas/public/public_training_data/mp_e_form/valid - train_path: /mnt/data_nas/public/public_training_data/mp_e_form/train - train_steps: 200000 - mp_gap: - intensive: true - mean: 1.2162048 - property_dim: 1 - std: 1.60209919 - test_path: /mnt/data_nas/public/public_training_data/mp_gap/valid - train_path: /mnt/data_nas/public/public_training_data/mp_gap/train - train_steps: 200000 - perovskites: - intensive: true - mean: 1.47318716 - property_dim: 1 - std: 0.74406216 - test_path: /mnt/data_nas/public/public_training_data/perovskites/valid - train_path: /mnt/data_nas/public/public_training_data/perovskites/train - train_steps: 100000 - phonons: - intensive: true - mean: 571.869 - property_dim: 1 - std: 481.02 - test_path: /mnt/data_nas/public/public_training_data/phonons/valid - train_path: /mnt/data_nas/public/public_training_data/phonons/train - train_steps: 100000 \ No newline at end of file diff --git a/lamstare/release/ood_test/OOD_DATASET.yml b/lamstare/release/ood_test/OOD_DATASET.yml index d43b16b..1b26e9a 100644 --- a/lamstare/release/ood_test/OOD_DATASET.yml +++ b/lamstare/release/ood_test/OOD_DATASET.yml @@ -113,6 +113,28 @@ OOD_TO_HEAD_MAP: force_weight: null virial_weight: null + BiarylDrugFragments: + head: Domains_Drug + filepath: + - /mnt/data_nas/zhangd/data/Drug_manyi_0810/data/Drug_like_torsion_0903/Rawdata_torsion_profile/BiarylDrugFragments + energy_weight: null + force_weight: null + virial_weight: null + genentech: + head: Domains_Drug + filepath: + - /mnt/data_nas/zhangd/data/Drug_manyi_0810/data/Drug_like_torsion_0903/Rawdata_torsion_profile/genentech + energy_weight: null + force_weight: null + virial_weight: null + raw_torsionnet500: + head: Domains_Drug + filepath: + - /mnt/data_nas/zhangd/data/Drug_manyi_0810/data/Drug_like_torsion_0903/Rawdata_torsion_profile/raw_torsionnet500 + energy_weight: null + force_weight: null + virial_weight: null + # DNPs_2023_Kr: # head: Domains_Alloy diff --git a/lamstare/release/ood_test/OOD_DATASET_v2.yml b/lamstare/release/ood_test/OOD_DATASET_v2.yml new file mode 100644 index 0000000..8d0570d --- /dev/null +++ b/lamstare/release/ood_test/OOD_DATASET_v2.yml @@ -0,0 +1,130 @@ +IS_MULTI_TASK: true +OOD_TO_HEAD_MAP: + ANI: + head: Domains_Drug # no path + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/ANI + energy_weight: 1 + force_weight: 1 + virial_weight: null + HEA25: + head: Domains_Alloy + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/HEA25 + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + HEA25S: + head: Domains_Alloy + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/HEA25S + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + HEMC_HEMB: + head: Domains_Alloy + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/HEMC_HEMB + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + MD22: + head: Domains_Drug + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/MD22 + energy_weight: 1 + force_weight: 1 + virial_weight: null + COLL_test: + head: Organic_Reactions + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/collision + energy_weight: 1 + force_weight: 1 + virial_weight: null + H_nature_2022: + head: Organic_Reactions + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/H_nature_2022/ + energy_weight: 1 + force_weight: 1 + virial_weight: null + REANN_CO2_Ni100: + head: ODAC23 + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/REANN_CO2_Ni100/ + energy_weight: 1 + force_weight: 1 + virial_weight: null + NequIP_NC_2022: + head: Domains_SSE_PBE + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/NequIP_NC_2022 + energy_weight: 1 + force_weight: 1 + virial_weight: null + Chig_AIMD: + head: Domains_Drug + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/AIMD_Chig + energy_weight: 1 + force_weight: 1 + virial_weight: null + CGM_MLP_NC2023: + head: OC20M + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/CGM_MLP + energy_weight: 1 + force_weight: 1 + virial_weight: null + Cu_MgO_catalysts: + head: OC20M + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/Cu_MgO_CO2/ + energy_weight: 1 + force_weight: 1 + virial_weight: null + Ca_batteries_CM2021: + head: Domains_Anode + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/Ca_batteries + energy_weight: 1 + force_weight: 1 + virial_weight: null + HPt_NC_2022: + head: OC20M + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/HPt_NC2022 + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + Si_ZEO22: + head: MP_traj_v024_alldata_mixu + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/Si_ZEO22 + energy_weight: 1 + force_weight: 1 + virial_weight: null + Sub_Alex_val: + head: MP_traj_v024_alldata_mixu + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/subalex_downsample_9k + energy_weight: 1 + force_weight: 1 + virial_weight: 1 + WBM: + head: MP_traj_v024_alldata_mixu + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/WBM_downsampled + energy_weight: 1 + force_weight: null + virial_weight: null + raw_torsionnet500: + head: Domains_Drug + filepath: + - /mnt/data_nas/public/OOD_test_data_v2/raw_torsionnet500 + energy_weight: 1 + force_weight: 1 + virial_weight: null + + \ No newline at end of file diff --git a/lamstare/release/ood_test/ood_data_std.csv b/lamstare/release/ood_test/ood_data_std.csv index adbda5f..92e9a4b 100644 --- a/lamstare/release/ood_test/ood_data_std.csv +++ b/lamstare/release/ood_test/ood_data_std.csv @@ -1,17 +1,19 @@ Dataset,energy_std,force_std,virial_std -ANI, 0.19963876946634762, 2.127641035225506, -HEA25S, 0.41014108988094833, 1.9903457009612417, -HEMC_PBE, 0.474762072996, 1.8364064815279475, 0.46794153820403517 -MD22, 0.007933722948483338, 1.1408107903449518, -COLL_train, 0.46905055129774204, 1.9436163958502823, -H_nature_2022, 0.3866962525278815, 2.0384989267502704, -REANN_CO2_Ni100, 0.007410380917403982, 0.6333957228116258, -Si_ZEO22, 0.3210517115888047, 1.272937929020927, -NequIP_NC_2022, 0.0066406378874763064, 0.9421629142611048, -Ca_batteries_CM2021, 0.018494248302507264, 0.051940686112690704, -Chig_AIMD, 0.012594134644011467, 0.8599492186937476, -CGM_MLP_NC2023, 0.9985195007897913, 2.0217020563415375, -HPt_NC_2022, 0.06153723768935137, 1.145242620074603, -Cu_MgO_catalysts, 0.014304435776269204, 0.18909695700558055, -Sub_Alex_val, 0.6379453799013669, 0.5079784121093517, 0.5688223832751108 -WBM, 0.37483169207295935, 0.0, +ANI, 0.19852160741109035, 2.134650444786246, +HEA25, 0.4086027291354181, 2.075184012071992, 2.065014356039771 +HEA25S, 0.4030134901622356, 1.5479359067976695, 1.4293255096528095 +HEMC_HEMB, 0.4750117425061965, 1.8089415904253994, 0.4589409203427954 +MD22, 0.007773825548398275, 1.1374624718556865, +COLL_test, 0.4690509429726113, 1.9436163951047407, +H_nature_2022, 0.3915623031231297, 2.0571296230388896, +REANN_CO2_Ni100, 0.011019047357696455, 0.6822029356434589, +NequIP_NC_2022, 0.007872460518687731, 0.9434954645234603, +Chig_AIMD, 0.012375194479857722, 0.8613627348206556, +CGM_MLP_NC2023, 0.9875093490539156, 2.022155629243275, +Cu_MgO_catalysts, 0.013837285997415875, 0.18923742282763548, +Ca_batteries_CM2021, 0.016540792734058157, 0.17821831827935788, +HPt_NC_2022, 0.04891938011948422, 1.1641152396504415, 0.27071525803447116 +Si_ZEO22, 0.03534121167926313, 1.2410267785352673, +Sub_Alex_val, 0.7749643377228371, 1.1503770816187873, 0.8678699239404154 +WBM, 0.3743104865117501 +raw_torsionnet500, 0.033879357854299824, 0.05088788030453556 diff --git a/lamstare/release/ood_test/push_data.py b/lamstare/release/ood_test/push_data.py index 92120a5..2b5c2c3 100644 --- a/lamstare/release/ood_test/push_data.py +++ b/lamstare/release/ood_test/push_data.py @@ -2,7 +2,7 @@ import yaml import os -with open(os.path.dirname(__file__)+"/OOD_DATASET.yml","r") as f: +with open(os.path.dirname(__file__)+"/OOD_DATASET_v2.yml","r") as f: yaml_dd = yaml.safe_load(f) WEIGHTS_E = {k:v['energy_weight'] for k, v in yaml_dd["OOD_TO_HEAD_MAP"].items()} @@ -12,31 +12,17 @@ if __name__ == "__main__": exp_paths = [ - # "/mnt/workspace/penganyang/experiments/0911_gelu_2M_RMSE_lr1e-3_1e-5_pref0220_10020_l6_attoff_batch128_4GPU_v024mixu" - # "/OOD-dataset-std", - - # "/mnt/data_nas/public/multitask/training_exps/1107_shareft_pref0021_1000100_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_shareft_240by6_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_shareft_pref0021_1000100_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1110_newdata_sharft_lr1e-3_1e-5_medium_l6_atton_37head_tanh_40GPU", - # "/1114_BASELINE_MACE", - # "/1114_BASELINE_7net", - # "/1114_BASELINE_Orb", - # "/1114_BASELINE_DP" - # "/mnt/data_nas/public/multitask/training_exps/1113_shareft_lr1e-3_1e-5_pref0220_10020_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1113_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_40GPU", - # "/mnt/data_nas/public/multitask/training_exps/1116_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_8GPU", - # "/mnt/data_nas/public/multitask/training_exps/1116_shareft_960by3_lr1e-3_1e-5_medium_l6_atton_37head_tanh_120GPU", - # "/mnt/workspace/public/multitask/training_exps/0920_dpa2_large_28head_MPTraj_gpu8_10M", - # "/mnt/workspace/public/multitask/training_exps/0920_dpa2_medium_28head_MPTraj_gpu8_10M", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_960by3_baseline", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_480by3", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by6", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_large_descp", - # "/mnt/data_nas/public/multitask/training_exps/1119_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3", - # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_192_48_12", - # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_192_48_32", - # "/mnt/data_nas/public/multitask/training_exps/1122_shareft_lr1e-3_1e-5_pref0021_1000100_24GUP_240by3_single_384_96_24", + # "/OOD_STD", + "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24", + "/mnt/data_nas/public/multitask/training_exps/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", + "/mnt/data_nas/public/multitask/training_exps/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3", + "/250120_BASELINE_Orb", + "/250120_BASELINE_Mattersim", + "/250120_BASELINE_MACE", + "/250120_BASELINE_7net", + "/250120_BASELINE_EqV2", + "/250120_BASELINE_DP", + "/250120_BASELINE_DPA3" ] diff --git a/lamstare/release/ood_test/submit_ood_test.py b/lamstare/release/ood_test/submit_ood_test.py index 8803c17..f012d59 100644 --- a/lamstare/release/ood_test/submit_ood_test.py +++ b/lamstare/release/ood_test/submit_ood_test.py @@ -14,7 +14,7 @@ def get_ood_to_head_map( - mapping_path: str = "OOD_DATASET.yml", + mapping_path: str = "OOD_DATASET_v2.yml", output_path: str = "./testood", overwrite: Optional[bool] = False, ) -> Tuple[Dict[str, str], bool]: @@ -63,7 +63,7 @@ def submit_ood_test( model_version: str, step: Optional[int] = None, is_multitask: bool = True, - mapping_path: str = "OOD_DATASET.yml", + mapping_path: str = "OOD_DATASET_v2.yml", output_path: str = "./testood", overwrite: Optional[bool] = False, ) -> None: @@ -84,6 +84,7 @@ def submit_ood_test( mapping, is_multitask = get_ood_to_head_map( mapping_path=mapping_path, output_path=output_path, overwrite=overwrite ) # also generates the dp test input files + is_multitask=False if step is None: step = get_latest_ckpt(exp_path) if is_multitask: @@ -103,7 +104,7 @@ def submit_ood_test( if USE_DLC: command = ( ". /mnt/data_nas/public/.bashrc; " - f"conda activate /mnt/data_nas/public/Miniconda/envs/{os.environ.get('CONDA_ENV','openlamdb')};" + f"conda activate /mnt/data_nas/public/Miniconda/envs/{os.environ.get('CONDA_ENV','openlam_db')};" "export PYTHONPATH=/mnt/data_nas/cc/LAMstare_new;" f"cd {Path(__file__).resolve().parent} ; " f"python3 run_ood_test.py {exp_path} {ood_dataset} {head} {model_version} {step} {testfile} {run_name}" @@ -141,7 +142,7 @@ def main(exp_path: str, freq: int = 200000, step: Optional[int] = None): submit_ood_test( exp_path=exp_path, model_version="autotest", - mapping_path=os.path.dirname(__file__) + "/OOD_DATASET.yml", + mapping_path=os.path.dirname(__file__) + "/OOD_DATASET_v2.yml", step=step, is_multitask=False, ) diff --git a/lamstare/release/property_test/PROPERTY_TEST.yml b/lamstare/release/property_test/PROPERTY_TEST.yml index 8c3cd50..9522f52 100644 --- a/lamstare/release/property_test/PROPERTY_TEST.yml +++ b/lamstare/release/property_test/PROPERTY_TEST.yml @@ -1,65 +1,533 @@ TASK_TO_HEAD_MAP: - dielectric: + # qm9_mu: + # property_name: dipole_moment + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_alpha: + # property_name: isotropic_polarizability + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_eps_homo: + # property_name: homo + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_eps_lumo: + # property_name: lumo + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_deta_eps: + # property_name: homo_lumo_gap + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_r2_abs: + # property_name: electronic_spatial_extent + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_zpve: + # property_name: zpve + # intensive: false + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_U_0: + # property_name: energy_U0 + # intensive: false + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_U: + # property_name: energy_U + # intensive: false + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_H: + # property_name: enthalpy_H + # intensive: false + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_G: + # property_name: free_energy + # intensive: false + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qm9_c_v: + # property_name: heat_capacity + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/test + # train_path: /mnt/data_nas/public/public_training_data/QM9_Property_new/train + # train_steps: 500000 + # qmof_bandgap: + # property_name: bandgap + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/QMOF_bandgap/valid + # train_path: /mnt/data_nas/public/public_training_data/QMOF_bandgap/train + # train_steps: 500000 + matbench_dielectric: + property_name: dielectric intensive: true - mean: 2.45697701 property_dim: 1 - std: 2.28836481 test_path: /mnt/data_nas/public/public_training_data/dielectric/valid train_path: /mnt/data_nas/public/public_training_data/dielectric/train - train_steps: 200 - jdft2d: + train_steps: 200000 + start_lr: 3e-4 + stop_lr: 1e-5 + matbench_dielectric_fold1: + property_name: dielectric + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/dielectric_1/valid + train_path: /mnt/data_nas/public/public_training_data/dielectric_1/train + train_steps: 200000 + start_lr: 3e-4 + stop_lr: 1e-5 + matbench_dielectric_fold2: + property_name: dielectric + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/dielectric_2/valid + train_path: /mnt/data_nas/public/public_training_data/dielectric_2/train + train_steps: 200000 + start_lr: 3e-4 + stop_lr: 1e-5 + matbench_dielectric_fold3: + property_name: dielectric + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/dielectric_3/valid + train_path: /mnt/data_nas/public/public_training_data/dielectric_3/train + train_steps: 200000 + start_lr: 3e-4 + stop_lr: 1e-5 + matbench_dielectric_fold4: + property_name: dielectric + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/dielectric_4/valid + train_path: /mnt/data_nas/public/public_training_data/dielectric_4/train + train_steps: 200000 + start_lr: 3e-4 + stop_lr: 1e-5 + matbench_jdft2d: + property_name: jdft2d intensive: true - mean: 117.0396 property_dim: 1 - std: 145.525 test_path: /mnt/data_nas/public/public_training_data/jdft2d/valid train_path: /mnt/data_nas/public/public_training_data/jdft2d/train train_steps: 100000 - log_gvrh: + start_lr: 1e-3 + stop_lr: 3e-8 + matbench_jdft2d_fold1: + property_name: jdft2d + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/jdft2d_1/valid + train_path: /mnt/data_nas/public/public_training_data/jdft2d_1/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 3e-8 + matbench_jdft2d_fold2: + property_name: jdft2d + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/jdft2d_2/valid + train_path: /mnt/data_nas/public/public_training_data/jdft2d_2/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 3e-8 + matbench_jdft2d_fold3: + property_name: jdft2d + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/jdft2d_3/valid + train_path: /mnt/data_nas/public/public_training_data/jdft2d_3/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 3e-8 + matbench_jdft2d_fold4: + property_name: jdft2d + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/jdft2d_4/valid + train_path: /mnt/data_nas/public/public_training_data/jdft2d_4/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 3e-8 + matbench_log_gvrh: + property_name: log_gvrh intensive: true - mean: 1.55292897 property_dim: 1 - std: 0.37077067 test_path: /mnt/data_nas/public/public_training_data/log_gvrh/valid train_path: /mnt/data_nas/public/public_training_data/log_gvrh/train train_steps: 100000 - log_kvrh: + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_gvrh_fold1: + property_name: log_gvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_gvrh_1/valid + train_path: /mnt/data_nas/public/public_training_data/log_gvrh_1/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_gvrh_fold2: + property_name: log_gvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_gvrh_2/valid + train_path: /mnt/data_nas/public/public_training_data/log_gvrh_2/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_gvrh_fold3: + property_name: log_gvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_gvrh_3/valid + train_path: /mnt/data_nas/public/public_training_data/log_gvrh_3/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_gvrh_fold4: + property_name: log_gvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_gvrh_4/valid + train_path: /mnt/data_nas/public/public_training_data/log_gvrh_4/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_kvrh: + property_name: log_kvrh intensive: true - mean: 1.8800296 property_dim: 1 - std: 0.36726444 test_path: /mnt/data_nas/public/public_training_data/log_kvrh/valid train_path: /mnt/data_nas/public/public_training_data/log_kvrh/train train_steps: 100000 - mp_e_form: + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_kvrh_fold1: + property_name: log_kvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_kvrh_1/valid + train_path: /mnt/data_nas/public/public_training_data/log_kvrh_1/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_kvrh_fold2: + property_name: log_kvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_kvrh_2/valid + train_path: /mnt/data_nas/public/public_training_data/log_kvrh_2/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_kvrh_fold3: + property_name: log_kvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_kvrh_3/valid + train_path: /mnt/data_nas/public/public_training_data/log_kvrh_3/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_log_kvrh_fold4: + property_name: log_kvrh + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/log_kvrh_4/valid + train_path: /mnt/data_nas/public/public_training_data/log_kvrh_4/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_mp_e_form: + property_name: mp_e_form intensive: true - mean: -1.4071425 property_dim: 1 - std: 1.16326513 test_path: /mnt/data_nas/public/public_training_data/mp_e_form/valid train_path: /mnt/data_nas/public/public_training_data/mp_e_form/train - train_steps: 200000 - mp_gap: + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_e_form_fold1: + property_name: mp_e_form + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_e_form_1/valid + train_path: /mnt/data_nas/public/public_training_data/mp_e_form_1/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_e_form_fold2: + property_name: mp_e_form + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_e_form_2/valid + train_path: /mnt/data_nas/public/public_training_data/mp_e_form_2/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_e_form_fold3: + property_name: mp_e_form + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_e_form_3/valid + train_path: /mnt/data_nas/public/public_training_data/mp_e_form_3/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_e_form_fold4: + property_name: mp_e_form + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_e_form_4/valid + train_path: /mnt/data_nas/public/public_training_data/mp_e_form_4/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_gap: + property_name: mp_gap intensive: true - mean: 1.2162048 property_dim: 1 - std: 1.60209919 test_path: /mnt/data_nas/public/public_training_data/mp_gap/valid train_path: /mnt/data_nas/public/public_training_data/mp_gap/train - train_steps: 200000 - perovskites: + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_gap_fold1: + property_name: mp_gap + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_gap_1/valid + train_path: /mnt/data_nas/public/public_training_data/mp_gap_1/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_gap_fold2: + property_name: mp_gap + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_gap_2/valid + train_path: /mnt/data_nas/public/public_training_data/mp_gap_2/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_gap_fold3: + property_name: mp_gap + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_gap_3/valid + train_path: /mnt/data_nas/public/public_training_data/mp_gap_3/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_mp_gap_fold4: + property_name: mp_gap + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/mp_gap_4/valid + train_path: /mnt/data_nas/public/public_training_data/mp_gap_4/train + train_steps: 500000 + start_lr: 1e-3 + stop_lr: 1e-6 + matbench_perovskites: + property_name: perovskites intensive: true - mean: 1.47318716 property_dim: 1 - std: 0.74406216 test_path: /mnt/data_nas/public/public_training_data/perovskites/valid train_path: /mnt/data_nas/public/public_training_data/perovskites/train train_steps: 100000 - phonons: + matbench_perovskites_fold1: + property_name: perovskites + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/perovskites_1/valid + train_path: /mnt/data_nas/public/public_training_data/perovskites_1/train + train_steps: 100000 + matbench_perovskites_fold2: + property_name: perovskites + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/perovskites_2/valid + train_path: /mnt/data_nas/public/public_training_data/perovskites_2/train + train_steps: 100000 + matbench_perovskites_fold3: + property_name: perovskites + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/perovskites_3/valid + train_path: /mnt/data_nas/public/public_training_data/perovskites_3/train + train_steps: 100000 + matbench_perovskites_fold4: + property_name: perovskites + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/perovskites_4/valid + train_path: /mnt/data_nas/public/public_training_data/perovskites_4/train + train_steps: 100000 + matbench_phonons: + property_name: phonons intensive: true - mean: 571.869 property_dim: 1 - std: 481.02 test_path: /mnt/data_nas/public/public_training_data/phonons/valid train_path: /mnt/data_nas/public/public_training_data/phonons/train - train_steps: 100000 \ No newline at end of file + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_phonons_fold1: + property_name: phonons + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/phonons_1/valid + train_path: /mnt/data_nas/public/public_training_data/phonons_1/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_phonons_fold2: + property_name: phonons + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/phonons_2/valid + train_path: /mnt/data_nas/public/public_training_data/phonons_2/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_phonons_fold3: + property_name: phonons + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/phonons_3/valid + train_path: /mnt/data_nas/public/public_training_data/phonons_3/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + matbench_phonons_fold4: + property_name: phonons + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/phonons_4/valid + train_path: /mnt/data_nas/public/public_training_data/phonons_4/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-4 + unconvbench_2d_exfoliation: + property_name: unconvbench_exfoliation + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_exfoliation/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_exfoliation/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-6 + unconvbench_2d_total_e: + property_name: unconvbench_2d_e_tot + intensive: false + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_2d_e_tot/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_2d_e_tot/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-5 + unconvbench_2d_bandgap: + property_name: unconvbench_2d_gap + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_2d_gap/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_2d_gap/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-5 + unconvbench_co2_adsp: + property_name: unconvbench_co2_adsp + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_co2_adsp/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_co2_adsp/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 3e-8 + unconvbench_qmof: + property_name: unconvbench_qmof + intensive: false + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_qmof/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_qmof/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-5 + unconvbench_supercon: + property_name: unconvbench_supercon + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_supercon/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_supercon/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-5 + unconvbench_defected: + property_name: unconvbench_defected + intensive: true + property_dim: 1 + test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_defected/valid + train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_defected/train + train_steps: 100000 + start_lr: 1e-3 + stop_lr: 1e-8 + # unconvbench_src_bulk: + # property_name: unconvbench_src_bulk + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_src_bulk/valid + # train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_src_bulk/train + # train_steps: 100000 + # unconvbench_bulk_s: + # property_name: unconvbench_bulk_s + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_s/valid + # train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_s/train + # train_steps: 100000 + # unconvbench_bulk_m: + # property_name: unconvbench_bulk_m + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_m/valid + # train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_m/train + # train_steps: 100000 + # unconvbench_bulk_l: + # property_name: unconvbench_bulk_l + # intensive: true + # property_dim: 1 + # test_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_l/valid + # train_path: /mnt/data_nas/public/public_training_data/Unconvbench_property/unconvbench_bulk_l/train + # train_steps: 100000 diff --git a/lamstare/release/property_test/run_property_test.py b/lamstare/release/property_test/run_property_test.py index 099f909..9db6c9b 100644 --- a/lamstare/release/property_test/run_property_test.py +++ b/lamstare/release/property_test/run_property_test.py @@ -15,14 +15,10 @@ def run_property_test( step: int, finetune_path: Path, run_name: str, - property_yml: str = "PROPERTY_TEST.yml", ) -> None: run_id = exp_path.split("/")[-1] # Get basename as id head_dptest_res = run_property_train_test(finetune_path, task_name, step) - with open(property_yml, 'r') as f: - tasks = yaml.load(f, Loader=yaml.FullLoader)["TASK_TO_HEAD_MAP"] - scalling_facotr = tasks[task_name]['std'] print(head_dptest_res) PropertyRecord( @@ -31,8 +27,8 @@ def run_property_test( model_version=model_version, task_name=task_name, step=step, - property_rmse=head_dptest_res["PROPERTY RMSE"]*scalling_facotr, - property_mae=head_dptest_res["PROPERTY MAE"]*scalling_facotr, + property_rmse=head_dptest_res["PROPERTY RMSE"], + property_mae=head_dptest_res["PROPERTY MAE"], ).insert() @@ -40,10 +36,10 @@ def run_property_test( def main(): logging.basicConfig(level=logging.DEBUG) - exp_path, task_name, model_version, step, finetune_path, run_name, property_yml = sys.argv[1:] + exp_path, task_name, model_version, step, finetune_path, run_name = sys.argv[1:] run_property_test( - exp_path, task_name, model_version, int(step), finetune_path, run_name, property_yml + exp_path, task_name, model_version, int(step), finetune_path, run_name ) diff --git a/lamstare/release/property_test/submit_property_test.py b/lamstare/release/property_test/submit_property_test.py index 20348a2..27d701b 100644 --- a/lamstare/release/property_test/submit_property_test.py +++ b/lamstare/release/property_test/submit_property_test.py @@ -29,15 +29,16 @@ def submit_property_test( finetune_path = prepare_property_finetune_folder(exp_path, task_name, step, property_yml, output_path) command = ( + "export NUM_WORKERS=0; " ". /mnt/data_nas/public/.bashrc; " f"conda activate /mnt/data_nas/public/Miniconda/envs/{os.environ.get('CONDA_ENV','openlam_db')};" - # "export PYTHONPATH=/mnt/data_nas/cc/LAMstare_new;" + "export PYTHONPATH=/mnt/data_nas/cc/LAMstare_new;" f"cd {Path(__file__).resolve().parent};" - f"python3 run_property_test.py {exp_path} {task_name} {model_version} {step} {finetune_path} {run_name} {property_yml}" + f"python3 run_property_test.py {exp_path} {task_name} {model_version} {step} {finetune_path} {run_name}" ).replace("workspace","data_nas") - + job_name = f"PROPERTY-{run_name}" - + if query_job_numbers(job_name): logging.warning(f"SKIPPED: {job_name} is already running.") else: @@ -48,15 +49,14 @@ def submit_property_test( logging.info(f"SKIPPED: {run_name} already exists.") else: logging.error(f"ERROR: {run_name} has multiple records, please check.") - break - + def main(exp_path: str, step: int): - + print(f"Running DPTEST for {exp_path} on ckpt-{step}...\n") submit_property_test( exp_path=exp_path, @@ -66,4 +66,6 @@ def main(exp_path: str, step: int): ) if __name__ == "__main__": - main("/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24", 1000000) \ No newline at end of file + # main("/mnt/data_nas/public/multitask/training_exps/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", 1000000) + # main("/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24",8000000) + main("/mnt/data_nas/public/multitask/training_exps/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3",3870000) \ No newline at end of file diff --git a/lamstare/utils/ase_dptest.py b/lamstare/utils/ase_dptest.py index bbe00de..4372345 100644 --- a/lamstare/utils/ase_dptest.py +++ b/lamstare/utils/ase_dptest.py @@ -5,6 +5,7 @@ import glob from pathlib import Path from lamstare.infra.ood_database import OODRecord +import torch def run_ase_dptest( calc, @@ -24,7 +25,7 @@ def run_ase_dptest( max_ele_num = 120 systems = [] - for path in testpath: + for path in testpath: systems.extend(glob.glob(f"{path}/*")) # check if the system is mixed type if len(glob.glob(systems[0] + '/**/real_atom_types.npy', recursive=True)) == 0: @@ -32,30 +33,33 @@ def run_ase_dptest( else: mix_type = True - + for filepth in systems: if mix_type: sys = dpdata.MultiSystems() sys.load_systems_from_file(filepth, fmt='deepmd/npy/mixed') else: sys = dpdata.LabeledSystem(filepth, fmt='deepmd/npy') - + for ls in sys: for frame in ls: - try: - atoms = frame.to_ase_structure()[0] - atoms.calc = calc - ff = atoms.get_forces() + atoms = frame.to_ase_structure()[0] + atoms.calc = calc + energy_predict = np.array(atoms.get_potential_energy()) + if not np.isnan(energy_predict): atomic_numbers = atoms.get_atomic_numbers() atom_num.append(np.bincount(atomic_numbers, minlength=max_ele_num)) - energy_predict = np.array(atoms.get_potential_energy()) + energy_pre.append(energy_predict) energy_lab.append(frame.data["energies"]) energy_err.append(energy_predict - frame.data["energies"]) - print(energy_err) - force_err.append(frame.data["forces"].squeeze(0) - np.array(ff)) - energy_err_per_atom.append(energy_err[-1]/force_err[-1].shape[0]) + energy_err_per_atom.append(energy_err[-1]/len(atoms)) + try: + ff = atoms.get_forces() + force_err.append(frame.data["forces"].squeeze(0) - np.array(ff)) + except: + pass try: stress = atoms.get_stress() stress_tensor = - np.array( @@ -67,10 +71,11 @@ def run_ase_dptest( virial_err_per_atom.append(virial_err[-1]/force_err[-1].shape[0]) except: pass - except: - pass - - + else: + from ase.io import write + write("/mnt/data_nas/penganyang/ani_cif.cif",atoms) + break + atom_num = np.array(atom_num) energy_err = np.array(energy_err) energy_pre = np.array(energy_pre) @@ -78,16 +83,21 @@ def run_ase_dptest( shift_bias, _, _, _ = np.linalg.lstsq(atom_num, energy_err, rcond=1e-10) unbiased_energy = energy_pre - (atom_num @ shift_bias.reshape(max_ele_num, -1)).reshape(-1) - energy_lab.squeeze() unbiased_energy_err_per_a = unbiased_energy / atom_num.sum(-1) - - + + res = { "Energy MAE": [np.mean(np.abs(np.stack(unbiased_energy)))], "Energy RMSE": [np.sqrt(np.mean(np.square(unbiased_energy)))], "Energy MAE/Natoms": [np.mean(np.abs(np.stack(unbiased_energy_err_per_a)))], "Energy RMSE/Natoms": [np.sqrt(np.mean(np.square(unbiased_energy_err_per_a)))], - "Force MAE": [np.mean(np.abs(np.concatenate(force_err)))], - "Force RMSE": [np.sqrt(np.mean(np.square(np.concatenate(force_err))))], } + if force_err: + res.update( + { + "Force MAE": [np.mean(np.abs(np.concatenate(force_err)))], + "Force RMSE": [np.sqrt(np.mean(np.square(np.concatenate(force_err))))], + } + ) if virial_err_per_atom != []: res.update( { @@ -96,16 +106,17 @@ def run_ase_dptest( "Virial MAE/Natoms": [np.mean(np.abs(np.stack(virial_err_per_atom)))], "Virial RMSE/Natoms": [np.sqrt(np.mean(np.square(np.stack(virial_err_per_atom))))]} ) - print(res) return res def main(model_name, testpath_mapping): + torch.set_default_dtype(torch.float32) + if model_name == "MACE": from mace.calculators import mace_mp CALC = mace_mp(model="medium", device="cuda", default_dtype="float64") elif model_name == "DP": from deepmd.calculator import DP - CALC = DP("/mnt/data_nas/penganyang/mptrj1113.pth") + CALC = DP("/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24/frezmptrj800w.pth") elif model_name == "Orb": from orb_models.forcefield import pretrained from orb_models.forcefield.calculator import ORBCalculator @@ -113,30 +124,36 @@ def main(model_name, testpath_mapping): CALC = ORBCalculator(orbff, device="cuda") elif model_name == "7net": from sevenn.sevennet_calculator import SevenNetCalculator - CALC = SevenNetCalculator("7net-0", device='cuda') + CALC = SevenNetCalculator("7net-0", device='cuda') elif model_name == "EqV2": from fairchem.core import OCPCalculator CALC = OCPCalculator( - model_name="EquiformerV2-31M-S2EF-OC20-All+MD", - local_cache="pretrained_models", - cpu=False, + checkpoint_path="eqV2_153M_omat_mp_salex.pt", + cpu=False ) + elif model_name == "Mattersim": + from mattersim.forcefield import MatterSimCalculator + CALC = MatterSimCalculator(load_path="MatterSim-v1.0.0-5M.pth", device="cuda") else: raise ValueError(f"Model {model_name} not supported.") - + for ood_name, testpath in mapping.items(): print(f"Processing {ood_name}, {testpath}") head_dptest_res = run_ase_dptest(CALC, testpath, ood_name) + if "Force MAE" not in head_dptest_res: + head_dptest_res["Force MAE"] = -1 + head_dptest_res["Force RMSE"] = -1 + if "Virial MAE" not in head_dptest_res: head_dptest_res["Virial MAE"] = -1 head_dptest_res["Virial RMSE"] = -1 head_dptest_res["Virial MAE/Natoms"] = -1 head_dptest_res["Virial RMSE/Natoms"] = -1 OODRecord( - run_id=f"1114_BASELINE_{model_name}", + run_id=f"250120_BASELINE_{model_name}", run_name=model_name+"#baseline#"+ood_name, - model_version="MPtrj", + model_version="Matbench-Discovery", ood_dataset=ood_name, step=1000000, head=model_name, @@ -151,20 +168,33 @@ def main(model_name, testpath_mapping): virial_mae_natoms=head_dptest_res["Virial MAE/Natoms"], virial_rmse_natoms=head_dptest_res["Virial RMSE/Natoms"], ).insert() - + # print(f"Finished {head_dptest_res}") + if __name__ == "__main__": import yaml - with open("/mnt/data_nas/public/multitask/LAMstare/lamstare/release/new_OOD_DATASET.yml","r") as f: + with open("/mnt/data_nas/cc/LAMstare_new/lamstare/release/ood_test/OOD_DATASET_v2.yml","r") as f: yaml_dd = yaml.safe_load(f) - - mapping = {k:v['filepath'] for k, v in yaml_dd["OOD_TO_HEAD_MAP"].items()} + + mapping = {k:v['filepath'] for k, v in yaml_dd["OOD_TO_HEAD_MAP"].items() if k in ["HEA25","HEA25S","HPt_NC_2022"]} + # mapping = {k:v['filepath'] for k, v in yaml_dd["OOD_TO_HEAD_MAP"].items() if k == "ANI"} + # mapping = { + # k: v + # for k, v in mapping.items() + # if k + # in [ + # # "COLL_train", # RuntimeError: The expanded size of the tensor (10) must match the existing size (11) at non-singleton dimension 0. Target sizes: [10]. Tensor sizes: [11] + # # "H_nature_2022", # RuntimeError: The expanded size of the tensor (2) must match the existing size (3) at non-singleton dimension 0. Target sizes: [2]. Tensor sizes: [3] + # "CGM_MLP_NC2023", # RuntimeError: The expanded size of the tensor (146) must match the existing size (147) at non-singleton dimension 0. Target sizes: [146]. Tensor sizes: [147] + # # "BiarylDrugFragments", + # # "genentech", + # # "raw_torsionnet500", + # ] + # } # main("DP", mapping) # main("Orb", mapping) # main("MACE", mapping) - mapping= {"WBM": mapping["WBM"]} - main("EqV2", mapping) - - - + main("Mattersim",mapping) + # main("7net",mapping) + # main("EqV2",mapping) diff --git a/lamstare/utils/dlc_submit.py b/lamstare/utils/dlc_submit.py index 88149be..8233d14 100644 --- a/lamstare/utils/dlc_submit.py +++ b/lamstare/utils/dlc_submit.py @@ -22,7 +22,7 @@ def submit_job_to_dlc(job_name:str, command:str): data_sources = os.environ["DATA_SOURCES"] workspace_id = os.environ["WORKSPACE_ID"] resource_id = os.environ["RESOURCE_ID"] - priority = 1 + priority = 9 worker_count = 1 worker_gpu = 1 worker_cpu = 12 diff --git a/lamstare/utils/dptest.py b/lamstare/utils/dptest.py index 7126c52..10e6a5d 100644 --- a/lamstare/utils/dptest.py +++ b/lamstare/utils/dptest.py @@ -59,52 +59,6 @@ def run_dptest( return result -## DEPRECATED - -# def run_single_head_dptest(exp_path:str, ckpt:int, head:str, test_file:Optional[str], ood_name:str) -> Dict[str,float]: -# logging.error("This function is deprecated. Please use run_dptest instead.") -# dptest_res = {} -# run_id=exp_path.split("/")[-1] # Get basename as id -# temp_file_name = f"{run_id}#{ckpt}#{head}" -# temp_file_dir=temp_file_path+run_id+ood_name+"/" - -# try: -# result_file = temp_file_dir+temp_file_name+".txt" -# print(result_file) -# if not os.path.exists(result_file): -# os.mkdir(temp_file_dir) -# script_path = os.path.join(os.path.dirname(__file__), "single_dptest.sh") -# args = [script_path, exp_path, str(ckpt), head, temp_file_dir, test_file] -# command = ( -# f'". /mnt/data_nas/public/.bashrc;' -# f"conda activate /mnt/data_nas/public/Miniconda/envs/{os.environ.get('CONDA_ENV','lamstare')};" -# f"cd /mnt/data_nas/cc/LAMstare/lamstare/release;" -# f"export TEMP_FILE_DIR={temp_file_dir};" -# ) -# command += " ".join(args) -# command += '"' -# print("Executing command: ", command) -# submit_job_to_dlc(f"OOD_TEST_{ood_name}_{run_id}", command) -# return None -# # print("Executing command: ", *args) -# # subprocess.run(args, check=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) -# else: -# # check if "weighted average of errors" is in the file -# with open(result_file, "r") as f: -# content = f.read() -# if "weighted average of errors" in content: -# print(f"job {temp_file_dir} already finished") -# dptest_res = extract_info_from_dptest_txt(head, result_file) -# # shutil.rmtree(temp_file_dir) -# return dptest_res -# else: -# print(f"job {temp_file_dir} ongoing") -# return None -# except FileExistsError: -# print(f"job {temp_file_dir} already submitted") -# return None -# except: -# print(f"Fail to test {temp_file_dir}") def extract_info_from_dptest_txt(dataset_name:str, filepath:Path|str, txt_type:str="standard") -> Dict[str,float]: diff --git a/lamstare/utils/plot.py b/lamstare/utils/plot.py index b37ed4c..92c6a21 100644 --- a/lamstare/utils/plot.py +++ b/lamstare/utils/plot.py @@ -16,7 +16,7 @@ # Plotting def fetch_lcurve(run_pth): """ - Given the absolute path to the experiment folder, return the dataframe containing learning curve information. + Given the absolute path to the experiment folder, return the dataframe containing learning curve information. """ logfile = Path(os.path.join(run_pth, "lcurve.out")) @@ -35,7 +35,7 @@ def get_tat_token(): url = "https://open.feishu.cn/open-apis/auth/v3/tenant_access_token/internal" headers = {"Content-Type": "application/json; charset=utf-8"} data = { - "app_id": app_id, + "app_id": app_id, "app_secret": app_secret, } response = requests.request("POST", url, headers=headers, json=data) @@ -48,10 +48,10 @@ def uploadImage(token, file): Upload an image to Feishu App to generate image_key for POST. """ url = "https://open.feishu.cn/open-apis/im/v1/images" - form = {"image_type": "message", "image": (open(file, "rb"))} + form = {"image_type": "message", "image": (open(file, "rb"))} multi_form = MultipartEncoder(form) headers = { - "Authorization": f"Bearer {token}", + "Authorization": f"Bearer {token}", } headers["Content-Type"] = multi_form.content_type response = requests.request("POST", url, headers=headers, data=multi_form) @@ -87,4 +87,3 @@ def sendimg(files: List[str], title:str): for file in files: img_keys.append(uploadImage(token, file)) send2bot(img_keys, title) - diff --git a/lamstare/utils/property.py b/lamstare/utils/property.py index bc79b90..4c0acc6 100644 --- a/lamstare/utils/property.py +++ b/lamstare/utils/property.py @@ -14,8 +14,8 @@ def get_property_json(params: dict): "descriptor": params.get("descriptor"), "fitting_net": { "type": "property", + "property_name":params.get("property_name"), "intensive": params.get("intensive"), - "bias_method": "no_bias", "task_dim": params.get("property_dim"), "neuron": [ 240, @@ -34,8 +34,8 @@ def get_property_json(params: dict): "learning_rate": { "type": "exp", "decay_steps": 500, - "start_lr": 0.001, - "stop_lr": 3.51e-08, + "start_lr": float(params.get("start_lr", 1e-3)), + "stop_lr": float(params.get("stop_lr", 1e-4)), "_comment": "that's all" }, "training": { @@ -49,14 +49,14 @@ def get_property_json(params: dict): "batch_size": 1, "_comment": "that's all" }, - "numb_steps": params.get("train_steps"), + "numb_steps": params.get("train_steps", 100000), "warmup_steps": 0, "gradient_max_norm": 5.0, "max_ckpt_keep": 10, "seed": 10, "disp_file": "lcurve.out", - "disp_freq": int(params.get("train_steps")//100), - "save_freq": int(params.get("train_steps")//4), + "disp_freq": int(params.get("train_steps", 100000)//100), + "save_freq": int(params.get("train_steps", 100000)//10), "_comment": "that's all", } } @@ -80,21 +80,21 @@ def prepare_property_finetune_folder(pretrain_exp_path:str, task_name:str, step: # I. prepare property finetune input.json with open(property_yaml, 'r') as f: tasks = yaml.load(f, Loader=yaml.FullLoader)["TASK_TO_HEAD_MAP"] - + with open(os.path.join(pretrain_exp_path,"input.json"), "r") as f: pretrain_config = json.load(f) - + params = tasks[task_name] if "loss_dict" in pretrain_config: # multitask input params.update({ - "descriptor": pretrain_config["model"]["shared_dict"]["dpa2_descriptor"], + "descriptor": pretrain_config["model"]["shared_dict"]["dpa3_descriptor"], "type_map": pretrain_config["model"]["shared_dict"]["type_map_all"] }) else: #single task input raise NotImplementedError finetune_config = get_property_json(params) - + with open(os.path.join(finetune_path,"input.json"), "w") as f: json.dump(finetune_config, f, indent=4) @@ -133,7 +133,7 @@ def run_property_train_test(finetune_path: str, task_name: str, step: int) -> di ret = os.system(command) if ret != 0: raise RuntimeError(f"Failed to freeze model {model}") - + # IV. Test model test_result = f"{finetune_path.split('/')[-1]}.txt" command = f"dp --pt test -m {f'finetuned_{task_name}.pth'} -f {f'{task_name}_valid.txt'} -l {test_result}" @@ -141,8 +141,7 @@ def run_property_train_test(finetune_path: str, task_name: str, step: int) -> di ret = os.system(command) if ret != 0: raise RuntimeError(f"Failed to test model {test_result}") - + # V. Extract results result = extract_info_from_dptest_txt(task_name, test_result,txt_type="property") return result - diff --git a/lamstare/utils/property_table.py b/lamstare/utils/property_table.py new file mode 100644 index 0000000..caf2297 --- /dev/null +++ b/lamstare/utils/property_table.py @@ -0,0 +1,194 @@ +from dotenv import load_dotenv # type: ignore +from lamstare.utils.plot import get_tat_token +import requests +from datetime import datetime +from collections import defaultdict +import json +from lamstare.infra.property_database import PropertyRecord +import numpy as np +from typing import List, Dict, Any +import os +import yaml +from pathlib import Path +import warnings + +load_dotenv() + +APP_TOKEN = "PpyPbMuxval4histOfDcjSoxnhc" +TABLE_ID = "tbl1rU5Md3tZx9cF" + + +def send2table( + data: Dict[str, Any], run_id: str, record_id=None, method: str = "post" +) -> None: + """ + This function sends data to the table in Feishu, either by creating a new record or updating an existing record. + """ + record_url = ( + "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/records" + % (APP_TOKEN, TABLE_ID) + ) + if method == "put": + assert record_id is not None, "record_id is required for put method" + record_url = f"{record_url}/{record_id}" + + headers = { + "Authorization": f"Bearer {get_tat_token()}", + "Content-Type": "application/json; charset=utf-8", + } + + data = { + "fields": { + "Experiment name": run_id, + "Update Time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + **{k:v for k,v in data.items() if not np.isnan(v)}, + } + } + if method == "post": + resp = requests.post(record_url, json=data, headers=headers) + elif method == "put": + resp = requests.put(record_url, json=data, headers=headers) + res = json.loads(resp.text) + print(res) + + +def fet_records_from_table(): + """ + This function fetches all records from the table in Feishu. returns a dictionary with the experiment name as key and the record_id as value. + """ + get_record_url = ( + "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/records" + % (APP_TOKEN, TABLE_ID) + ) + headers = { + "Authorization": f"Bearer {get_tat_token()}", + "Content-Type": "application/json; charset=utf-8", + } + resp = requests.get(get_record_url, headers=headers) + res = json.loads(resp.text) + + if res["data"]["total"] == 0: + return {} + return { + item["fields"]["Experiment name"]: item["record_id"] + for item in res["data"]["items"] + } + +def merge_similar_keys(data): + grouped = defaultdict(list) + + # Iterate through the keys in the dictionary + for key in data.keys(): + # Extract the prefix (everything before '_fold' or '_mae') + prefix = key.split('_fold')[0] if '_fold' in key else key.split('_mae')[0] + grouped[f"{prefix}_mae"].append(data[key]) + + return grouped + +def fetch_ood_res(exp_path: str) -> dict: + run_id = exp_path.split("/")[-1] # Get basename as id + runs = PropertyRecord.query_by_run(run_id) + data = {} + columns = fetch_column() + columns = [dd['field_name'] for dd in columns ] + for run in runs: + data[f"{run.task_name}_mae"] = ( + run.property_mae if run.property_mae != -1 else np.nan + ) + + # process batch average + avg_fold_data = merge_similar_keys({k:v for k, v in data.items()}) + avg_fold_data = {k: np.mean(v) for k, v in avg_fold_data.items()} + + return avg_fold_data + + + +def main(exp_paths:List[str]): + run2record = fet_records_from_table() + for exp_path in exp_paths: + data = fetch_ood_res(exp_path) + + run_id=exp_path.split("/")[-1] + if run_id in run2record: + method = "put" + else: + method = "post" + send2table(data, run_id, record_id=run2record.get(run_id), method=method) + + + +def fetch_column(): + url = ( + "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/fields" + % (APP_TOKEN, TABLE_ID) + ) + headers = { + "Authorization": f"Bearer {get_tat_token()}", + "Content-Type": "application/json; charset=utf-8", + } + resp = requests.get(url, headers=headers) + res = json.loads(resp.text)['data']['items'] + return res + +def delete_column(): + res = fetch_column() + for fid in res: + url = ( + "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/fields/%s" + % (APP_TOKEN, TABLE_ID, fid['field_id']) + ) + headers = { + "Authorization": f"Bearer {get_tat_token()}", + "Content-Type": "application/json; charset=utf-8", + } + resp = requests.delete(url, headers=headers) + +def add_column(): + cur_columns = [col['field_name'] for col in fetch_column()] + with open(Path(os.path.dirname(__file__)).parent / "release/property_test/PROPERTY_TEST.yml","r") as f: + yaml_dd = yaml.safe_load(f) + + url = ( + "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/fields" + % (APP_TOKEN, TABLE_ID) + ) + headers = { + "Authorization": f"Bearer {get_tat_token()}", + "Content-Type": "application/json; charset=utf-8", + } + + data = { + "field_name": "Update Time", + "type": 5, + } + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) + + + for ood in yaml_dd["TASK_TO_HEAD_MAP"]: + data = { + "field_name": f"{ood}_mae", + "type": 2, + "property": { + "formatter": "0.0000" + } + } + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) + + +if __name__ == "__main__": + # fetch_column() + # delete_column() + # add_column() + # push_weights() + main([ + "/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3", + # "/1126_prod_shareft_120GUP_240by3_single_384_96_24", + # "/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", + # "/prod_shareft_120GUP_240by3_single_384_96_24", + # "/dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1", + # "/dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3" + ]) + pass diff --git a/lamstare/utils/submit_cronjob.py b/lamstare/utils/submit_cronjob.py index 3c8103b..beda7f7 100644 --- a/lamstare/utils/submit_cronjob.py +++ b/lamstare/utils/submit_cronjob.py @@ -1,16 +1,23 @@ import logging -from lamstare.release.submit_ood_test import main as main_ood +from lamstare.release.ood_test.submit_ood_test import main as main_ood from lamstare.experiments.run_test import main as main_ind if __name__ == "__main__": logging.basicConfig(level=logging.DEBUG) - freq = 200000 - step = None + freq = 400000 + step = 500000 # export CONDA_ENV=openlam_db # os.environ["CONDA_ENV"] = "openlam_db" - for exp_path in [ + for exp_path, freq in { + # "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24": 400000, + # "/mnt/data_nas/public/multitask/training_exps/0103_dpa3a_shareft_restart_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1":200000, + # "/mnt/data_nas/public/multitask/training_exps/0105_dpa3a_shareft_384_96_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_120GPU_240by3":400000, + # "/mnt/data_nas/public/multitask/training_exps/0115_dpa3a_shareft_128_64_32_scp1_e1a_tanh_rc6_120_arc_4_30_l6_64GPU_240by3_float32":400000, + # "/mnt/data_nas/public/multitask/training_exps/1225_dpa3a_shareft_rc6_120_arc_4_30_l6_120GPU_240by3_384_96_32_comp1":400000, + # "/mnt/workspace/public/multitask/training_exps/N0130_dpa3a_shareft_128_64_32_scp1_e1a_cdsilu10_rc6_120_arc_4_30_l6_64GPU_240by3_float32":200000, + "/mnt/workspace/public/multitask/training_exps/0202_dpa3a_shareft_256_128_32_scp1_e1a_csilu10_rc6_120_arc_4_30_l9_104GPU_240by3":200000, - "/mnt/data_nas/public/multitask/training_exps/1126_prod_shareft_120GUP_240by3_single_384_96_24" - ]: + + }.items(): main_ind(exp_path, freq, step) main_ood(exp_path, freq, step) diff --git a/lamstare/utils/table.py b/lamstare/utils/table.py index 43cf671..072e879 100644 --- a/lamstare/utils/table.py +++ b/lamstare/utils/table.py @@ -8,13 +8,14 @@ from typing import List, Dict, Any import os import yaml +from pathlib import Path +import warnings load_dotenv() APP_TOKEN = os.environ.get("TABLE_APP_TOKEN") TABLE_ID = os.environ.get("TABLE_ID") - def send2table( data: Dict[str, Any], run_id: str, record_id=None, method: str = "post" ) -> None: @@ -38,7 +39,7 @@ def send2table( "fields": { "Experiment name": run_id, "Update Time": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), - **{k:v for k,v in data.items() if not np.isnan(v)}, + **{k:v for k,v in data.items() if (v and not np.isnan(v))}, } } if method == "post": @@ -46,6 +47,7 @@ def send2table( elif method == "put": resp = requests.put(record_url, json=data, headers=headers) res = json.loads(resp.text) + print(res) def fet_records_from_table(): @@ -103,7 +105,7 @@ def fetch_ood_res(exp_path: str, weights_e: dict, weights_f: dict, weights_v: di data["Weighted_v_rmse"] = cal_weighted_log_mean(rmse_v, weights_v) - # path = "/mnt/workspace/public/multitask/LAMstare/ood_data_std_v.txt" + # path = "/mnt/data_nas/penganyang/ood_data_std_v.txt" # with open(path, "r") as f: # contents = f.read() # data = {} @@ -122,7 +124,6 @@ def fetch_ood_res(exp_path: str, weights_e: dict, weights_f: dict, weights_v: di # data["Weighted_e_rmse"] = cal_weighted_log_mean(rmse_e, weights_e) # data["Weighted_f_rmse"] = cal_weighted_log_mean(rmse_f, weights_f) # data["Weighted_v_rmse"] = cal_weighted_log_mean(rmse_v, weights_v) - # print(data) return data @@ -137,14 +138,15 @@ def cal_weighted_log_mean(rmses: dict, weights: dict): assert rmses.shape == weights.shape weighted_log_rmse = np.log(rmses) * weights - weighted_log_rmse_mean = np.nanmean(weighted_log_rmse) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=RuntimeWarning) + weighted_log_rmse_mean = np.nanmean(weighted_log_rmse) weighted_rmse_mean = np.exp(weighted_log_rmse_mean) return weighted_rmse_mean def main(exp_paths:List[str], weights_e:dict, weights_f:dict, weights_v:dict): run2record = fet_records_from_table() - for exp_path in exp_paths: data = fetch_ood_res(exp_path, weights_e, weights_f, weights_v) run_id=exp_path.split("/")[-1] @@ -156,7 +158,7 @@ def main(exp_paths:List[str], weights_e:dict, weights_f:dict, weights_v:dict): def push_weights(): - with open(os.path.dirname(__file__) + "/../release/OOD_DATASET.yml", "r") as f: + with open(Path(os.path.dirname(__file__)).parent / "release/ood_test/OOD_DATASET_v2.yml","r") as f: yaml_dd = yaml.safe_load(f) run2record = fet_records_from_table() @@ -175,7 +177,7 @@ def push_weights(): } send2table(data, "Weight", record_id=run2record.get("Weight"), method=method) -def delete_column(): +def fetch_column(): url = ( "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/fields" % (APP_TOKEN, TABLE_ID) @@ -185,8 +187,12 @@ def delete_column(): "Content-Type": "application/json; charset=utf-8", } resp = requests.get(url, headers=headers) - res = json.loads(resp.text) - for fid in res['data']['items']: + res = json.loads(resp.text)['data']['items'] + return res + +def delete_column(): + res = fetch_column() + for fid in res: url = ( "https://open.feishu.cn/open-apis/bitable/v1/apps/%s/tables/%s/fields/%s" % (APP_TOKEN, TABLE_ID, fid['field_id']) @@ -198,8 +204,8 @@ def delete_column(): resp = requests.delete(url, headers=headers) def add_column(): - - with open(os.path.dirname(__file__) + "/../release/OOD_DATASET.yml","r") as f: + cur_columns = [col['field_name'] for col in fetch_column()] + with open(Path(os.path.dirname(__file__)).parent / "release/ood_test/OOD_DATASET_v2.yml","r") as f: yaml_dd = yaml.safe_load(f) url = ( @@ -215,7 +221,8 @@ def add_column(): "field_name": "Update Time", "type": 5, } - resp = requests.post(url, json=data,headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) for ood in yaml_dd["OOD_TO_HEAD_MAP"]: @@ -226,7 +233,8 @@ def add_column(): "formatter": "0.0000" } } - resp = requests.post(url, json=data,headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) data = { "field_name": f"{ood}_f_rmse", "type": 2, @@ -234,7 +242,8 @@ def add_column(): "formatter": "0.0000" } } - resp = requests.post(url, json=data,headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) data = { "field_name": f"{ood}_v_rmse", "type": 2, @@ -251,7 +260,8 @@ def add_column(): "formatter": "0.0000" } } - resp = requests.post(url, json=data,headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) data = { "field_name": "Weighted_f_rmse", "type": 2, @@ -259,7 +269,8 @@ def add_column(): "formatter": "0.0000" } } - resp = requests.post(url, json=data,headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data,headers=headers) data = { "field_name": "Weighted_v_rmse", "type": 2, @@ -267,11 +278,13 @@ def add_column(): "formatter": "0.0000" } } - resp = requests.post(url, json=data, headers=headers) + if data['field_name'] not in cur_columns: + resp = requests.post(url, json=data, headers=headers) if __name__ == "__main__": + # fetch_column() # delete_column() # add_column() - push_weights() + # push_weights() pass