transfer the norm_func to main.py and easier for custom design

Author: mantle2048

.gitignore

@@ -5,6 +5,7 @@ tests/
 exps_cached/
 diagnosis/
 figs/
+exps/
 __pycache__/
 
 # C extensions

Makefile

@@ -22,3 +22,12 @@ imporve:
 
 efficiency:
 	python main.py type=sample_efficiency_curve
+
+## Delete all compiled Python files
+clean:
+	find . -type f -name "*.py[co]" -delete
+	find . -type d -name "__pycache__" -delete
+
+## Lint using flake8
+lint:
+	flake8 src

main.py

@@ -20,6 +20,19 @@
 CONFIDENCE = 0.68
 
 
+def random_score_norm_func(task: str, scores: List):
+    random_score = {
+        'HalfCheetah-v4': -290.0479832104089,
+        'Ant-v4': -55.14243068976598,
+        'Walker2d-v4': 2.5912887180069686,
+        'Humanoid-v4': 120.45141735893694
+    }
+    scores = np.array(scores)
+    nume = scores - random_score[task]
+    deno = np.max(scores) - random_score[task]
+    return nume / deno
+
+
 def create_diagnosis(
     n_epoch: int = 200,
     epoch_len: int = 5000,
@@ -103,7 +116,11 @@ def OG(metric_val): return \
         save_fig(fig, f'metric_curve_{task.lower()}', fig_dir)
 
     algo_scores, normalized_algo_scores = \
-        read_and_norm_algo_scores(diagnosis_dir, algos, 'all')
+        read_and_norm_algo_scores(
+            diagnosis_dir, algos,
+            milestone='all',
+            norm_func=random_score_norm_func
+        )
 
     scores, cis = \
         rly.get_interval_estimates(
@@ -131,7 +148,10 @@ def metric_value(
 ):
 
     algo_scores, normalized_algo_scores = \
-        read_and_norm_algo_scores(diagnosis_dir, algos, milestone)
+        read_and_norm_algo_scores(
+            diagnosis_dir, algos, milestone,
+            norm_func=random_score_norm_func,
+        )
 
     aggregate_func_mapper = {
         'Mean': metrics.aggregate_mean,
@@ -180,7 +200,11 @@ def performance_profiles(
     for i, milestone in enumerate(milestones):
 
         algo_scores, normalized_algo_scores = \
-            read_and_norm_algo_scores(diagnosis_dir, algos, milestone)
+            read_and_norm_algo_scores(
+                diagnosis_dir, algos, milestone,
+                norm_func=random_score_norm_func
+            )
+
         perf_prof, perf_prof_cis = \
             rly.create_performance_profile(
                 normalized_algo_scores, tau,
@@ -220,7 +244,10 @@ def probability_of_improvement(
     **kwargs,
 ):
     algo_scores, normalized_algo_scores = \
-        read_and_norm_algo_scores(diagnosis_dir, algos, milestone)
+        read_and_norm_algo_scores(
+            diagnosis_dir, algos, milestone,
+            norm_func=random_score_norm_func
+        )
 
     pairs = generate_pairs(algos)
 
@@ -250,7 +277,11 @@ def sample_efficiency_curve(
     **kwargs,
 ):
     algo_scores, normalized_algo_scores = \
-        read_and_norm_algo_scores(diagnosis_dir, algos, 'all')
+        read_and_norm_algo_scores(
+            diagnosis_dir, algos, 'all',
+            norm_func=random_score_norm_func
+        )
+
     steps = np.array(steps) - 1
     normalized_algo_steps_scores_dict = {algo: scores[:, :, steps] for algo, scores
                                          in normalized_algo_scores.items()}
@@ -294,7 +325,10 @@ def overall_ranks(
 
     for i, milestone in enumerate(milestones):
         algo_scores, normalized_algo_scores = \
-            read_and_norm_algo_scores(diagnosis_dir, algos, milestone)
+            read_and_norm_algo_scores(
+                diagnosis_dir, algos, milestone,
+                norm_func=random_score_norm_func
+            )
 
         # num_task * (num_algo * num_algo)
         rank_matrix = \

rlplot/plot_helpers.py

@@ -262,49 +262,6 @@ def generate_pairs(elements):
     return pairs[::-1]
 
 
-random_score = {
-
-    'HalfCheetah-v4': -290.0479832104089,
-    'Ant-v4': -55.14243068976598,
-    'Walker2d-v4': 2.5912887180069686,
-    'Humanoid-v4': 120.45141735893694,
-
-}
-
-
-def random_score_norm_func(task: str, scores: List):
-    scores = np.array(scores)
-    nume = scores - random_score[task]
-    deno = np.max(scores) - random_score[task]
-    return nume / deno
-
-
-def normalized_scores(
-    task: str,
-    scores: Union[np.ndarray, List],
-    norm_func: Callable
-):
-    algos = list(scores.keys())
-    envs = list(scores[algos[0]].keys())
-    num_runs = scores[algos[0]][envs[0]].shape[0]
-    env_scores = {env: [] for env in envs}
-    for algo in algos:
-        for env in envs:
-            env_scores[env] += scores[algo][env].tolist()
-    normalized_env_scores = {}
-    for env in envs:
-        normalized_env_scores[env] = norm_func(env, env_scores[env])
-    normalized_scores = {}
-    start, end = 0, num_runs
-    for algo in algos:
-        normalized_scores[algo] = {}
-        for env in envs:
-            normalized_scores[algo][env] = normalized_env_scores[env][start:end]
-        start += num_runs
-        end += num_runs
-    return normalized_scores
-
-
 def convert_to_matrix(score_dict, sort=False):
     if sort:
         keys = sorted(list(score_dict.keys()))
@@ -324,8 +281,8 @@ def read_milestone_from_yaml(
 
 
 def read_and_norm_algo_scores(
-    dir, algos, milestone='1m',
-    norm_func=random_score_norm_func,
+    dir, algos, milestone: str,
+    norm_func: Callable,
 ):
     algo_scores = {
         algo: read_milestone_from_yaml(dir, algo, milestone)

rlplot/plot_utils.py

@@ -8,6 +8,7 @@
 
 # yanked and modified from https://github.com/google-research/rliable/blob/master/rliable/plot_utils.py
 
+
 def _non_linear_scaling(performance_profiles,
                         tau_list,
                         xticklabels=None,
@@ -714,3 +715,29 @@ def plot_overall_ranks(
 #     # fig.subplots_adjust(hspace=0.25)
 #
 #     save_fig(fig, save_name, save_dir)
+
+
+# def normalized_scores(
+#     task: str,
+#     scores: Union[np.ndarray, List],
+#     norm_func: Callable
+# ):
+#     algos = list(scores.keys())
+#     envs = list(scores[algos[0]].keys())
+#     num_runs = scores[algos[0]][envs[0]].shape[0]
+#     env_scores = {env: [] for env in envs}
+#     for algo in algos:
+#         for env in envs:
+#             env_scores[env] += scores[algo][env].tolist()
+#     normalized_env_scores = {}
+#     for env in envs:
+#         normalized_env_scores[env] = norm_func(env, env_scores[env])
+#     normalized_scores = {}
+#     start, end = 0, num_runs
+#     for algo in algos:
+#         normalized_scores[algo] = {}
+#         for env in envs:
+#             normalized_scores[algo][env] = normalized_env_scores[env][start:end]
+#         start += num_runs
+#         end += num_runs
+#     return normalized_scores