yeehaw

Author: cinjon

eval2.py

@@ -1,5 +1,6 @@
 import numpy as np
 import pandas as pd
+import matplotlib.pyplot as plt
 from scipy.interpolate import interp1d
 import os
 
@@ -119,13 +120,13 @@ def average_recall_vs_nr_proposals(proposals,
         recall[ridx, :] = matches.sum(axis=0) / positives.sum()
 
     # Recall is averaged.
-    recall = recall.mean(axis=0)
+    avg_recall = recall.mean(axis=0)
 
     # Get the average number of proposals per video.
     proposals_per_video = pcn_lst * (
         float(proposals.shape[0]) / video_lst.shape[0])
 
-    return recall, proposals_per_video
+    return recall, avg_recall, proposals_per_video
 
 
 def recall_vs_tiou_thresholds(proposals,
@@ -210,14 +211,80 @@ def recall_vs_tiou_thresholds(proposals,
     return recall, tiou_thresholds
 
 
+def plot_metric(opt,
+                average_nr_proposals,
+                average_recall,
+                recall,
+                tiou_thresholds=np.linspace(0.5, 1.0, 11)):
+
+    fn_size = 14
+    plt.figure(num=None, figsize=(12, 8))
+    ax = plt.subplot(1, 1, 1)
+
+    colors = [
+        'k', 'r', 'yellow', 'b', 'c', 'm', 'b', 'pink', 'lawngreen', 'indigo'
+    ]
+    area_under_curve = np.zeros_like(tiou_thresholds)
+    for i in range(recall.shape[0]):
+        area_under_curve[i] = np.trapz(recall[i], average_nr_proposals)
+
+    for idx, tiou in enumerate(tiou_thresholds[::2]):
+        ax.plot(average_nr_proposals,
+                recall[2 * idx, :],
+                color=colors[idx + 1],
+                label="tiou=[" + str(tiou) + "], area=" +
+                str(int(area_under_curve[2 * idx] * 100) / 100.),
+                linewidth=4,
+                linestyle='--',
+                marker=None)
+    # Plots Average Recall vs Average number of proposals.
+    ax.plot(
+        average_nr_proposals,
+        average_recall,
+        color=colors[0],
+        label="tiou = 0.5:0.05:1.0," + " area=" +
+        str(int(np.trapz(average_recall, average_nr_proposals) * 100) / 100.),
+        linewidth=4,
+        linestyle='-',
+        marker=None)
+
+    handles, labels = ax.get_legend_handles_labels()
+    ax.legend([handles[-1]] + handles[:-1], [labels[-1]] + labels[:-1],
+              loc='best')
+
+    plt.ylabel('Average Recall', fontsize=fn_size)
+    plt.xlabel('Average Number of Proposals per Video', fontsize=fn_size)
+    plt.grid(b=True, which="both")
+    plt.ylim([0, 1.0])
+    plt.setp(plt.axes().get_xticklabels(), fontsize=fn_size)
+    plt.setp(plt.axes().get_yticklabels(), fontsize=fn_size)
+    #plt.show()
+    save_path = os.path.join(opt['postprocessed_results_dir'], 'evaluation_result.jpg')
+    plt.savefig(save_path)
+
+    
 def evaluation_proposal(opt):
-    bsn_results = pd.read_csv(os.path.join(opt['postprocessed_results_dir'], 'thumos14_results.csv'))
+    if 'thumos' in opt['dataset']:
+        bsn_results = pd.read_csv(os.path.join(opt['postprocessed_results_dir'], 'thumos14_results.csv'))
+    elif 'gymnastics' in opt['dataset']:
+        bsn_results = pd.read_csv(os.path.join(opt['postprocessed_results_dir'], 'gym_results.csv'))        
     ground_truth = pd.read_csv(opt['video_info'])
 
     # Computes average recall vs average number of proposals.
-    average_recall, average_nr_proposals = average_recall_vs_nr_proposals(
+    recall, average_recall, average_nr_proposals = average_recall_vs_nr_proposals(
         bsn_results, ground_truth)
-
-    print(average_nr_proposals.shape)
+    area_under_curve = np.trapz(average_recall, average_nr_proposals)
     f = interp1d(average_nr_proposals, average_recall, axis=0)
-    print(f(50), f(100), f(200), f(500), f(1000))
+    interp_results = [(k, f(k)) for k in [50, 100, 200, 500, 1000]]
+    interp_str = ', '.join(['%d: %.4f' % (k, v) for k, v in interp_results])
+
+    with open(os.path.join(opt['postprocessed_results_dir'], 'output.txt'), 'w') as f:
+        f.write('[RESULTS] Performance on %s proposal task.\n' % opt['dataset'])
+        f.write('\tArea Under the AR vs AN curve: {}%\n'.format(
+            100. * float(area_under_curve) / average_nr_proposals[-1]))
+        f.write('Interpolation results: %s\n' % interp_str)
+    
+    plot_metric(opt, average_nr_proposals, average_recall, recall)
+
+    
+    

gen_postprocessed_results_jobs.py

@@ -27,20 +27,21 @@
 for pem_results_subdir in os.listdir(pem_results_dir):
     counter = int(regex.match(pem_results_subdir).groups()[0])
     job = pemrun(find_counter=counter)
+    job['do_eval_after_postprocessing'] = True
+    job['num_gpus'] = num_gpus
+    job['num_cpus'] = 48
+    job['gb'] = 64
+    job['time'] = 1
+    job['module'] = 'Post_processing'
 
     name = job['name']
     for ckpt_subdir in os.listdir(os.path.join(pem_results_dir, pem_results_subdir)):
         _job = deepcopy(job)
-        _job['module'] = 'Post_processing'
         dirkey = '%s/%s' % (pem_results_subdir, ckpt_subdir)
         _job['postprocessed_results_dir'] = os.path.join(postprocessed_results_dir, dirkey)
         _job['pem_inference_results_dir'] = os.path.join(pem_results_dir, dirkey)
         if 'thumos' in _job['dataset']:
             _job['video_info'] = _job['video_info'].replace('Full_Annotation.csv', 'thumos14_test_groundtruth.csv')
         _job['name'] = '2019.09.18.%s.%s' % (pem_results_subdir, ckpt_subdir)
-        _job['num_gpus'] = num_gpus
-        _job['num_cpus'] = 48
-        _job['gb'] = 64
-        _job['time'] = 4
 
         func(_job, counter, email, code_directory)

gen_tem_results_jobs.py

@@ -25,8 +25,6 @@
 
 for ckpt_subdir in os.listdir(ckpt_directory):
     counter = int(regex.match(ckpt_subdir).groups()[0])
-    if counter not in [195]:
-        continue
 
     _job = run(find_counter=counter)
     _job['num_gpus'] = 8
@@ -39,12 +37,7 @@
     _job['checkpoint_path'] = os.path.join(ckpt_directory, ckpt_subdir)
     _job['tem_results_subset'] = 'full'
     name = _job['name']
-    for ckpt_epoch in [5, 15, 20]:
-        if counter == 195:
-            if ckpt_epoch < 20:
-                continue
-            ckpt_epoch = 19
-            
+    for ckpt_epoch in [5, 8]:
         _job['checkpoint_epoch'] = ckpt_epoch
         _job['name'] = '%s.ckpt%d' % (name, ckpt_epoch)
         print(ckpt_subdir, counter)

opts.py

@@ -40,19 +40,21 @@ def parse_opt():
     # TEM model settings
     parser.add_argument('--tem_feat_dim', type=int, default=400)
     parser.add_argument('--tem_hidden_dim', type=int, default=512)
-    parser.add_argument('--tem_nonlinear_factor', type=int, default=0.01)
+    parser.add_argument('--tem_nonlinear_factor', type=float, default=0.01)
+    parser.add_argument('--tem_reset_params', action='store_true')
 
     # PEM model settings
     parser.add_argument('--pem_feat_dim', type=int, default=32)
     parser.add_argument('--pem_hidden_dim', type=int, default=256)
 
     # TEM Training settings
     parser.add_argument('--tem_training_lr', type=float, default=0.001)
-    parser.add_argument('--tem_weight_decay', type=float, default=0.0001)
-    parser.add_argument('--tem_lr_penalty', type=float, default=0.0)
+    parser.add_argument('--tem_weight_decay', type=float, default=0.0)
+    parser.add_argument('--tem_l2_loss', type=float, default=0.005)
     parser.add_argument('--tem_epoch', type=int, default=30) # NOTE: was 20
     parser.add_argument('--tem_step_size', type=int, default=7)
     parser.add_argument('--tem_step_gamma', type=float, default=0.1) # 0.1
+    parser.add_argument('--tem_lr_milestones', type=str, default='5') 
     parser.add_argument('--tem_batch_size', type=int, default=16)
     parser.add_argument('--tem_match_thres', type=float, default=0.5)
     parser.add_argument('--tem_compute_loss_interval', type=float, default=20)    
@@ -64,6 +66,7 @@ def parse_opt():
     parser.add_argument('--pem_nonlinear_factor', type=int, default=0.1)
     parser.add_argument('--pem_training_lr', type=float, default=0.01)
     parser.add_argument('--pem_weight_decay', type=float, default=0.00001)
+    parser.add_argument('--pem_l2_loss', type=float, default=0.000025)    
     parser.add_argument('--pem_epoch', type=int, default=20)
     parser.add_argument('--pem_step_size', type=int, default=10)
     parser.add_argument('--pem_step_gamma', type=float, default=0.1)
@@ -100,6 +103,7 @@ def parse_opt():
     # Post processing
     parser.add_argument('--post_process_top_K', type=int, default=100)
     parser.add_argument('--post_process_thread', type=int, default=8)
+    parser.add_argument('--do_eval_after_postprocessing', action='store_true')    
     parser.add_argument('--soft_nms_alpha', type=float, default=0.75)
     parser.add_argument('--soft_nms_low_thres', type=float, default=0.65)
     parser.add_argument('--soft_nms_high_thres', type=float, default=0.9)

pgm.py

@@ -65,11 +65,16 @@ def generate_proposals(opt, video_list, video_data):
             print('Skipping %s because %s is not a path.' % (video_name, results_path))
             skipped_paths.append(results_path)
             continue
+
+        anno_df_ = anno_df[anno_df.video == video_name]
 
         tdf = pd.read_csv(results_path)
         start_scores = tdf.start.values[:]
         end_scores = tdf.end.values[:]
-        frame_list = tdf.frames.values[:]
+        try:
+            frame_list = tdf.frames.values[:]
+        except Exception as e:
+            frame_list = tdf.frame.values[:]
 
         start_bins = np.zeros(len(start_scores))
         start_bins[[0, -1]] = 1
@@ -95,15 +100,14 @@ def generate_proposals(opt, video_list, video_data):
         xmin_score_list = []
         xmax_list = []
         xmax_score_list = []
-        for index in range(len(start_bins)):
+        for index in range(len(start_scores)):
             if start_bins[index] == 1:
                 xmin_list.append(int(frame_list[index]))
                 xmin_score_list.append(start_scores[index])
             if end_bins[index] == 1:
                 xmax_list.append(int(frame_list[index]))
                 xmax_score_list.append(end_scores[index])
 
-        print('Doing new_props')
         new_props = []
         for ii in range(len(xmax_list)):
             if ii % 5000 == 0:
@@ -133,7 +137,6 @@ def generate_proposals(opt, video_list, video_data):
         # print('saving preliminary to %s' % path)
         # new_df.to_csv(path, index=False)
 
-        print('Doing gt max')
         if video_dict is not None:
             video_info = video_dict[video_name]
             video_fps = video_info['fps']
@@ -146,25 +149,29 @@ def generate_proposals(opt, video_list, video_data):
 
                 gt_xmins.append(annos[idx]["segment"][0] * video_fps)
                 gt_xmaxs.append(annos[idx]["segment"][1] * video_fps)
-        elif anno_df is not None:
-            gt_xmins = anno_df.startFrame.values[:]
-            gt_xmaxs = anno_df.endFrame.values[:]
-            
-        print('GT Xmins and Xmaxs')
+        elif anno_df_ is not None:
+            gt_xmins = anno_df_.startFrame.values[:]
+            gt_xmaxs = anno_df_.endFrame.values[:]
+
+        # Ok, so all of these gt_xmins and gt_xmaxs are the same ...
+        # ... As are the xmin and xmax values in the DFs.
+        
         new_iou_list = []
         match_xmin_list = []
         match_xmax_list = []
-        print('Doing iou and xmin lists.')        
         for j in range(len(new_df)):
             tmp_new_iou = list(
-                iou_with_anchors(new_df.xmin.values[j],
-                                 new_df.xmax.values[j], gt_xmins, gt_xmaxs))
+                iou_with_anchors(
+                    new_df.xmin.values[j],
+                    new_df.xmax.values[j],
+                    gt_xmins,
+                    gt_xmaxs)
+            )
             new_iou_list.append(max(tmp_new_iou))
             match_xmin_list.append(gt_xmins[tmp_new_iou.index(max(tmp_new_iou))])
             match_xmax_list.append(gt_xmaxs[tmp_new_iou.index(max(tmp_new_iou))])
 
         new_ioa_list = []
-        print('Doing ioa max')        
         for j in range(len(new_df)):
             tmp_new_ioa = max(
                 ioa_with_anchors(new_df.xmin.values[j],
@@ -179,8 +186,7 @@ def generate_proposals(opt, video_list, video_data):
         print('saving to %s' % path)
         new_df.to_csv(path, index=False)
         print('Video %s took %.4f time' % (video_name, time.time() - start_time))
-    print('Total time was %.4f' % (time.time() - start_time))
-    print(skipped_paths)
+    print('Total time was %.4f' % (time.time() - start_time), skipped_paths)
 
 
 def getDatasetDict(opt):
@@ -211,11 +217,10 @@ def bookend_zeros(arr, num):
     return np.concatenate([np.zeros([num]), arr, np.zeros([num])])
 
 
-def generate_features_repr(opt, video_list, video_dict):
+def generate_features(opt, video_list, video_dict):
     num_sample_start = opt["num_sample_start"]
     num_sample_end = opt["num_sample_end"]
     num_sample_action = opt["num_sample_action"]
-    num_sample_interpld = opt["num_sample_interpld"]
     num_videoframes = opt["num_videoframes"]
     skip_videoframes = opt["skip_videoframes"]
     bookend_num = int(num_videoframes / skip_videoframes)
@@ -234,6 +239,10 @@ def generate_features_repr(opt, video_list, video_dict):
             print("NOT generating features for %s because features don't exist." % video_name)
             continue        
         adf = pd.read_csv(tem_path)
+        try:
+            adf_frames = adf.frames.values[:]
+        except Exception as e:
+            adf_frames = adf.frame.values[:]
 
         proposals_path = os.path.join(proposals_dir, '%s.proposals.csv' % video_name)
         if not os.path.exists(proposals_path):
@@ -247,7 +256,7 @@ def generate_features_repr(opt, video_list, video_dict):
         score_end = bookend_zeros(adf.end.values[:], bookend_num)
         score_start = bookend_zeros(adf.start.values[:], bookend_num)
         # 
-        snippets = [skip_videoframes*i - normalizer for i in range(bookend_num)] + list(adf.frames.values[:]) + [skip_videoframes*i + skip_videoframes + adf.frames.values[:][-1] for i in range(bookend_num)]
+        snippets = [skip_videoframes*i - normalizer for i in range(bookend_num)] + list(adf_frames) + [skip_videoframes*i + skip_videoframes + adf_frames[-1] for i in range(bookend_num)]
         print('Computing the interp1ds')
         f_action = interp1d(snippets, score_action, axis=0)
         f_start = interp1d(snippets, score_start, axis=0)
@@ -282,7 +291,11 @@ def generate_features_repr(opt, video_list, video_dict):
 
             #action
             plen_action = (xmax - xmin) / (num_sample_action - 1)
-            tmp_x_new = [xmin_0 + plen_action * ii for ii in range(num_sample_action)]
+            
+            # I originall had the following (see xmin_0)
+            # tmp_x_new = [xmin_0 + plen_action * ii for ii in range(num_sample_action)]
+            # But they have this:
+            tmp_x_new = [xmin + plen_action * ii for ii in range(num_sample_action)]            
             tmp_y_new_action = f_action(tmp_x_new)
             tmp_y_new_action = np.reshape(tmp_y_new_action, [-1])
 
@@ -303,6 +316,8 @@ def PGM_proposal_generation(opt):
     if 'thumos' in opt['dataset']:
         video_data = pd.read_csv(os.path.join(opt["video_info"], 'Full_Annotation.csv'))
         video_list = sorted(list(set(video_data.video.values[:])))
+        # video_list = [k for k in video_list if 'video_validation_0000053' in k]
+        # print(video_list)
     else:
         video_data = load_json(opt["video_anno"])
         video_list = sorted(video_data.keys())  #[:199]
@@ -348,7 +363,7 @@ def PGM_feature_generation(opt):
     # NOTE: change this back.
     # video_list = [k for k in video_list if '12.18.18' in k or '12.5.18' in k]
 
-    func = generate_features_repr
+    func = generate_features
     num_videos = len(video_list)
     num_threads = min(num_videos, opt['pgm_thread'])
     num_videos_per_thread = int(num_videos / opt["pgm_thread"])

post_processing.py

@@ -107,7 +107,7 @@ def video_post_process(opt, video_list, video_dict):
         df = df.sort_values(by="score", ascending=False)
         video_info = video_dict[video_name]
         video_duration = float(
-            video_info["duration_frame"] / 16 *
+            video_intfo["duration_frame"] / 16 *
             16) / video_info["duration_frame"] * video_info["duration_second"]
         proposal_list = []
 

post_processing2.py

@@ -102,7 +102,7 @@ def BSN_post_processing(opt):
     pem_inference_results = opt['pem_inference_results_dir']
     for num, video_name in enumerate(videoNameList):
         if num % 25 == 0:
-            print(num, len(videoNameList), name)
+            print(num, len(videoNameList), video_name)
         videoAnno = annoDf[annoDf["video-name"] == video_name]
         videoFrame = videoAnno["video-frames"].values[0]
         try:
@@ -130,5 +130,9 @@ def BSN_post_processing(opt):
     output_dir = opt['postprocessed_results_dir']
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
-    outfile = os.path.join(output_dir, 'thumos14_results.csv')
+
+    if 'thumos' in opt['dataset']:
+        outfile = os.path.join(output_dir, 'thumos14_results.csv')        
+    elif 'gymnastics' in opt['dataset']:
+        outfile = os.path.join(output_dir, 'gym_results.csv')        
     outDf.to_csv(outfile, index=False)