updated.'

Author: cinjon

dataset.py

@@ -490,6 +490,7 @@ def __init__(self, proposals, frame_list, max_zero_weight=0.25):
 
             if percent < max_zero_weight:
                 # We don't care if there aren't many zeros.
+                weights.append(1)
                 continue
 
             # Otherwise, we roughly want there to be 10% zeros at most.
@@ -525,8 +526,6 @@ def __init__(self, opt, subset="train"):
     def _exists(self, video_name):
         pgm_proposals_path = os.path.join(self.opt['pgm_proposals_dir'], '%s.proposals.csv' % video_name)
         pgm_features_path = os.path.join(self.opt['pgm_features_dir'], '%s.features.npy' % video_name)
-        print(pgm_proposals_path)
-        print(pgm_features_path)
         return os.path.exists(pgm_proposals_path) and os.path.exists(pgm_features_path)
 
     def _getDatasetDict(self):
@@ -536,16 +535,18 @@ def _getDatasetDict(self):
         for i in range(len(anno_df)):
             video_name = anno_df.video.values[i]
             video_info = anno_database[video_name]
-            video_subset = anno_df.subset.values[i]
+            
+            if 'thumos' in self.opt['dataset']:
+                video_subset = video_name.split('_')[1].replace('validation', 'train')
+            else:
+                video_subset = anno_df.subset.values[i]
+                
             if self.subset == "full":
                 self.video_dict[video_name] = video_info
             if self.subset in video_subset:
                 self.video_dict[video_name] = video_info
         self.video_list = sorted(self.video_dict.keys())
         self.video_list = [k for k in self.video_list if self._exists(k)]
-        print('\n***\n')
-        print(self.subset)
-        print(self.video_list)
 
         if self.opt['pem_do_index']:
             self.features = {}
@@ -554,15 +555,19 @@ def _getDatasetDict(self):
             for video_name in self.video_list:
                 pgm_proposals_path = os.path.join(self.opt['pgm_proposals_dir'], '%s.proposals.csv' % video_name)
                 pgm_features_path = os.path.join(self.opt['pgm_features_dir'], '%s.features.npy' % video_name)
-                pdf = pd.read_csv(pgm_proposals_path)
-                pdf = pdf.sort_values(by="score", ascending=False)
+                pdf = pd.read_csv(pgm_proposals_path)                    
                 video_feature = np.load(pgm_features_path)
-                video_feature = video_feature[pdf[:self.top_K].index]
                 pre_count = len(pdf)
-                pdf = pdf[:self.top_K]
-                print(video_name, pre_count, len(pdf), video_feature.shape)
-                print('Num zeros in match_iou: ', len(pdf[pdf.match_iou == 0]))
-                print('')
+                if self.top_K is not None:
+                    try:
+                        pdf = pdf.sort_values(by="score", ascending=False)
+                    except KeyError:
+                        pdf['score'] = pdf.xmin_score * pdf.xmax_score
+                        pdf = pdf.sort_values(by="score", ascending=False)
+                    pdf = pdf[:self.top_K]
+                    video_feature = video_feature[pdf.index]
+                    
+                print(video_name, pre_count, len(pdf), video_feature.shape, pgm_proposals_path, pgm_features_path)
                 self.proposals[video_name] = pdf
                 self.features[video_name] = video_feature
                 self.indices.extend([(video_name, i) for i in range(len(pdf))])
@@ -600,10 +605,11 @@ def __getitem__(self, index):
             # ***
             pdf = pdf.sort_values(by="score", ascending=False)
             # ***
-            pdf = pdf[:self.top_K]
-            
             video_feature = np.load(pgm_features_path)
-            video_feature = video_feature[:self.top_K, :]
+            if self.top_K is not None:
+                pdf = pdf[:self.top_K]
+                video_feature = video_feature[:self.top_K, :]
+                
             video_feature = torch.Tensor(video_feature)
 
             if self.mode == "train":

gen_pgm_results_jobs.py

@@ -23,16 +23,22 @@
 
 for tem_results_subdir in os.listdir(tem_results_dir):
     counter = int(regex.match(tem_results_subdir).groups()[0])
+    if counter in [301]:
+        continue
+    
+    print(tem_results_dir, tem_results_subdir, counter)
     job = run(find_counter=counter)
 
     name = job['name']
     for ckpt_subdir in os.listdir(os.path.join(tem_results_dir, tem_results_subdir)):
         _job = deepcopy(job)
+        if 'thumos' in _job['dataset']:
+            _job['video_anno'] = os.path.join(_job['video_info'], 'thumos_anno_action.json')
         _job['num_gpus'] = 0
         _job['num_cpus'] = 48
         _job['pgm_thread'] = 40
         _job['gb'] = 64
-        _job['time'] = 6 # what time should this be?
+        _job['time'] = 5 # how long?
         _job['tem_results_dir'] = os.path.join(tem_results_dir, tem_results_subdir, ckpt_subdir)
 
         propdir = os.path.join(pgm_proposals_dir, tem_results_subdir, ckpt_subdir)

gen_tem_results_jobs.py

@@ -28,6 +28,9 @@
 
 for ckpt_subdir in os.listdir(ckpt_directory):
     counter = int(regex.match(ckpt_subdir).groups()[0])
+    if counter not in [195]:
+        continue
+
     _job = run(find_counter=counter)
     _job['num_gpus'] = 8
     _job['num_cpus'] = 8 * 10
@@ -40,6 +43,11 @@
     _job['tem_results_subset'] = 'full'
     name = _job['name']
     for ckpt_epoch in [5, 15, 20]:
+        if counter == 195:
+            if ckpt_epoch < 20:
+                continue
+            ckpt_epoch = 19
+            
         _job['checkpoint_epoch'] = ckpt_epoch
         _job['name'] = '%s.ckpt%d' % (name, ckpt_epoch)
         print(ckpt_subdir, counter)

loss_function.py

@@ -90,9 +90,9 @@ def PEM_loss_function(anchors_iou, match_iou, opt):
 
     iou_weights = u_hmask + u_smmask + u_slmask
     iou_loss = F.smooth_l1_loss(anchors_iou, match_iou.squeeze())
-    print('LOSS')
-    print(iou_loss.shape, iou_weights.shape)
-    print(iou_weights)
+    # print('LOSS')
+    # print(iou_loss.shape, iou_weights.shape)
+    # print(iou_weights)
     iou_loss = torch.sum(iou_loss * iou_weights) / (1e-6 + torch.sum(iou_weights))
 
     return {'iou_loss': iou_loss}

main.py

@@ -89,7 +89,7 @@ def train_TEM(data_loader, model, optimizer, epoch, global_step, comet_exp, opt)
 
     print('Count: ', count)
     epoch_sums, epoch_avg = compute_metrics(epoch_sums, loss, count)
-    epoch_avg['current_l2'] = sum([W.norm(2) for W in model.module.parameters()])
+    epoch_avg['current_l2'] = sum([W.norm(2) for W in model.module.parameters()]).cpu().detach().numpy()
     steps_per_second = (n_iter+1) / (time.time() - start)
     epoch_avg['steps_per_second'] = steps_per_second
     print('\n***End of Epoch %d***\nS/S %.3f, Global Step %d, Local Step %d / %d.' % (epoch, steps_per_second, global_step, n_iter, len(data_loader)))
@@ -241,7 +241,6 @@ def test_PEM(data_loader, model, epoch, global_step, comet_exp, opt):
 def BSN_Train_TEM(opt):
     if opt['do_representation']:
         model = TEM(opt)
-        img_loading_func = get_img_loader(opt)
         partial_load(opt['representation_checkpoint'], model)
         for param in model.representation_model.parameters():
             param.requires_grad = False
@@ -258,6 +257,7 @@ def BSN_Train_TEM(opt):
                            weight_decay=opt["tem_weight_decay"])
 
     if opt['dataset'] == 'gymnastics':
+        img_loading_func = get_img_loader(opt)
         train_data_set = GymnasticsDataSet(opt, subset=opt['tem_train_subset'], img_loading_func=img_loading_func, overlap_windows=True)
         train_sampler = GymnasticsSampler(train_data_set.video_dict, train_data_set.frame_list)
         test_data_set = GymnasticsDataSet(opt, subset="test", img_loading_func=img_loading_func)
@@ -323,10 +323,10 @@ def BSN_Train_TEM(opt):
         comet_exp.set_name(opt['name'])
 
     for epoch in range(opt["tem_epoch"]):
-        # test_TEM(test_loader, model, epoch, global_step, comet_exp, opt)
         global_step = train_TEM(train_loader, model, optimizer, epoch, global_step, comet_exp, opt)
         scheduler.step()
-    test_TEM(test_loader, model, epoch, global_step, comet_exp, opt)        
+        test_TEM(test_loader, model, epoch, global_step, comet_exp, opt)
+    # test_TEM(test_loader, model, epoch, global_step, comet_exp, opt)        
 
 
 def BSN_Train_PEM(opt):
@@ -344,7 +344,6 @@ def collate_fn(batch):
         batch_iou = torch.cat([x[1] for x in batch])
         return batch_data, batch_iou
 
-
     train_dataset = ProposalDataSet(opt, subset="train")
     train_sampler = ProposalSampler(train_dataset.proposals, train_dataset.indices, max_zero_weight=opt['pem_max_zero_weight'])
 
@@ -396,17 +395,17 @@ def collate_fn(batch):
         comet_exp.set_name(opt['name'])    
 
     for epoch in range(opt["pem_epoch"]):
+        test_PEM(test_loader, model, epoch, global_step, comet_exp, opt)
         global_step = train_PEM(train_loader, model, optimizer, epoch, global_step, comet_exp, opt)
         scheduler.step()
-        test_PEM(test_loader, model, epoch, global_step, comet_exp, opt)
+    test_PEM(test_loader, model, epoch, global_step, comet_exp, opt)
 
 
 def BSN_inference_TEM(opt):
     output_dir = os.path.join(opt['tem_results_dir'], opt['checkpoint_path'].split('/')[-1])
     print(sorted(opt.items()))
 
     model = TEM(opt)
-    img_loading_func = get_img_loader(opt)
     checkpoint_epoch = opt['checkpoint_epoch']
     if checkpoint_epoch is not None:
         checkpoint_path = os.path.join(opt['checkpoint_path'], 'tem_checkpoint.%d.pth' % checkpoint_epoch)
@@ -429,6 +428,7 @@ def BSN_inference_TEM(opt):
     model.eval()
 
     if opt['dataset'] == 'gymnastics':
+        img_loading_func = get_img_loader(opt)
         dataset = GymnasticsDataSet(opt, subset=opt['tem_results_subset'], img_loading_func=img_loading_func)
     elif opt['dataset'] == 'thumosfeatures':
         feature_dirs = opt['feature_dirs'].split(',')
@@ -488,11 +488,8 @@ def BSN_inference_TEM(opt):
                 current_data[0].extend(batch_action[batch_idx])
                 current_data[1].extend(batch_start[batch_idx])
                 current_data[2].extend(batch_end[batch_idx])
-                # NOTE: Will this work ?
-                ###
                 current_data[3].extend(anchor_xmin)
                 current_data[4].extend(anchor_xmax)
-                ###
                 current_data[5].extend(list(frames))
             else:
                 batch_video = video[batch_idx]