[benchmark] Fix detector not being reset between runs

Author: Breakthrough

benchmarks/README.md

@@ -27,16 +27,16 @@ python evaluate.py -d BBC --detector detect-content
 ```
 
 ### Result
-The performance is computed as recall, precision, f1, and elapsed time. 
+The performance is computed as recall, precision, f1, and elapsed time.
 The following results indicate that ContentDetector achieves the highest performance on the BBC dataset.
 
 |      Detector     | Recall | Precision |   F1  | Elapsed time (second) |
 |:-----------------:|:------:|:---------:|:-----:|:---------------------:|
-|  AdaptiveDetector |  7.80  |   96.18   | 14.44 |         25.75         |
-|  ContentDetector  |  84.52 |   88.77   | 86.59 |         25.50         |
-|    HashDetector   |  8.57  |   80.27   | 15.48 |         23.78         |
-| HistogramDetector |  8.22  |   70.82   | 14.72 |         18.60         |
-| ThresholdDetector |  0.00  |    0.00   |  0.00 |         18.95         |
+|  AdaptiveDetector |  87.52 |   97.21   | 92.11 |         27.84         |
+|  ContentDetector  |  85.23 |   89.53   | 87.33 |         26.46         |
+|    HashDetector   |  92.96 |   76.27   | 83.79 |         16.26         |
+| HistogramDetector |  90.55 |   72.76   | 80.68 |         16.13         |
+| ThresholdDetector |  0.00  |   0.00    |  0.00 |         18.95         |
 
 ## Citation
 ### BBC

benchmarks/bbc_dataset.py

@@ -1,5 +1,6 @@
-import os
 import glob
+import os
+
 
 class BBCDataset:
     """
@@ -8,14 +9,19 @@ class BBCDataset:
     The dataset consists of 11 videos (BBC/videos/bbc_01.mp4 to BBC/videos/bbc_11.mp4).
     The annotated scenes are provided in corresponding files (BBC/fixed/[i]-scenes.txt).
     """
+
     def __init__(self, dataset_dir: str):
-        self._video_files = [file for file in sorted(glob.glob(os.path.join(dataset_dir, 'videos', '*.mp4')))]
-        self._scene_files = [file for file in sorted(glob.glob(os.path.join(dataset_dir, 'fixed', '*-scenes.txt')))]
-        assert (len(self._video_files) == len(self._scene_files))
+        self._video_files = [
+            file for file in sorted(glob.glob(os.path.join(dataset_dir, "videos", "*.mp4")))
+        ]
+        self._scene_files = [
+            file for file in sorted(glob.glob(os.path.join(dataset_dir, "fixed", "*.txt")))
+        ]
+        assert len(self._video_files) == len(self._scene_files)
         for video_file, scene_file in zip(self._video_files, self._scene_files):
-            video_id = os.path.basename(video_file).replace('bbc_', '').split('.')[0]
-            scene_id = os.path.basename(scene_file).split('-')[0]
-            assert (video_id == scene_id)
+            video_id = os.path.basename(video_file).replace("bbc_", "").split(".")[0]
+            scene_id = os.path.basename(scene_file).split("_")[0]
+            assert video_id == scene_id
 
     def __getitem__(self, index):
         video_file = self._video_files[index]

benchmarks/benchmark.py

@@ -1,51 +1,82 @@
-import time
 import argparse
+import time
+
 from bbc_dataset import BBCDataset
 from evaluator import Evaluator
-
 from tqdm import tqdm
-from scenedetect import detect
-from scenedetect import AdaptiveDetector, ContentDetector, HashDetector, HistogramDetector, ThresholdDetector
 
-def _load_detector(detector_name: str):
+from scenedetect import (
+    AdaptiveDetector,
+    ContentDetector,
+    HashDetector,
+    HistogramDetector,
+    ThresholdDetector,
+    detect,
+)
+
+
+def make_detector(detector_name: str):
     detector_map = {
-        'detect-adaptive': AdaptiveDetector(),
-        'detect-content': ContentDetector(),
-        'detect-hash': HashDetector(),
-        'detect-hist': HistogramDetector(),
-        'detect-threshold': ThresholdDetector(),
+        "detect-adaptive": AdaptiveDetector(),
+        "detect-content": ContentDetector(),
+        "detect-hash": HashDetector(),
+        "detect-hist": HistogramDetector(),
+        "detect-threshold": ThresholdDetector(),
     }
     return detector_map[detector_name]
 
-def _detect_scenes(detector, dataset):
+
+def _detect_scenes(detector_type: str, dataset):
     pred_scenes = {}
     for video_file, scene_file in tqdm(dataset):
         start = time.time()
+        detector = make_detector(detector_type)
         pred_scene_list = detect(video_file, detector)
         elapsed = time.time() - start
-
-        pred_scenes[scene_file] = {
-            'video_file': video_file,
-            'elapsed': elapsed,
-            'pred_scenes': [scene[1].frame_num for scene in pred_scene_list]
+        scenes = {
+            scene_file: {
+                "video_file": video_file,
+                "elapsed": elapsed,
+                "pred_scenes": [scene[1].frame_num for scene in pred_scene_list],
+            }
         }
+        result = Evaluator().evaluate_performance(scenes)
+        print(f"{video_file} results:")
+        print(
+            "Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}\n".format(
+                result["recall"], result["precision"], result["f1"], result["elapsed"]
+            )
+        )
+        pred_scenes.update(scenes)
 
     return pred_scenes
 
-def main(args):
-    dataset = BBCDataset('BBC')
-    detector = _load_detector(args.detector)
-    pred_scenes = _detect_scenes(detector, dataset)
-    evaluator = Evaluator()
-    result = evaluator.evaluate_performance(pred_scenes)
 
-    print('Detector: {} Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}'
-          .format(args.detector, result['recall'], result['precision'], result['f1'], result['elapsed']))
+def main(args):
+    pred_scenes = _detect_scenes(detector_type=args.detector, dataset=BBCDataset("BBC"))
+    result = Evaluator().evaluate_performance(pred_scenes)
+    print("Overall Results:")
+    print(
+        "Detector: {} Recall: {:.2f}, Precision: {:.2f}, F1: {:.2f} Elapsed time: {:.2f}".format(
+            args.detector, result["recall"], result["precision"], result["f1"], result["elapsed"]
+        )
+    )
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Benchmarking PySceneDetect performance.')
-    parser.add_argument('--detector', type=str, choices=['detect-adaptive', 'detect-content', 'detect-hash', 'detect-hist', 'detect-threshold'], 
-                        default='detect-content', help='Detector name. Implemented detectors are listed: https://www.scenedetect.com/docs/latest/cli.html')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Benchmarking PySceneDetect performance.")
+    parser.add_argument(
+        "--detector",
+        type=str,
+        choices=[
+            "detect-adaptive",
+            "detect-content",
+            "detect-hash",
+            "detect-hist",
+            "detect-threshold",
+        ],
+        default="detect-content",
+        help="Detector name. Implemented detectors are listed: https://www.scenedetect.com/docs/latest/cli.html",
+    )
     args = parser.parse_args()
-    main(args)
+    main(args)

benchmarks/evaluator.py

@@ -1,35 +1,38 @@
 from statistics import mean
 
+
 class Evaluator:
     def __init__(self):
         pass
-    
+
     def _load_scenes(self, scene_filename):
         with open(scene_filename) as f:
-            gt_scene_list = [x.strip().split('\t')[1] for x in f.readlines()]
+            gt_scene_list = [x.strip().split("\t")[1] for x in f.readlines()]
             gt_scene_list = [int(x) + 1 for x in gt_scene_list]
         return gt_scene_list
 
     def evaluate_performance(self, pred_scenes):
         total_correct = 0
         total_pred = 0
         total_gt = 0
+        assert pred_scenes
 
         for scene_file, pred in pred_scenes.items():
             gt_scene_list = self._load_scenes(scene_file)
-            pred_list = pred['pred_scenes']
+            pred_list = pred["pred_scenes"]
             total_correct += len(set(pred_list) & set(gt_scene_list))
             total_pred += len(pred_list)
             total_gt += len(gt_scene_list)
 
+        assert total_pred, pred_scenes
         recall = total_correct / total_gt
         precision = total_correct / total_pred
         f1 = 2 * recall * precision / (recall + precision) if (recall + precision) != 0 else 0
-        avg_elapsed = mean([x['elapsed'] for x in pred_scenes.values()])
+        avg_elapsed = mean([x["elapsed"] for x in pred_scenes.values()])
         result = {
-            'recall': recall * 100,
-            'precision': precision * 100,
-            'f1': f1 * 100,
-            'elapsed': avg_elapsed
+            "recall": recall * 100,
+            "precision": precision * 100,
+            "f1": f1 * 100,
+            "elapsed": avg_elapsed,
         }
-        return result
+        return result