From fc54c47ed6be1ee856fe14549d0a89b5535dc0c5 Mon Sep 17 00:00:00 2001
From: Roman Donchenko <roman@cvat.ai>
Date: Thu, 21 Mar 2024 17:06:00 +0200
Subject: [PATCH] quality_control: correct the per-class accuracy formula
 (#7640)

<!-- Raise an issue to propose your change
(https://github.com/opencv/cvat/issues).
It helps to avoid duplication of efforts from multiple independent
contributors.
Discuss your ideas with maintainers to be sure that changes will be
approved and merged.
Read the [Contribution
guide](https://opencv.github.io/cvat/docs/contributing/). -->

<!-- Provide a general summary of your changes in the Title above -->

### Motivation and context
<!-- Why is this change required? What problem does it solve? If it
fixes an open
issue, please link to the issue here. Describe your changes in detail,
add
screenshots. -->
The current formula used to calculate `ConfusionMatrix.accuracy` is, in
fact, not accuracy, but the Jaccard index. Replace it with the correct
formula.

Since the Jaccard index is a useful metric in its own right, calculate
it too, but save it in another attribute of `ConfusionMatrix`.

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->
Manual testing.

### Checklist
<!-- Go over all the following points, and put an `x` in all the boxes
that apply.
If an item isn't applicable for some reason, then ~~explicitly
strikethrough~~ the whole
line. If you don't do that, GitHub will show incorrect progress for the
pull request.
If you're unsure about any of these, don't hesitate to ask. We're here
to help! -->
- [x] I submit my changes into the `develop` branch
- [x] I have created a changelog fragment <!-- see top comment in
CHANGELOG.md -->
- ~~[ ] I have updated the documentation accordingly~~
- ~~[ ] I have added tests to cover my changes~~
- ~~[ ] I have linked related issues (see [GitHub docs](

https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword))~~
- ~~[ ] I have increased versions of npm packages if it is necessary

([cvat-canvas](https://github.com/opencv/cvat/tree/develop/cvat-canvas#versioning),

[cvat-core](https://github.com/opencv/cvat/tree/develop/cvat-core#versioning),

[cvat-data](https://github.com/opencv/cvat/tree/develop/cvat-data#versioning)
and

[cvat-ui](https://github.com/opencv/cvat/tree/develop/cvat-ui#versioning))~~

### License

- [x] I submit _my code changes_ under the same [MIT License](
https://github.com/opencv/cvat/blob/develop/LICENSE) that covers the
project.
  Feel free to contact the maintainers if that's a concern.

---------

Co-authored-by: Maxim Zhiltsov <zhiltsov.max35@gmail.com>
---
 .../20240319_183656_roman_accuracy_jaccard.md |    5 +
 cvat/apps/quality_control/quality_reports.py  |   15 +-
 tests/python/rest_api/test_quality_control.py |   11 +
 tests/python/shared/assets/cvat_db/data.json  | 1716 +++++++++++++----
 .../shared/assets/quality_conflicts.json      |  710 ++++++-
 .../python/shared/assets/quality_reports.json |   62 +-
 6 files changed, 2189 insertions(+), 330 deletions(-)
 create mode 100644 changelog.d/20240319_183656_roman_accuracy_jaccard.md

diff --git a/changelog.d/20240319_183656_roman_accuracy_jaccard.md b/changelog.d/20240319_183656_roman_accuracy_jaccard.md
new file mode 100644
index 000000000000..18df9aff59a8
--- /dev/null
+++ b/changelog.d/20240319_183656_roman_accuracy_jaccard.md
@@ -0,0 +1,5 @@
+### Fixed
+
+- Corrected the formula for per-class accuracy in quality reports;
+  the old formula is now exposed as the `jaccard_index` key
+  (<https://github.com/opencv/cvat/pull/7640>)
diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py
index 42714788a39b..c715851f1c19 100644
--- a/cvat/apps/quality_control/quality_reports.py
+++ b/cvat/apps/quality_control/quality_reports.py
@@ -218,6 +218,7 @@ class ConfusionMatrix(_Serializable):
     precision: np.array
     recall: np.array
     accuracy: np.array
+    jaccard_index: Optional[np.array]
 
     @property
     def axes(self):
@@ -240,6 +241,9 @@ def from_dict(cls, d: dict):
             precision=np.asarray(d["precision"]),
             recall=np.asarray(d["recall"]),
             accuracy=np.asarray(d["accuracy"]),
+            # This field didn't exist at first, so it might not be present
+            # in old serialized instances.
+            jaccard_index=np.asarray(d["jaccard_index"]) if "jaccard_index" in d else None,
         )
 
 
@@ -1934,17 +1938,23 @@ def _generate_annotations_summary(
         matched_ann_counts = np.diag(confusion_matrix)
         ds_ann_counts = np.sum(confusion_matrix, axis=1)
         gt_ann_counts = np.sum(confusion_matrix, axis=0)
+        total_annotations_count = np.sum(confusion_matrix)
 
-        label_accuracies = _arr_div(
+        label_jaccard_indices = _arr_div(
             matched_ann_counts, ds_ann_counts + gt_ann_counts - matched_ann_counts
         )
         label_precisions = _arr_div(matched_ann_counts, ds_ann_counts)
         label_recalls = _arr_div(matched_ann_counts, gt_ann_counts)
+        label_accuracies = (
+            total_annotations_count  # TP + TN + FP + FN
+            - (ds_ann_counts - matched_ann_counts)  # - FP
+            - (gt_ann_counts - matched_ann_counts)  # - FN
+            # ... = TP + TN
+        ) / (total_annotations_count or 1)
 
         valid_annotations_count = np.sum(matched_ann_counts)
         missing_annotations_count = np.sum(confusion_matrix[cls._UNMATCHED_IDX, :])
         extra_annotations_count = np.sum(confusion_matrix[:, cls._UNMATCHED_IDX])
-        total_annotations_count = np.sum(confusion_matrix)
         ds_annotations_count = np.sum(ds_ann_counts[: cls._UNMATCHED_IDX])
         gt_annotations_count = np.sum(gt_ann_counts[: cls._UNMATCHED_IDX])
 
@@ -1961,6 +1971,7 @@ def _generate_annotations_summary(
                 precision=label_precisions,
                 recall=label_recalls,
                 accuracy=label_accuracies,
+                jaccard_index=label_jaccard_indices,
             ),
         )
 
diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py
index 58df82b1b8f5..96bdf6cb4fbe 100644
--- a/tests/python/rest_api/test_quality_control.py
+++ b/tests/python/rest_api/test_quality_control.py
@@ -1202,3 +1202,14 @@ def test_settings_affect_metrics(
 
         new_report = self.create_quality_report(admin_user, task_id)
         assert new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"]
+
+    def test_old_report_can_be_loaded(self, admin_user, quality_reports):
+        report = min((r for r in quality_reports if r["task_id"]), key=lambda r: r["id"])
+        assert report["created_date"] < "2024"
+
+        with make_api_client(admin_user) as api_client:
+            (report_data, _) = api_client.quality_api.retrieve_report_data(report["id"])
+
+        # This report should have been created before the Jaccard index was included.
+        for d in [report_data["comparison_summary"], *report_data["frame_results"].values()]:
+            assert d["annotations"]["confusion_matrix"]["jaccard_index"] is None
diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json
index abffd52e0757..bc4500e9edad 100644
--- a/tests/python/shared/assets/cvat_db/data.json
+++ b/tests/python/shared/assets/cvat_db/data.json
@@ -10974,6 +10974,32 @@
     "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}"
   }
 },
+{
+  "model": "quality_control.qualityreport",
+  "pk": 5,
+  "fields": {
+    "job": null,
+    "task": 22,
+    "parent": null,
+    "created_date": "2024-03-21T11:16:21.845Z",
+    "target_last_updated": "2023-11-24T15:23:30.045Z",
+    "gt_last_updated": "2023-11-24T15:18:55.216Z",
+    "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.8478260869565217,0.6086956521739131,0.9565217391304348,0.5434782608695652],\"jaccard_index\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.825,0.7,0.95,0.625],\"jaccard_index\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}"
+  }
+},
+{
+  "model": "quality_control.qualityreport",
+  "pk": 6,
+  "fields": {
+    "job": 27,
+    "task": null,
+    "parent": 5,
+    "created_date": "2024-03-21T11:16:21.847Z",
+    "target_last_updated": "2023-11-24T15:23:30.269Z",
+    "gt_last_updated": "2023-11-24T15:18:55.216Z",
+    "data": "{\"parameters\":{\"included_annotation_types\":[\"bbox\",\"points\",\"mask\",\"polygon\",\"polyline\",\"skeleton\"],\"compare_attributes\":true,\"ignored_attributes\":[],\"iou_threshold\":0.4,\"low_overlap_threshold\":0.8,\"oks_sigma\":0.09,\"line_thickness\":0.01,\"compare_line_orientation\":true,\"line_orientation_threshold\":0.1,\"compare_groups\":true,\"group_match_threshold\":0.5,\"check_covered_annotations\":true,\"object_visibility_threshold\":0.05,\"panoptic_comparison\":true},\"comparison_summary\":{\"frame_share\":0.2727272727272727,\"frames\":[0,1,2],\"conflict_count\":40,\"warning_count\":16,\"error_count\":24,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":10,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6},\"annotations\":{\"valid_count\":22,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,8,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.6774193548387096,0.5,0.0],\"accuracy\":[0.8478260869565217,0.6086956521739131,0.9565217391304348,0.5434782608695652],\"jaccard_index\":[0.0,0.5384615384615384,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.4782608695652174,\"precision\":0.6111111111111112,\"recall\":0.6285714285714286},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":10,\"extra_count\":11,\"total_count\":46,\"ds_count\":36,\"gt_count\":35,\"mean_iou\":0.18567508032031,\"accuracy\":0.5434782608695652},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"frame_count\":3,\"mean_conflict_count\":13.333333333333334},\"frame_results\":{\"0\":{\"conflicts\":[{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":91,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":118,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":88,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":102,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":68,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"},{\"obj_id\":98,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":141,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"},{\"obj_id\":132,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"low_overlap\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":107,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":121,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":122,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":131,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":70,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":76,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":93,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":74,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":95,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":73,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":96,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":64,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":66,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":94,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"extra_annotation\",\"annotation_ids\":[{\"obj_id\":142,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"skeleton\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":92,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":119,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":89,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"points\"},{\"obj_id\":123,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_label\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"},{\"obj_id\":97,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":0,\"type\":\"mismatching_direction\",\"annotation_ids\":[{\"obj_id\":67,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"polyline\"},{\"obj_id\":99,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"covered_annotation\",\"annotation_ids\":[{\"obj_id\":69,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_attributes\",\"annotation_ids\":[{\"obj_id\":65,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":101,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":82,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":103,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":81,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":104,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":83,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":105,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":87,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":106,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":80,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":111,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"},{\"frame_id\":0,\"type\":\"mismatching_groups\",\"annotation_ids\":[{\"obj_id\":77,\"job_id\":27,\"type\":\"shape\",\"shape_type\":\"rectangle\"},{\"obj_id\":114,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"warning\"}],\"annotations\":{\"valid_count\":22,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,2,0,3],[1,21,0,7],[0,0,1,1],[1,2,1,0]],\"precision\":[0.0,0.7241379310344828,0.5,0.0],\"recall\":[0.0,0.84,0.5,0.0],\"accuracy\":[0.825,0.7,0.95,0.625],\"jaccard_index\":[0.0,0.6363636363636364,0.3333333333333333,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.55,\"precision\":0.6111111111111112,\"recall\":0.7586206896551724},\"annotation_components\":{\"shape\":{\"valid_count\":25,\"missing_count\":4,\"extra_count\":11,\"total_count\":40,\"ds_count\":36,\"gt_count\":29,\"mean_iou\":0.55702524096093,\"accuracy\":0.625},\"label\":{\"valid_count\":22,\"invalid_count\":3,\"total_count\":25,\"accuracy\":0.88}},\"conflict_count\":34,\"warning_count\":16,\"error_count\":18,\"conflicts_by_type\":{\"low_overlap\":7,\"missing_annotation\":4,\"extra_annotation\":11,\"mismatching_label\":3,\"mismatching_direction\":1,\"covered_annotation\":1,\"mismatching_attributes\":1,\"mismatching_groups\":6}},\"1\":{\"conflicts\":[{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":130,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"rectangle\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":128,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"points\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":124,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polygon\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":125,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"ellipse\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":127,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"mask\"}],\"severity\":\"error\"},{\"frame_id\":1,\"type\":\"missing_annotation\",\"annotation_ids\":[{\"obj_id\":129,\"job_id\":28,\"type\":\"shape\",\"shape_type\":\"polyline\"}],\"severity\":\"error\"}],\"annotations\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,6,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[1.0,0.0,1.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":6,\"extra_count\":0,\"total_count\":6,\"ds_count\":0,\"gt_count\":6,\"mean_iou\":0.0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":6,\"warning_count\":0,\"error_count\":6,\"conflicts_by_type\":{\"missing_annotation\":6}},\"2\":{\"conflicts\":[],\"annotations\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"confusion_matrix\":{\"labels\":[\"dog\",\"cat\",\"skele\",\"unmatched\"],\"rows\":[[0,0,0,0],[0,0,0,0],[0,0,0,0],[0,0,0,0]],\"precision\":[0.0,0.0,0.0,0.0],\"recall\":[0.0,0.0,0.0,0.0],\"accuracy\":[0.0,0.0,0.0,0.0],\"jaccard_index\":[0.0,0.0,0.0,0.0],\"axes\":{\"cols\":\"gt\",\"rows\":\"ds\"}},\"accuracy\":0.0,\"precision\":0.0,\"recall\":0.0},\"annotation_components\":{\"shape\":{\"valid_count\":0,\"missing_count\":0,\"extra_count\":0,\"total_count\":0,\"ds_count\":0,\"gt_count\":0,\"mean_iou\":0,\"accuracy\":0.0},\"label\":{\"valid_count\":0,\"invalid_count\":0,\"total_count\":0,\"accuracy\":0.0}},\"conflict_count\":0,\"warning_count\":0,\"error_count\":0,\"conflicts_by_type\":{}}}}"
+  }
+},
 {
   "model": "quality_control.annotationconflict",
   "pk": 1,
@@ -11745,230 +11771,1257 @@
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 1,
+  "model": "quality_control.annotationconflict",
+  "pk": 78,
   "fields": {
-    "conflict": 1,
-    "obj_id": 65,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 2,
+  "model": "quality_control.annotationconflict",
+  "pk": 79,
   "fields": {
-    "conflict": 1,
-    "obj_id": 101,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 3,
+  "model": "quality_control.annotationconflict",
+  "pk": 80,
   "fields": {
-    "conflict": 2,
-    "obj_id": 91,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 4,
+  "model": "quality_control.annotationconflict",
+  "pk": 81,
   "fields": {
-    "conflict": 2,
-    "obj_id": 118,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 5,
+  "model": "quality_control.annotationconflict",
+  "pk": 82,
   "fields": {
-    "conflict": 3,
-    "obj_id": 88,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 6,
+  "model": "quality_control.annotationconflict",
+  "pk": 83,
   "fields": {
-    "conflict": 3,
-    "obj_id": 102,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 7,
+  "model": "quality_control.annotationconflict",
+  "pk": 84,
   "fields": {
-    "conflict": 4,
-    "obj_id": 68,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "polygon"
+    "report": 6,
+    "frame": 0,
+    "type": "low_overlap",
+    "severity": "warning"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 8,
+  "model": "quality_control.annotationconflict",
+  "pk": 85,
   "fields": {
-    "conflict": 4,
-    "obj_id": 98,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "polygon"
+    "report": 6,
+    "frame": 0,
+    "type": "missing_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 9,
+  "model": "quality_control.annotationconflict",
+  "pk": 86,
   "fields": {
-    "conflict": 5,
-    "obj_id": 67,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "polyline"
+    "report": 6,
+    "frame": 0,
+    "type": "missing_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 10,
+  "model": "quality_control.annotationconflict",
+  "pk": 87,
   "fields": {
-    "conflict": 5,
-    "obj_id": 99,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "polyline"
+    "report": 6,
+    "frame": 0,
+    "type": "missing_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 11,
+  "model": "quality_control.annotationconflict",
+  "pk": 88,
   "fields": {
-    "conflict": 6,
-    "obj_id": 92,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "missing_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 12,
+  "model": "quality_control.annotationconflict",
+  "pk": 89,
   "fields": {
-    "conflict": 6,
-    "obj_id": 119,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 13,
+  "model": "quality_control.annotationconflict",
+  "pk": 90,
   "fields": {
-    "conflict": 7,
-    "obj_id": 107,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 14,
+  "model": "quality_control.annotationconflict",
+  "pk": 91,
   "fields": {
-    "conflict": 8,
-    "obj_id": 121,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 15,
+  "model": "quality_control.annotationconflict",
+  "pk": 92,
   "fields": {
-    "conflict": 9,
-    "obj_id": 122,
-    "job_id": 28,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 16,
+  "model": "quality_control.annotationconflict",
+  "pk": 93,
   "fields": {
-    "conflict": 10,
-    "obj_id": 70,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 17,
+  "model": "quality_control.annotationconflict",
+  "pk": 94,
   "fields": {
-    "conflict": 11,
-    "obj_id": 76,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "rectangle"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 18,
+  "model": "quality_control.annotationconflict",
+  "pk": 95,
   "fields": {
-    "conflict": 12,
-    "obj_id": 93,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "points"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 19,
+  "model": "quality_control.annotationconflict",
+  "pk": 96,
   "fields": {
-    "conflict": 13,
-    "obj_id": 74,
-    "job_id": 27,
-    "type": "shape",
-    "shape_type": "polygon"
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
   }
 },
 {
-  "model": "quality_control.annotationid",
-  "pk": 20,
+  "model": "quality_control.annotationconflict",
+  "pk": 97,
   "fields": {
-    "conflict": 14,
-    "obj_id": 95,
-    "job_id": 27,
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 98,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 99,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "extra_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 100,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_label",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 101,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_label",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 102,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_label",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 103,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_direction",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 104,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "covered_annotation",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 105,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_attributes",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 106,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 107,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 108,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 109,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 110,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 111,
+  "fields": {
+    "report": 6,
+    "frame": 0,
+    "type": "mismatching_groups",
+    "severity": "warning"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 112,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 113,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 114,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 115,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 116,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationconflict",
+  "pk": 117,
+  "fields": {
+    "report": 6,
+    "frame": 1,
+    "type": "missing_annotation",
+    "severity": "error"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 1,
+  "fields": {
+    "conflict": 1,
+    "obj_id": 65,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 2,
+  "fields": {
+    "conflict": 1,
+    "obj_id": 101,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 3,
+  "fields": {
+    "conflict": 2,
+    "obj_id": 91,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 4,
+  "fields": {
+    "conflict": 2,
+    "obj_id": 118,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 5,
+  "fields": {
+    "conflict": 3,
+    "obj_id": 88,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 6,
+  "fields": {
+    "conflict": 3,
+    "obj_id": 102,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 7,
+  "fields": {
+    "conflict": 4,
+    "obj_id": 68,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 8,
+  "fields": {
+    "conflict": 4,
+    "obj_id": 98,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 9,
+  "fields": {
+    "conflict": 5,
+    "obj_id": 67,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 10,
+  "fields": {
+    "conflict": 5,
+    "obj_id": 99,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 11,
+  "fields": {
+    "conflict": 6,
+    "obj_id": 92,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 12,
+  "fields": {
+    "conflict": 6,
+    "obj_id": 119,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 13,
+  "fields": {
+    "conflict": 7,
+    "obj_id": 107,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 14,
+  "fields": {
+    "conflict": 8,
+    "obj_id": 121,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 15,
+  "fields": {
+    "conflict": 9,
+    "obj_id": 122,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 16,
+  "fields": {
+    "conflict": 10,
+    "obj_id": 70,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 17,
+  "fields": {
+    "conflict": 11,
+    "obj_id": 76,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 18,
+  "fields": {
+    "conflict": 12,
+    "obj_id": 93,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 19,
+  "fields": {
+    "conflict": 13,
+    "obj_id": 74,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 20,
+  "fields": {
+    "conflict": 14,
+    "obj_id": 95,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 21,
+  "fields": {
+    "conflict": 15,
+    "obj_id": 73,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 22,
+  "fields": {
+    "conflict": 16,
+    "obj_id": 96,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "ellipse"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 23,
+  "fields": {
+    "conflict": 17,
+    "obj_id": 64,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 24,
+  "fields": {
+    "conflict": 18,
+    "obj_id": 66,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 25,
+  "fields": {
+    "conflict": 19,
+    "obj_id": 94,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 26,
+  "fields": {
+    "conflict": 20,
+    "obj_id": 89,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 27,
+  "fields": {
+    "conflict": 20,
+    "obj_id": 123,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 28,
+  "fields": {
+    "conflict": 21,
+    "obj_id": 92,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 29,
+  "fields": {
+    "conflict": 21,
+    "obj_id": 119,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 30,
+  "fields": {
+    "conflict": 22,
+    "obj_id": 69,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "mask"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 31,
+  "fields": {
+    "conflict": 22,
+    "obj_id": 97,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 32,
+  "fields": {
+    "conflict": 23,
+    "obj_id": 67,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 33,
+  "fields": {
+    "conflict": 23,
+    "obj_id": 99,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 34,
+  "fields": {
+    "conflict": 24,
+    "obj_id": 69,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "mask"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 35,
+  "fields": {
+    "conflict": 25,
+    "obj_id": 65,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 36,
+  "fields": {
+    "conflict": 25,
+    "obj_id": 101,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 37,
+  "fields": {
+    "conflict": 26,
+    "obj_id": 82,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 38,
+  "fields": {
+    "conflict": 26,
+    "obj_id": 103,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 39,
+  "fields": {
+    "conflict": 27,
+    "obj_id": 81,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 40,
+  "fields": {
+    "conflict": 27,
+    "obj_id": 104,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 41,
+  "fields": {
+    "conflict": 28,
+    "obj_id": 83,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 42,
+  "fields": {
+    "conflict": 28,
+    "obj_id": 105,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 43,
+  "fields": {
+    "conflict": 29,
+    "obj_id": 87,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 44,
+  "fields": {
+    "conflict": 29,
+    "obj_id": 106,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 45,
+  "fields": {
+    "conflict": 30,
+    "obj_id": 80,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 46,
+  "fields": {
+    "conflict": 30,
+    "obj_id": 111,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 47,
+  "fields": {
+    "conflict": 31,
+    "obj_id": 77,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 48,
+  "fields": {
+    "conflict": 31,
+    "obj_id": 114,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 49,
+  "fields": {
+    "conflict": 32,
+    "obj_id": 130,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 50,
+  "fields": {
+    "conflict": 33,
+    "obj_id": 128,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 51,
+  "fields": {
+    "conflict": 34,
+    "obj_id": 124,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 52,
+  "fields": {
+    "conflict": 35,
+    "obj_id": 125,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "ellipse"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 53,
+  "fields": {
+    "conflict": 36,
+    "obj_id": 127,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "mask"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 54,
+  "fields": {
+    "conflict": 37,
+    "obj_id": 129,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 55,
+  "fields": {
+    "conflict": 38,
+    "obj_id": 65,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 56,
+  "fields": {
+    "conflict": 38,
+    "obj_id": 101,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 57,
+  "fields": {
+    "conflict": 39,
+    "obj_id": 91,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 58,
+  "fields": {
+    "conflict": 39,
+    "obj_id": 118,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 59,
+  "fields": {
+    "conflict": 40,
+    "obj_id": 88,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 60,
+  "fields": {
+    "conflict": 40,
+    "obj_id": 102,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 61,
+  "fields": {
+    "conflict": 41,
+    "obj_id": 68,
+    "job_id": 27,
     "type": "shape",
     "shape_type": "polygon"
   }
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 21,
+  "pk": 62,
   "fields": {
-    "conflict": 15,
+    "conflict": 41,
+    "obj_id": 98,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 63,
+  "fields": {
+    "conflict": 42,
+    "obj_id": 67,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 64,
+  "fields": {
+    "conflict": 42,
+    "obj_id": 99,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "polyline"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 65,
+  "fields": {
+    "conflict": 43,
+    "obj_id": 141,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "skeleton"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 66,
+  "fields": {
+    "conflict": 43,
+    "obj_id": 132,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "skeleton"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 67,
+  "fields": {
+    "conflict": 44,
+    "obj_id": 92,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 68,
+  "fields": {
+    "conflict": 44,
+    "obj_id": 119,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 69,
+  "fields": {
+    "conflict": 45,
+    "obj_id": 107,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 70,
+  "fields": {
+    "conflict": 46,
+    "obj_id": 121,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 71,
+  "fields": {
+    "conflict": 47,
+    "obj_id": 122,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 72,
+  "fields": {
+    "conflict": 48,
+    "obj_id": 131,
+    "job_id": 28,
+    "type": "shape",
+    "shape_type": "skeleton"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 73,
+  "fields": {
+    "conflict": 49,
+    "obj_id": 70,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 74,
+  "fields": {
+    "conflict": 50,
+    "obj_id": 76,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "rectangle"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 75,
+  "fields": {
+    "conflict": 51,
+    "obj_id": 93,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "points"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 76,
+  "fields": {
+    "conflict": 52,
+    "obj_id": 74,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 77,
+  "fields": {
+    "conflict": 53,
+    "obj_id": 95,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "polygon"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 78,
+  "fields": {
+    "conflict": 54,
     "obj_id": 73,
     "job_id": 27,
     "type": "shape",
@@ -11977,9 +13030,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 22,
+  "pk": 79,
   "fields": {
-    "conflict": 16,
+    "conflict": 55,
     "obj_id": 96,
     "job_id": 27,
     "type": "shape",
@@ -11988,9 +13041,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 23,
+  "pk": 80,
   "fields": {
-    "conflict": 17,
+    "conflict": 56,
     "obj_id": 64,
     "job_id": 27,
     "type": "shape",
@@ -11999,9 +13052,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 24,
+  "pk": 81,
   "fields": {
-    "conflict": 18,
+    "conflict": 57,
     "obj_id": 66,
     "job_id": 27,
     "type": "shape",
@@ -12010,9 +13063,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 25,
+  "pk": 82,
   "fields": {
-    "conflict": 19,
+    "conflict": 58,
     "obj_id": 94,
     "job_id": 27,
     "type": "shape",
@@ -12021,10 +13074,21 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 26,
+  "pk": 83,
   "fields": {
-    "conflict": 20,
-    "obj_id": 89,
+    "conflict": 59,
+    "obj_id": 142,
+    "job_id": 27,
+    "type": "shape",
+    "shape_type": "skeleton"
+  }
+},
+{
+  "model": "quality_control.annotationid",
+  "pk": 84,
+  "fields": {
+    "conflict": 60,
+    "obj_id": 92,
     "job_id": 27,
     "type": "shape",
     "shape_type": "points"
@@ -12032,10 +13096,10 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 27,
+  "pk": 85,
   "fields": {
-    "conflict": 20,
-    "obj_id": 123,
+    "conflict": 60,
+    "obj_id": 119,
     "job_id": 28,
     "type": "shape",
     "shape_type": "points"
@@ -12043,10 +13107,10 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 28,
+  "pk": 86,
   "fields": {
-    "conflict": 21,
-    "obj_id": 92,
+    "conflict": 61,
+    "obj_id": 89,
     "job_id": 27,
     "type": "shape",
     "shape_type": "points"
@@ -12054,10 +13118,10 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 29,
+  "pk": 87,
   "fields": {
-    "conflict": 21,
-    "obj_id": 119,
+    "conflict": 61,
+    "obj_id": 123,
     "job_id": 28,
     "type": "shape",
     "shape_type": "points"
@@ -12065,9 +13129,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 30,
+  "pk": 88,
   "fields": {
-    "conflict": 22,
+    "conflict": 62,
     "obj_id": 69,
     "job_id": 27,
     "type": "shape",
@@ -12076,9 +13140,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 31,
+  "pk": 89,
   "fields": {
-    "conflict": 22,
+    "conflict": 62,
     "obj_id": 97,
     "job_id": 28,
     "type": "shape",
@@ -12087,9 +13151,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 32,
+  "pk": 90,
   "fields": {
-    "conflict": 23,
+    "conflict": 63,
     "obj_id": 67,
     "job_id": 27,
     "type": "shape",
@@ -12098,9 +13162,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 33,
+  "pk": 91,
   "fields": {
-    "conflict": 23,
+    "conflict": 63,
     "obj_id": 99,
     "job_id": 28,
     "type": "shape",
@@ -12109,9 +13173,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 34,
+  "pk": 92,
   "fields": {
-    "conflict": 24,
+    "conflict": 64,
     "obj_id": 69,
     "job_id": 27,
     "type": "shape",
@@ -12120,9 +13184,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 35,
+  "pk": 93,
   "fields": {
-    "conflict": 25,
+    "conflict": 65,
     "obj_id": 65,
     "job_id": 27,
     "type": "shape",
@@ -12131,9 +13195,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 36,
+  "pk": 94,
   "fields": {
-    "conflict": 25,
+    "conflict": 65,
     "obj_id": 101,
     "job_id": 28,
     "type": "shape",
@@ -12142,9 +13206,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 37,
+  "pk": 95,
   "fields": {
-    "conflict": 26,
+    "conflict": 66,
     "obj_id": 82,
     "job_id": 27,
     "type": "shape",
@@ -12153,9 +13217,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 38,
+  "pk": 96,
   "fields": {
-    "conflict": 26,
+    "conflict": 66,
     "obj_id": 103,
     "job_id": 28,
     "type": "shape",
@@ -12164,9 +13228,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 39,
+  "pk": 97,
   "fields": {
-    "conflict": 27,
+    "conflict": 67,
     "obj_id": 81,
     "job_id": 27,
     "type": "shape",
@@ -12175,9 +13239,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 40,
+  "pk": 98,
   "fields": {
-    "conflict": 27,
+    "conflict": 67,
     "obj_id": 104,
     "job_id": 28,
     "type": "shape",
@@ -12186,9 +13250,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 41,
+  "pk": 99,
   "fields": {
-    "conflict": 28,
+    "conflict": 68,
     "obj_id": 83,
     "job_id": 27,
     "type": "shape",
@@ -12197,9 +13261,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 42,
+  "pk": 100,
   "fields": {
-    "conflict": 28,
+    "conflict": 68,
     "obj_id": 105,
     "job_id": 28,
     "type": "shape",
@@ -12208,9 +13272,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 43,
+  "pk": 101,
   "fields": {
-    "conflict": 29,
+    "conflict": 69,
     "obj_id": 87,
     "job_id": 27,
     "type": "shape",
@@ -12219,9 +13283,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 44,
+  "pk": 102,
   "fields": {
-    "conflict": 29,
+    "conflict": 69,
     "obj_id": 106,
     "job_id": 28,
     "type": "shape",
@@ -12230,9 +13294,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 45,
+  "pk": 103,
   "fields": {
-    "conflict": 30,
+    "conflict": 70,
     "obj_id": 80,
     "job_id": 27,
     "type": "shape",
@@ -12241,9 +13305,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 46,
+  "pk": 104,
   "fields": {
-    "conflict": 30,
+    "conflict": 70,
     "obj_id": 111,
     "job_id": 28,
     "type": "shape",
@@ -12252,9 +13316,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 47,
+  "pk": 105,
   "fields": {
-    "conflict": 31,
+    "conflict": 71,
     "obj_id": 77,
     "job_id": 27,
     "type": "shape",
@@ -12263,9 +13327,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 48,
+  "pk": 106,
   "fields": {
-    "conflict": 31,
+    "conflict": 71,
     "obj_id": 114,
     "job_id": 28,
     "type": "shape",
@@ -12274,9 +13338,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 49,
+  "pk": 107,
   "fields": {
-    "conflict": 32,
+    "conflict": 72,
     "obj_id": 130,
     "job_id": 28,
     "type": "shape",
@@ -12285,9 +13349,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 50,
+  "pk": 108,
   "fields": {
-    "conflict": 33,
+    "conflict": 73,
     "obj_id": 128,
     "job_id": 28,
     "type": "shape",
@@ -12296,9 +13360,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 51,
+  "pk": 109,
   "fields": {
-    "conflict": 34,
+    "conflict": 74,
     "obj_id": 124,
     "job_id": 28,
     "type": "shape",
@@ -12307,9 +13371,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 52,
+  "pk": 110,
   "fields": {
-    "conflict": 35,
+    "conflict": 75,
     "obj_id": 125,
     "job_id": 28,
     "type": "shape",
@@ -12318,9 +13382,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 53,
+  "pk": 111,
   "fields": {
-    "conflict": 36,
+    "conflict": 76,
     "obj_id": 127,
     "job_id": 28,
     "type": "shape",
@@ -12329,9 +13393,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 54,
+  "pk": 112,
   "fields": {
-    "conflict": 37,
+    "conflict": 77,
     "obj_id": 129,
     "job_id": 28,
     "type": "shape",
@@ -12340,9 +13404,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 55,
+  "pk": 113,
   "fields": {
-    "conflict": 38,
+    "conflict": 78,
     "obj_id": 65,
     "job_id": 27,
     "type": "shape",
@@ -12351,9 +13415,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 56,
+  "pk": 114,
   "fields": {
-    "conflict": 38,
+    "conflict": 78,
     "obj_id": 101,
     "job_id": 28,
     "type": "shape",
@@ -12362,9 +13426,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 57,
+  "pk": 115,
   "fields": {
-    "conflict": 39,
+    "conflict": 79,
     "obj_id": 91,
     "job_id": 27,
     "type": "shape",
@@ -12373,9 +13437,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 58,
+  "pk": 116,
   "fields": {
-    "conflict": 39,
+    "conflict": 79,
     "obj_id": 118,
     "job_id": 28,
     "type": "shape",
@@ -12384,9 +13448,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 59,
+  "pk": 117,
   "fields": {
-    "conflict": 40,
+    "conflict": 80,
     "obj_id": 88,
     "job_id": 27,
     "type": "shape",
@@ -12395,9 +13459,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 60,
+  "pk": 118,
   "fields": {
-    "conflict": 40,
+    "conflict": 80,
     "obj_id": 102,
     "job_id": 28,
     "type": "shape",
@@ -12406,9 +13470,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 61,
+  "pk": 119,
   "fields": {
-    "conflict": 41,
+    "conflict": 81,
     "obj_id": 68,
     "job_id": 27,
     "type": "shape",
@@ -12417,9 +13481,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 62,
+  "pk": 120,
   "fields": {
-    "conflict": 41,
+    "conflict": 81,
     "obj_id": 98,
     "job_id": 28,
     "type": "shape",
@@ -12428,9 +13492,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 63,
+  "pk": 121,
   "fields": {
-    "conflict": 42,
+    "conflict": 82,
     "obj_id": 67,
     "job_id": 27,
     "type": "shape",
@@ -12439,9 +13503,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 64,
+  "pk": 122,
   "fields": {
-    "conflict": 42,
+    "conflict": 82,
     "obj_id": 99,
     "job_id": 28,
     "type": "shape",
@@ -12450,9 +13514,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 65,
+  "pk": 123,
   "fields": {
-    "conflict": 43,
+    "conflict": 83,
     "obj_id": 141,
     "job_id": 27,
     "type": "shape",
@@ -12461,9 +13525,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 66,
+  "pk": 124,
   "fields": {
-    "conflict": 43,
+    "conflict": 83,
     "obj_id": 132,
     "job_id": 28,
     "type": "shape",
@@ -12472,9 +13536,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 67,
+  "pk": 125,
   "fields": {
-    "conflict": 44,
+    "conflict": 84,
     "obj_id": 92,
     "job_id": 27,
     "type": "shape",
@@ -12483,9 +13547,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 68,
+  "pk": 126,
   "fields": {
-    "conflict": 44,
+    "conflict": 84,
     "obj_id": 119,
     "job_id": 28,
     "type": "shape",
@@ -12494,9 +13558,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 69,
+  "pk": 127,
   "fields": {
-    "conflict": 45,
+    "conflict": 85,
     "obj_id": 107,
     "job_id": 28,
     "type": "shape",
@@ -12505,9 +13569,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 70,
+  "pk": 128,
   "fields": {
-    "conflict": 46,
+    "conflict": 86,
     "obj_id": 121,
     "job_id": 28,
     "type": "shape",
@@ -12516,9 +13580,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 71,
+  "pk": 129,
   "fields": {
-    "conflict": 47,
+    "conflict": 87,
     "obj_id": 122,
     "job_id": 28,
     "type": "shape",
@@ -12527,9 +13591,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 72,
+  "pk": 130,
   "fields": {
-    "conflict": 48,
+    "conflict": 88,
     "obj_id": 131,
     "job_id": 28,
     "type": "shape",
@@ -12538,9 +13602,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 73,
+  "pk": 131,
   "fields": {
-    "conflict": 49,
+    "conflict": 89,
     "obj_id": 70,
     "job_id": 27,
     "type": "shape",
@@ -12549,9 +13613,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 74,
+  "pk": 132,
   "fields": {
-    "conflict": 50,
+    "conflict": 90,
     "obj_id": 76,
     "job_id": 27,
     "type": "shape",
@@ -12560,9 +13624,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 75,
+  "pk": 133,
   "fields": {
-    "conflict": 51,
+    "conflict": 91,
     "obj_id": 93,
     "job_id": 27,
     "type": "shape",
@@ -12571,9 +13635,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 76,
+  "pk": 134,
   "fields": {
-    "conflict": 52,
+    "conflict": 92,
     "obj_id": 74,
     "job_id": 27,
     "type": "shape",
@@ -12582,9 +13646,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 77,
+  "pk": 135,
   "fields": {
-    "conflict": 53,
+    "conflict": 93,
     "obj_id": 95,
     "job_id": 27,
     "type": "shape",
@@ -12593,9 +13657,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 78,
+  "pk": 136,
   "fields": {
-    "conflict": 54,
+    "conflict": 94,
     "obj_id": 73,
     "job_id": 27,
     "type": "shape",
@@ -12604,9 +13668,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 79,
+  "pk": 137,
   "fields": {
-    "conflict": 55,
+    "conflict": 95,
     "obj_id": 96,
     "job_id": 27,
     "type": "shape",
@@ -12615,9 +13679,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 80,
+  "pk": 138,
   "fields": {
-    "conflict": 56,
+    "conflict": 96,
     "obj_id": 64,
     "job_id": 27,
     "type": "shape",
@@ -12626,9 +13690,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 81,
+  "pk": 139,
   "fields": {
-    "conflict": 57,
+    "conflict": 97,
     "obj_id": 66,
     "job_id": 27,
     "type": "shape",
@@ -12637,9 +13701,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 82,
+  "pk": 140,
   "fields": {
-    "conflict": 58,
+    "conflict": 98,
     "obj_id": 94,
     "job_id": 27,
     "type": "shape",
@@ -12648,9 +13712,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 83,
+  "pk": 141,
   "fields": {
-    "conflict": 59,
+    "conflict": 99,
     "obj_id": 142,
     "job_id": 27,
     "type": "shape",
@@ -12659,9 +13723,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 84,
+  "pk": 142,
   "fields": {
-    "conflict": 60,
+    "conflict": 100,
     "obj_id": 92,
     "job_id": 27,
     "type": "shape",
@@ -12670,9 +13734,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 85,
+  "pk": 143,
   "fields": {
-    "conflict": 60,
+    "conflict": 100,
     "obj_id": 119,
     "job_id": 28,
     "type": "shape",
@@ -12681,9 +13745,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 86,
+  "pk": 144,
   "fields": {
-    "conflict": 61,
+    "conflict": 101,
     "obj_id": 89,
     "job_id": 27,
     "type": "shape",
@@ -12692,9 +13756,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 87,
+  "pk": 145,
   "fields": {
-    "conflict": 61,
+    "conflict": 101,
     "obj_id": 123,
     "job_id": 28,
     "type": "shape",
@@ -12703,9 +13767,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 88,
+  "pk": 146,
   "fields": {
-    "conflict": 62,
+    "conflict": 102,
     "obj_id": 69,
     "job_id": 27,
     "type": "shape",
@@ -12714,9 +13778,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 89,
+  "pk": 147,
   "fields": {
-    "conflict": 62,
+    "conflict": 102,
     "obj_id": 97,
     "job_id": 28,
     "type": "shape",
@@ -12725,9 +13789,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 90,
+  "pk": 148,
   "fields": {
-    "conflict": 63,
+    "conflict": 103,
     "obj_id": 67,
     "job_id": 27,
     "type": "shape",
@@ -12736,9 +13800,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 91,
+  "pk": 149,
   "fields": {
-    "conflict": 63,
+    "conflict": 103,
     "obj_id": 99,
     "job_id": 28,
     "type": "shape",
@@ -12747,9 +13811,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 92,
+  "pk": 150,
   "fields": {
-    "conflict": 64,
+    "conflict": 104,
     "obj_id": 69,
     "job_id": 27,
     "type": "shape",
@@ -12758,9 +13822,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 93,
+  "pk": 151,
   "fields": {
-    "conflict": 65,
+    "conflict": 105,
     "obj_id": 65,
     "job_id": 27,
     "type": "shape",
@@ -12769,9 +13833,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 94,
+  "pk": 152,
   "fields": {
-    "conflict": 65,
+    "conflict": 105,
     "obj_id": 101,
     "job_id": 28,
     "type": "shape",
@@ -12780,9 +13844,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 95,
+  "pk": 153,
   "fields": {
-    "conflict": 66,
+    "conflict": 106,
     "obj_id": 82,
     "job_id": 27,
     "type": "shape",
@@ -12791,9 +13855,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 96,
+  "pk": 154,
   "fields": {
-    "conflict": 66,
+    "conflict": 106,
     "obj_id": 103,
     "job_id": 28,
     "type": "shape",
@@ -12802,9 +13866,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 97,
+  "pk": 155,
   "fields": {
-    "conflict": 67,
+    "conflict": 107,
     "obj_id": 81,
     "job_id": 27,
     "type": "shape",
@@ -12813,9 +13877,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 98,
+  "pk": 156,
   "fields": {
-    "conflict": 67,
+    "conflict": 107,
     "obj_id": 104,
     "job_id": 28,
     "type": "shape",
@@ -12824,9 +13888,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 99,
+  "pk": 157,
   "fields": {
-    "conflict": 68,
+    "conflict": 108,
     "obj_id": 83,
     "job_id": 27,
     "type": "shape",
@@ -12835,9 +13899,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 100,
+  "pk": 158,
   "fields": {
-    "conflict": 68,
+    "conflict": 108,
     "obj_id": 105,
     "job_id": 28,
     "type": "shape",
@@ -12846,9 +13910,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 101,
+  "pk": 159,
   "fields": {
-    "conflict": 69,
+    "conflict": 109,
     "obj_id": 87,
     "job_id": 27,
     "type": "shape",
@@ -12857,9 +13921,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 102,
+  "pk": 160,
   "fields": {
-    "conflict": 69,
+    "conflict": 109,
     "obj_id": 106,
     "job_id": 28,
     "type": "shape",
@@ -12868,9 +13932,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 103,
+  "pk": 161,
   "fields": {
-    "conflict": 70,
+    "conflict": 110,
     "obj_id": 80,
     "job_id": 27,
     "type": "shape",
@@ -12879,9 +13943,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 104,
+  "pk": 162,
   "fields": {
-    "conflict": 70,
+    "conflict": 110,
     "obj_id": 111,
     "job_id": 28,
     "type": "shape",
@@ -12890,9 +13954,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 105,
+  "pk": 163,
   "fields": {
-    "conflict": 71,
+    "conflict": 111,
     "obj_id": 77,
     "job_id": 27,
     "type": "shape",
@@ -12901,9 +13965,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 106,
+  "pk": 164,
   "fields": {
-    "conflict": 71,
+    "conflict": 111,
     "obj_id": 114,
     "job_id": 28,
     "type": "shape",
@@ -12912,9 +13976,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 107,
+  "pk": 165,
   "fields": {
-    "conflict": 72,
+    "conflict": 112,
     "obj_id": 130,
     "job_id": 28,
     "type": "shape",
@@ -12923,9 +13987,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 108,
+  "pk": 166,
   "fields": {
-    "conflict": 73,
+    "conflict": 113,
     "obj_id": 128,
     "job_id": 28,
     "type": "shape",
@@ -12934,9 +13998,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 109,
+  "pk": 167,
   "fields": {
-    "conflict": 74,
+    "conflict": 114,
     "obj_id": 124,
     "job_id": 28,
     "type": "shape",
@@ -12945,9 +14009,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 110,
+  "pk": 168,
   "fields": {
-    "conflict": 75,
+    "conflict": 115,
     "obj_id": 125,
     "job_id": 28,
     "type": "shape",
@@ -12956,9 +14020,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 111,
+  "pk": 169,
   "fields": {
-    "conflict": 76,
+    "conflict": 116,
     "obj_id": 127,
     "job_id": 28,
     "type": "shape",
@@ -12967,9 +14031,9 @@
 },
 {
   "model": "quality_control.annotationid",
-  "pk": 112,
+  "pk": 170,
   "fields": {
-    "conflict": 77,
+    "conflict": 117,
     "obj_id": 129,
     "job_id": 28,
     "type": "shape",
diff --git a/tests/python/shared/assets/quality_conflicts.json b/tests/python/shared/assets/quality_conflicts.json
index 44afa16d4266..00f6e8c05709 100644
--- a/tests/python/shared/assets/quality_conflicts.json
+++ b/tests/python/shared/assets/quality_conflicts.json
@@ -1,8 +1,716 @@
 {
-  "count": 77,
+  "count": 117,
   "next": null,
   "previous": null,
   "results": [
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 129,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 117,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 127,
+          "shape_type": "mask",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 116,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 125,
+          "shape_type": "ellipse",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 115,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 124,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 114,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 128,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 113,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 130,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 1,
+      "id": 112,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 77,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 114,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 111,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 80,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 111,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 110,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 87,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 106,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 109,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 83,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 105,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 108,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 81,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 104,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 107,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 82,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 103,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 106,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_groups"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 65,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 101,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 105,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_attributes"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 69,
+          "shape_type": "mask",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 104,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "covered_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 67,
+          "shape_type": "polyline",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 99,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 103,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "mismatching_direction"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 69,
+          "shape_type": "mask",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 97,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 102,
+      "report_id": 6,
+      "severity": "error",
+      "type": "mismatching_label"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 89,
+          "shape_type": "points",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 123,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 101,
+      "report_id": 6,
+      "severity": "error",
+      "type": "mismatching_label"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 92,
+          "shape_type": "points",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 119,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 100,
+      "report_id": 6,
+      "severity": "error",
+      "type": "mismatching_label"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 142,
+          "shape_type": "skeleton",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 99,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 94,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 98,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 66,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 97,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 64,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 96,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 96,
+          "shape_type": "ellipse",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 95,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 73,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 94,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 95,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 93,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 74,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 92,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 93,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 91,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 76,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 90,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 70,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 89,
+      "report_id": 6,
+      "severity": "error",
+      "type": "extra_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 131,
+          "shape_type": "skeleton",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 88,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 122,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 87,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 121,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 86,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 28,
+          "obj_id": 107,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 85,
+      "report_id": 6,
+      "severity": "error",
+      "type": "missing_annotation"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 92,
+          "shape_type": "points",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 119,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 84,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 141,
+          "shape_type": "skeleton",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 132,
+          "shape_type": "skeleton",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 83,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 67,
+          "shape_type": "polyline",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 99,
+          "shape_type": "polyline",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 82,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 68,
+          "shape_type": "polygon",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 98,
+          "shape_type": "polygon",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 81,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 88,
+          "shape_type": "points",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 102,
+          "shape_type": "points",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 80,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 91,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 118,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 79,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
+    {
+      "annotation_ids": [
+        {
+          "job_id": 27,
+          "obj_id": 65,
+          "shape_type": "rectangle",
+          "type": "shape"
+        },
+        {
+          "job_id": 28,
+          "obj_id": 101,
+          "shape_type": "rectangle",
+          "type": "shape"
+        }
+      ],
+      "frame": 0,
+      "id": 78,
+      "report_id": 6,
+      "severity": "warning",
+      "type": "low_overlap"
+    },
     {
       "annotation_ids": [
         {
diff --git a/tests/python/shared/assets/quality_reports.json b/tests/python/shared/assets/quality_reports.json
index 6beef693db72..ea9d806233af 100644
--- a/tests/python/shared/assets/quality_reports.json
+++ b/tests/python/shared/assets/quality_reports.json
@@ -1,5 +1,5 @@
 {
-  "count": 4,
+  "count": 6,
   "next": null,
   "previous": null,
   "results": [
@@ -122,6 +122,66 @@
       "target": "job",
       "target_last_updated": "2023-11-24T15:23:30.269000Z",
       "task_id": null
+    },
+    {
+      "created_date": "2024-03-21T11:16:21.845000Z",
+      "gt_last_updated": "2023-11-24T15:18:55.216000Z",
+      "id": 5,
+      "job_id": null,
+      "parent_id": null,
+      "summary": {
+        "conflict_count": 40,
+        "conflicts_by_type": {
+          "covered_annotation": 1,
+          "extra_annotation": 11,
+          "low_overlap": 7,
+          "mismatching_attributes": 1,
+          "mismatching_direction": 1,
+          "mismatching_groups": 6,
+          "mismatching_label": 3,
+          "missing_annotation": 10
+        },
+        "ds_count": 36,
+        "error_count": 24,
+        "frame_count": 3,
+        "frame_share": 0.2727272727272727,
+        "gt_count": 35,
+        "valid_count": 22,
+        "warning_count": 16
+      },
+      "target": "task",
+      "target_last_updated": "2023-11-24T15:23:30.045000Z",
+      "task_id": 22
+    },
+    {
+      "created_date": "2024-03-21T11:16:21.847000Z",
+      "gt_last_updated": "2023-11-24T15:18:55.216000Z",
+      "id": 6,
+      "job_id": 27,
+      "parent_id": 5,
+      "summary": {
+        "conflict_count": 40,
+        "conflicts_by_type": {
+          "covered_annotation": 1,
+          "extra_annotation": 11,
+          "low_overlap": 7,
+          "mismatching_attributes": 1,
+          "mismatching_direction": 1,
+          "mismatching_groups": 6,
+          "mismatching_label": 3,
+          "missing_annotation": 10
+        },
+        "ds_count": 36,
+        "error_count": 24,
+        "frame_count": 3,
+        "frame_share": 0.2727272727272727,
+        "gt_count": 35,
+        "valid_count": 22,
+        "warning_count": 16
+      },
+      "target": "job",
+      "target_last_updated": "2023-11-24T15:23:30.269000Z",
+      "task_id": null
     }
   ]
 }
\ No newline at end of file