diff --git a/examples/egosuite_evaluation/evaluate.py b/examples/egosuite_evaluation/evaluate.py index 216cb2d..b35acea 100644 --- a/examples/egosuite_evaluation/evaluate.py +++ b/examples/egosuite_evaluation/evaluate.py @@ -65,6 +65,7 @@ ) SCHEMA_VERSION = 1 +PROJECTED_HAND_LABEL_TYPE = "projected-hand-joints" EXPECTED_HAND_JOINT_COUNT = 21 DEFAULT_RUNS_DIRECTORY = Path("data/egosuite-evaluation/runs") DEFAULT_LABELS_DIRECTORY = Path("data/egosuite-evaluation/labels") @@ -158,6 +159,22 @@ def load_projected_hand_label_report( ) from error if not isinstance(report_payload, dict): raise ValueError(f"projected-hand label report {report_path} must contain a JSON object") + schema_version = report_payload.get("schema_version") + if ( + not isinstance(schema_version, int) + or isinstance(schema_version, bool) + or schema_version != SCHEMA_VERSION + ): + raise ValueError( + f"projected-hand label report {report_path} field 'schema_version' has value " + f"{schema_version!r}; supported value is {SCHEMA_VERSION}" + ) + label_type = report_payload.get("label_type") + if not isinstance(label_type, str) or label_type != PROJECTED_HAND_LABEL_TYPE: + raise ValueError( + f"projected-hand label report {report_path} field 'label_type' has value " + f"{label_type!r}; supported value is {PROJECTED_HAND_LABEL_TYPE!r}" + ) raw_frame_records = report_payload.get("frames") if not isinstance(raw_frame_records, list): raise ValueError(f"projected-hand label report {report_path} must contain a 'frames' array") @@ -938,7 +955,7 @@ def write_label_report( source_uri_by_path = _source_uri_by_path(tuple(labels_by_source)) report = { "schema_version": SCHEMA_VERSION, - "label_type": "projected-hand-joints", + "label_type": PROJECTED_HAND_LABEL_TYPE, "camera_view": camera_view.value, "frame_stride": frame_stride, "limit_per_episode": limit_per_episode, diff --git a/examples/egosuite_evaluation/tests/test_evaluation.py b/examples/egosuite_evaluation/tests/test_evaluation.py index 6eee407..c0f9c9f 100644 --- a/examples/egosuite_evaluation/tests/test_evaluation.py +++ b/examples/egosuite_evaluation/tests/test_evaluation.py @@ -269,31 +269,39 @@ def test_pipeline_registers_projected_hand_visibility_as_an_hflow_check() -> Non def test_saved_label_report_selects_exact_frames_for_a_canonical_episode(tmp_path: Path) -> None: source_path = tmp_path / "episode-123.mcap" report_path = tmp_path / "labels.json" - report_path.write_text( - json.dumps( - { - "frames": [ - { - "source_path": str(source_path), - "source_episode": "episode-123", - "camera_view": "head-left", - "frame_index": frame_index, - "left_in_frame_joint_count": 21, - "right_in_frame_joint_count": 21 if frame_index == 8 else 0, - "expected_hand_count": 2 if frame_index == 8 else 1, - "left_hand_issue_reasons": [], - "right_hand_issue_reasons": ["occlusion"] if frame_index == 8 else [], - } - for frame_index in (8, 3) - ] - } + labels = [ + ProjectedHandFrameLabel( + source_path=source_path, + source_episode="episode-123", + camera_view=CameraView.HEAD_LEFT, + frame_index=frame_index, + left_in_frame_joint_count=21, + right_in_frame_joint_count=21 if frame_index == 8 else 0, + expected_hand_count=2 if frame_index == 8 else 1, + left_hand_issue_reasons=(), + right_hand_issue_reasons=("occlusion",) if frame_index == 8 else (), ) + for frame_index in (8, 3) + ] + write_label_report( + {source_path: labels}, + camera_view=CameraView.HEAD_LEFT, + frame_stride=30, + limit_per_episode=None, + episode_count=None, + samples_per_episode=None, + samples_per_hand_count=None, + sample_seed=42, + output_path=report_path, ) + report_payload = json.loads(report_path.read_text()) + recorded_source_uri = report_payload["frames"][0]["source_uri"] + assert isinstance(recorded_source_uri, str) label_report = load_projected_hand_label_report(report_path) selected_labels = labels_for_pipeline_episode( tmp_path / "episode-123.canonical.mcap", - {"source_uri": "run-a/episode-123.mcap"}, + {"source_uri": recorded_source_uri}, label_report, ) @@ -302,11 +310,96 @@ def test_saved_label_report_selects_exact_frames_for_a_canonical_episode(tmp_pat assert selected_labels[1].right_hand_issue_reasons == ("occlusion",) +@pytest.mark.parametrize( + ("envelope", "field_name", "found_value", "supported_value"), + [ + pytest.param( + {"label_type": "projected-hand-joints"}, + "schema_version", + "None", + "1", + id="missing-schema-version", + ), + pytest.param( + {"schema_version": "1", "label_type": "projected-hand-joints"}, + "schema_version", + "'1'", + "1", + id="non-integer-schema-version", + ), + pytest.param( + {"schema_version": 0, "label_type": "projected-hand-joints"}, + "schema_version", + "0", + "1", + id="older-schema-version", + ), + # bool subclasses int, so True passes an isinstance(_, int) check and + # compares equal to 1. Only the explicit bool guard rejects it. + pytest.param( + {"schema_version": True, "label_type": "projected-hand-joints"}, + "schema_version", + "True", + "1", + id="bool-schema-version", + ), + pytest.param( + {"schema_version": 2, "label_type": "projected-hand-joints"}, + "schema_version", + "2", + "1", + id="future-schema-version", + ), + pytest.param( + {"schema_version": 1}, + "label_type", + "None", + "'projected-hand-joints'", + id="missing-label-type", + ), + pytest.param( + {"schema_version": 1, "label_type": 1}, + "label_type", + "1", + "'projected-hand-joints'", + id="non-string-label-type", + ), + pytest.param( + {"schema_version": 1, "label_type": "bounding-boxes"}, + "label_type", + "'bounding-boxes'", + "'projected-hand-joints'", + id="unsupported-label-type", + ), + ], +) +def test_saved_label_report_rejects_unsupported_envelope_before_frames( + tmp_path: Path, + envelope: dict[str, object], + field_name: str, + found_value: str, + supported_value: str, +) -> None: + report_path = tmp_path / "labels.json" + report_path.write_text(json.dumps({**envelope, "frames": [None]})) + + with pytest.raises(ValueError) as error: + load_projected_hand_label_report(report_path) + + message = str(error.value) + assert str(report_path) in message + assert repr(field_name) in message + assert f"value {found_value}" in message + assert f"supported value is {supported_value}" in message + + def test_legacy_label_report_rejects_one_basename_for_multiple_sources(tmp_path: Path) -> None: report_path = tmp_path / "labels.json" report_path.write_text( json.dumps( { + "schema_version": 1, + "label_type": "projected-hand-joints", "frames": [ { "source_path": str(tmp_path / run_name / "episode.mcap"), @@ -320,7 +413,7 @@ def test_legacy_label_report_rejects_one_basename_for_multiple_sources(tmp_path: "right_hand_issue_reasons": [], } for run_name, frame_index in (("run-a", 3), ("run-b", 8)) - ] + ], } ) ) @@ -338,6 +431,8 @@ def test_saved_label_report_matches_same_named_sources_by_canonical_provenance( report_path.write_text( json.dumps( { + "schema_version": 1, + "label_type": "projected-hand-joints", "frames": [ { "source_path": str(source_path), @@ -355,7 +450,7 @@ def test_saved_label_report_matches_same_named_sources_by_canonical_provenance( (first_source_path, "run-a/episode.mcap", 1), (second_source_path, "run-b/episode.mcap", 2), ) - ] + ], } ) ) @@ -387,6 +482,8 @@ def test_saved_label_report_rejects_missing_or_unrelated_canonical_provenance( report_path.write_text( json.dumps( { + "schema_version": 1, + "label_type": "projected-hand-joints", "frames": [ { "source_path": str(source_path), @@ -400,7 +497,7 @@ def test_saved_label_report_rejects_missing_or_unrelated_canonical_provenance( "left_hand_issue_reasons": [], "right_hand_issue_reasons": [], } - ] + ], } ) )