diff --git a/ir_datasets/datasets/trec_tot.py b/ir_datasets/datasets/trec_tot.py index f7c8af67..de64fa5d 100644 --- a/ir_datasets/datasets/trec_tot.py +++ b/ir_datasets/datasets/trec_tot.py @@ -35,8 +35,20 @@ def default_text(self): return self.title + ' ' + self.text +class TipOfTheTongueTestQuery(NamedTuple): + query_id: str + domain: str + title: str + text: str + sentence_annotations: List[Dict[str, str]] + + def default_text(self): + return self.title + ' ' + self.text + + QUERY_MAP = {'query_id': 'id', 'url': 'url', 'domain': 'domain', 'title': 'title', 'text': 'text', 'sentence_annotations': 'sentence_annotations'} +QUERY_MAP_TEST = {'query_id': 'id', 'domain': 'domain', 'title': 'title', 'text': 'text', 'sentence_annotations': 'sentence_annotations'} def _init(): documentation = YamlDocumentation(f'docs/{NAME}.yaml') @@ -64,6 +76,16 @@ def _init(): documentation(f'2023/{s}'), ) ir_datasets.registry.register(f'{NAME}/2023/{s}', subsets[f'2023/{s}']) + + # Test dataset outside of the loop above as no qrels available at the moment and queries come from different file + s = 'test' + test_dlc = dlc['test-queries'] + subsets[f'2023/{s}'] = Dataset( + docs_2023_handler, + JsonlQueries(Cache(ZipExtract(test_dlc, f'test/queries.jsonl'), base_path/f'2023/{s}/queries.jsonl'), query_cls=TipOfTheTongueTestQuery, mapping=QUERY_MAP_TEST, lang='en'), + documentation(f'2023/{s}'), + ) + ir_datasets.registry.register(f'{NAME}/2023/{s}', subsets[f'2023/{s}']) return base, subsets diff --git a/ir_datasets/etc/downloads.json b/ir_datasets/etc/downloads.json index 5994c0e0..cca341d2 100644 --- a/ir_datasets/etc/downloads.json +++ b/ir_datasets/etc/downloads.json @@ -5812,6 +5812,11 @@ "url": "https://surfdrive.surf.nl/files/index.php/s/FaEK4xc6Xp2JcAJ/download", "expected_md5": "f84fe82cb80e3ee1072576c8d6c4a417", "cache_path": "trec-tot.zip" + }, + "test-queries": { + "url": "https://surfdrive.surf.nl/files/index.php/s/qQCQ1iRZvj712kv/download", + "expected_md5": "756b77a2725f0960defb3a9b12d95334", + "cache_path": "trec-tot-test-queries.zip" } }, diff --git a/test/integration/trec_tot.py b/test/integration/trec_tot.py index 7301df30..98c51c6e 100644 --- a/test/integration/trec_tot.py +++ b/test/integration/trec_tot.py @@ -1,7 +1,7 @@ import re import unittest from ir_datasets.formats import TrecQrel -from ir_datasets.datasets.trec_tot import TipOfTheTongueDoc, TipOfTheTongueQuery +from ir_datasets.datasets.trec_tot import TipOfTheTongueDoc, TipOfTheTongueQuery, TipOfTheTongueTestQuery from .base import DatasetIntegrationTest import ir_datasets @@ -28,6 +28,13 @@ def test_test_tip_of_the_tongue_queries_dev(self): 149: TipOfTheTongueQuery('521', 'https://irememberthismovie.com/ghost-story/', 'movie', 'Ghost story', 'There’s an old one I got to see just once as a kid. Dude takes a bet to survive one nite in a mansion. So, he’s there thinking he’s alone and all these people show up. Remember two real well. A girl he falls I love with and some scientist. Scientist talking at one point shows experiment where he’s talking about how long something can live, cuts the head off a snake, tells dude to try to pick up snake head and dude almost gets bit. Another scene he’s with the girl in bed laying his head in her chest and jumps up worried cause he can’t hear her heartbeat. End of show he walks out stands by the mansion gates and it slams on him with the spikes stabbing him in the back. He loses the bet but gets the girl', [{'id': 1, 'text': 'There’s an old one I got to see just once as a kid.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': True, 'situational_evidence': False}}}, {'id': 2, 'text': 'Dude takes a bet to survive one nite in a mansion.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 3, 'text': 'So, he’s there thinking he’s alone and all these people show up.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 4, 'text': 'Remember two real well.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': False, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 5, 'text': 'A girl he falls I love with and some scientist.', 'labels': {'opinion': False, 'emotion': False, 'hedging': True, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 6, 'text': 'Scientist talking at one point shows experiment where he’s talking about how long something can live, cuts the head off a snake, tells dude to try to pick up snake head and dude almost gets bit.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 7, 'text': 'Another scene he’s with the girl in bed laying his head in her chest and jumps up worried cause he can’t hear her heartbeat.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 8, 'text': 'End of show he walks out stands by the mansion gates and it slams on him with the spikes stabbing him in the back.', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': True, 'plot': False, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': True, 'object': True, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}, {'id': 9, 'text': 'He loses the bet but gets the girl', 'labels': {'opinion': False, 'emotion': False, 'hedging': False, 'social': False, 'comparison_relative': False, 'search': False, 'movie': {'music_compare': False, 'origin_movie': False, 'origin_language': False, 'category': False, 'character': True, 'production_visual': False, 'music_specific': False, 'location_specific': False, 'production_audio': False, 'scene': False, 'plot': True, 'person_fictional': False, 'negation': False, 'genre_audience': False, 'person_real': False, 'production_camera_angle': False, 'origin_actor': False, 'location_type': False, 'object': False, 'timeframe_singular': False, 'quote': False, 'genre_traditional_tone': False, 'release_date': False, 'timeframe_plural': False}, 'context': {'physical_medium': False, 'physical_user_location': False, 'situational_count': False, 'cross_media': False, 'situational_witness': False, 'temporal': False, 'situational_evidence': False}}}]), }) + def test_test_tip_of_the_tongue_queries_dev(self): + self._test_queries('trec-tot/2023/test', count=150, items={ + 0: TipOfTheTongueTestQuery('519', 'movie', 'redbox mystery driving me insane', 'Ok, so my hubby, brother-in-law, and I rented a movie from redbox in the mid 2000\u2019s (around the time streaming was becoming big) and I gotta say, we were glad we only paid a buck for it. This movie was so bad we talked and joked through the entire thing like the robots out of MST3K. But it bothers me that none of us can remember the name of the movie or what it was about. The only thing any of us really remember was it took place is completely or mostly in a third world country and near the end this woman, American I think, was standing either topless or naked in a bunch of mud and was surrounded by young boys. Most of whom were somewhere around 12-14 years old. And they start covering her in mud and as they do so, they are groping her breasts and ass. Not sure if they were supposed to, but they were. I think this scene stuck with us because it was about the best part of the movie and the only thing even relatively exciting that happened. We kept watching hoping that it would get better at some point. That someone didn\u2019t actually, from start to finish, make a movie that terrible. I hope someone can help, because I would really like that know what that movie was supposed to be about and what it was called. I thank you all in advance.', [{"id": 1, "text": "Ok, so my hubby, brother-in-law, and I rented a movie from redbox in the mid 2000\u2019s (around the time streaming was becoming big) and I gotta say, we were glad we only paid a buck for it.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": True, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": True, "temporal": True, "situational_evidence": True}}}, {"id": 2, "text": "This movie was so bad we talked and joked through the entire thing like the robots out of MST3K.", "labels": {"opinion": True, "emotion": False, "hedging": False, "social": False, "comparison_relative": True, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": True, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 3, "text": "But it bothers me that none of us can remember the name of the movie or what it was about.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": True, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 4, "text": "The only thing any of us really remember was it took place is completely or mostly in a third world country and near the end this woman, American I think, was standing either topless or naked in a bunch of mud and was surrounded by young boys.", "labels": {"opinion": False, "emotion": False, "hedging": True, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": True, "object": True, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 5, "text": "Most of whom were somewhere around 12-14 years old.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 6, "text": "And they start covering her in mud and as they do so, they are groping her breasts and ass.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 7, "text": "Not sure if they were supposed to, but they were.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 8, "text": "I think this scene stuck with us because it was about the best part of the movie and the only thing even relatively exciting that happened.", "labels": {"opinion": True, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 9, "text": "We kept watching hoping that it would get better at some point.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": True, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 10, "text": "That someone didn\u2019t actually, from start to finish, make a movie that terrible.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 11, "text": "I hope someone can help, because I would really like that know what that movie was supposed to be about and what it was called.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 12, "text": "I thank you all in advance.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": True, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}]), + 9: TipOfTheTongueTestQuery('505', 'movie', '(Warning: Violent gore content ! ) Scene from a movie I can\u2019t remember', 'Two guys meet with some maffia or a gang in a house. They get separated and one of them is let into a big dining hall. After they let him wait for a couple of minutes, they bring in a big trash can with a foor sticking out. He then realises its the guy he came with (his friend possibly), and he freaks out and even throws up if I\u2019m not mistaken. Please help me remember the name of this movie! Thanks ?', [{"id": 1, "text": "Two guys meet with some maffia or a gang in a house.", "labels": {"opinion": False, "emotion": False, "hedging": True, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": True, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 2, "text": "They get separated and one of them is let into a big dining hall.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": True, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 3, "text": "After they let him wait for a couple of minutes, they bring in a big trash can with a foor sticking out.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": True, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 4, "text": "He then realises its the guy he came with (his friend possibly), and he freaks out and even throws up if I\u2019m not mistaken.", "labels": {"opinion": False, "emotion": False, "hedging": True, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 5, "text": "Please help me remember the name of this movie!", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 6, "text": "Thanks ?", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": True, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}]), + 149: TipOfTheTongueTestQuery('565', 'movie', '70s film', 'Judging by the quality, I\u2019d say this film was made in the 70s. It\u2019s about a large American family who all come back home to attend the funeral of the youngest member, a guy who killed himself. After the funeral, which happens early in the film, they all go home, and from there it\u2019s basically one of those films where everyone\u2019s thinking about their life choices, etc. There\u2019s a bit of tension because they\u2019ve all got their own lives and live in different states so they don\u2019t see each other a lot.', [{"id": 1, "text": "Judging by the quality, I\u2019d say this film was made in the 70s.", "labels": {"opinion": True, "emotion": False, "hedging": True, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": True, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 2, "text": "It\u2019s about a large American family who all come back home to attend the funeral of the youngest member, a guy who killed himself.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": True, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": False, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 3, "text": "After the funeral, which happens early in the film, they all go home, and from there it\u2019s basically one of those films where everyone\u2019s thinking about their life choices, etc.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": True, "character": False, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": True, "plot": False, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": True, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": True, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}, {"id": 4, "text": "There\u2019s a bit of tension because they\u2019ve all got their own lives and live in different states so they don\u2019t see each other a lot.", "labels": {"opinion": False, "emotion": False, "hedging": False, "social": False, "comparison_relative": False, "search": False, "movie": {"music_compare": False, "origin_movie": False, "origin_language": False, "category": False, "character": True, "production_visual": False, "music_specific": False, "location_specific": False, "production_audio": False, "scene": False, "plot": True, "person_fictional": False, "negation": False, "genre_audience": False, "person_real": False, "production_camera_angle": False, "origin_actor": False, "location_type": True, "object": False, "timeframe_singular": False, "quote": False, "genre_traditional_tone": False, "release_date": False, "timeframe_plural": False}, "context": {"physical_medium": False, "physical_user_location": False, "situational_count": False, "cross_media": False, "situational_witness": False, "temporal": False, "situational_evidence": False}}}]), + }) + def test_test_tip_of_the_tongue_qrels_train(self): self._test_qrels('trec-tot/2023/train', count=150, items={ 0: TrecQrel('763', '16742289', 1, '0'),