tasks: add any-to-any (#1346)

Since these models are becoming more popular I added the task page

---------

Co-authored-by: vb <[email protected]>

Author: merveenoyan

packages/tasks/src/tasks/any-to-any/about.md

@@ -0,0 +1,58 @@
+## Use Cases
+
+### Embodied Agents
+
+Any-to-any models can help embodied agents operate in multi-sensory environments, such as video games or physical robots. The model can take in an image or video of a scene, text prompts, and audio, and respond by generating text, actions, predict next frames, or generate speech commands.
+
+### Real-time Accessibility Systems
+
+Vision-language based any-to-any models can be used aid visually impaired people. A real-time on-device any-to-any model can take a real-world video stream from wearable glasses, and describe the scene in audio (e.g., "A person in a red coat is walking toward you") or provide real-time closed captions and environmental sound cues.
+
+### Multimodal Content Creation
+
+One can use any-to-any models to generate multimodal content. For example, given a video and an outline, the model can generate speech, better videos, or a descriptive blog post. Moreover, these models can sync narration timing with visual transitions.
+
+## Inference
+
+You can infer with any-to-any models using transformers. Below is an example to infer Qwen2.5-Omni-7B model, make sure to check the model you're inferring with.
+
+```python
+import soundfile as sf
+from transformers import Qwen2_5OmniModel, Qwen2_5OmniProcessor
+from qwen_omni_utils import process_mm_info
+
+model = Qwen2_5OmniModel.from_pretrained("Qwen/Qwen2.5-Omni-7B", torch_dtype="auto", device_map="auto")
+
+processor = Qwen2_5OmniProcessor.from_pretrained("Qwen/Qwen2.5-Omni-7B")
+
+conversation = [
+    {
+        "role": "system",
+        "content": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.",
+    },
+    {
+        "role": "user",
+        "content": [
+            {"type": "video", "video": "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw.mp4"},
+        ],
+    },
+]
+
+USE_AUDIO_IN_VIDEO = True
+
+text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
+audios, images, videos = process_mm_info(conversation, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+inputs = processor(text=text, audios=audios, images=images, videos=videos, return_tensors="pt", padding=True, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+inputs = inputs.to(model.device).to(model.dtype)
+
+# Inference: Generation of the output text and audio
+text_ids, audio = model.generate(**inputs, use_audio_in_video=USE_AUDIO_IN_VIDEO)
+
+text = processor.batch_decode(text_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)
+print(text)
+sf.write(
+    "output.wav",
+    audio.reshape(-1).detach().cpu().numpy(),
+    samplerate=24000,
+)
+```

packages/tasks/src/tasks/any-to-any/data.ts

@@ -0,0 +1,66 @@
+import type { TaskDataCustom } from "../index.js";
+
+const taskData: TaskDataCustom = {
+	datasets: [
+		{
+			description: "A dataset with multiple modality input and output pairs.",
+			id: "PKU-Alignment/align-anything",
+		},
+	],
+	demo: {
+		inputs: [
+			{   
+				filename: "any-to-any-input.jpg",
+				type: "img",
+			},
+			{
+				label: "Text Prompt",
+				content: "Describe the position of the bee in detail.",
+				type: "text",
+			},
+		],
+		outputs: [
+			{
+				label: "Generated Text",
+				content:
+					"The place in the picture is Osaka Castle, located in Osaka, Japan. Osaka Castle is a historic castle that was originally built in the 16th century by Toyotomi Hideyoshi, a powerful warlord of the time. It is one of the most famous landmarks in Osaka and is known for its distinctive white walls and black roof tiles. The castle has been rebuilt several times over the centuries and is now a popular tourist attraction, offering visitors a glimpse into Japan's rich history and culture.",
+				type: "text",
+			},
+            {   
+				filename: "any-to-any-output.wav",
+				type: "audio",
+			},
+		],
+	},
+	metrics: [],
+	models: [
+		{
+			description: "Strong model that can take in video, audio, image, text and output text and natural speech.",
+			id: "Qwen/Qwen2.5-Omni-7B",
+		},
+		{
+			description: "Robust model that can take in image and text and generate image and text.",
+			id: "deepseek-ai/Janus-Pro-7B",
+		},
+		{
+			description: "Any-to-any model with speech, video, audio, image and text understanding capabilities.",
+			id: "openbmb/MiniCPM-o-2_6",
+		},
+		{
+			description: "A model that can understand image and text and generate image and text.",
+			id: "EPFL-VILAB/4M-21_XL",
+		},
+	],
+	spaces: [
+		{
+			description: "An application to chat with an any-to-any (image & text) model.",
+			id: "deepseek-ai/Janus-Pro-7B",
+		},
+	],
+	summary:
+		"Any-to-any models can understand two or more modalities and output two or more modalities.",
+	widgetModels: [],
+	youtubeId: "",
+};
+
+export default taskData;

packages/tasks/src/tasks/index.ts

@@ -1,6 +1,7 @@
 import type { PipelineType } from "../pipelines.js";
 import { PIPELINE_DATA } from "../pipelines.js";
 
+import anyToAny from "./any-to-any/data.js";
 import audioClassification from "./audio-classification/data.js";
 import audioToAudio from "./audio-to-audio/data.js";
 import automaticSpeechRecognition from "./automatic-speech-recognition/data.js";
@@ -198,7 +199,7 @@ function getData(type: PipelineType, partialTaskData: TaskDataCustom = placehold
 // Tasks that call getData() without the second argument will
 // have a "placeholder" page.
 export const TASKS_DATA: Record<PipelineType, TaskData | undefined> = {
-	"any-to-any": getData("any-to-any", placeholder),
+	"any-to-any": getData("any-to-any", anyToAny),
 	"audio-classification": getData("audio-classification", audioClassification),
 	"audio-to-audio": getData("audio-to-audio", audioToAudio),
 	"audio-text-to-text": getData("audio-text-to-text", placeholder),