Merge pull request #53759 from nextcloud/enh/noid/taskpro-audio-chat

[TaskProcessing] Add audio-to-audio chat task type

Author: marcelklehr

lib/composer/composer/autoload_classmap.php

@@ -842,6 +842,7 @@
     'OCP\\TaskProcessing\\ShapeDescriptor' => $baseDir . '/lib/public/TaskProcessing/ShapeDescriptor.php',
     'OCP\\TaskProcessing\\ShapeEnumValue' => $baseDir . '/lib/public/TaskProcessing/ShapeEnumValue.php',
     'OCP\\TaskProcessing\\Task' => $baseDir . '/lib/public/TaskProcessing/Task.php',
+    'OCP\\TaskProcessing\\TaskTypes\\AudioToAudioChat' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/AudioToAudioChat.php',
     'OCP\\TaskProcessing\\TaskTypes\\AudioToText' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/AudioToText.php',
     'OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ContextAgentInteraction.php',
     'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php',

lib/composer/composer/autoload_static.php

@@ -883,6 +883,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
         'OCP\\TaskProcessing\\ShapeDescriptor' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/ShapeDescriptor.php',
         'OCP\\TaskProcessing\\ShapeEnumValue' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/ShapeEnumValue.php',
         'OCP\\TaskProcessing\\Task' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/Task.php',
+        'OCP\\TaskProcessing\\TaskTypes\\AudioToAudioChat' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/AudioToAudioChat.php',
         'OCP\\TaskProcessing\\TaskTypes\\AudioToText' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/AudioToText.php',
         'OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ContextAgentInteraction.php',
         'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php',

lib/private/TaskProcessing/Manager.php

@@ -589,6 +589,7 @@ private function _getTaskTypes(): array {
 			\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID => \OCP\Server::get(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::class),
 			\OCP\TaskProcessing\TaskTypes\TextToTextProofread::ID => \OCP\Server::get(\OCP\TaskProcessing\TaskTypes\TextToTextProofread::class),
 			\OCP\TaskProcessing\TaskTypes\TextToSpeech::ID => \OCP\Server::get(\OCP\TaskProcessing\TaskTypes\TextToSpeech::class),
+			\OCP\TaskProcessing\TaskTypes\AudioToAudioChat::ID => \OCP\Server::get(\OCP\TaskProcessing\TaskTypes\AudioToAudioChat::class),
 		];
 
 		foreach ($context->getTaskProcessingTaskTypes() as $providerServiceRegistration) {

lib/public/TaskProcessing/TaskTypes/AudioToAudioChat.php

@@ -0,0 +1,112 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * SPDX-FileCopyrightText: 2025 Nextcloud GmbH and Nextcloud contributors
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+
+namespace OCP\TaskProcessing\TaskTypes;
+
+use OCP\IL10N;
+use OCP\L10N\IFactory;
+use OCP\TaskProcessing\EShapeType;
+use OCP\TaskProcessing\ITaskType;
+use OCP\TaskProcessing\ShapeDescriptor;
+
+/**
+ * This is the task processing task type for text chat
+ * @since 32.0.0
+ */
+class AudioToAudioChat implements ITaskType {
+	/**
+	 * @since 32.0.0
+	 */
+	public const ID = 'core:audio2audio:chat';
+
+	private IL10N $l;
+
+	/**
+	 * @param IFactory $l10nFactory
+	 * @since 32.0.0
+	 */
+	public function __construct(
+		IFactory $l10nFactory,
+	) {
+		$this->l = $l10nFactory->get('lib');
+	}
+
+
+	/**
+	 * @inheritDoc
+	 * @since 32.0.0
+	 */
+	public function getName(): string {
+		return $this->l->t('Audio chat');
+	}
+
+	/**
+	 * @inheritDoc
+	 * @since 32.0.0
+	 */
+	public function getDescription(): string {
+		return $this->l->t('Voice chat with the assistant');
+	}
+
+	/**
+	 * @return string
+	 * @since 32.0.0
+	 */
+	public function getId(): string {
+		return self::ID;
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 32.0.0
+	 */
+	public function getInputShape(): array {
+		return [
+			'system_prompt' => new ShapeDescriptor(
+				$this->l->t('System prompt'),
+				$this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'),
+				EShapeType::Text
+			),
+			'input' => new ShapeDescriptor(
+				$this->l->t('Chat voice message'),
+				$this->l->t('Describe a task that you want the assistant to do or ask a question'),
+				EShapeType::Audio
+			),
+			'history' => new ShapeDescriptor(
+				$this->l->t('Chat history'),
+				$this->l->t('The history of chat messages before the current message, starting with a message by the user'),
+				EShapeType::ListOfTexts
+			)
+		];
+	}
+
+	/**
+	 * @return ShapeDescriptor[]
+	 * @since 32.0.0
+	 */
+	public function getOutputShape(): array {
+		return [
+			'input_transcript' => new ShapeDescriptor(
+				$this->l->t('Input transcript'),
+				$this->l->t('Transcription of the audio input'),
+				EShapeType::Text,
+			),
+			'output' => new ShapeDescriptor(
+				$this->l->t('Response voice message'),
+				$this->l->t('The generated voice response as part of the conversation'),
+				EShapeType::Audio
+			),
+			'output_transcript' => new ShapeDescriptor(
+				$this->l->t('Output transcript'),
+				$this->l->t('Transcription of the audio output'),
+				EShapeType::Text,
+			),
+		];
+	}
+}