Skip to content

Commit a17fb07

Browse files
lukasdotcomlukasdotcom
committed
Feat: Add MultimodalChatWithTools and MultimodalContextAgentInteraction
Signed-off-by: Lukas Schaefer <lukas@lschaefer.xyz>
1 parent fbb559d commit a17fb07

5 files changed

Lines changed: 279 additions & 0 deletions

File tree

‎lib/composer/composer/autoload_classmap.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -971,6 +971,8 @@
971971
'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php',
972972
'OCP\\TaskProcessing\\TaskTypes\\GenerateEmoji' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/GenerateEmoji.php',
973973
'OCP\\TaskProcessing\\TaskTypes\\ImageToTextOpticalCharacterRecognition' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/ImageToTextOpticalCharacterRecognition.php',
974+
'OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php',
975+
'OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php',
974976
'OCP\\TaskProcessing\\TaskTypes\\TextToImage' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToImage.php',
975977
'OCP\\TaskProcessing\\TaskTypes\\TextToSpeech' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToSpeech.php',
976978
'OCP\\TaskProcessing\\TaskTypes\\TextToText' => $baseDir . '/lib/public/TaskProcessing/TaskTypes/TextToText.php',

‎lib/composer/composer/autoload_static.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,6 +1012,8 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
10121012
'OCP\\TaskProcessing\\TaskTypes\\ContextWrite' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ContextWrite.php',
10131013
'OCP\\TaskProcessing\\TaskTypes\\GenerateEmoji' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/GenerateEmoji.php',
10141014
'OCP\\TaskProcessing\\TaskTypes\\ImageToTextOpticalCharacterRecognition' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/ImageToTextOpticalCharacterRecognition.php',
1015+
'OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php',
1016+
'OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php',
10151017
'OCP\\TaskProcessing\\TaskTypes\\TextToImage' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToImage.php',
10161018
'OCP\\TaskProcessing\\TaskTypes\\TextToSpeech' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToSpeech.php',
10171019
'OCP\\TaskProcessing\\TaskTypes\\TextToText' => __DIR__ . '/../../..' . '/lib/public/TaskProcessing/TaskTypes/TextToText.php',

‎lib/private/TaskProcessing/Manager.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@
7979
use OCP\TaskProcessing\TaskTypes\ContextWrite;
8080
use OCP\TaskProcessing\TaskTypes\GenerateEmoji;
8181
use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition;
82+
use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools;
83+
use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction;
8284
use OCP\TaskProcessing\TaskTypes\TextToImage;
8385
use OCP\TaskProcessing\TaskTypes\TextToSpeech;
8486
use OCP\TaskProcessing\TaskTypes\TextToText;
@@ -696,13 +698,15 @@ private function _getTaskTypes(): array {
696698
GenerateEmoji::ID => Server::get(GenerateEmoji::class),
697699
TextToTextChangeTone::ID => Server::get(TextToTextChangeTone::class),
698700
TextToTextChatWithTools::ID => Server::get(TextToTextChatWithTools::class),
701+
MultimodalChatWithTools::ID => Server::get(MultimodalChatWithTools::class),
699702
ContextAgentInteraction::ID => Server::get(ContextAgentInteraction::class),
700703
TextToTextProofread::ID => Server::get(TextToTextProofread::class),
701704
TextToTextReformatParagraphs::ID => Server::get(TextToTextReformatParagraphs::class),
702705
TextToSpeech::ID => Server::get(TextToSpeech::class),
703706
AudioToAudioChat::ID => Server::get(AudioToAudioChat::class),
704707
AudioToAudioTranslate::ID => Server::get(AudioToAudioTranslate::class),
705708
ContextAgentAudioInteraction::ID => Server::get(ContextAgentAudioInteraction::class),
709+
MultimodalContextAgentInteraction::ID => Server::get(MultimodalContextAgentInteraction::class),
706710
AnalyzeImages::ID => Server::get(AnalyzeImages::class),
707711
ImageToTextOpticalCharacterRecognition::ID => Server::get(ImageToTextOpticalCharacterRecognition::class),
708712
];

‎lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCP\TaskProcessing\TaskTypes;
11+
12+
use OCP\IL10N;
13+
use OCP\L10N\IFactory;
14+
use OCP\TaskProcessing\EShapeType;
15+
use OCP\TaskProcessing\ITaskType;
16+
use OCP\TaskProcessing\ShapeDescriptor;
17+
18+
/**
19+
* This is the task processing task type for invoking multimodal Chat-enabled LLMs with tool call support
20+
* @since 35.0.0
21+
*/
22+
class MultimodalChatWithTools implements ITaskType {
23+
/**
24+
* @since 35.0.0
25+
*/
26+
public const ID = 'core:text2text:multimodal-chatwithtools';
27+
28+
private IL10N $l;
29+
30+
/**
31+
* @param IFactory $l10nFactory
32+
* @since 35.0.0
33+
*/
34+
public function __construct(
35+
IFactory $l10nFactory,
36+
) {
37+
$this->l = $l10nFactory->get('lib');
38+
}
39+
40+
/**
41+
* @inheritDoc
42+
* @since 35.0.0
43+
*/
44+
#[\Override]
45+
public function getName(): string {
46+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
47+
return $this->l->t('Chat with tools using attachments');
48+
}
49+
50+
/**
51+
* @inheritDoc
52+
* @since 35.0.0
53+
*/
54+
#[\Override]
55+
public function getDescription(): string {
56+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
57+
return $this->l->t('Chat with the language model with tool calling support and attachments.');
58+
}
59+
60+
/**
61+
* @return string
62+
* @since 35.0.0
63+
*/
64+
#[\Override]
65+
public function getId(): string {
66+
return self::ID;
67+
}
68+
69+
/**
70+
* @return ShapeDescriptor[]
71+
* @since 35.0.0
72+
*/
73+
#[\Override]
74+
public function getInputShape(): array {
75+
return [
76+
'system_prompt' => new ShapeDescriptor(
77+
$this->l->t('System prompt'),
78+
$this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'),
79+
EShapeType::Text
80+
),
81+
'input' => new ShapeDescriptor(
82+
$this->l->t('Chat message'),
83+
$this->l->t('Describe a task that you want the assistant to do or ask a question'),
84+
EShapeType::Text
85+
),
86+
'input_attachments' => new ShapeDescriptor(
87+
$this->l->t('Input attachments'),
88+
$this->l->t('Files to send along with the chat message.'),
89+
EShapeType::ListOfFiles
90+
),
91+
'tool_message' => new ShapeDescriptor(
92+
$this->l->t('Tool message'),
93+
$this->l->t('The result of tool calls in the last interaction'),
94+
EShapeType::Text
95+
),
96+
'history' => new ShapeDescriptor(
97+
$this->l->t('Chat history'),
98+
$this->l->t('The history of chat messages before the current message, starting with a message by the user'),
99+
EShapeType::ListOfTexts
100+
),
101+
// See https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools for the format
102+
'tools' => new ShapeDescriptor(
103+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
104+
$this->l->t('Available tools'),
105+
$this->l->t('The available tools in JSON format'),
106+
EShapeType::Text
107+
),
108+
];
109+
}
110+
111+
/**
112+
* @return ShapeDescriptor[]
113+
* @since 35.0.0
114+
*/
115+
#[\Override]
116+
public function getOutputShape(): array {
117+
return [
118+
'output' => new ShapeDescriptor(
119+
$this->l->t('Generated response'),
120+
$this->l->t('The response from the chat model'),
121+
EShapeType::Text
122+
),
123+
'output_attachments' => new ShapeDescriptor(
124+
$this->l->t('Output attachments'),
125+
$this->l->t('Files generated by the model.'),
126+
EShapeType::ListOfFiles
127+
),
128+
'tool_calls' => new ShapeDescriptor(
129+
$this->l->t('Tool calls'),
130+
$this->l->t('Tools call instructions from the model in JSON format'),
131+
EShapeType::Text
132+
),
133+
];
134+
}
135+
}

‎lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCP\TaskProcessing\TaskTypes;
11+
12+
use OCP\IL10N;
13+
use OCP\L10N\IFactory;
14+
use OCP\TaskProcessing\EShapeType;
15+
use OCP\TaskProcessing\IInternalTaskType;
16+
use OCP\TaskProcessing\ShapeDescriptor;
17+
18+
/**
19+
* This is the task processing task type for multimodal Context Agent interaction
20+
* @since 35.0.0
21+
*/
22+
class MultimodalContextAgentInteraction implements IInternalTaskType {
23+
/**
24+
* @since 35.0.0
25+
*/
26+
public const ID = 'core:contextagent:multimodal-interaction';
27+
28+
private IL10N $l;
29+
30+
/**
31+
* @param IFactory $l10nFactory
32+
* @since 35.0.0
33+
*/
34+
public function __construct(
35+
IFactory $l10nFactory,
36+
) {
37+
$this->l = $l10nFactory->get('lib');
38+
}
39+
40+
/**
41+
* @inheritDoc
42+
* @since 35.0.0
43+
*/
44+
#[\Override]
45+
public function getName(): string {
46+
return 'ContextAgent multimodal'; // We do not translate this
47+
}
48+
49+
/**
50+
* @inheritDoc
51+
* @since 35.0.0
52+
*/
53+
#[\Override]
54+
public function getDescription(): string {
55+
return $this->l->t('Chat with an agent using attachments');
56+
}
57+
58+
/**
59+
* @return string
60+
* @since 35.0.0
61+
*/
62+
#[\Override]
63+
public function getId(): string {
64+
return self::ID;
65+
}
66+
67+
/**
68+
* @return ShapeDescriptor[]
69+
* @since 35.0.0
70+
*/
71+
#[\Override]
72+
public function getInputShape(): array {
73+
return [
74+
'input' => new ShapeDescriptor(
75+
$this->l->t('Chat message'),
76+
$this->l->t('A chat message to send to the agent.'),
77+
EShapeType::Text
78+
),
79+
'input_attachments' => new ShapeDescriptor(
80+
$this->l->t('Input attachments'),
81+
$this->l->t('Files to send along with the chat message.'),
82+
EShapeType::ListOfFiles
83+
),
84+
'confirmation' => new ShapeDescriptor(
85+
$this->l->t('Confirmation'),
86+
$this->l->t('Whether to confirm previously requested actions: 0 for denial and 1 for confirmation.'),
87+
EShapeType::Number
88+
),
89+
'conversation_token' => new ShapeDescriptor(
90+
$this->l->t('Conversation token'),
91+
$this->l->t('A token representing the conversation.'),
92+
EShapeType::Text
93+
),
94+
'memories' => new ShapeDescriptor(
95+
$this->l->t('Memories'),
96+
$this->l->t('Memories to send to the agent.'),
97+
EShapeType::ListOfTexts
98+
),
99+
];
100+
}
101+
102+
/**
103+
* @return ShapeDescriptor[]
104+
* @since 35.0.0
105+
*/
106+
#[\Override]
107+
public function getOutputShape(): array {
108+
return [
109+
'output' => new ShapeDescriptor(
110+
$this->l->t('Generated response'),
111+
$this->l->t('The response from the chat model.'),
112+
EShapeType::Text
113+
),
114+
'output_attachments' => new ShapeDescriptor(
115+
$this->l->t('Output attachments'),
116+
$this->l->t('Files generated by the agent.'),
117+
EShapeType::ListOfFiles
118+
),
119+
'conversation_token' => new ShapeDescriptor(
120+
$this->l->t('The new conversation token'),
121+
$this->l->t('Send this along with the next interaction.'),
122+
EShapeType::Text
123+
),
124+
'actions' => new ShapeDescriptor(
125+
$this->l->t('Requested actions by the agent'),
126+
$this->l->t('Actions that the agent would like to carry out in JSON format.'),
127+
EShapeType::Text
128+
),
129+
'sources' => new ShapeDescriptor(
130+
$this->l->t('Sources'),
131+
$this->l->t('Sources of the generated response.'),
132+
EShapeType::ListOfTexts
133+
),
134+
];
135+
}
136+
}

0 commit comments

Comments
 (0)