Skip to content

Commit fc6b6ed

Browse files
lukasdotcomlukasdotcom
committed
Feat: Add MultimodalChatWithTools and MultimodalContextAgentInteraction
Signed-off-by: Lukas Schaefer <lukas@lschaefer.xyz>
1 parent a256d7e commit fc6b6ed

3 files changed

Lines changed: 266 additions & 0 deletions

File tree

‎lib/private/TaskProcessing/Manager.php‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@
7676
use OCP\TaskProcessing\TaskTypes\AudioToTextSubtitles;
7777
use OCP\TaskProcessing\TaskTypes\ContextAgentAudioInteraction;
7878
use OCP\TaskProcessing\TaskTypes\ContextAgentInteraction;
79+
use OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction;
7980
use OCP\TaskProcessing\TaskTypes\ContextWrite;
8081
use OCP\TaskProcessing\TaskTypes\GenerateEmoji;
8182
use OCP\TaskProcessing\TaskTypes\ImageToTextOpticalCharacterRecognition;
@@ -84,6 +85,7 @@
8485
use OCP\TaskProcessing\TaskTypes\TextToText;
8586
use OCP\TaskProcessing\TaskTypes\TextToTextChangeTone;
8687
use OCP\TaskProcessing\TaskTypes\TextToTextChat;
88+
use OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools;
8789
use OCP\TaskProcessing\TaskTypes\TextToTextChatWithTools;
8890
use OCP\TaskProcessing\TaskTypes\TextToTextFormalization;
8991
use OCP\TaskProcessing\TaskTypes\TextToTextHeadline;
@@ -696,13 +698,15 @@ private function _getTaskTypes(): array {
696698
GenerateEmoji::ID => Server::get(GenerateEmoji::class),
697699
TextToTextChangeTone::ID => Server::get(TextToTextChangeTone::class),
698700
TextToTextChatWithTools::ID => Server::get(TextToTextChatWithTools::class),
701+
MultimodalChatWithTools::ID => Server::get(MultimodalChatWithTools::class),
699702
ContextAgentInteraction::ID => Server::get(ContextAgentInteraction::class),
700703
TextToTextProofread::ID => Server::get(TextToTextProofread::class),
701704
TextToTextReformatParagraphs::ID => Server::get(TextToTextReformatParagraphs::class),
702705
TextToSpeech::ID => Server::get(TextToSpeech::class),
703706
AudioToAudioChat::ID => Server::get(AudioToAudioChat::class),
704707
AudioToAudioTranslate::ID => Server::get(AudioToAudioTranslate::class),
705708
ContextAgentAudioInteraction::ID => Server::get(ContextAgentAudioInteraction::class),
709+
MultimodalContextAgentInteraction::ID => Server::get(MultimodalContextAgentInteraction::class),
706710
AnalyzeImages::ID => Server::get(AnalyzeImages::class),
707711
ImageToTextOpticalCharacterRecognition::ID => Server::get(ImageToTextOpticalCharacterRecognition::class),
708712
];

‎lib/public/TaskProcessing/TaskTypes/MultimodalChatWithTools.php‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCP\TaskProcessing\TaskTypes;
11+
12+
use OCP\IL10N;
13+
use OCP\L10N\IFactory;
14+
use OCP\TaskProcessing\EShapeType;
15+
use OCP\TaskProcessing\IInternalTaskType;
16+
use OCP\TaskProcessing\ITaskType;
17+
use OCP\TaskProcessing\ShapeDescriptor;
18+
19+
/**
20+
* This is the task processing task type for invoking multimodal Chat-enabled LLMs with tool call support
21+
* @since 35.0.0
22+
*/
23+
class MultimodalChatWithTools implements ITaskType {
24+
/**
25+
* @since 35.0.0
26+
*/
27+
public const ID = 'core:text2text:multimodal-chatwithtools';
28+
29+
private IL10N $l;
30+
31+
/**
32+
* @param IFactory $l10nFactory
33+
* @since 35.0.0
34+
*/
35+
public function __construct(
36+
IFactory $l10nFactory,
37+
) {
38+
$this->l = $l10nFactory->get('lib');
39+
}
40+
41+
/**
42+
* @inheritDoc
43+
* @since 35.0.0
44+
*/
45+
#[\Override]
46+
public function getName(): string {
47+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
48+
return $this->l->t('Chat with tools using attachments');
49+
}
50+
51+
/**
52+
* @inheritDoc
53+
* @since 35.0.0
54+
*/
55+
#[\Override]
56+
public function getDescription(): string {
57+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
58+
return $this->l->t('Chat with the language model with tool calling support and attachments.');
59+
}
60+
61+
/**
62+
* @return string
63+
* @since 35.0.0
64+
*/
65+
#[\Override]
66+
public function getId(): string {
67+
return self::ID;
68+
}
69+
70+
/**
71+
* @return ShapeDescriptor[]
72+
* @since 35.0.0
73+
*/
74+
#[\Override]
75+
public function getInputShape(): array {
76+
return [
77+
'system_prompt' => new ShapeDescriptor(
78+
$this->l->t('System prompt'),
79+
$this->l->t('Define rules and assumptions that the assistant should follow during the conversation.'),
80+
EShapeType::Text
81+
),
82+
'input' => new ShapeDescriptor(
83+
$this->l->t('Chat message'),
84+
$this->l->t('Describe a task that you want the assistant to do or ask a question'),
85+
EShapeType::Text
86+
),
87+
'input_attachments' => new ShapeDescriptor(
88+
$this->l->t('Input attachments'),
89+
$this->l->t('Files to send along with the chat message.'),
90+
EShapeType::ListOfFiles
91+
),
92+
'tool_message' => new ShapeDescriptor(
93+
$this->l->t('Tool message'),
94+
$this->l->t('The result of tool calls in the last interaction'),
95+
EShapeType::Text
96+
),
97+
'history' => new ShapeDescriptor(
98+
$this->l->t('Chat history'),
99+
$this->l->t('The history of chat messages before the current message, starting with a message by the user'),
100+
EShapeType::ListOfTexts
101+
),
102+
// See https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools for the format
103+
'tools' => new ShapeDescriptor(
104+
// TRANSLATORS Tool calling, also known as function calling, is a structured way to give LLMs the ability to make requests back to the application that called it. You define the tools you want to make available to the model, and the model will make tool requests to your app as necessary to fulfill the prompts you give it.
105+
$this->l->t('Available tools'),
106+
$this->l->t('The available tools in JSON format'),
107+
EShapeType::Text
108+
),
109+
];
110+
}
111+
112+
/**
113+
* @return ShapeDescriptor[]
114+
* @since 35.0.0
115+
*/
116+
#[\Override]
117+
public function getOutputShape(): array {
118+
return [
119+
'output' => new ShapeDescriptor(
120+
$this->l->t('Generated response'),
121+
$this->l->t('The response from the chat model'),
122+
EShapeType::Text
123+
),
124+
'output_attachments' => new ShapeDescriptor(
125+
$this->l->t('Output attachments'),
126+
$this->l->t('Files generated by the model.'),
127+
EShapeType::ListOfFiles
128+
),
129+
'tool_calls' => new ShapeDescriptor(
130+
$this->l->t('Tool calls'),
131+
$this->l->t('Tools call instructions from the model in JSON format'),
132+
EShapeType::Text
133+
),
134+
];
135+
}
136+
}

‎lib/public/TaskProcessing/TaskTypes/MultimodalContextAgentInteraction.php‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
<?php
2+
3+
declare(strict_types=1);
4+
5+
/**
6+
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
7+
* SPDX-License-Identifier: AGPL-3.0-or-later
8+
*/
9+
10+
namespace OCP\TaskProcessing\TaskTypes;
11+
12+
use OCP\IL10N;
13+
use OCP\L10N\IFactory;
14+
use OCP\TaskProcessing\EShapeType;
15+
use OCP\TaskProcessing\IInternalTaskType;
16+
use OCP\TaskProcessing\ShapeDescriptor;
17+
18+
/**
19+
* This is the task processing task type for multimodal Context Agent interaction
20+
* @since 35.0.0
21+
*/
22+
class MultimodalContextAgentInteraction implements IInternalTaskType {
23+
/**
24+
* @since 35.0.0
25+
*/
26+
public const ID = 'core:contextagent:multimodal-interaction';
27+
28+
private IL10N $l;
29+
30+
/**
31+
* @param IFactory $l10nFactory
32+
* @since 35.0.0
33+
*/
34+
public function __construct(
35+
IFactory $l10nFactory,
36+
) {
37+
$this->l = $l10nFactory->get('lib');
38+
}
39+
40+
/**
41+
* @inheritDoc
42+
* @since 35.0.0
43+
*/
44+
#[\Override]
45+
public function getName(): string {
46+
return 'ContextAgent multimodal'; // We do not translate this
47+
}
48+
49+
/**
50+
* @inheritDoc
51+
* @since 35.0.0
52+
*/
53+
#[\Override]
54+
public function getDescription(): string {
55+
return $this->l->t('Chat with an agent using attachments');
56+
}
57+
58+
/**
59+
* @return string
60+
* @since 35.0.0
61+
*/
62+
#[\Override]
63+
public function getId(): string {
64+
return self::ID;
65+
}
66+
67+
/**
68+
* @return ShapeDescriptor[]
69+
* @since 35.0.0
70+
*/
71+
#[\Override]
72+
public function getInputShape(): array {
73+
return [
74+
'input' => new ShapeDescriptor(
75+
$this->l->t('Chat message'),
76+
$this->l->t('A chat message to send to the agent.'),
77+
EShapeType::Text
78+
),
79+
'input_attachments' => new ShapeDescriptor(
80+
$this->l->t('Input attachments'),
81+
$this->l->t('Files to send along with the chat message.'),
82+
EShapeType::ListOfFiles
83+
),
84+
'confirmation' => new ShapeDescriptor(
85+
$this->l->t('Confirmation'),
86+
$this->l->t('Whether to confirm previously requested actions: 0 for denial and 1 for confirmation.'),
87+
EShapeType::Number
88+
),
89+
'conversation_token' => new ShapeDescriptor(
90+
$this->l->t('Conversation token'),
91+
$this->l->t('A token representing the conversation.'),
92+
EShapeType::Text
93+
),
94+
];
95+
}
96+
97+
/**
98+
* @return ShapeDescriptor[]
99+
* @since 35.0.0
100+
*/
101+
#[\Override]
102+
public function getOutputShape(): array {
103+
return [
104+
'output' => new ShapeDescriptor(
105+
$this->l->t('Generated response'),
106+
$this->l->t('The response from the chat model.'),
107+
EShapeType::Text
108+
),
109+
'output_attachments' => new ShapeDescriptor(
110+
$this->l->t('Output attachments'),
111+
$this->l->t('Files generated by the agent.'),
112+
EShapeType::ListOfFiles
113+
),
114+
'conversation_token' => new ShapeDescriptor(
115+
$this->l->t('The new conversation token'),
116+
$this->l->t('Send this along with the next interaction.'),
117+
EShapeType::Text
118+
),
119+
'actions' => new ShapeDescriptor(
120+
$this->l->t('Requested actions by the agent'),
121+
$this->l->t('Actions that the agent would like to carry out in JSON format.'),
122+
EShapeType::Text
123+
),
124+
];
125+
}
126+
}

0 commit comments

Comments
 (0)