Skip to content

Commit 1b16198

Browse files
lukasdotcomlukasdotcom
committed
Support context agent for multimodal chat
Signed-off-by: Lukas Schaefer <lukas@lschaefer.xyz>
1 parent 7a83ef5 commit 1b16198

4 files changed

Lines changed: 82 additions & 16 deletions

File tree

‎lib/Listener/BeforeTemplateRenderedListener.php‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -72,12 +72,15 @@ public function handle(Event $event): void {
7272
$this->initialStateService->provideInitialState('contextChatIndexingComplete', $indexingComplete);
7373
$this->initialStateService->provideInitialState('contextAgentToolSources', $this->assistantService->informationSources);
7474
$this->initialStateService->provideInitialState('audio_chat_available', $this->assistantService->isAudioChatAvailable());
75-
$multimodalChatAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes());
76-
$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
7775
$autoplayAudioChat = $this->config->getUserValue($this->userId, Application::APP_ID, 'autoplay_audio_chat', '1') === '1';
7876
$this->initialStateService->provideInitialState('autoplay_audio_chat', $autoplayAudioChat);
7977
$agencyAvailable = class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\ContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes());
8078
$this->initialStateService->provideInitialState('agency_available', $agencyAvailable);
79+
80+
$multimodalChatAvailable = $agencyAvailable
81+
? (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypes()))
82+
: (class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalChatWithTools') && array_key_exists(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypes()));
83+
$this->initialStateService->provideInitialState('multimodal_chat_available', $multimodalChatAvailable);
8184
}
8285
if (class_exists(\OCA\Viewer\Event\LoadViewer::class)) {
8386
$this->eventDispatcher->dispatchTyped(new \OCA\Viewer\Event\LoadViewer());

‎lib/Listener/ChattyLLMTaskListener.php‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,12 +132,13 @@ public function handle(Event $event): void {
132132
// the task is not an audio one, but we might still need to Tts the answer
133133
// if it is a response to a ContextAgentInteraction confirmation that was asked about an audio message
134134
$this->runTtsIfNeeded($sessionId, $message, $taskTypeId, $task->getUserId());
135-
if ($isMultimodalChat) {
135+
if ($isMultimodalChat || $isMultimodalAgencyChat) {
136136
$attachments = $taskOutput['output_attachments'] ?? [];
137-
$attachments = array_map(function ($attachment) {
137+
$attachments = array_map(function ($attachment) use ($task) {
138138
return [
139139
'type' => 'File',
140140
'file_id' => $attachment,
141+
'ocp_task_id' => $task->getId(),
141142
];
142143
}, $attachments);
143144
$message->setAttachments(json_encode($attachments));

‎lib/Service/AssistantService.php‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -772,7 +772,7 @@ private function extractFileIdsFromTask(Task $task): array {
772772
/** @var int|list<int> $inputSlot */
773773
$inputSlot = $task->getInput()[$key];
774774
if (is_array($inputSlot)) {
775-
$ids += $inputSlot;
775+
$ids = array_merge($ids, $inputSlot);
776776
} else {
777777
$ids[] = $inputSlot;
778778
}
@@ -791,7 +791,7 @@ private function extractFileIdsFromTask(Task $task): array {
791791
}
792792
}
793793
}
794-
return array_values($ids);
794+
return $ids;
795795
}
796796

797797
/**

‎lib/Service/ChatService.php‎

Lines changed: 72 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -385,6 +385,13 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
385385
// audio agency
386386
$fileId = $audioAttachment['file_id'];
387387
$taskId = $this->scheduleAgencyAudioTask($userId, $fileId, $agencyConfirm, $lastConversationToken, $sessionId, $lastUserMessage->getId());
388+
} elseif ($this->isMultimodalContextAgentAvailable()) {
389+
// multimodal agency
390+
$prompt = $lastUserMessage->getContent();
391+
$inputAttachments = array_map(static function (array $attachment) {
392+
return $attachment['file_id'];
393+
}, $lastAttachments);
394+
$taskId = $this->scheduleAgencyMultimodalTask($userId, $prompt, $agencyConfirm, $lastConversationToken, $sessionId, $inputAttachments);
388395
} else {
389396
// classic agency
390397
$prompt = $lastUserMessage->getContent();
@@ -434,19 +441,15 @@ public function scheduleMessageGeneration(?string $userId, int $sessionId, int $
434441
} else {
435442
if ($this->isMultimodalChatAvailable()) {
436443
// for a multimodal chat also attachments need to be added to the history
437-
$assistantService = $this->assistantService;
438-
$historyMessages = array_map(static function (Message $message) use ($userId, $assistantService) {
444+
$historyMessages = array_map(static function (Message $message) {
439445
$attachments = $message->jsonSerialize()['attachments'];
440446
// Attachments that were generated need to be saved in the user's files so they are accessible to provider
441-
$content = array_map(static function (array $attachment) use ($userId, $assistantService, $message) {
442-
if ($message->getRole() === Message::ROLE_ASSISTANT) {
443-
$info = $assistantService->saveOutputFile($userId, $message->getOcpTaskId(), $attachment['file_id']);
444-
return [
445-
'type' => 'file',
446-
'file_id' => $info['fileId'],
447-
];
447+
$content = array_map(static function (array $attachment) {
448+
$newAttachment = ['type' => 'file', 'file_id' => $attachment['file_id']];
449+
if (isset($attachment['ocp_task_id'])) {
450+
$newAttachment['ocp_task_id'] = $attachment['ocp_task_id'];
448451
}
449-
return ['type' => 'file', 'file_id' => $attachment['file_id']];
452+
return $newAttachment;
450453
}, $attachments);
451454
$content[] = [
452455
'type' => 'text',
@@ -591,6 +594,13 @@ public function isMultimodalChatAvailable(): bool {
591594
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalChatWithTools::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
592595
}
593596

597+
public function isMultimodalContextAgentAvailable(): bool {
598+
if (!class_exists('OCP\\TaskProcessing\\TaskTypes\\MultimodalContextAgentInteraction')) {
599+
return false;
600+
}
601+
return in_array(\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID, $this->taskProcessingManager->getAvailableTaskTypeIds());
602+
}
603+
594604

595605
private function getAudioHistory(array $history): array {
596606
// history is a list of JSON strings
@@ -799,6 +809,58 @@ private function scheduleAgencyTask(
799809
return $task->getId() ?? 0;
800810
}
801811

812+
/**
813+
* Schedule a multimodal agency chat task
814+
*
815+
* @param list<int> $inputAttachments
816+
* @throws BadRequestException
817+
* @throws InternalException
818+
*/
819+
private function scheduleAgencyMultimodalTask(
820+
?string $userId,
821+
string $content,
822+
int $confirmation,
823+
string $conversationToken,
824+
int $sessionId,
825+
array $inputAttachments,
826+
): int {
827+
$customId = 'chatty-llm:' . $sessionId;
828+
$this->checkIfSessionIsThinking($userId, $customId);
829+
$taskInput = [
830+
'input' => $content,
831+
'input_attachments' => $inputAttachments,
832+
'confirmation' => $confirmation,
833+
'conversation_token' => $conversationToken,
834+
];
835+
/** @psalm-suppress UndefinedClass */
836+
if (isset($this->taskProcessingManager->getAvailableTaskTypes()[\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID]['optionalInputShape']['memories'])) {
837+
$taskInput['memories'] = $this->sessionSummaryService->getMemories($userId);
838+
}
839+
/** @psalm-suppress UndefinedClass */
840+
$task = new Task(
841+
\OCP\TaskProcessing\TaskTypes\MultimodalContextAgentInteraction::ID,
842+
$taskInput,
843+
Application::APP_ID . ':chatty-llm',
844+
$userId,
845+
$customId
846+
);
847+
/** @psalm-suppress UndefinedMethod */
848+
$task->setPreferStreaming(true);
849+
try {
850+
$this->taskProcessingManager->scheduleTask($task);
851+
} catch (PreConditionNotMetException $e) {
852+
throw new BadRequestException('pre_condition_not_met', previous: $e);
853+
} catch (\OCP\TaskProcessing\Exception\UnauthorizedException $e) {
854+
throw new BadRequestException('unauthorized', previous: $e);
855+
} catch (ValidationException $e) {
856+
throw new BadRequestException('validation_failed', previous: $e);
857+
} catch (\OCP\TaskProcessing\Exception\Exception $e) {
858+
$this->logger->error($e->getMessage(), ['exception' => $e]);
859+
throw new InternalException(previous: $e);
860+
}
861+
return $task->getId() ?? 0;
862+
}
863+
802864
/**
803865
* Schedule an audio chat task
804866
* @throws BadRequestException

0 commit comments

Comments
 (0)