From 7e8c1b238aa4870ae372d8394a8a2b9fa33015c5 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Fri, 8 Aug 2025 11:14:10 -0400 Subject: [PATCH 1/4] feat: better sticker generator hiding prompt with seperate task type Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 4 + lib/TaskProcessing/TextToStickerProvider.php | 109 +++++++++++++++++++ lib/TaskProcessing/TextToStickerTaskType.php | 72 ++++++++++++ src/components/TaskTypeSelect.vue | 2 +- src/stickerGeneration.js | 9 +- 5 files changed, 189 insertions(+), 7 deletions(-) create mode 100644 lib/TaskProcessing/TextToStickerProvider.php create mode 100644 lib/TaskProcessing/TextToStickerTaskType.php diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 82e8f205d..d90ca163c 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -31,6 +31,8 @@ use OCA\Assistant\Reference\Text2StickerProvider; use OCA\Assistant\TaskProcessing\AudioToAudioChatProvider; use OCA\Assistant\TaskProcessing\ContextAgentAudioInteractionProvider; +use OCA\Assistant\TaskProcessing\TextToStickerProvider; +use OCA\Assistant\TaskProcessing\TextToStickerTaskType; use OCA\Files\Event\LoadAdditionalScriptsEvent; use OCP\AppFramework\App; use OCP\AppFramework\Bootstrap\IBootContext; @@ -94,6 +96,8 @@ public function register(IRegistrationContext $context): void { if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')) { $context->registerTaskProcessingProvider(ContextAgentAudioInteractionProvider::class); } + $context->registerTaskProcessingTaskType(TextToStickerTaskType::class); + $context->registerTaskProcessingProvider(TextToStickerProvider::class); } public function boot(IBootContext $context): void { diff --git a/lib/TaskProcessing/TextToStickerProvider.php b/lib/TaskProcessing/TextToStickerProvider.php new file mode 100644 index 000000000..3ea95d835 --- /dev/null +++ b/lib/TaskProcessing/TextToStickerProvider.php @@ -0,0 +1,109 @@ +l->t('Assistant fallback'); + } + + public function getTaskTypeId(): string { + return TextToStickerTaskType::ID; + } + + public function getExpectedRuntime(): int { + return 60; + } + + public function getInputShapeEnumValues(): array { + return []; + } + + public function getInputShapeDefaults(): array { + return []; + } + + + public function getOptionalInputShape(): array { + return []; + } + + public function getOptionalInputShapeEnumValues(): array { + return []; + } + + public function getOptionalInputShapeDefaults(): array { + return []; + } + + public function getOutputShapeEnumValues(): array { + return []; + } + + public function getOptionalOutputShape(): array { + return []; + } + + public function getOptionalOutputShapeEnumValues(): array { + return []; + } + + public function process(?string $userId, array $input, callable $reportProgress): array { + if (!isset($input['input']) || !is_string($input['input'])) { + throw new RuntimeException('Invalid prompt'); + } + $input = $input['input']; + + // Generate Image with custom prompt + try { + $task = new Task( + TextToImage::ID, + [ + 'input' => $this->l->t('cartoon, neutral background, sticker of %1$s', [$input]), + 'numberOfImages' => 1 + ], + Application::APP_ID . ':internal', + $userId, + ); + $taskOutput = $this->taskProcessingService->runTaskProcessingTask($task); + $images = $taskOutput['images']; + if (empty($images)) { + throw new RuntimeException('No images generated'); + } + $outputImage = $this->taskProcessingService->getOutputFileContent($images[0]); + return ['image' => $outputImage]; + } catch (Exception $e) { + $this->logger->warning('Generating image failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('Generating image failed with: ' . $e->getMessage()); + } + } +} diff --git a/lib/TaskProcessing/TextToStickerTaskType.php b/lib/TaskProcessing/TextToStickerTaskType.php new file mode 100644 index 000000000..18ac6957c --- /dev/null +++ b/lib/TaskProcessing/TextToStickerTaskType.php @@ -0,0 +1,72 @@ +l->t('Generate sticker'); + } + + /** + * @inheritDoc + */ + public function getDescription(): string { + return $this->l->t('Generate sticker from text'); + } + + /** + * @return string + */ + public function getId(): string { + return self::ID; + } + + /** + * @return ShapeDescriptor[] + */ + public function getInputShape(): array { + return [ + 'input' => new ShapeDescriptor( + $this->l->t('Prompt'), + $this->l->t('Describe the sticker you would like to create'), + EShapeType::Text, + ), + ]; + } + + /** + * @return ShapeDescriptor[] + */ + public function getOutputShape(): array { + return [ + 'image' => new ShapeDescriptor( + $this->l->t('Output stickers'), + $this->l->t('The generated stickers'), + EShapeType::Image, + ), + ]; + } +} diff --git a/src/components/TaskTypeSelect.vue b/src/components/TaskTypeSelect.vue index 4ccabc5a6..83749fcf8 100644 --- a/src/components/TaskTypeSelect.vue +++ b/src/components/TaskTypeSelect.vue @@ -217,7 +217,7 @@ export default { return 'translate' } else if (id.startsWith('richdocuments')) { return 'generate' - } else if (id.includes('image')) { + } else if (id.includes('image') || id.includes('sticker')) { return 'image' } else if (id.includes('audio') || id.includes('speech')) { return 'audio' diff --git a/src/stickerGeneration.js b/src/stickerGeneration.js index 37e9c0b19..cd0bbd25d 100644 --- a/src/stickerGeneration.js +++ b/src/stickerGeneration.js @@ -11,14 +11,11 @@ registerCustomPickerElement('assistant_sticker_generation', async (el, { provide const app = createApp( ImageResultCustomPickerElement, { - inputs: { - input: t('assistant', 'cartoon, neutral background, sticker of '), - }, providerId, accessible, - taskType: 'core:text2image', - outputKey: 'images', - multipleImages: true, + taskType: 'assistant:text2sticker', + outputKey: 'image', + multipleImages: false, }, ) app.mixin({ methods: { t, n } }) From 9f56c610e1cf08b88d49e734bb979f98780ad29d Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 11 Aug 2025 10:11:22 -0400 Subject: [PATCH 2/4] hide sticker task when no text2img Signed-off-by: Lukas Schaefer --- src/components/AssistantTextProcessingForm.vue | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/components/AssistantTextProcessingForm.vue b/src/components/AssistantTextProcessingForm.vue index 300db3c0a..70d6e57d6 100644 --- a/src/components/AssistantTextProcessingForm.vue +++ b/src/components/AssistantTextProcessingForm.vue @@ -281,9 +281,13 @@ export default { return null }, sortedTaskTypes() { - const filteredTaskTypes = this.taskTypeIdList !== null + let filteredTaskTypes = this.taskTypeIdList !== null ? this.taskTypes.slice().filter(t => this.taskTypeIdList.find(tt => tt === t.id)) : this.taskTypes.slice() + const hasImageTakType = filteredTaskTypes.find(tt => tt.id === 'core:text2image') + if (!hasImageTakType) { + filteredTaskTypes = filteredTaskTypes.filter(tt => tt.id !== 'assistant:text2sticker') + } return filteredTaskTypes.sort((a, b) => { const prioA = a.priority From 1b2becdf28e393b7fa7da83dcc02646013ded329 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 18 Aug 2025 10:43:27 -0400 Subject: [PATCH 3/4] fix typos and change image to sticker Signed-off-by: Lukas Schaefer --- lib/TaskProcessing/TextToStickerProvider.php | 8 ++++---- lib/TaskProcessing/TextToStickerTaskType.php | 2 +- src/components/AssistantTextProcessingForm.vue | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/TaskProcessing/TextToStickerProvider.php b/lib/TaskProcessing/TextToStickerProvider.php index 3ea95d835..77bfc5d70 100644 --- a/lib/TaskProcessing/TextToStickerProvider.php +++ b/lib/TaskProcessing/TextToStickerProvider.php @@ -33,7 +33,7 @@ public function getId(): string { } public function getName(): string { - return $this->l->t('Assistant fallback'); + return $this->l->t('Assistant'); } public function getTaskTypeId(): string { @@ -97,13 +97,13 @@ public function process(?string $userId, array $input, callable $reportProgress) $taskOutput = $this->taskProcessingService->runTaskProcessingTask($task); $images = $taskOutput['images']; if (empty($images)) { - throw new RuntimeException('No images generated'); + throw new RuntimeException('No sticker generated'); } $outputImage = $this->taskProcessingService->getOutputFileContent($images[0]); return ['image' => $outputImage]; } catch (Exception $e) { - $this->logger->warning('Generating image failed with: ' . $e->getMessage(), ['exception' => $e]); - throw new RuntimeException('Generating image failed with: ' . $e->getMessage()); + $this->logger->warning('Generating sticker failed with: ' . $e->getMessage(), ['exception' => $e]); + throw new RuntimeException('Generating sticker failed with: ' . $e->getMessage()); } } } diff --git a/lib/TaskProcessing/TextToStickerTaskType.php b/lib/TaskProcessing/TextToStickerTaskType.php index 18ac6957c..ae0b9386d 100644 --- a/lib/TaskProcessing/TextToStickerTaskType.php +++ b/lib/TaskProcessing/TextToStickerTaskType.php @@ -64,7 +64,7 @@ public function getOutputShape(): array { return [ 'image' => new ShapeDescriptor( $this->l->t('Output stickers'), - $this->l->t('The generated stickers'), + $this->l->t('The generated sticker'), EShapeType::Image, ), ]; diff --git a/src/components/AssistantTextProcessingForm.vue b/src/components/AssistantTextProcessingForm.vue index 70d6e57d6..d9ba90d87 100644 --- a/src/components/AssistantTextProcessingForm.vue +++ b/src/components/AssistantTextProcessingForm.vue @@ -284,8 +284,8 @@ export default { let filteredTaskTypes = this.taskTypeIdList !== null ? this.taskTypes.slice().filter(t => this.taskTypeIdList.find(tt => tt === t.id)) : this.taskTypes.slice() - const hasImageTakType = filteredTaskTypes.find(tt => tt.id === 'core:text2image') - if (!hasImageTakType) { + const hasImageTaskType = filteredTaskTypes.find(tt => tt.id === 'core:text2image') + if (!hasImageTaskType) { filteredTaskTypes = filteredTaskTypes.filter(tt => tt.id !== 'assistant:text2sticker') } From 18c9f1265cccedc9e8d515b47d76fcd75afacbd8 Mon Sep 17 00:00:00 2001 From: Lukas Schaefer Date: Mon, 18 Aug 2025 11:46:07 -0400 Subject: [PATCH 4/4] enable sticker task only when sticker picker is enabled Signed-off-by: Lukas Schaefer --- lib/AppInfo/Application.php | 11 +++++++++-- lib/Settings/Admin.php | 4 ++++ src/components/AssistantTextProcessingForm.vue | 6 +----- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index d90ca163c..0c7e677cc 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -41,6 +41,7 @@ use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\AppFramework\Http\Events\BeforeTemplateRenderedEvent; use OCP\Collaboration\Reference\RenderReferenceEvent; +use OCP\IAppConfig; use OCP\Security\CSP\AddContentSecurityPolicyEvent; use OCP\TaskProcessing\Events\TaskFailedEvent; use OCP\TaskProcessing\Events\TaskSuccessfulEvent; @@ -54,8 +55,12 @@ class Application extends App implements IBootstrap { public const CHAT_USER_INSTRUCTIONS = 'This is a conversation in a specific language between the user and you, Nextcloud Assistant. You are a kind, polite and helpful AI that helps the user to the best of its abilities. If you do not understand something, you will ask for clarification. Detect the language that the user is using. Make sure to use the same language in your response. Do not mention the language explicitly.'; public const CHAT_USER_INSTRUCTIONS_TITLE = 'Above is a chat session in a specific language between the user and you, Nextcloud Assistant. Generate a suitable title summarizing the conversation in the same language. Output only the title in plain text, nothing else.'; + private IAppConfig $appConfig; public function __construct(array $urlParams = []) { parent::__construct(self::APP_ID, $urlParams); + + $container = $this->getContainer(); + $this->appConfig = $container->get(IAppConfig::class); } public function register(IRegistrationContext $context): void { @@ -96,8 +101,10 @@ public function register(IRegistrationContext $context): void { if (class_exists('OCP\\TaskProcessing\\TaskTypes\\ContextAgentAudioInteraction')) { $context->registerTaskProcessingProvider(ContextAgentAudioInteractionProvider::class); } - $context->registerTaskProcessingTaskType(TextToStickerTaskType::class); - $context->registerTaskProcessingProvider(TextToStickerProvider::class); + if ($this->appConfig->getValueString(Application::APP_ID, 'text_to_sticker_picker_enabled', '1') === '1') { + $context->registerTaskProcessingTaskType(TextToStickerTaskType::class); + $context->registerTaskProcessingProvider(TextToStickerProvider::class); + } } public function boot(IBootContext $context): void { diff --git a/lib/Settings/Admin.php b/lib/Settings/Admin.php index f65f6f38f..b410b5afa 100644 --- a/lib/Settings/Admin.php +++ b/lib/Settings/Admin.php @@ -43,6 +43,10 @@ public function getForm(): TemplateResponse { $freePromptPickerEnabled = $this->appConfig->getValueString(Application::APP_ID, 'free_prompt_picker_enabled', '1') === '1'; $textToImagePickerEnabled = $this->appConfig->getValueString(Application::APP_ID, 'text_to_image_picker_enabled', '1') === '1'; $textToStickerPickerEnabled = $this->appConfig->getValueString(Application::APP_ID, 'text_to_sticker_picker_enabled', '1') === '1'; + if ($textToStickerPickerEnabled && !$textToImageAvailable) { + $this->appConfig->setValueString(Application::APP_ID, 'text_to_sticker_picker_enabled', '0'); + $textToStickerPickerEnabled = false; + } $speechToTextEnabled = $this->appConfig->getValueString(Application::APP_ID, 'speech_to_text_picker_enabled', '1') === '1'; $chattyLLMUserInstructions = $this->appConfig->getValueString(Application::APP_ID, 'chat_user_instructions', Application::CHAT_USER_INSTRUCTIONS) ?: Application::CHAT_USER_INSTRUCTIONS; diff --git a/src/components/AssistantTextProcessingForm.vue b/src/components/AssistantTextProcessingForm.vue index d9ba90d87..300db3c0a 100644 --- a/src/components/AssistantTextProcessingForm.vue +++ b/src/components/AssistantTextProcessingForm.vue @@ -281,13 +281,9 @@ export default { return null }, sortedTaskTypes() { - let filteredTaskTypes = this.taskTypeIdList !== null + const filteredTaskTypes = this.taskTypeIdList !== null ? this.taskTypes.slice().filter(t => this.taskTypeIdList.find(tt => tt === t.id)) : this.taskTypes.slice() - const hasImageTaskType = filteredTaskTypes.find(tt => tt.id === 'core:text2image') - if (!hasImageTaskType) { - filteredTaskTypes = filteredTaskTypes.filter(tt => tt.id !== 'assistant:text2sticker') - } return filteredTaskTypes.sort((a, b) => { const prioA = a.priority