From 68eab7289fb6872a2d05ac274b74c8537190e5a6 Mon Sep 17 00:00:00 2001 From: Jonas Buchholz Date: Thu, 27 Nov 2025 17:39:46 +0100 Subject: [PATCH 1/3] add option for async translation in deepl api client --- integreat_cms/cms/views/bulk_action_views.py | 2 +- .../utils/machine_translation_api_client.py | 6 +- integreat_cms/deepl_api/deepl_api_client.py | 84 +++++++++++++++---- 3 files changed, 75 insertions(+), 17 deletions(-) diff --git a/integreat_cms/cms/views/bulk_action_views.py b/integreat_cms/cms/views/bulk_action_views.py index f8d8dcd91e..47549e6f2f 100644 --- a/integreat_cms/cms/views/bulk_action_views.py +++ b/integreat_cms/cms/views/bulk_action_views.py @@ -223,7 +223,7 @@ def post( to_translate, ) api_client = language_node.mt_provider.api_client(request, self.form) - api_client.translate_queryset(to_translate, language_node.slug) + api_client.translate_queryset(to_translate, language_node.slug, translate_async=True) # Let the base view handle the redirect return super().post(request, *args, **kwargs) diff --git a/integreat_cms/core/utils/machine_translation_api_client.py b/integreat_cms/core/utils/machine_translation_api_client.py index 75989dc2cc..d1433abeec 100644 --- a/integreat_cms/core/utils/machine_translation_api_client.py +++ b/integreat_cms/core/utils/machine_translation_api_client.py @@ -88,7 +88,7 @@ def reset(self) -> None: self.failed_because_exceeds_limit = [] @abstractmethod - def invoke_translation_api(self) -> None: + def invoke_translation_api(self, translate_asyn: bool = False) -> None: """ Translate all content objects stored in self.queryset. Needs to be implemented by subclasses of MachineTranslationApiClient. @@ -119,12 +119,14 @@ def translate_queryset( self, queryset: list[Event] | (list[Page] | list[POI]), language_slug: str, + translate_async: bool = False, ) -> None: """ This function translates a content queryset via DeepL :param queryset: The content QuerySet :param language_slug: The target language slug + :param translate_async: Whether to use asynchronous translations """ if not queryset: return @@ -158,7 +160,7 @@ def translate_queryset( self.filter_exceeds_limit() # Provider-API-spcific implementation - self.invoke_translation_api() + self.invoke_translation_api(translate_async) # Update remaining budget of the region region.mt_budget_used += sum( diff --git a/integreat_cms/deepl_api/deepl_api_client.py b/integreat_cms/deepl_api/deepl_api_client.py index f1ce4c1d40..22b4239918 100644 --- a/integreat_cms/deepl_api/deepl_api_client.py +++ b/integreat_cms/deepl_api/deepl_api_client.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from celery import shared_task, group from html import unescape from typing import TYPE_CHECKING @@ -24,6 +25,56 @@ logger = logging.getLogger(__name__) +def chunks(seq, size: int = 10): + """ + Helper function to split ids list into chunks + """ + if size <= 0: + raise ValueError("Chunk size must be > 0") + for i in range(0, len(seq), size): + yield seq[i:i + size] + + +@shared_task +def translate_queryset_async(ids_chunk, app_label, model_name, source_language_slug, target_language_key): + translator = deepl.Translator( + auth_key=settings.DEEPL_AUTH_KEY, + server_url=settings.DEEPL_API_URL, + ) + Model = apps.get_model(app_label, model_name) + qs = Model.objects.filter(pk__in=ids_chunk) + + + deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") + + for content_object in qs: + data = { + "status": content_object.source_translation.status, + "machine_translated": True, + "currently_in_translation": False, + "title": unescape(content_object.source_translation.title), + } + + for attr, attr_val in content_object.translatable_attributes: + translate_attr(data, attr, attr_val, translator, deepl_config, source_language_slug, + target_language_key) + + +def translate_attr(data, attr, attr_val, translator, deepl_config, source_language_slug, target_language_key): + # data has to be unescaped for DeepL to recognize Umlaute + glossary = deepl_config.get_glossary( + source_language_slug, + target_language_key, + ) + logger.debug("Used glossary for translation: %s", glossary) + data[attr] = translator.translate_text( + unescape(attr_val), + source_lang=source_language_slug, + target_lang=target_language_key, + tag_handling="html", + glossary=glossary, + ) + class DeepLApiClient(MachineTranslationApiClient): """ @@ -68,10 +119,27 @@ def get_target_language_key(target_language: Language) -> str: return code return "" - def invoke_translation_api(self) -> None: + def invoke_translation_api(self, translate_async: bool = False) -> None: """ Translate all content objects stored in self.queryset using DeepL. """ + if translate_async: + app_label = self.queryset[0]._meta.app_label + model_name = self.queryset[0]._meta.model_name + ids = list(x.id for x in self.queryset) + job = group( + translate_queryset_async.s( + ids_chunk, + app_label=app_label, + model_name=model_name, + source_language_slug=self.source_language.slug, + target_language_key=self.target_language_key, + ) + for ids_chunk in chunks(ids) + ) + job.apply_async() + return + deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") for content_object in self.queryset: @@ -84,19 +152,7 @@ def invoke_translation_api(self) -> None: for attr, attr_val in content_object.translatable_attributes: try: - # data has to be unescaped for DeepL to recognize Umlaute - glossary = deepl_config.get_glossary( - self.source_language.slug, - self.target_language_key, - ) - logger.debug("Used glossary for translation: %s", glossary) - data[attr] = self.translator.translate_text( - unescape(attr_val), - source_lang=self.source_language.slug, - target_lang=self.target_language_key, - tag_handling="html", - glossary=glossary, - ) + translate_attr(data, attr, attr_val, self.translator, deepl_config, self.source_language.slug, self.target_language_key) except DeepLException: messages.error( self.request, From 3a169a7cea40eb2c778c935a7896df91b964b823 Mon Sep 17 00:00:00 2001 From: Jonas Buchholz Date: Thu, 27 Nov 2025 17:55:41 +0100 Subject: [PATCH 2/3] ruff --- integreat_cms/cms/views/bulk_action_views.py | 4 +- integreat_cms/deepl_api/deepl_api_client.py | 43 ++++++++++++++++---- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/integreat_cms/cms/views/bulk_action_views.py b/integreat_cms/cms/views/bulk_action_views.py index 47549e6f2f..19441cd9fd 100644 --- a/integreat_cms/cms/views/bulk_action_views.py +++ b/integreat_cms/cms/views/bulk_action_views.py @@ -223,7 +223,9 @@ def post( to_translate, ) api_client = language_node.mt_provider.api_client(request, self.form) - api_client.translate_queryset(to_translate, language_node.slug, translate_async=True) + api_client.translate_queryset( + to_translate, language_node.slug, translate_async=True + ) # Let the base view handle the redirect return super().post(request, *args, **kwargs) diff --git a/integreat_cms/deepl_api/deepl_api_client.py b/integreat_cms/deepl_api/deepl_api_client.py index 22b4239918..9d396dd919 100644 --- a/integreat_cms/deepl_api/deepl_api_client.py +++ b/integreat_cms/deepl_api/deepl_api_client.py @@ -1,11 +1,11 @@ from __future__ import annotations import logging -from celery import shared_task, group from html import unescape from typing import TYPE_CHECKING import deepl +from celery import group, shared_task from deepl.exceptions import DeepLException from django.apps import apps from django.conf import settings @@ -25,6 +25,7 @@ logger = logging.getLogger(__name__) + def chunks(seq, size: int = 10): """ Helper function to split ids list into chunks @@ -32,11 +33,13 @@ def chunks(seq, size: int = 10): if size <= 0: raise ValueError("Chunk size must be > 0") for i in range(0, len(seq), size): - yield seq[i:i + size] + yield seq[i : i + size] @shared_task -def translate_queryset_async(ids_chunk, app_label, model_name, source_language_slug, target_language_key): +def translate_queryset_async( + ids_chunk, app_label, model_name, source_language_slug, target_language_key +): translator = deepl.Translator( auth_key=settings.DEEPL_AUTH_KEY, server_url=settings.DEEPL_API_URL, @@ -44,7 +47,6 @@ def translate_queryset_async(ids_chunk, app_label, model_name, source_language_s Model = apps.get_model(app_label, model_name) qs = Model.objects.filter(pk__in=ids_chunk) - deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") for content_object in qs: @@ -56,11 +58,26 @@ def translate_queryset_async(ids_chunk, app_label, model_name, source_language_s } for attr, attr_val in content_object.translatable_attributes: - translate_attr(data, attr, attr_val, translator, deepl_config, source_language_slug, - target_language_key) + translate_attr( + data, + attr, + attr_val, + translator, + deepl_config, + source_language_slug, + target_language_key, + ) -def translate_attr(data, attr, attr_val, translator, deepl_config, source_language_slug, target_language_key): +def translate_attr( + data, + attr, + attr_val, + translator, + deepl_config, + source_language_slug, + target_language_key, +): # data has to be unescaped for DeepL to recognize Umlaute glossary = deepl_config.get_glossary( source_language_slug, @@ -126,7 +143,7 @@ def invoke_translation_api(self, translate_async: bool = False) -> None: if translate_async: app_label = self.queryset[0]._meta.app_label model_name = self.queryset[0]._meta.model_name - ids = list(x.id for x in self.queryset) + ids = [x.id for x in self.queryset] job = group( translate_queryset_async.s( ids_chunk, @@ -152,7 +169,15 @@ def invoke_translation_api(self, translate_async: bool = False) -> None: for attr, attr_val in content_object.translatable_attributes: try: - translate_attr(data, attr, attr_val, self.translator, deepl_config, self.source_language.slug, self.target_language_key) + translate_attr( + data, + attr, + attr_val, + self.translator, + deepl_config, + self.source_language.slug, + self.target_language_key, + ) except DeepLException: messages.error( self.request, From 43525b0d3276c7dde6fd5be133faa1e0fe24f09a Mon Sep 17 00:00:00 2001 From: Jonas Buchholz Date: Fri, 28 Nov 2025 18:09:08 +0100 Subject: [PATCH 3/3] save async translations --- integreat_cms/deepl_api/deepl_api_client.py | 38 ++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/integreat_cms/deepl_api/deepl_api_client.py b/integreat_cms/deepl_api/deepl_api_client.py index 9d396dd919..2ee4ea787b 100644 --- a/integreat_cms/deepl_api/deepl_api_client.py +++ b/integreat_cms/deepl_api/deepl_api_client.py @@ -10,6 +10,7 @@ from django.apps import apps from django.conf import settings from django.contrib import messages +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from ..core.utils.machine_translation_api_client import MachineTranslationApiClient @@ -38,13 +39,20 @@ def chunks(seq, size: int = 10): @shared_task def translate_queryset_async( - ids_chunk, app_label, model_name, source_language_slug, target_language_key + ids_chunk, + app_label, + model_name, + source_language_slug, + target_language_key, + form_module, + form_name, ): translator = deepl.Translator( auth_key=settings.DEEPL_AUTH_KEY, server_url=settings.DEEPL_API_URL, ) Model = apps.get_model(app_label, model_name) + form_class = import_string(f"{form_module}.{form_name}") qs = Model.objects.filter(pk__in=ids_chunk) deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") @@ -67,6 +75,32 @@ def translate_queryset_async( source_language_slug, target_language_key, ) + save_translation_async(content_object, data, form_class, target_language_key) + + +def save_translation_async(content_object, data, form_class, target_language_key): + content_translation_form = form_class( + data=data, + instance=content_object.existing_target_translation, + additional_instance_attributes={ + "language": target_language_key, + content_object.source_translation.foreign_field(): content_object, + }, + ) + + # Validate content translation + if content_translation_form.is_valid(): + content_translation_form.save() + # Revert "currently in translation" value of all versions + if content_object.existing_target_translation: + if settings.REDIS_CACHE: + content_object.existing_target_translation.all_versions.invalidated_update( + currently_in_translation=False, + ) + else: + content_object.existing_target_translation.all_versions.update( + currently_in_translation=False, + ) def translate_attr( @@ -151,6 +185,8 @@ def invoke_translation_api(self, translate_async: bool = False) -> None: model_name=model_name, source_language_slug=self.source_language.slug, target_language_key=self.target_language_key, + form_module=self.form_class.__module__, + form_name=self.form_class.__name__, ) for ids_chunk in chunks(ids) )