diff --git a/integreat_cms/cms/views/bulk_action_views.py b/integreat_cms/cms/views/bulk_action_views.py index f8d8dcd91e..19441cd9fd 100644 --- a/integreat_cms/cms/views/bulk_action_views.py +++ b/integreat_cms/cms/views/bulk_action_views.py @@ -223,7 +223,9 @@ def post( to_translate, ) api_client = language_node.mt_provider.api_client(request, self.form) - api_client.translate_queryset(to_translate, language_node.slug) + api_client.translate_queryset( + to_translate, language_node.slug, translate_async=True + ) # Let the base view handle the redirect return super().post(request, *args, **kwargs) diff --git a/integreat_cms/core/utils/machine_translation_api_client.py b/integreat_cms/core/utils/machine_translation_api_client.py index 75989dc2cc..d1433abeec 100644 --- a/integreat_cms/core/utils/machine_translation_api_client.py +++ b/integreat_cms/core/utils/machine_translation_api_client.py @@ -88,7 +88,7 @@ def reset(self) -> None: self.failed_because_exceeds_limit = [] @abstractmethod - def invoke_translation_api(self) -> None: + def invoke_translation_api(self, translate_asyn: bool = False) -> None: """ Translate all content objects stored in self.queryset. Needs to be implemented by subclasses of MachineTranslationApiClient. @@ -119,12 +119,14 @@ def translate_queryset( self, queryset: list[Event] | (list[Page] | list[POI]), language_slug: str, + translate_async: bool = False, ) -> None: """ This function translates a content queryset via DeepL :param queryset: The content QuerySet :param language_slug: The target language slug + :param translate_async: Whether to use asynchronous translations """ if not queryset: return @@ -158,7 +160,7 @@ def translate_queryset( self.filter_exceeds_limit() # Provider-API-spcific implementation - self.invoke_translation_api() + self.invoke_translation_api(translate_async) # Update remaining budget of the region region.mt_budget_used += sum( diff --git a/integreat_cms/deepl_api/deepl_api_client.py b/integreat_cms/deepl_api/deepl_api_client.py index f1ce4c1d40..2ee4ea787b 100644 --- a/integreat_cms/deepl_api/deepl_api_client.py +++ b/integreat_cms/deepl_api/deepl_api_client.py @@ -5,10 +5,12 @@ from typing import TYPE_CHECKING import deepl +from celery import group, shared_task from deepl.exceptions import DeepLException from django.apps import apps from django.conf import settings from django.contrib import messages +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from ..core.utils.machine_translation_api_client import MachineTranslationApiClient @@ -25,6 +27,106 @@ logger = logging.getLogger(__name__) +def chunks(seq, size: int = 10): + """ + Helper function to split ids list into chunks + """ + if size <= 0: + raise ValueError("Chunk size must be > 0") + for i in range(0, len(seq), size): + yield seq[i : i + size] + + +@shared_task +def translate_queryset_async( + ids_chunk, + app_label, + model_name, + source_language_slug, + target_language_key, + form_module, + form_name, +): + translator = deepl.Translator( + auth_key=settings.DEEPL_AUTH_KEY, + server_url=settings.DEEPL_API_URL, + ) + Model = apps.get_model(app_label, model_name) + form_class = import_string(f"{form_module}.{form_name}") + qs = Model.objects.filter(pk__in=ids_chunk) + + deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") + + for content_object in qs: + data = { + "status": content_object.source_translation.status, + "machine_translated": True, + "currently_in_translation": False, + "title": unescape(content_object.source_translation.title), + } + + for attr, attr_val in content_object.translatable_attributes: + translate_attr( + data, + attr, + attr_val, + translator, + deepl_config, + source_language_slug, + target_language_key, + ) + save_translation_async(content_object, data, form_class, target_language_key) + + +def save_translation_async(content_object, data, form_class, target_language_key): + content_translation_form = form_class( + data=data, + instance=content_object.existing_target_translation, + additional_instance_attributes={ + "language": target_language_key, + content_object.source_translation.foreign_field(): content_object, + }, + ) + + # Validate content translation + if content_translation_form.is_valid(): + content_translation_form.save() + # Revert "currently in translation" value of all versions + if content_object.existing_target_translation: + if settings.REDIS_CACHE: + content_object.existing_target_translation.all_versions.invalidated_update( + currently_in_translation=False, + ) + else: + content_object.existing_target_translation.all_versions.update( + currently_in_translation=False, + ) + + +def translate_attr( + data, + attr, + attr_val, + translator, + deepl_config, + source_language_slug, + target_language_key, +): + # data has to be unescaped for DeepL to recognize Umlaute + glossary = deepl_config.get_glossary( + source_language_slug, + target_language_key, + ) + logger.debug("Used glossary for translation: %s", glossary) + data[attr] = translator.translate_text( + unescape(attr_val), + source_lang=source_language_slug, + target_lang=target_language_key, + tag_handling="html", + glossary=glossary, + ) + + class DeepLApiClient(MachineTranslationApiClient): """ DeepL API client to automatically translate selected objects. @@ -68,10 +170,29 @@ def get_target_language_key(target_language: Language) -> str: return code return "" - def invoke_translation_api(self) -> None: + def invoke_translation_api(self, translate_async: bool = False) -> None: """ Translate all content objects stored in self.queryset using DeepL. """ + if translate_async: + app_label = self.queryset[0]._meta.app_label + model_name = self.queryset[0]._meta.model_name + ids = [x.id for x in self.queryset] + job = group( + translate_queryset_async.s( + ids_chunk, + app_label=app_label, + model_name=model_name, + source_language_slug=self.source_language.slug, + target_language_key=self.target_language_key, + form_module=self.form_class.__module__, + form_name=self.form_class.__name__, + ) + for ids_chunk in chunks(ids) + ) + job.apply_async() + return + deepl_config: DeepLApiClientConfig = apps.get_app_config("deepl_api") for content_object in self.queryset: @@ -84,19 +205,15 @@ def invoke_translation_api(self) -> None: for attr, attr_val in content_object.translatable_attributes: try: - # data has to be unescaped for DeepL to recognize Umlaute - glossary = deepl_config.get_glossary( + translate_attr( + data, + attr, + attr_val, + self.translator, + deepl_config, self.source_language.slug, self.target_language_key, ) - logger.debug("Used glossary for translation: %s", glossary) - data[attr] = self.translator.translate_text( - unescape(attr_val), - source_lang=self.source_language.slug, - target_lang=self.target_language_key, - tag_handling="html", - glossary=glossary, - ) except DeepLException: messages.error( self.request,