From b86f8bee34a41709733afc2c86c3283f7d235920 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 11:54:00 -0300 Subject: [PATCH 01/38] cria app xml_manager --- core/utils/__init__.py | 0 xml_manager/__init__.py | 0 xml_manager/migrations/__init__.py | 0 xml_manager/tests.py | 3 +++ 4 files changed, 3 insertions(+) create mode 100644 core/utils/__init__.py create mode 100644 xml_manager/__init__.py create mode 100644 xml_manager/migrations/__init__.py create mode 100644 xml_manager/tests.py diff --git a/core/utils/__init__.py b/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xml_manager/__init__.py b/xml_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xml_manager/migrations/__init__.py b/xml_manager/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xml_manager/tests.py b/xml_manager/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/xml_manager/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. From 277704312ec10f1d4ef9ad7f237426a274e447be Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 11:58:00 -0300 Subject: [PATCH 02/38] Adiciona urls para enviar e processar xmls --- xml_manager/urls.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 xml_manager/urls.py diff --git a/xml_manager/urls.py b/xml_manager/urls.py new file mode 100644 index 0000000..7979f50 --- /dev/null +++ b/xml_manager/urls.py @@ -0,0 +1,8 @@ +from django.urls import path +from . import views + + +urlpatterns = [ + path('upload/', views.upload_xml_page, name='upload_xml_page'), + path('process/', views.process_xml, name='process_xml'), +] From 2e328bff2ceb36703fe6537bfafd25f654e6a751 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 11:58:32 -0300 Subject: [PATCH 03/38] cria view para receber xml e disparar processamento --- xml_manager/views.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 xml_manager/views.py diff --git a/xml_manager/views.py b/xml_manager/views.py new file mode 100644 index 0000000..0bb51a3 --- /dev/null +++ b/xml_manager/views.py @@ -0,0 +1,28 @@ +from django.shortcuts import render, redirect +from django.urls import reverse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST +from django.contrib import messages + +from .tasks import task_process_xml_document +from .models import XMLDocument + + +def upload_xml_page(request): + return render(request, "xml_manager/process_form.html") + + +@require_POST +@csrf_exempt +def process_xml(request): + xml_file = request.FILES.get('xml_file') + if not xml_file: + messages.error(request, "Arquivo não enviado.") + return redirect(reverse('upload_xml_page')) + + xml_obj = XMLDocument.objects.create(xml_file=xml_file) + + task_process_xml_document.delay(xml_obj.id) + + messages.success(request, "Arquivo enviado para processamento!") + return redirect(reverse('upload_xml_page')) From 1538b3a627030bcc65051e5f05866db0cc88be22 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 11:58:53 -0300 Subject: [PATCH 04/38] =?UTF-8?q?adiciona=20m=C3=B3dulo=20admin=20e=20apps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/admin.py | 3 +++ xml_manager/apps.py | 6 ++++++ 2 files changed, 9 insertions(+) create mode 100644 xml_manager/admin.py create mode 100644 xml_manager/apps.py diff --git a/xml_manager/admin.py b/xml_manager/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/xml_manager/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/xml_manager/apps.py b/xml_manager/apps.py new file mode 100644 index 0000000..1292960 --- /dev/null +++ b/xml_manager/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class XmlManagerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'xml_manager' From df202bd181403a7c422b511a7b961e0aa124a15d Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:02:45 -0300 Subject: [PATCH 05/38] =?UTF-8?q?adiciona=20utils.py=20a=20partir=20de=20o?= =?UTF-8?q?utro=20reposit=C3=B3rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/utils/utils.py | 94 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 core/utils/utils.py diff --git a/core/utils/utils.py b/core/utils/utils.py new file mode 100644 index 0000000..57205e1 --- /dev/null +++ b/core/utils/utils.py @@ -0,0 +1,94 @@ +import logging +import re +import requests + +from django.contrib.auth import get_user_model +from langcodes import standardize_tag, tag_is_valid +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + + +logger = logging.getLogger(__name__) +User = get_user_model() + + +def language_iso(code): + code = re.split(r"-|_", code)[0] if code else "" + if tag_is_valid(code): + return standardize_tag(code) + return "" + + +class RetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +class NonRetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +@retry( + retry=retry_if_exception_type(RetryableError), + wait=wait_exponential(multiplier=1, min=1, max=5), + stop=stop_after_attempt(5), +) +def fetch_data(url, headers=None, json=False, timeout=2, verify=True): + """ + Get the resource with HTTP + Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, + then up to 10 seconds, then 10 seconds afterwards + Args: + url: URL address + headers: HTTP headers + json: True|False + verify: Verify the SSL. + Returns: + Return a requests.response object. + Except: + Raise a RetryableError to retry. + """ + + try: + logger.info("Fetching the URL: %s" % url) + response = requests.get(url, headers=headers, timeout=timeout, verify=verify) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + logger.error("Erro fetching the content: %s, retry..., erro: %s" % (url, exc)) + raise RetryableError(exc) from exc + except ( + requests.exceptions.InvalidSchema, + requests.exceptions.MissingSchema, + requests.exceptions.InvalidURL, + ) as exc: + raise NonRetryableError(exc) from exc + try: + response.raise_for_status() + except requests.HTTPError as exc: + if 400 <= exc.response.status_code < 500: + raise NonRetryableError(exc) from exc + elif 500 <= exc.response.status_code < 600: + logger.error( + "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) + ) + raise RetryableError(exc) from exc + else: + raise + + return response.content if not json else response.json() + + +def _get_user(request, username=None, user_id=None): + try: + return User.objects.get(pk=request.user_id) + except AttributeError: + if user_id: + return User.objects.get(pk=user_id) + if username: + return User.objects.get(username=username) From ced61ce291c5fee3de807599c5f801fea13daa9a Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:03:10 -0300 Subject: [PATCH 06/38] adiciona template para processar e listar documentos (customizados) --- .../templates/xml_manager/process_form.html | 16 +++++++ .../xml_manager/xmldocument_list.html | 47 +++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 xml_manager/templates/xml_manager/process_form.html create mode 100644 xml_manager/templates/xml_manager/xmldocument_list.html diff --git a/xml_manager/templates/xml_manager/process_form.html b/xml_manager/templates/xml_manager/process_form.html new file mode 100644 index 0000000..858cd9d --- /dev/null +++ b/xml_manager/templates/xml_manager/process_form.html @@ -0,0 +1,16 @@ +{% extends "wagtailadmin/base.html" %} + +{% block content %} +
+

Processar Arquivo XML

+

Envie um arquivo XML para validar seu conteúdo. Após o envio, serão gerados automaticamente os arquivos HTML e PDF correspondentes.

+
+ {% csrf_token %} +
+ + +
+ +
+
+{% endblock %} \ No newline at end of file diff --git a/xml_manager/templates/xml_manager/xmldocument_list.html b/xml_manager/templates/xml_manager/xmldocument_list.html new file mode 100644 index 0000000..e26933e --- /dev/null +++ b/xml_manager/templates/xml_manager/xmldocument_list.html @@ -0,0 +1,47 @@ +{% extends "wagtailadmin/generic/index.html" %} + +{% block content %} +
+

XML Documents

+ + + + + + + + + + + + + {% for obj in xmldocument.object_list %} + + + + + + + + + {% endfor %} + +
Arquivo XMLValidationExceptionsPDFHTMLEnviado em
{{ obj.xml_file.name }} + {% if obj.validation_file %} + Download + {% endif %} + + {% if obj.exceptions_file %} + Download + {% endif %} + + {% if obj.pdf_file %} + Download + {% endif %} + + {% if obj.html_file %} + Download + {% endif %} + {{ obj.uploaded_at }}
+
+{% endblock %} \ No newline at end of file From 4e65b8baaec5f00a99bd96e334a6e5e38a698441 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:03:34 -0300 Subject: [PATCH 07/38] cria modelo para gerenciar arquivos XML (um para um) - a corrigir --- xml_manager/models.py | 39 +++++++++++++++++++++++++ xml_manager/wagtail_hooks.py | 55 ++++++++++++++++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 xml_manager/models.py create mode 100644 xml_manager/wagtail_hooks.py diff --git a/xml_manager/models.py b/xml_manager/models.py new file mode 100644 index 0000000..7a21411 --- /dev/null +++ b/xml_manager/models.py @@ -0,0 +1,39 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class XMLDocument(models.Model): + xml_file = models.FileField( + upload_to='xml_manager/xml/', + verbose_name=_("XML File"), + help_text=_("Upload an XML file for processing.") + ) + validation_file = models.FileField( + upload_to='xml_manager/csv/', + blank=True, + null=True, + verbose_name=_("Validation File") + ) + exceptions_file = models.FileField( + upload_to='xml_manager/exceptions/', + blank=True, + null=True, + verbose_name=_("Exceptions File") + ) + pdf_file = models.FileField( + upload_to='xml_manager/pdf/', + blank=True, + null=True, + verbose_name=_("PDF File") + ) + html_file = models.FileField( + upload_to='xml_manager/html/', + blank=True, + null=True, + verbose_name=_("HTML File") + ) + uploaded_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("Uploaded At"), + help_text=_("The date and time when the file was uploaded.") + ) diff --git a/xml_manager/wagtail_hooks.py b/xml_manager/wagtail_hooks.py new file mode 100644 index 0000000..c41676c --- /dev/null +++ b/xml_manager/wagtail_hooks.py @@ -0,0 +1,55 @@ +from django.urls import path, include +from django.utils.translation import gettext_lazy as _ +from wagtail import hooks +from wagtail.admin.menu import MenuItem +from wagtail.snippets.models import register_snippet +from wagtail.snippets.views.snippets import SnippetViewSet + +from config.menu import get_menu_order + +from .models import XMLDocument +from . import urls + + +@hooks.register('register_admin_urls') +def register_admin_urls(): + return [ + path('xml-manager/', include(urls)), + ] + + +class XMLDocumentSnippetViewSet(SnippetViewSet): + model = XMLDocument + verbose_name = _("XML Document") + verbose_name_plural = _("XML Documents") + icon = "folder-open-inverse" + menu_name = "xml_manager" + menu_label = _("XML Document") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = True + + list_display = ( + "xml_file", + "validation_file", + "exceptions_file", + "pdf_file", + "html_file", + "uploaded_at", + ) + list_filter = ( + ) + search_fields = ( + "xml_file", + ) + + +register_snippet(XMLDocumentSnippetViewSet) + +@hooks.register('register_admin_menu_item') +def register_process_form_menu_item(): + return MenuItem( + _('Process XML'), + '/admin/xml-manager/upload/', + icon_name='form', + order=get_menu_order("xml_manager") + 1, + ) From 8f3c5bd2f1984aa59d04893711621f4cc772ddbe Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:04:11 -0300 Subject: [PATCH 08/38] cria task para processar xml - chama processos de validate, pdf e html --- xml_manager/tasks.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 xml_manager/tasks.py diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py new file mode 100644 index 0000000..8428083 --- /dev/null +++ b/xml_manager/tasks.py @@ -0,0 +1,34 @@ +import logging +import os +import tempfile + +from packtools import data_checker + +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ + +from config import celery_app +from core.utils.utils import _get_user + +from .models import XMLDocument + + +User = get_user_model() + + +@celery_app.task(bind=True, name=_('Process XML Document'), timelimit=-1) +def task_process_xml_document(self, xml_id, user_id=None, username=None): + user = _get_user(self.request, username=username, user_id=user_id) + + try: + xml_document = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML document with ID {xml_id} does not exist.') + return False + + logging.info(f'Processing XML file {xml_document.xml_file.name}.') + task_validate_xml_file.delay(xml_id, user_id=user_id, username=username) + task_generate_pdf_file.delay(xml_id, user_id=user_id, username=username) + task_generate_html_file.delay(xml_id, user_id=user_id, username=username) + + return True From c94f4d4a2c74c8a079a1f0b4904c8f18b5466b22 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:04:25 -0300 Subject: [PATCH 09/38] cria task para validar xml --- xml_manager/tasks.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 8428083..0aeef93 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -32,3 +32,35 @@ def task_process_xml_document(self, xml_id, user_id=None, username=None): task_generate_html_file.delay(xml_id, user_id=user_id, username=username) return True + + +@celery_app.task(bind=True, name=_('Validate XML Document'), timelimit=-1) +def task_validate_xml_file(self, xml_id, user_id=None, username=None): + user = _get_user(self.request, username=username, user_id=user_id) + + try: + xml_file = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + logging.info(f'Starting XML validation for XML file {xml_file.xml_file.name}.') + params = {} + + temp_dir = tempfile.gettempdir() + base_filename = os.path.splitext(os.path.basename(xml_file.xml_file.name))[0] + + path_csv = os.path.join(temp_dir, f"{base_filename}.validation.csv") + path_exceptions = os.path.join(temp_dir, f"{base_filename}.exceptions.json") + + try: + validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_file.xml_file.path) + validator.validate(params=params, csv_per_xml=False) + except Exception as e: + logging.error(f'Error during XML validation: {e}') + return False + + xml_file.validation_file = path_csv + xml_file.exceptions_file = path_exceptions + xml_file.save() + logging.info(f'XML validation completed successfully for {xml_file.xml_file.name}.') From 50a5dd00b60ad8f387c8096953474776e8fe1811 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 18 Aug 2025 12:04:53 -0300 Subject: [PATCH 10/38] =?UTF-8?q?cria=20tasks=20para=20gerar=20pdf=20e=20h?= =?UTF-8?q?tml=20(h=C3=A1=20ToDos)=20-=20a=20completar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/tasks.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 0aeef93..43c9130 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -64,3 +64,31 @@ def task_validate_xml_file(self, xml_id, user_id=None, username=None): xml_file.exceptions_file = path_exceptions xml_file.save() logging.info(f'XML validation completed successfully for {xml_file.xml_file.name}.') + + +@celery_app.task(bind=True, name=_('Generate HTML File'), timelimit=-1) +def task_generate_pdf_file(self, xml_id, user_id=None, username=None): + user = _get_user(self.request, username=username, user_id=user_id) + + try: + xml_file = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + logging.info(f'Starting PDF generation for XML file {xml_file.xml_file.name}.') + # TODO: Implement PDF generation logic here + + +@celery_app.task(bind=True, name=_('Generate HTML File'), timelimit=-1) +def task_generate_html_file(self, xml_id, user_id=None, username=None): + user = _get_user(self.request, username=username, user_id=user_id) + + try: + xml_file = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + logging.info(f'Starting HTML generation for XML file {xml_file.xml_file.name}.') + # TODO: Implement HTML generation logic here From 321ef806fd73017fb1f39859a7e50e1a38749592 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare <158627036+pitangainnovare@users.noreply.github.com> Date: Mon, 18 Aug 2025 12:40:32 -0300 Subject: [PATCH 11/38] Update xml_manager/views.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove anotação que desabilitava proteção CSRF Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- xml_manager/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xml_manager/views.py b/xml_manager/views.py index 0bb51a3..f959537 100644 --- a/xml_manager/views.py +++ b/xml_manager/views.py @@ -13,7 +13,6 @@ def upload_xml_page(request): @require_POST -@csrf_exempt def process_xml(request): xml_file = request.FILES.get('xml_file') if not xml_file: From 1d88857043f465f0bf0be8bd82b5270361f699ae Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 21:50:08 -0300 Subject: [PATCH 12/38] =?UTF-8?q?remove=20anota=C3=A7=C3=A3o=20name=20em?= =?UTF-8?q?=20defini=C3=A7=C3=A3o=20de=20tasks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/tasks.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 43c9130..51ea7a3 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -16,7 +16,7 @@ User = get_user_model() -@celery_app.task(bind=True, name=_('Process XML Document'), timelimit=-1) +@celery_app.task(bind=True, timelimit=-1) def task_process_xml_document(self, xml_id, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) @@ -34,7 +34,7 @@ def task_process_xml_document(self, xml_id, user_id=None, username=None): return True -@celery_app.task(bind=True, name=_('Validate XML Document'), timelimit=-1) +@celery_app.task(bind=True, timelimit=-1) def task_validate_xml_file(self, xml_id, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) @@ -66,7 +66,7 @@ def task_validate_xml_file(self, xml_id, user_id=None, username=None): logging.info(f'XML validation completed successfully for {xml_file.xml_file.name}.') -@celery_app.task(bind=True, name=_('Generate HTML File'), timelimit=-1) +@celery_app.task(bind=True, timelimit=-1) def task_generate_pdf_file(self, xml_id, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) @@ -80,7 +80,7 @@ def task_generate_pdf_file(self, xml_id, user_id=None, username=None): # TODO: Implement PDF generation logic here -@celery_app.task(bind=True, name=_('Generate HTML File'), timelimit=-1) +@celery_app.task(bind=True, timelimit=-1) def task_generate_html_file(self, xml_id, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) From 8d7749ace401147f4cc8bfce74919819bf362da1 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 22:39:51 -0300 Subject: [PATCH 13/38] =?UTF-8?q?define=20=5Fget=5Fuser=20(similar=20ao=20?= =?UTF-8?q?upload)=20e=20muda=20lugar=20em=20que=20=C3=A9=20chamado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/tasks.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 51ea7a3..b451e0f 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -16,10 +16,18 @@ User = get_user_model() +def _get_user(request, username=None, user_id=None): + try: + return User.objects.get(pk=request.user_id) + except AttributeError: + if user_id: + return User.objects.get(pk=user_id) + if username: + return User.objects.get(username=username) + + @celery_app.task(bind=True, timelimit=-1) def task_process_xml_document(self, xml_id, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - try: xml_document = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: @@ -36,14 +44,14 @@ def task_process_xml_document(self, xml_id, user_id=None, username=None): @celery_app.task(bind=True, timelimit=-1) def task_validate_xml_file(self, xml_id, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - try: xml_file = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False + user = _get_user(self.request, username=username, user_id=user_id) + logging.info(f'Starting XML validation for XML file {xml_file.xml_file.name}.') params = {} @@ -67,28 +75,28 @@ def task_validate_xml_file(self, xml_id, user_id=None, username=None): @celery_app.task(bind=True, timelimit=-1) -def task_generate_pdf_file(self, xml_id, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - +def task_generate_pdf_file(self, xml_id, user_id=None, username=None): try: xml_file = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False + user = _get_user(self.request, username=username, user_id=user_id) + logging.info(f'Starting PDF generation for XML file {xml_file.xml_file.name}.') # TODO: Implement PDF generation logic here @celery_app.task(bind=True, timelimit=-1) -def task_generate_html_file(self, xml_id, user_id=None, username=None): - user = _get_user(self.request, username=username, user_id=user_id) - +def task_generate_html_file(self, xml_id, user_id=None, username=None): try: xml_file = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False + user = _get_user(self.request, username=username, user_id=user_id) + logging.info(f'Starting HTML generation for XML file {xml_file.xml_file.name}.') # TODO: Implement HTML generation logic here From 9134e57e4be7052b74662f18bf1cff4a3af957d6 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 22:40:04 -0300 Subject: [PATCH 14/38] apaga utils.py --- core/utils/utils.py | 94 --------------------------------------------- 1 file changed, 94 deletions(-) delete mode 100644 core/utils/utils.py diff --git a/core/utils/utils.py b/core/utils/utils.py deleted file mode 100644 index 57205e1..0000000 --- a/core/utils/utils.py +++ /dev/null @@ -1,94 +0,0 @@ -import logging -import re -import requests - -from django.contrib.auth import get_user_model -from langcodes import standardize_tag, tag_is_valid -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_exponential, -) - - -logger = logging.getLogger(__name__) -User = get_user_model() - - -def language_iso(code): - code = re.split(r"-|_", code)[0] if code else "" - if tag_is_valid(code): - return standardize_tag(code) - return "" - - -class RetryableError(Exception): - """Recoverable error without having to modify the data state on the client - side, e.g. timeouts, errors from network partitioning, etc. - """ - - -class NonRetryableError(Exception): - """Recoverable error without having to modify the data state on the client - side, e.g. timeouts, errors from network partitioning, etc. - """ - - -@retry( - retry=retry_if_exception_type(RetryableError), - wait=wait_exponential(multiplier=1, min=1, max=5), - stop=stop_after_attempt(5), -) -def fetch_data(url, headers=None, json=False, timeout=2, verify=True): - """ - Get the resource with HTTP - Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, - then up to 10 seconds, then 10 seconds afterwards - Args: - url: URL address - headers: HTTP headers - json: True|False - verify: Verify the SSL. - Returns: - Return a requests.response object. - Except: - Raise a RetryableError to retry. - """ - - try: - logger.info("Fetching the URL: %s" % url) - response = requests.get(url, headers=headers, timeout=timeout, verify=verify) - except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: - logger.error("Erro fetching the content: %s, retry..., erro: %s" % (url, exc)) - raise RetryableError(exc) from exc - except ( - requests.exceptions.InvalidSchema, - requests.exceptions.MissingSchema, - requests.exceptions.InvalidURL, - ) as exc: - raise NonRetryableError(exc) from exc - try: - response.raise_for_status() - except requests.HTTPError as exc: - if 400 <= exc.response.status_code < 500: - raise NonRetryableError(exc) from exc - elif 500 <= exc.response.status_code < 600: - logger.error( - "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) - ) - raise RetryableError(exc) from exc - else: - raise - - return response.content if not json else response.json() - - -def _get_user(request, username=None, user_id=None): - try: - return User.objects.get(pk=request.user_id) - except AttributeError: - if user_id: - return User.objects.get(pk=user_id) - if username: - return User.objects.get(username=username) From c4009b21e8f5442222a9ab8da899add0658ad206 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 22:40:14 -0300 Subject: [PATCH 15/38] acrescenta requester.py --- core/utils/requester.py | 162 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 core/utils/requester.py diff --git a/core/utils/requester.py b/core/utils/requester.py new file mode 100644 index 0000000..be2732d --- /dev/null +++ b/core/utils/requester.py @@ -0,0 +1,162 @@ +import logging +import requests + +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + + +logger = logging.getLogger(__name__) + + +class RetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +class NonRetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +def _add_param(params, name, value): + if value: + params[name] = value + return params + + +@retry( + retry=retry_if_exception_type(RetryableError), + wait=wait_exponential(multiplier=1, min=1, max=5), + stop=stop_after_attempt(5), +) +def post_data( + url, + auth=None, + data=None, + files=None, + headers=None, + json=False, + timeout=2, + verify=True, +): + """ + Post data with HTTP + Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, + then up to 10 seconds, then 10 seconds afterwards + Args: + url: URL address + files: files + headers: HTTP headers + json: True|False + verify: Verify the SSL. + Returns: + Return a requests.response object. + Except: + Raise a RetryableError to retry. + """ + try: + params = dict( + headers=headers, + timeout=timeout, + verify=verify, + ) + params = _add_param(params, "auth", auth) + params = _add_param(params, "files", files) + params = _add_param(params, "data", data) + response = requests.post(url, **params) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + logger.error("Erro posting data (timeout=%s): %s, retry..., erro: %s" % (timeout, url, exc)) + raise RetryableError(exc) from exc + except ( + requests.exceptions.InvalidSchema, + requests.exceptions.MissingSchema, + requests.exceptions.InvalidURL, + ) as exc: + raise NonRetryableError(exc) from exc + try: + response.raise_for_status() + except requests.HTTPError as exc: + if response := is_http_error_json_response(json, response): + return response + if 400 <= exc.response.status_code < 500: + raise NonRetryableError(exc) from exc + elif 500 <= exc.response.status_code < 600: + logger.error( + "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) + ) + raise RetryableError(exc) from exc + else: + raise + + return response.content if not json else response.json() + + +def is_http_error_json_response(json, response): + """ + Algumas API, por exemplo, opac_5, retornam a mensagem de erro em formato + JSON + """ + if not json: + return + + try: + return response.json() + except Exception as json_error: + return + + +@retry( + retry=retry_if_exception_type(RetryableError), + wait=wait_exponential(multiplier=1, min=1, max=5), + stop=stop_after_attempt(5), +) +def fetch_data(url, params=None, headers=None, json=False, timeout=2, verify=True): + """ + Get the resource with HTTP + Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, + then up to 10 seconds, then 10 seconds afterwards + Args: + url: URL address + headers: HTTP headers + json: True|False + verify: Verify the SSL. + Returns: + Return a requests.response object. + Except: + Raise a RetryableError to retry. + """ + + try: + logger.info("Fetching the URL: %s %s" % (url, params)) + response = requests.get( + url, params=params, headers=headers, timeout=timeout, verify=verify + ) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + logger.error("Erro fetching the content: %s, retry..., erro: %s" % (url, exc)) + raise RetryableError(exc) from exc + except ( + requests.exceptions.InvalidSchema, + requests.exceptions.MissingSchema, + requests.exceptions.InvalidURL, + ) as exc: + raise NonRetryableError(exc) from exc + try: + response.raise_for_status() + except requests.HTTPError as exc: + if 400 <= exc.response.status_code < 500: + raise NonRetryableError(exc) from exc + elif 500 <= exc.response.status_code < 600: + logger.error( + "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) + ) + raise RetryableError(exc) from exc + else: + raise + + return response.content if not json else response.json() From c794bad6909ed9a8e861c7d52f29247d697816d3 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 23:26:36 -0300 Subject: [PATCH 16/38] inclui xml_manager em lista de apps de settings.py --- config/settings/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/settings/base.py b/config/settings/base.py index 365a1d1..f3de397 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -80,7 +80,8 @@ "core_settings", "django_celery_beat", "tracker", - "reference" + "reference", + "xml_manager", ] INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + WAGTAIL From 7be9c346243cc1bb27d147b0d2e1b16d7a3c8376 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 23:27:15 -0300 Subject: [PATCH 17/38] =?UTF-8?q?inclui=20depend=C3=AAncias=20packtools=20?= =?UTF-8?q?e=20langdetect?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/base.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requirements/base.txt b/requirements/base.txt index f548288..e12f8d1 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -41,3 +41,11 @@ whitenoise==6.6.0 # https://github.com/evansd/whitenoise # ------------------------------------------------------------------------------ huggingface-hub==0.26.1 llama-cpp-python==0.3.14 + +# Packtools +# ------------------------------------------------------------------------------ +git+https://git@github.com/scieloorg/packtools@4.11.20#egg=packtools + +# Langdetect +# ------------------------------------------------------------------------------ +langdetect~=1.0.9 \ No newline at end of file From 318ab055a9f6c5b23d4ae8125d4194ec84f30495 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Sun, 24 Aug 2025 23:27:48 -0300 Subject: [PATCH 18/38] Cria modelos XMLDocumentPDF e XMLDocumentHTML e altera XMLDocument --- xml_manager/migrations/0001_initial.py | 45 ++++++++++++++++++++++++++ xml_manager/models.py | 34 ++++++++++++++----- 2 files changed, 71 insertions(+), 8 deletions(-) create mode 100644 xml_manager/migrations/0001_initial.py diff --git a/xml_manager/migrations/0001_initial.py b/xml_manager/migrations/0001_initial.py new file mode 100644 index 0000000..fd2b460 --- /dev/null +++ b/xml_manager/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 5.0.8 on 2025-08-25 02:26 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='XMLDocument', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xml_file', models.FileField(help_text='Upload an XML file for processing.', upload_to='xml_manager/xml/', verbose_name='XML File')), + ('validation_file', models.FileField(blank=True, null=True, upload_to='xml_manager/csv/', verbose_name='Validation File')), + ('exceptions_file', models.FileField(blank=True, null=True, upload_to='xml_manager/exceptions/', verbose_name='Exceptions File')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, help_text='The date and time when the file was uploaded.', verbose_name='Uploaded At')), + ], + ), + migrations.CreateModel( + name='XMLDocumentHTML', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('html_file', models.FileField(upload_to='xml_manager/html/', verbose_name='HTML File')), + ('language', models.CharField(default='pt', help_text='Language code or name', max_length=32, verbose_name='Language')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='Uploaded At')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='htmls', to='xml_manager.xmldocument', verbose_name='XML Document')), + ], + ), + migrations.CreateModel( + name='XMLDocumentPDF', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pdf_file', models.FileField(upload_to='xml_manager/pdf/', verbose_name='PDF File')), + ('language', models.CharField(default='pt', help_text='Language code or name', max_length=32, verbose_name='Language')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='Uploaded At')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pdfs', to='xml_manager.xmldocument', verbose_name='XML Document')), + ], + ), + ] diff --git a/xml_manager/models.py b/xml_manager/models.py index 7a21411..6882f5a 100644 --- a/xml_manager/models.py +++ b/xml_manager/models.py @@ -20,20 +20,38 @@ class XMLDocument(models.Model): null=True, verbose_name=_("Exceptions File") ) + uploaded_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("Uploaded At"), + help_text=_("The date and time when the file was uploaded.") + ) + + +class XMLDocumentPDF(models.Model): + xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="pdfs", verbose_name=_("XML Document")) pdf_file = models.FileField( upload_to='xml_manager/pdf/', - blank=True, - null=True, verbose_name=_("PDF File") ) + language = models.CharField( + max_length=32, + default="pt", + verbose_name=_("Language"), + help_text=_("Language code or name") + ) + uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) + + +class XMLDocumentHTML(models.Model): + xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="htmls", verbose_name=_("XML Document")) html_file = models.FileField( upload_to='xml_manager/html/', - blank=True, - null=True, verbose_name=_("HTML File") ) - uploaded_at = models.DateTimeField( - auto_now_add=True, - verbose_name=_("Uploaded At"), - help_text=_("The date and time when the file was uploaded.") + language = models.CharField( + max_length=32, + default="pt", + verbose_name=_("Language"), + help_text=_("Language code or name") ) + uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) From 76c87fc537f9b4850ef3660dd10ce58360f0b269 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:20:28 -0300 Subject: [PATCH 19/38] organiza menu de apps --- config/menu.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/menu.py b/config/menu.py index c014971..8dc074e 100644 --- a/config/menu.py +++ b/config/menu.py @@ -1,4 +1,7 @@ WAGTAIL_MENU_APPS_ORDER = [ + "xml_manager", + "reference", + "tracker", "django_celery_beat", ] From d7359b78f6d941eade6efc2deeaf35854f649d34 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:20:52 -0300 Subject: [PATCH 20/38] =?UTF-8?q?Cria=20choices=20relacionados=20a=20ocorr?= =?UTF-8?q?=C3=AAncia=20de=20eventos=20ao=20processar=20XML?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tracker/choices.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tracker/choices.py b/tracker/choices.py index 268789c..e70c3da 100644 --- a/tracker/choices.py +++ b/tracker/choices.py @@ -28,3 +28,17 @@ (PROGRESS_STATUS_PENDING, _("Pending")), (PROGRESS_STATUS_IGNORED, _("ignored")), ) + +XML_DOCUMENT_PARSING_ERROR = "XPE" +XML_DOCUMENT_VALIDATION_ERROR = "VDE" +XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR = "CDE" +XML_DOCUMENT_CONVERSION_TO_PDF_ERROR = "CPE" +XML_DOCUMENT_CONVERSION_TO_TEX_ERROR = "CTE" + +XML_DOCUMENT_EVENT = [ + (XML_DOCUMENT_PARSING_ERROR, _("XML Parsing Error")), + (XML_DOCUMENT_VALIDATION_ERROR, _("XML Validation Error")), + (XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, _("XML Conversion to DOCX Error")), + (XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, _("XML Conversion to PDF Error")), + (XML_DOCUMENT_CONVERSION_TO_TEX_ERROR, _("XML Conversion to TEX Error")), +] From e781091e22c3ab6af9d820240284d9649b55c076 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:21:10 -0300 Subject: [PATCH 21/38] =?UTF-8?q?exclui=20exce=C3=A7=C3=B5es=20n=C3=A3o=20?= =?UTF-8?q?usadas=20e=20cria=20outras?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tracker/exceptions.py | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/tracker/exceptions.py b/tracker/exceptions.py index 5d0f7dc..571c194 100644 --- a/tracker/exceptions.py +++ b/tracker/exceptions.py @@ -1,26 +1,6 @@ -class ProcEventCreateError(Exception): +class GeneralEventCreateError(Exception): ... -class UnexpectedEventCreateError(Exception): - ... - - -class EventCreateError(Exception): - ... - - -class EventReportCreateError(Exception): - ... - - -class EventReportSaveFileError(Exception): - ... - - -class EventReportCreateError(Exception): - ... - - -class EventReportDeleteEventsError(Exception): - ... +class XMLDocumentEventCreateError(Exception): + ... \ No newline at end of file From a4e79abaf4e740766924fe84f1da8152ec7653c8 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:22:08 -0300 Subject: [PATCH 22/38] refatora tracker para conter eventos associados ao document_xml e gerais --- tracker/migrations/0002_xmldocumentevent.py | 35 ++++ ...e_unexpectedevent_generalevent_and_more.py | 32 +++ ...alter_xmldocumentevent_options_and_more.py | 22 ++ tracker/models.py | 192 ++++++------------ tracker/tasks.py | 13 +- tracker/wagtail_hooks.py | 52 ++++- 6 files changed, 200 insertions(+), 146 deletions(-) create mode 100644 tracker/migrations/0002_xmldocumentevent.py create mode 100644 tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py create mode 100644 tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py diff --git a/tracker/migrations/0002_xmldocumentevent.py b/tracker/migrations/0002_xmldocumentevent.py new file mode 100644 index 0000000..4df8637 --- /dev/null +++ b/tracker/migrations/0002_xmldocumentevent.py @@ -0,0 +1,35 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:04 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0001_initial'), + ('xml_manager', '0002_alter_xmldocument_options_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='XMLDocumentEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Creation date')), + ('updated', models.DateTimeField(auto_now=True, verbose_name='Last update date')), + ('error_type', models.CharField(blank=True, choices=[('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error')], max_length=3, null=True, verbose_name='Error Type')), + ('data', models.JSONField(default=dict, verbose_name='Data')), + ('message', models.TextField(blank=True, null=True, verbose_name='Message')), + ('handled', models.BooleanField(default=False, verbose_name='Handled')), + ('creator', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_creator', to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('updated_by', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_last_mod_user', to=settings.AUTH_USER_MODEL, verbose_name='Updater')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='xml_manager.xmldocument')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py b/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py new file mode 100644 index 0000000..5ee4d28 --- /dev/null +++ b/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0002_xmldocumentevent'), + ] + + operations = [ + migrations.RenameModel( + old_name='UnexpectedEvent', + new_name='GeneralEvent', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_excepti_7389c9_idx', + old_name='tracker_une_excepti_47ede4_idx', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_item_77b66a_idx', + old_name='tracker_une_item_795336_idx', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_action_1474c9_idx', + old_name='tracker_une_action_55cd4b_idx', + ), + ] diff --git a/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py b/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py new file mode 100644 index 0000000..9c9ec14 --- /dev/null +++ b/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0003_rename_unexpectedevent_generalevent_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='xmldocumentevent', + options={'verbose_name': 'XML Document Event', 'verbose_name_plural': 'XML Document Events'}, + ), + migrations.AlterField( + model_name='xmldocumentevent', + name='error_type', + field=models.CharField(blank=True, choices=[('XPE', 'XML Parsing Error'), ('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error')], max_length=3, null=True, verbose_name='Error Type'), + ), + ] diff --git a/tracker/models.py b/tracker/models.py index 6786920..ae74c50 100644 --- a/tracker/models.py +++ b/tracker/models.py @@ -2,23 +2,76 @@ import logging import traceback import uuid -from datetime import datetime -from django.core.files.base import ContentFile from django.db import models from django.utils.translation import gettext_lazy as _ from core.models import CommonControlField +from xml_manager.models import XMLDocument from tracker import choices + from .exceptions import ( - EventCreateError, - EventReportCreateError, - EventReportSaveFileError, - UnexpectedEventCreateError + GeneralEventCreateError, + XMLDocumentEventCreateError, ) -class UnexpectedEvent(models.Model): +class XMLDocumentEvent(CommonControlField): + xml_document = models.ForeignKey( + XMLDocument, + on_delete=models.CASCADE, + null=False, + blank=False, + db_index=True, + ) + error_type = models.CharField( + _("Error Type"), + choices=choices.XML_DOCUMENT_EVENT, + max_length=3, + null=True, + blank=True, + ) + data = models.JSONField( + _("Data"), + default=dict, + ) + message = models.TextField( + _("Message"), + null=True, + blank=True, + ) + handled = models.BooleanField( + _("Handled"), + default=False + ) + + @classmethod + def create(cls, xml_document, error_type, data, message, save=False): + try: + obj = cls() + obj.xml_document = xml_document + obj.error_type = error_type + obj.data = data + obj.message = message + if save: + obj.save() + + return obj + + except Exception as exc: + raise XMLDocumentEventCreateError( + f"Unable to create XMLDocumentEvent. EXCEPTION {exc}" + ) + + def __str__(self): + return f"{self.data} - {self.message}" + + class Meta: + verbose_name = _("XML Document Event") + verbose_name_plural = _("XML Document Events") + + +class GeneralEvent(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created = models.DateTimeField(verbose_name=_("Creation date"), auto_now_add=True) exception_type = models.TextField(_("Exception Type"), null=True, blank=True) @@ -45,6 +98,8 @@ class Meta: models.Index(fields=["action"]), ] ordering = ["-created"] + verbose_name = _("General Event") + verbose_name_plural = _("General Events") def __str__(self): if self.item or self.action: @@ -92,125 +147,6 @@ def create( obj.save() return obj except Exception as exc: - raise UnexpectedEventCreateError( - f"Unable to create unexpected event ({exception} {exc_traceback}). EXCEPTION {exc}" - ) - - -class Event(CommonControlField): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - message = models.TextField(_("Message"), null=True, blank=True) - message_type = models.CharField( - _("Message type"), - choices=choices.EVENT_MSG_TYPE, - max_length=16, - null=True, - blank=True, - ) - detail = models.JSONField(null=True, blank=True) - unexpected_event = models.ForeignKey( - UnexpectedEvent, on_delete=models.SET_NULL, null=True, blank=True - ) - - class Meta: - abstract = True - indexes = [ - models.Index(fields=["message_type"]), - ] - - @property - def data(self): - d = {} - d["created"] = self.created.isoformat() - d["user"] = self.user.username - d.update( - dict( - message=self.message, message_type=self.message_type, detail=self.detail - ) - ) - if self.unexpected_event: - d.update(self.unexpected_event.data) - return d - - @classmethod - def create( - cls, - user=None, - message_type=None, - message=None, - e=None, - exc_traceback=None, - detail=None, - ): - try: - obj = cls() - obj.creator = user - obj.message = message - obj.message_type = message_type - obj.detail = detail - obj.save() - - if e: - logging.exception(f"{message}: {e}") - obj.unexpected_event = UnexpectedEvent.create( - exception=e, - exc_traceback=exc_traceback, - ) - obj.save() - except Exception as exc: - raise EventCreateError( - f"Unable to create Event ({message} {e}). EXCEPTION: {exc}" - ) - return obj - - -def tracker_file_directory_path(instance, filename): - # file will be uploaded to MEDIA_ROOT/user_/ - - d = datetime.utcnow() - return f"tracker/{d.year}/{d.month}/{d.day}/{filename}" - - -class EventReport(CommonControlField): - file = models.FileField( - upload_to=tracker_file_directory_path, null=True, blank=True - ) - - class Meta: - abstract = True - - def save_file(self, events, ext=None): - if not events: - return - try: - ext = ".json" - content = json.dumps(list([item.data for item in events])) - name = datetime.utcnow().isoformat() + ext - self.file.save(name, ContentFile(content)) - self.delete_events(events) - except Exception as e: - raise EventReportSaveFileError( - f"Unable to save EventReport.file ({name}). Exception: {e}" - ) - - def delete_events(self, events): - for item in events: - try: - item.unexpected_event.delete() - except: - pass - try: - item.delete() - except: - pass - - @classmethod - def create(cls, user): - try: - obj = cls() - obj.creator = user - obj.save() - except Exception as e: - raise EventReportCreateError( - f"Unable to create EventReport. Exception: {e}" + raise GeneralEventCreateError( + f"Unable to create general event ({exception} {exc_traceback}). EXCEPTION {exc}" ) diff --git a/tracker/tasks.py b/tracker/tasks.py index af422b8..9f9154f 100644 --- a/tracker/tasks.py +++ b/tracker/tasks.py @@ -1,16 +1,13 @@ -# tasks.py -import logging -import sys from datetime import datetime from config import celery_app -from .models import UnexpectedEvent +from .models import GeneralEvent -@celery_app.task(bind=True, name="cleanup_unexpected_events") -def delete_unexpected_events(self, exception_type, start_date=None, end_date=None, user_id=None, username=None): +@celery_app.task(bind=True) +def delete_general_events(self, exception_type, start_date=None, end_date=None, user_id=None, username=None): """ - Delete UnexpectedEvent records based on exception type and optional date range. + Delete GeneralEvent records based on exception type and optional date range. """ filters = {} @@ -24,4 +21,4 @@ def delete_unexpected_events(self, exception_type, start_date=None, end_date=Non end_date = datetime.fromisoformat(end_date) filters['created__lte'] = end_date - UnexpectedEvent.objects.filter(**filters).delete() + GeneralEvent.objects.filter(**filters).delete() diff --git a/tracker/wagtail_hooks.py b/tracker/wagtail_hooks.py index e1b82e0..8c87445 100644 --- a/tracker/wagtail_hooks.py +++ b/tracker/wagtail_hooks.py @@ -7,14 +7,45 @@ from config.menu import get_menu_order -from .models import UnexpectedEvent +from .models import XMLDocumentEvent, GeneralEvent -class UnexpectedEventModelAdmin(ModelAdmin): - model = UnexpectedEvent +class XMLDocumentEventModelAdmin(ModelAdmin): + model = XMLDocumentEvent inspect_view_enabled = True - menu_label = _("Unexpected Events") - menu_icon = "folder" + menu_label = _("XML Document Events") + menu_icon = "warning" + menu_order = 100 + add_to_settings_menu = False + exclude_from_explorer = False + list_per_page = 10 + + list_display = ( + "xml_document", + "error_type", + "data", + "message", + "created", + ) + list_filter = ("error_type", ) + search_fields = ( + "message", + "data", + ) + inspect_view_fields = ( + "xml_document", + "error_type", + "data", + "message", + "created", + ) + + +class GeneralEventModelAdmin(ModelAdmin): + model = GeneralEvent + inspect_view_enabled = True + menu_label = _("General Events") + menu_icon = "warning" menu_order = 200 add_to_settings_menu = False exclude_from_explorer = False @@ -44,11 +75,12 @@ class UnexpectedEventModelAdmin(ModelAdmin): "created", ) -class UnexpectedEventModelAdminGroup(ModelAdminGroup): - menu_icon = "folder" - menu_label = _("Unexpected errors") + +class EventModelAdminGroup(ModelAdminGroup): + menu_icon = "warning" + menu_label = _("Unexpected Events") menu_order = get_menu_order("tracker") - items = (UnexpectedEventModelAdmin) + items = (GeneralEventModelAdmin, XMLDocumentEventModelAdmin) -modeladmin_register(UnexpectedEventModelAdmin) +modeladmin_register(EventModelAdminGroup) From e03184576ad13903220a3bb6dbf02ac004ddc42e Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:22:50 -0300 Subject: [PATCH 23/38] =?UTF-8?q?melhora=20forma=20de=20processar=20XML=20?= =?UTF-8?q?-=20remove=20views/templates=20desnecess=C3=A1rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/xml_manager/process_form.html | 16 ------- .../xml_manager/xmldocument_list.html | 47 ------------------- xml_manager/urls.py | 5 +- xml_manager/views.py | 29 ++++-------- 4 files changed, 10 insertions(+), 87 deletions(-) delete mode 100644 xml_manager/templates/xml_manager/process_form.html delete mode 100644 xml_manager/templates/xml_manager/xmldocument_list.html diff --git a/xml_manager/templates/xml_manager/process_form.html b/xml_manager/templates/xml_manager/process_form.html deleted file mode 100644 index 858cd9d..0000000 --- a/xml_manager/templates/xml_manager/process_form.html +++ /dev/null @@ -1,16 +0,0 @@ -{% extends "wagtailadmin/base.html" %} - -{% block content %} -
-

Processar Arquivo XML

-

Envie um arquivo XML para validar seu conteúdo. Após o envio, serão gerados automaticamente os arquivos HTML e PDF correspondentes.

-
- {% csrf_token %} -
- - -
- -
-
-{% endblock %} \ No newline at end of file diff --git a/xml_manager/templates/xml_manager/xmldocument_list.html b/xml_manager/templates/xml_manager/xmldocument_list.html deleted file mode 100644 index e26933e..0000000 --- a/xml_manager/templates/xml_manager/xmldocument_list.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends "wagtailadmin/generic/index.html" %} - -{% block content %} -
-

XML Documents

- - - - - - - - - - - - - {% for obj in xmldocument.object_list %} - - - - - - - - - {% endfor %} - -
Arquivo XMLValidationExceptionsPDFHTMLEnviado em
{{ obj.xml_file.name }} - {% if obj.validation_file %} - Download - {% endif %} - - {% if obj.exceptions_file %} - Download - {% endif %} - - {% if obj.pdf_file %} - Download - {% endif %} - - {% if obj.html_file %} - Download - {% endif %} - {{ obj.uploaded_at }}
-
-{% endblock %} \ No newline at end of file diff --git a/xml_manager/urls.py b/xml_manager/urls.py index 7979f50..226389b 100644 --- a/xml_manager/urls.py +++ b/xml_manager/urls.py @@ -3,6 +3,5 @@ urlpatterns = [ - path('upload/', views.upload_xml_page, name='upload_xml_page'), - path('process/', views.process_xml, name='process_xml'), -] + path("process//", views.process_xml_pk, name="process_xml_pk"), +] \ No newline at end of file diff --git a/xml_manager/views.py b/xml_manager/views.py index f959537..8f67642 100644 --- a/xml_manager/views.py +++ b/xml_manager/views.py @@ -1,27 +1,14 @@ -from django.shortcuts import render, redirect -from django.urls import reverse -from django.views.decorators.csrf import csrf_exempt -from django.views.decorators.http import require_POST from django.contrib import messages +from django.contrib.admin.views.decorators import staff_member_required +from django.shortcuts import redirect, get_object_or_404 from .tasks import task_process_xml_document from .models import XMLDocument -def upload_xml_page(request): - return render(request, "xml_manager/process_form.html") - - -@require_POST -def process_xml(request): - xml_file = request.FILES.get('xml_file') - if not xml_file: - messages.error(request, "Arquivo não enviado.") - return redirect(reverse('upload_xml_page')) - - xml_obj = XMLDocument.objects.create(xml_file=xml_file) - - task_process_xml_document.delay(xml_obj.id) - - messages.success(request, "Arquivo enviado para processamento!") - return redirect(reverse('upload_xml_page')) +@staff_member_required +def process_xml_pk(request, pk): + obj = get_object_or_404(XMLDocument, pk=pk) + task_process_xml_document.delay(obj.id) + messages.success(request, f"XML {obj.id} enviado para processamento") + return redirect("/admin/snippets/xml_manager/xmldocument/") From 16299bec263c115507b99d4297d1826d1e7b3e8b Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:23:38 -0300 Subject: [PATCH 24/38] refatora models para ter xmldocument, xmldocument_pdf e xmldocument_html --- xml_manager/models.py | 50 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/xml_manager/models.py b/xml_manager/models.py index 6882f5a..2534978 100644 --- a/xml_manager/models.py +++ b/xml_manager/models.py @@ -1,5 +1,6 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +from wagtail.admin.panels import FieldPanel class XMLDocument(models.Model): @@ -9,13 +10,13 @@ class XMLDocument(models.Model): help_text=_("Upload an XML file for processing.") ) validation_file = models.FileField( - upload_to='xml_manager/csv/', + upload_to='xml_manager/validation/', blank=True, null=True, verbose_name=_("Validation File") ) exceptions_file = models.FileField( - upload_to='xml_manager/exceptions/', + upload_to='xml_manager/validation/', blank=True, null=True, verbose_name=_("Exceptions File") @@ -26,6 +27,17 @@ class XMLDocument(models.Model): help_text=_("The date and time when the file was uploaded.") ) + panels = [ + FieldPanel("xml_file"), + ] + + def __str__(self): + return f"{self.xml_file.name}" + + class Meta: + verbose_name = _("XML Document") + verbose_name_plural = _("XML Documents") + class XMLDocumentPDF(models.Model): xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="pdfs", verbose_name=_("XML Document")) @@ -41,6 +53,23 @@ class XMLDocumentPDF(models.Model): ) uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) + def __str__(self): + return f"PDF for {self.xml_document.xml_file.name} ({self.language})" + + class Meta: + verbose_name = _("XML Document PDF") + verbose_name_plural = _("XML Document PDFs") + + @classmethod + def create(cls, xml_document, pdf_file, language="pt"): + pdf_instance = cls( + xml_document=xml_document, + pdf_file=pdf_file, + language=language + ) + pdf_instance.save() + return pdf_instance + class XMLDocumentHTML(models.Model): xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="htmls", verbose_name=_("XML Document")) @@ -55,3 +84,20 @@ class XMLDocumentHTML(models.Model): help_text=_("Language code or name") ) uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) + + def __str__(self): + return f"HTML for {self.xml_document.xml_file.name} ({self.language})" + + class Meta: + verbose_name = _("XML Document HTML") + verbose_name_plural = _("XML Document HTMLs") + + @classmethod + def create(cls, xml_document, html_file, language="pt"): + html_instance = cls( + xml_document=xml_document, + html_file=html_file, + language=language + ) + html_instance.save() + return html_instance From 5b48f5da1aa964efb0041778c09fb7104f555ae8 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:24:06 -0300 Subject: [PATCH 25/38] melhora wagtail hooks para apresentar os modelos de documentxml* --- xml_manager/wagtail_hooks.py | 124 ++++++++++++++++++++++++++++------- 1 file changed, 100 insertions(+), 24 deletions(-) diff --git a/xml_manager/wagtail_hooks.py b/xml_manager/wagtail_hooks.py index c41676c..d57720f 100644 --- a/xml_manager/wagtail_hooks.py +++ b/xml_manager/wagtail_hooks.py @@ -1,21 +1,54 @@ +import os + from django.urls import path, include +from django.urls import reverse +from django.utils.html import format_html from django.utils.translation import gettext_lazy as _ from wagtail import hooks -from wagtail.admin.menu import MenuItem +from wagtail.admin.ui.tables import Column from wagtail.snippets.models import register_snippet -from wagtail.snippets.views.snippets import SnippetViewSet +from wagtail.snippets.views.snippets import SnippetViewSet, SnippetViewSetGroup from config.menu import get_menu_order -from .models import XMLDocument +from .models import XMLDocument, XMLDocumentPDF, XMLDocumentHTML from . import urls -@hooks.register('register_admin_urls') -def register_admin_urls(): - return [ - path('xml-manager/', include(urls)), - ] +class FileNameColumn(Column): + """ + Coluna que mostra apenas o nome do arquivo de um FileField. + """ + def get_value(self, instance): + val = super().get_value(instance) + if not val: + return "-" + return os.path.basename(getattr(val, "name", str(val))) + + +class LinkColumn(Column): + """ + Column que recebe um FileField (FieldFile) e renderiza um . + Cria '-' se não houver arquivo. + """ + def get_value(self, instance): + val = super().get_value(instance) + if not val: + return "-" + name = os.path.basename(getattr(val, "name", str(val))) + try: + url = val.url + except Exception: + return name + return format_html('{}', url, name) + + +class ActionColumn(Column): + def get_value(self, instance): + url = reverse("process_xml_pk", args=[instance.pk]) + return format_html( + 'Processar', url + ) class XMLDocumentSnippetViewSet(SnippetViewSet): @@ -26,30 +59,73 @@ class XMLDocumentSnippetViewSet(SnippetViewSet): menu_name = "xml_manager" menu_label = _("XML Document") menu_order = get_menu_order("xml_manager") - add_to_admin_menu = True + add_to_admin_menu = False list_display = ( "xml_file", - "validation_file", - "exceptions_file", - "pdf_file", - "html_file", + LinkColumn("validation_file", label=_("Validation file")), + LinkColumn("exceptions_file", label=_("Exceptions file")), "uploaded_at", + ActionColumn("actions", label=_("Ações")), ) - list_filter = ( + + search_fields = ("xml_file",) + + +class XMLDocumentPDFSnippetViewSet(SnippetViewSet): + model = XMLDocumentPDF + verbose_name = _("XML Document PDF") + verbose_name_plural = _("XML Document PDFs") + icon = "doc-full" + menu_name = "xml_manager" + menu_label = _("XML Document PDF") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = False + + list_display = ( + "xml_document", + "pdf_file", ) - search_fields = ( - "xml_file", + + search_fields = ("pdf_file",) + + +class XMLDocumentHTMLSnippetViewSet(SnippetViewSet): + model = XMLDocumentHTML + verbose_name = _("XML Document HTML") + verbose_name_plural = _("XML Document HTMLs") + icon = "doc-full" + menu_name = "xml_manager" + menu_label = _("XML Document HTML") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = False + + list_display = ( + "xml_document", + "html_file", ) + search_fields = ("html_file",) -register_snippet(XMLDocumentSnippetViewSet) -@hooks.register('register_admin_menu_item') -def register_process_form_menu_item(): - return MenuItem( - _('Process XML'), - '/admin/xml-manager/upload/', - icon_name='form', - order=get_menu_order("xml_manager") + 1, + +class XMLDocumentSnippetViewSetGroup(SnippetViewSetGroup): + menu_name = 'xml_manager' + menu_label = _("XML Manager") + menu_icon = "folder-open-inverse" + menu_order = get_menu_order("xml_manager") + items = ( + XMLDocumentSnippetViewSet, + XMLDocumentHTMLSnippetViewSet, + XMLDocumentPDFSnippetViewSet, ) + + +register_snippet(XMLDocumentSnippetViewSetGroup) + + +@hooks.register('register_admin_urls') +def register_admin_urls(): + return [ + path('xml-manager/', include(urls)), + ] From 8404fa5a6e2b3f571c4a988cacd131b2ddc4a560 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:24:40 -0300 Subject: [PATCH 26/38] melhora tasks e implementa processamento de xml para pdf --- xml_manager/tasks.py | 124 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 106 insertions(+), 18 deletions(-) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index b451e0f..59af1a2 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -3,14 +3,30 @@ import tempfile from packtools import data_checker +from packtools.sps.formats.pdf.pipeline import docx +from packtools.sps.formats.pdf.utils import file_utils +from packtools.sps.utils import xml_utils +from django.core.files import File +from django.conf import settings from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ from config import celery_app -from core.utils.utils import _get_user - -from .models import XMLDocument +from tracker.choices import ( + XML_DOCUMENT_PARSING_ERROR, + XML_DOCUMENT_VALIDATION_ERROR, + XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, + XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + XML_DOCUMENT_CONVERSION_TO_TEX_ERROR, +) +from tracker.models import XMLDocumentEvent + +from .models import ( + XMLDocument, + XMLDocumentHTML, + XMLDocumentPDF, +) User = get_user_model() @@ -45,47 +61,119 @@ def task_process_xml_document(self, xml_id, user_id=None, username=None): @celery_app.task(bind=True, timelimit=-1) def task_validate_xml_file(self, xml_id, user_id=None, username=None): try: - xml_file = XMLDocument.objects.get(id=xml_id) + xml_document = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False user = _get_user(self.request, username=username, user_id=user_id) - logging.info(f'Starting XML validation for XML file {xml_file.xml_file.name}.') + logging.info(f'Starting XML validation for XML file {xml_document.xml_file.name}.') params = {} - temp_dir = tempfile.gettempdir() - base_filename = os.path.splitext(os.path.basename(xml_file.xml_file.name))[0] + base_filename = os.path.splitext(os.path.basename(xml_document.xml_file.name))[0] - path_csv = os.path.join(temp_dir, f"{base_filename}.validation.csv") - path_exceptions = os.path.join(temp_dir, f"{base_filename}.exceptions.json") + validation_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation') + exceptions_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation') + os.makedirs(validation_dir, exist_ok=True) + os.makedirs(exceptions_dir, exist_ok=True) + + path_csv = os.path.join(validation_dir, f"{base_filename}.validation.csv") + path_exceptions = os.path.join(exceptions_dir, f"{base_filename}.exceptions.json") try: - validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_file.xml_file.path) + validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_document.xml_file.path) validator.validate(params=params, csv_per_xml=False) except Exception as e: logging.error(f'Error during XML validation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_VALIDATION_ERROR, + data={}, + message=str(e), + save=True, + ) return False - xml_file.validation_file = path_csv - xml_file.exceptions_file = path_exceptions - xml_file.save() - logging.info(f'XML validation completed successfully for {xml_file.xml_file.name}.') + with open(path_csv, 'rb') as f_csv: + xml_document.validation_file.save(os.path.basename(path_csv), File(f_csv), save=False) + + with open(path_exceptions, 'rb') as f_exc: + xml_document.exceptions_file.save(os.path.basename(path_exceptions), File(f_exc), save=False) + + xml_document.save() + logging.info(f'XML validation completed successfully for {xml_document.xml_file.name}.') + return True @celery_app.task(bind=True, timelimit=-1) def task_generate_pdf_file(self, xml_id, user_id=None, username=None): try: - xml_file = XMLDocument.objects.get(id=xml_id) + xml_document = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False user = _get_user(self.request, username=username, user_id=user_id) - - logging.info(f'Starting PDF generation for XML file {xml_file.xml_file.name}.') - # TODO: Implement PDF generation logic here + + try: + xml_tree = xml_utils.get_xml_tree(xml_document.xml_file.path) + except Exception as e: + logging.error(f'Error parsing XML file: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_PARSING_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + + # TODO: Iterate over available languages + + logging.info(f'Starting PDF generation for XML file {xml_document.xml_file.name}.') + try: + base_docx_layout = os.path.join(settings.MEDIA_ROOT, 'layout.docx') + document = docx.pipeline_docx(xml_tree, data={'base_layout': base_docx_layout}) + except Exception as e: + logging.error(f'Error during PDF generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + + pdf_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'pdf') + os.makedirs(pdf_dir, exist_ok=True) + + # DOCX intermediário + temp_dir = tempfile.gettempdir() + base_filename = os.path.splitext(os.path.basename(xml_document.xml_file.name))[0] + path_intermediate_docx = os.path.join(temp_dir, f"{base_filename}.docx") + document.save(path_intermediate_docx) + + path_pdf = os.path.join(pdf_dir, f"{base_filename}.pdf") + try: + file_utils.convert_docx_to_pdf(path_intermediate_docx, path_pdf) + except Exception as e: + logging.error(f'Error during DOCX to PDF conversion: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + + with open(path_pdf, 'rb') as f_pdf: + pdf_instance = XMLDocumentPDF(xml_document=xml_document, language="pt") + pdf_instance.pdf_file.save(os.path.basename(path_pdf), File(f_pdf), save=True) + + return True @celery_app.task(bind=True, timelimit=-1) From 8c59d477c76bcac6babd2ae1ffb22f25ed01e4e7 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Mon, 25 Aug 2025 15:24:55 -0300 Subject: [PATCH 27/38] =?UTF-8?q?migra=C3=A7=C3=B5es=20de=20xml=5Fmanager?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...0002_alter_xmldocument_options_and_more.py | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 xml_manager/migrations/0002_alter_xmldocument_options_and_more.py diff --git a/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py b/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py new file mode 100644 index 0000000..3eb3bf9 --- /dev/null +++ b/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('xml_manager', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='xmldocument', + options={'verbose_name': 'XML Document', 'verbose_name_plural': 'XML Documents'}, + ), + migrations.AlterModelOptions( + name='xmldocumenthtml', + options={'verbose_name': 'XML Document HTML', 'verbose_name_plural': 'XML Document HTMLs'}, + ), + migrations.AlterModelOptions( + name='xmldocumentpdf', + options={'verbose_name': 'XML Document PDF', 'verbose_name_plural': 'XML Document PDFs'}, + ), + migrations.AlterField( + model_name='xmldocument', + name='exceptions_file', + field=models.FileField(blank=True, null=True, upload_to='xml_manager/validation/', verbose_name='Exceptions File'), + ), + migrations.AlterField( + model_name='xmldocument', + name='validation_file', + field=models.FileField(blank=True, null=True, upload_to='xml_manager/validation/', verbose_name='Validation File'), + ), + ] From ae6f6574bfac5fd736e7d672b630461ab94d0f8c Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:00:42 -0300 Subject: [PATCH 28/38] adiciona layout padrao docx --- compose/local/django/Dockerfile | 3 +++ compose/production/django/Dockerfile | 3 +++ docx_layouts/layout.docx | Bin 0 -> 21380 bytes 3 files changed, 6 insertions(+) create mode 100644 docx_layouts/layout.docx diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index b37d495..e668957 100755 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -44,6 +44,9 @@ RUN python -m pip install --upgrade pip setuptools wheel # Requirements are installed here to ensure they will be cached. COPY ./requirements . +# Create application directory to hold DOCX layouts +COPY ./docx_layouts . + # Update pip RUN python -m pip install --upgrade pip diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 99e938c..a2189c1 100755 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -19,6 +19,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # Requirements are installed here to ensure they will be cached. COPY ./requirements . +# Create application directory to hold DOCX layouts +COPY ./docx_layouts . + # Update pip RUN python -m pip install --upgrade pip diff --git a/docx_layouts/layout.docx b/docx_layouts/layout.docx new file mode 100644 index 0000000000000000000000000000000000000000..25b4aeae9c68f47dcf88740ce454036ff6d51b98 GIT binary patch literal 21380 zcmeFXWprH4vLz^HW{Vjumc?i>Gh579VrFJ$W@ct)wwRe&vMh#{zwh3j`(F3E(>?QN z)|#%Ys&!6PM&!=3Gve&XDmh6|Ff<@2C@3JpR7rK9e|f-u9_!c{S=!Un{dupP?Uev! zM1t6M&(=?McWzP^GOX_mnMoJ(^og0cgh-bXE7I87l!IxLQ|ZNLemXiLDcHb$>vAhj zz=H%YZ}ScF(>v?S*>-$r=m?f>Hq|k$E)=>w)~_0q8G#Xree_A06w^LRBMG#dv%l3C zV}!?Qbe#j;X*0WIcsuv8o>jN1^lNCODRIhq64Io_hNsXNQfi84A1nM(>?`DJ z^y|maQ$mR(^S05<^A+KxyUB6vWUpe3BpoLVxTI57fk6i9#t%h91oh>QTcP3!+>`sX z0<+z!r}4RE1tk|l8{gbcK1TZpVOZK0RocyFE0@CXAeVwft(${#-82) zU}r?fh@PVAK7+f6+zo zR^@EPy0M_#0Dk0XVAmIyAf#)U-#}%5+mtcvOnOThfud|cNa22MsIhK1Yq!Gg;uHxU zjO~Hu7jZOuXc%eAQK(&&i@^(xfI6VOXYLE!7NwbmrPY72mWFu60h zxm%6KwwSAw0HI zBY;iAOs?K@q@@p!t@n%sGZ6VvZ#({u*URO{lsKb=Xfvav!BCQA-<*3^-E`H75&zao zuZpuyVaoc07Ua6t8lbC7U{|jQH~{;bicOB}LHCE^E65*4@b{0x6oG+&!v8HJNdL-+ zo{h~PLNetwE&2)3ytb;L>XwE`rRQLWm{{+Xn)>RT0E4IEdtyrSN%vdRtQ7$~GDuU- zH(aMg*$Zc;LBvO?C>0AZ5)Qz@Nq4>#m8`m*V3>hm z2NLrx01PayP5$8@xQ6Hn8f#HcKo`JgZArM=EPPHm+*5G{JvR3_U?H)ska@hewznVf z=xI7vN308j0o)SQ$5P|mGtFnCtQf1#U|Z@6X49(@PW<6FEKts3SrKMrCcGYqTBDtT z51M=#2euf&Z!K}BE}Nc{VOp`Gpu=?B$2cv+@d@baoYi_dq;sJu^%+OPjI!X>L_7LZ zzuKyXxk+4>&sJULq63TK*`d#j9}h;-@OpkGYLu9{oU7XTK#Zw8 zANDgKcyojD-qoJ#rkYu20mE$bGChO-;Q_|Y5JTdV2eE(41M0u>VBl!)0I>Rtim6(d zJ_fYFosD9sl{F>E*x80~b}prdn9U|&d%%cvxTiab-(^%V0k6y)&zoQc(h9P9D?JSr z5G_0c1ufvMkX^iYbXylcAm;$$YrE6Z4-)W@QQ+{egccmdXy%2>Q`_*#90qhG!T^OV zTLqS=slL?nUEp|5r3LwwK-4E<500ms({FE*2!>l%kDCA@cdL9alQqmg^V(Si@=5+g zee?PHuOlA!^IKK!Fex)Yh!*(p3oW3t@Zy*~ZFU7K zekJW0NDSbu4+~Z`R5T zkE!R5coc2%{W3-Z3kB%M&7u~Ygv;!$0YsoG(!`D#$Pr%P1u%z}!=Wc-dlbFmE+i7| zGT&=nIF@%iZ+FBEVCe{8@Y)A!iob>O!(!T^F+IB&kaB7IDFP4B-}3Ph{F8r0`XNdf$NOY)&i{ZBY9j^*Ie?{!V^hXu*)Y>$`>?!Cwl`PezMGV(^=&bJg&aVnud^2 zl1RslNL^V;HS+9gbysFmx3%dzj@v)_BHm(LgwjSe`j9+aPZP&3>nL3Kvx5}pa%AHo zINeeH%OX{krytBJdJ!q!abIl`&FNHvu5L!@+ZX0C21i;f0y@JIpSXt2HDc-DJbQyy zkqXwFl3`8LS+!e+I|&MPr&N&S`Fx!vm@|>ah1z!-GSSWFgenWrMDYF=dv+)?10CVO z&9VIzXkY)SroOD%R*<$$i>LP3V5m#94V=-+Nc#pxscKb$PHwIl&4&5gOpRZ>jT8)- zcnfG;WR-c>`Sp7G7#}|iWBLHk+gyrIgD*eurp`a^?(ZqG-_m;wX>!zV)=fbjsBXl+ zQ;3Ho=Mc9JLYkYENw=)N8z8(H?XbmBoaqZ2iDNuu6GYzNiedAI;Ii*~oO)PncNOx` z`jOokDCqd;8b&Lmm;}6Mhahf4Bm+Qd{-j#Ju*jmdkxjPNpqUL zLYIohoCjIweIuSd<9*TDn24(1YHQ`zwmCjDD1fxVZ9A;?lEJ}8Ui<1r>_;J;r4y_* zRPgwAYwq?9rAw|yhpgI@P?{bDQfTgSgEdo>i){4y*-L?I28Uaky#AbXU1*QX#;k*l z+96|VC`gCvHwygHSTGSoZV9fxf13Y|L0Xywo6rr`xd0FySXMI$-Hy>0Jr783-;&KK zjC$zFqrUnW?lWeFA0vT<5y4WnjKk8B!xys7SBYyne}deJ{oY#XGjU}={JSOj=O*&k zk~B5aGc>Ye{1e@U@zd7+3~)nN?l1}Nuu5wzxwG13Z>`d&_dsS)NVK6O6g|)PunFIn zq|r6$Yh&KtURQ|5e;3CIO#XzKp$a`hhHf#Afz~S6f4i`~=Lb0}nJNLBRIEEns!(Mc zC#PbD>rh(*Q&7m2`AXB8=TrnHSQFF!1vHUA3wG#*-wnK|z+sA1Z#n!>x8=SSS5E$j zuNDe14!%ah_M%T1Z6kPu%yF}fHn3Jy!?V=8a=*4Jb+!?B0uE$v@fnZO^|-ub29!LC z9B$nXA~1E9LvS^t;nmvEUMx!J1(ahs#U1DEo-^!d@aGqRrb54{hArL&bS{%5F6WMfzF|PQsaSbih@G|Um^ADY$neT;FU7v{-66&^q_E_aJ zO`L%KFL7cF05}-gG5tGE%pd+UC#ahAp?`7GJN`TPf8@mKf!&?Z*HOtt$?uSg!$xwk zJl!aM9|4qpk<~XrsRVUgqN*g5su$_&zV#44SE zl3&?s0ivT2YK1LMJH%m^g8OltfQ6*KWz@>|O0UX&s_ImQe{%AL(zW6roUr`GiNYUF z5?{>??cqK-f&ZJ6yT3RA|5r{ZJ~?SqM%$v9lbLwL+<>K8&$ajmCw9i4oV@)f8TSkE&c1zx#^jXboAt2E@4tiZ@O9zYOD5cHf$u|j;9 zom(z9n`E%B%n~=u;AryRd)OP}9-TfHtE>ReNLrYyRKX_sA~(ev>(mben#&)){_;%^ zqOr&^7QH@qdpR^OX@g*fVeP8>cJ@Hd~g6P2~<-YJ^SLHkE`Df28cH;1~17b@MA^<)M8)hdN|C&rqV{B=I)jm|NN;StP>Vei+ek14l{cFmvsG5eJGfnKEjdo9J4v96(M`m|j4dR^ z>bqkDS3_sInMQ_U<>6z#bFD#?wF5;W+;Hu6eCh4Q*$7wA=R@J*INJ{EU6=;NLo)xUxNp}cULA+Yh2@(!^7 zjX-4nKY{rFg+To8D$3r-!NJVh#QqO0jS2$RpOt9G1A6RLOoCuIG068sIZbpoDDIvC zoEk%ST6aNg?JY7i^hO2x61FwwcINuC(ewVaeIO!phrkvBV`5;{+*xbYkV!X2tZ>pe z2MKHkxe$cB6zMB+D}3J&b;cb8xss$UL*h>Yd3s%{UgGlmU%D=V|4AbxWY1RmscCeUL4%?Zm?zD`}1u zL&J~Xjx}zJk#0-;ReHqX66U|C<>V_6=v~fRpo-fM^A5lxyfgKX=pQ(1PqzK40TP$q zPJE9}=|i9GGwq%y(si*7TXHK`bL*7Zo%IR;)Gb|@Z9}BrDuFT0a+|A%?0H=`aPn9+ z>K|0|Cew4jf&Gb?5xO%rtrbRlx z1oeEV5t-`zXC(uLVaH~WgEC^@>p~z_-II_ol}ZZd{u38&i8mRP`t+3oV@JLYScR%e zt0g3F7^$^jO0%($;a;e``ge6A_rf>DHddch0o`?tZcwslPjqy*o`IhMzM4}x42qm4 zV(IlsTalJgqp_ymgpfG)fc!0O(VGtTw(ScE^G7in>wF#(r9&Y_%dv*AT~VXe$S>ac z?nHrH4*tK$dYE>QioLQL<(5ha5H7aY6>q zl4d#gjG&aM!A4QwXL^)P4JvIm`9rF`$jpy^jd6tv$#0l4O2kp<@vCAst3-zA>GMqg%mll zaz0JJFOn-I9KZAc)$Uc;t0L7{Mm+YgNRoM|6In9u97ys~s-`VN|Cw6i+$gjaFnM^a z{8LUKSiQoDbJ#V!Yc%KqcIUJt751{{ebUvw#)0#sc{<$y*aGhzOlx=!x`4IM`>Bnz zx4-csM^tm|a@y1BH{3(=YXjSa>j!^4;7>;E8kH@P`s70L-xgTt|K!5K)X2(+{;&7H zVq`&83oyrk=6R&5=4CSFyl8$A0D+DOMat^cQ@KZK9ECb)od1D8ES_#irO63_UAq6g zx4w6_=MI<)AI}ctcdk(BNT4~-;=p1Q9}gI}4D8;p+{`j4U>N}uqFRBuO6f~JW5cYO z=b*u2I#i(53$(8+CnE|=Gm|qh8Z1zk9B%`%3UU;Q!@h*KA__TTN2cKBbC=I0X7ob$uRm8-Lt;r8Uc0a8pOELwCYoI%W>+W&!)DI{xA`&9m`<4pSw;?nFGrdzIE~TS)8IKw~cQL#C`!Jl@jG{0UgCX-*785)7mntZ(K8~Fv<0j zBKRFX3`BM~!4^FkVv6^%-kNZqgv_Bn))S(X_FA%{$K=*w*!#Km-` zo()dbpJPmkzr+S>*Ry;hel;#4NNZGy$uFlzBp>EpGfr34X!J`}reQDa0(*I8n(`gg z5vRTSi3+D!4w9K)#F#s1LA}a0M9yYCR*YK4`1$}|jrrT-P&$(``(x~YWbT|OpACfR zDZVCK{o}7qY%c3>Nb8^b4Ux_P?O)rfvz5j|rj=E{pTcL*^9?60~4Y@{|*AKpiOR<`Xlb^zE_ z8!BRN3>i;Euy{sHv;FQ00)LASCl2+B2bhT9);=LGpfIT*5z z5@<#QEZ|JpGSyJ=eSS5*j`;6(wusOh`-|?&I|pImc3bV`x46`|r3xG-^ZF?*lxFj0 zBa_fODg&Ym;EqN_)Z@7EwnlVbWc>Ls`!-4tTCu z8dQ9p8d7D&+sRfK{%wp1k@AkIN%wOr*TnH+y~YS)Xxkl_i(Lq2OxOZ%=+d5{Use98 zX_q;N@p$F>b*v>ZTg*&gI`~L+2=9f!cdPtk-1}_Hr;!qC527NIt4cQj`A(x3Dw=;7 zoiyafT=Od?LB>6d`Pm}z*Y_%ZM{aMFLg~niuVPQ$4S)YQvE9k5oBWyfWT5{YuZaIl zd-e{lmPUU`^ovYQ=UuV5;jITno7Wlu59{t``!{z>P)#09%{i_Go|gy#I_Ah;*p_k& z-ZzQwK62V4l0wvqVfP#xRt#Yi5lRE}a)*2$Z+!1AT8&=O&Fu%qZ#zYA#qH@T9a#X3 z@!v~`Y~AlnZ+b3ss~09`gO-iz!vxk{!|mxGkH5NKUz!P=U#`ksmLzz)H{TvF`X|S| zKJND}F09{Cx~H7HS33sJ_B)Rz?;V?Dy*9i$-Ur60XzAEqy@P5%02hi{TB!MoWt{(j}?*wJ(LqmZb&{sM3X;3Hjf zPrpc(9bd5f;IWvHK68BElhs*Dzc@Ebykj++p3IK#qGSDK`^xREbK!hay_VhlRQ&{4 z@h(dL?X~oLzh}v&L&vwi4oI#q(Ax6yXw|t2>mIJ(c=u|~K-W&Yg6THQ{vb2+nDtpQ znoWv60Y~s|qVIUE`SCS;Y0rz4d8*4ky~*WadLPj%yOMWZ=;ux1u+~%Kg5}%qM~#m6 zohABYPxj8c1eT-zNz0f1+?^_lDFD4EnC~csAbk^$lF@*Wbn2&Eu1-wr|1d zKPkO$BwyQJTYh}qy)$Ni-+4A_4f$X*o2zbjvGscME?P+#t7hNosE+@Le)hKN9-;$wbgiKRr|*eFYO#kGUzSQjhs@Wm=c{5@d2fmQ~mcDA!_T z`Obu93vZ^CXW$)L^+x7)pM1Qgqp#~7n)^m3a+tgfI6U7!i>UJOOBGh->y=6?&(=L0 zS&(H=+&!g8`C{`MHKM=T%kSpu^P|fIFMZx|FB9H5#VpY7uMg!duTQKFCFA{GXGbee z_l{id?q-GJDU39&eaR2SJk6z6+eKb00JgJgBW8IpOXJ(;d!{v7=Ln+X+z&Fcf<#L>X)bK^u^@n6^~H__6AKp+U~A%O&#sJ9Cg-3 zRlGZkrH_Vn#`?DE)kF)-qo-zaqJX||Jra8~T^{`^u-ltZ9AT$!0{J30sIpB`zKr1!g1!b4)u=CcFYo}kxfRMse#THP#;j4VOal>qGS>&8qc8R?j;DwO!&=>6aB4-K3 zjvfXNd?=KQ9I#K>1fD@Q800foxQwK*Y4`+70;p4AxFxp$rw7zEUEh+sMG~`u{iq0m zCjtj7A}L&pR2vKgA^EWxfHAt%H^um|SwK;_)CYYYXs+vn0Ty9Iu8k|VKl@^6UmIKf z!^rrb)_)z$1M&kzko?&!VEvp1o()kyTGal>?9OPT2(2 z+4QJa|5Q1swY(vpNjO%NBwKz!u4()$laV!sR`UJK;LKn`%X(?&T({-TsaA`!)3T|W z_No8vp?Tj8Fu1_G^0_p_a9lBh(T`6KW)X`cO%HIAZ~4Yn(=;dF`L=HFtXh{n+}A4} z)+VXlwdx>GU7J=1gJ}xaDx&C**w!yU)u^ju-1W=!EN8JF2`KUrbiV$ zfxLeQyq_-g=xmI=E*w0<5`)|azMn4j=yZ;~lI`Sdi#d*Y@r0|4Nvn+HGQwTDXsgEKD7T(e5qyd(%`nLW!uUKP@de>rIIqs z7*b$>^uz*D_AR5K8_&#}OZ)dyo0$mJ~Z7co)8?X?iynP>-(pxK)*Dtx?CqK2e+v9ZG>ATmEz^7$@p46nq3^E@eV z!$An)Ieb_e%x>j3L)T9C(r<&B(XkN-dh0hH*Huie#L{s?k2YrW#GgP-Y{%&mzzS&3 zLrnaoVdvVJHemqEE;QYm!u`gp87JglAieAn0IgP2djj*3Ygw;6&G64D9nI1(^IK|w z%+1zW3>m%5>HNL?_46m2rip{kJM~H3@Yy=qhg9)XV-$VSX+!z);idFOy9{@Rc{21{ z_PaolwK41I;Nx5qOx*AgH~!W}=$hsuj^~oi^R;`&##VKbj_g+P`6epC#o^k)8h(@P zxuUI1@j&mUmf*_KU`CJ#K1+gU!kC%quzygUEE)&=h1-nWy*zetE)OF17_ZMi2tGDdkIs&)iHdB)m~K;Sq~cMguOyD_3C%y@-6Uw6dfOuVHbMf}8nEO!le- zblHVqkU33ZNfRPRS!F`O>Vi?}FF#W~u4(VxJiYke-!CvIc1SU=)Qq#vdCnN6jRyT_O1%W7o0DQ)ubXrruuL_s%T2N> zdf^{uiLZoMHg2PbNHMN!1X`1NC4AaMl5< zfmfCg!Z#$A0q-#@fou)w|Hue(i-}~voBlj~3h95(2vSLk9oYA_EY!T7jxOe)3L-!( zjGIwQGYy|^qPA{0VGcc zLhdHhXJv=!bY<~k(|5$Fr%rDoL=Q!>KG1pjw=h#%RXJ;nBj4=4(qVlS%{PNY09dfLvlvW}hmYd!e7z~Rj#}sAqgNeI zy%q(V)O_HIXK>-z#;neb<}}GIg3N!{ zxp}yoSjsf49X>cF;G5B3^l7>8Q`GXz-2*?l=ELLrA;%3Q#Z{=O8fv(_wwz`qG)MKa zkJ369r!4+@$z+kJ`}}0MEPHZ^_w)PYdV$N?R9nr}plJgxPl znW%XfM0S$w&3#OcWd4Y2F|J=+vSzgRQ}tr<@HqbfFbDpYNLYC2V-!2Q_6q6=PUWu} z2@hLtR1Mk1eTCwJjRW`wv*NnBg)zJ_?fu^ecJF8im4_S|w$WxdzsNlD3O!XtGwUVl zO%$}vU89}paY6cRp}7HgTVLl&&9hZ*9EO6_KBs2}#O%idXbJYC16{N`i^xh`b}U1taPqg=BqZ|yZ* zL4oV#GWbY~?R`=+Ts-=9b=h?mVIawO;U^Zjb7M>nkj#oHZP=NsxDA6cIQp)CtdM zy38)c5rzNfaGMWScwB?!Xm;IC(Bc6t%VIeQ#K~VyiwpWNyY<>*hi1yEL-x znD9WKZVK$?JB&R#pw?PIF61MA-wncPo;#6qo>~kC6%W5BC-si0Z}MUS-pvq{1cPTu zIg4xD9_DwszjbS+ete{RDLz*wl5_z7?&JZ@ICq20F89-de6K#6=i>H*$+>^cIGH;A zu(jq96B=(Ykk8DS*>gzwaL6!+?JVU4cFfi{(W5vpFuY2z zhVwBjI8d{(M$Up$Au!Q&R@rnjdahAVK?{sxd#{_fPWDj2NFMcVVCpv(rL;z;0HaCg z#G=l0UPdsS%7jkLgYvAwy}UOk9C=&dV?Po+mkVew9hn$ppAZw0H=!G3tWAcmZZdTN zs9xSgFdtGs{%&P2UErVZZxOr+S1pLf9bN`xx+`QBzqV zNf=u!_9DUBLl{5LN0~p*v57bF&N)XNNTmh_a0_ORZO%da#0 zUtLj6LNt`;k~ZqJI?VzTei*qiv@QqT0BuV)6Yu23D7SlngW5o z=N(4dt4V&(GTG&81VY%Ea_RWgWlX4$uvs3(Rmt3MbX|j~NDwo*9UP&->Gz?pwoTF9 z)=gW5sV{e;=$L4STWhAIKYA#i2E?1oq+IYr&m#$xuhxMb*?Wny2GGRuyoF#p z`Qo6aU~M7j-fu3!O;H&OiyizRH0KuWrIYIpyWuB|+O!%(?Qt7t zI~5TOQ3WK~jy)YEwc+a$mIltDLwV@t*Gha5#I^Ya)?!ti4-w>Fo_QbZ>e*w$U!NQ~ zbR_z8sF_6)5yCX}<&G6=tm(Z$zW#WXQJmdKn845|jHmDw{K-@?w?@r;{rFQ~oiazQ z%-F-jIPU33OC3u(5~hWuqaD^O+(|M18B3-EVqH($*mblLJ@m&WI-l_rL! zEd|El^OuVLeT`Sf@$Qc?+y~lhh(fk#L6X-yv-4KWmQKrwi-Ysg#ih3n9?xU5ChZ13 z>!yzmj>-DDZ@1lvp0`&wPJp}7k!52;LuO5;*B13u&uc(BV3pv<3R&j;Zw3RFzI%2<*l1DbPHrb|j zpqA*1+zQ4?nzDKX5OLKytBBc!a{c^4C=AO-E$K}m!KvYZW+sLTM&588?+Mvo?(=@; zs6Xm6SOR_VS7L+!W%|kHfaPhN9JF~DU?APcsPe5L#30imx^u_&>zOE%0Dc_9lhM<( zV*PD(LX>yxgxsZ$DVc*x=A${~b@@DZ@!od(uIxev{RXQz)OV*QD#V1kpv&R|w3$vx z?->+cws%O5^EQ%V4GRv*(?0p;u>%#=LbR&nKK0HseV8i+mJzh8)ndF1l~&Y#9-i3tTJ76RL04%tswr%mH+6$9-9D@wD{QOUI+hsqW+SD>^>a z_gonhf$giZ=Af31+4C^-=6t2dEel>S#)K*_;u8r;OuGHdxA7tY!=sO|3T0k|*w+f4 z!8&13hc$0^rlOBq{&P!Ay|wV}QF!C5l>{0Ozmx<7I*4+IQkcivqTo3pR3h~KeDUJH z>gx`{IY+>sfJ8Ut*2EOg@Fa*wvIN@}ea)3UK323N??voIg!Ed@w=lXUSyz&Ge+-?k z&l1j>snbVBZ#YjeL7`=4y*AJtf%9U43xwiBUM&zw>t|O?+AhYww*T84mXAaQNfv@{ z$4bw~FFOkL@#lYGjMwmwv3q5CwvYCu{0RIA5MFEAVwoPH0ly&8UL7)fqLa|WOLB~c zx#LY!Ykdho1&y!s^(ciNPx7SA3pgT}ji9|wf%zy*P+pNmOoiZ&(B7WR%ZFF76*B>A$(6bg`Z`Sm5qlYxxnOsXG4B~}#Ht{t z4zfg3Uxfp=l7lFu`HZBh*YjVcE?V8BzOeuey%CB=Zd*XV96A!y^33Wl7={-p7vuf* zS)1AWGP^_u5u|$`EcyATSna*heLc7L-6J2GgLiOH9cwqiM|3zZtqq(@S84>@p1TBc zs9vaEIOGWBj6)JN!S09Ms}uaz)xs)WDC6X|ftYS6R?n2XVNkk;A7utdGbpZ+x}QFv zFtGf@ol4xZ!Yq7s%iJ&nbOUsQpvaY{qD~9wfLh2tI7GfPfC@6WACLvT3fGLXzwLaN zJlb6#>;ku+!}(s*gCtcayZKA$2P~MP@&+Lo2^g~wLurq*5=A5^zn%~BqQBUuHP`)L z*89mJ>TJrFIt);UMQj1OzCN1k^xc+0p2V5d& zbAQjNhJ04o`9KJbi=hnd-j~fI75YdJ34KrBLCNzb;Q9U#B8yM-yp->KGKz^e+fT%U z>wS`aXA|;H6^Zi3)=wt%!Rz+7r@Ly=kJ)@3PSQak1*!L0K5<6mH$`l+XF!l-$h+qq z)CLK!YrwHi^n2kjvZEMygi_CI;RjWz&p&zv{O4X2dX8c~QDDbnK2vL(NMnjJqzQ;W z$pUWwIUZ$o@a-(>0%w7z*lKWr=dl$4>VS#@Wi0p@<-6-YktZw(x}J=B^aZka$~|i7 zTql()l)91VulOh;DtFb^MgU19XtRM5%TptocwvZ;%45sP2arQmAE8JVUdM|f?UG8U z4TSjd&)cWHkl7EmBFgckOtU|HzvC2?8VC`KnDqy1hk1_t79`p=PsGBDtp=Qs2PP^8 z${Z7x424x{HKG%PD@%c>*d+mn+5NiMVLON8Cw^wBa}6!AE(8@0VtC<$HcL=bm%o(Z(9CPX+LJLt7gT0xC; zwIYI*q@g7gjodbmn8iw_9r0&Pu?J=Z#AJc=LM_FrP=WLbQGyn@XWQzWvHBQ%Fl%E4 zki-Si&!VI+V#0hZtMd%|^j2|s1V4?98de0t)5Y{qD5$^B9eUEg$#=nK?HSJ&oqi%B zo-Mi~9jNld*SIi?k3|s7m8PYMnYCtbC^dpm5N zyS&Auen^K-D(jch4TVaY{nh7Id1LvK4n^gUjJ1%RQ_kW9bQRIzCY{gj^1F;&@A_1D zQcmusu5H`-ppG@gWRR!-<*pd*MmUWV^bLSX_hss(=o!S+G($5jla-T+f&DlULWWYh z8oLiYPyIKisF1o_0vS*83BUT%H_wpfKW*UVBA3fr7QYp}$$Q|SsJ79hYP0`XZ$bwb z1IW?0Y4e-2FI!jFJ~*e+fVSH?8IRaek2VYn8<@_Ywu{7u$L(}EK3J&co2T(;H@Cdn zzhuab<-F>rcZc7_uX%;t)q|s>^9X`R?MKh^7>Dnv5JaE53)_qN*^=1@B417r*#>UF zzLW*hvqe_&c!&$LcLdYVEVc*HUk@!|cJT*?ZA&6{6%>agWO2g$mJR9r(*}^OIWvVO zW~pkq1qO9IILs_BRxCI@4OM4xgMYf;gIIN1+=Bo-JXc^HN6L=vP-$Uzwc$RHQgUUlo@Rhbm&;H`pi5$cRmM%>bl#Obld&joyhU= zvf1(9yMLDlLCK>^zX3zFyd_ zRsKDYMSZ^?tH~LVq>@5bYCvzH5n1vKosUt9#st z4GHbqkaoxt@>CYi;ig8#qp=f_SSfO5ng9-FgEiL^7C1!0X@Z?n8{6mDleQWZW2(4_ z5Zu9$C52z~iU;|QT%0~c%}CMl;*lrQx!JlcvxXrMXOc_1P*)7gX7;l<90J*!GnCH_ zjGD*aS2RZ&4V9{#a-aGY6b@wJQ=Nb}o~NLeBz?gF`Nm)qjv&f(bI{7!T~b%NhbTFaG+IMD9cn zo*Tp}FHb^UVt2d#2+Na`($8cq|?N}p247aNO-L=IlC1Z)ftJbuMrJQt!aLA97wn#8}sqTTXsK%=bP3HmBruQg+y7E4hMzSEFhF4vm1@?D-r!`8`%_f&=IDLc&3M{y+k-7 zUTYDT!br6-cld1So4@dlqx?#Zi5ET2&jlBj(4yq6{m}#)YVDYlf?8}!(;eE65!sn9 z&N;>FKrU5W8yCvXL3%BJA681tErE@6VX?gRtAh2ptO#GY5pFlXOyw_taeFze3&cXI5z@FBZb}$Xa0u2{;j>(|1r^;mHdz{uJlKF)s_C+FFlhbXsPa^He-CcEr zt?I~FwAXcYOl4`mEdnhxiWvN<3V;H97MV?3l?q`@Ny$4u!%;|?O1Usa*-W;X8vbJ+ zR@2t;wd^xke#fHHl_`A2it*M#epy19!bDlav{GSqQoE;iVv}Zeln=TRUYYW! zn(u2r%Nv!n)=wg;4X%q5I5qngQ}cm{vi{SY7|_pgkX07rpfWS_^u}!t-_d|T^sNCs zrH~1c$n`h;L^#17Qoij2+?TS?bK{e6OzIEjiuLJ33U&zO99(Fx)(URc0W8d9rUPI+ z^Aa;VW>oExGjlIDLN$=F#XOo;eok|05;MMJrW9oblae#vmZA&5B2#i!^3RN*Ik|g9 zouS>Bq`X@=qA6lj|E2S+*`Lmst$ARP$vN%kdL4COcv;km{V;BvdIZN|${y5JIhia; zC=R;NtD#+zzfc?|6WJ*RNXSj^2l&aSoI;4=G4cHD!~5YOZJ2gz2p zFf=x!K)ne)8j6F9MPFUF&k9B^wU1tNKDg?%;N0h~%C%XqH2!W38uR+47Hl_tS5E90 z;c8D@R=nQ|qw=t)t(>AjOJbI>AAs3cR|;`Rjq#c|BCsQMDrJ!-W$0E;;sf?>Tv`$yj>6oh?kjEG z9MuTkL@LcaFlK9~IBS({@+uIi_Z5TBm@mkSl^H2mCel5c-T*)AYQM~nQvX#y{7eiW?GQcz?x95m1JDGSZ39-5YN z*@VnZTURE%PTejustHPeS`sRNQhY0|vqZZMa;H?_0tgP1eWT@KEhzfbe+u@rc5?qj zhI4jCl`F@mkiWuT03l8JeB;hBkugDov^#-duUV+CE=atKI7Msfpj zAM$P8M^sJsFUX4_Y2ZZ=N{g&fvx{b`6mGuJPNzKhvU76!CCrVw@ znAeSE3ht_nAZ?f_RGy1TYC0u9COJX4V($39AP!|V^-v|e-Yiyt*KI-) z-%r5Qu4cXKWqjaC-<;e1k7K(D^F>d(Yfp5_%9aZ!z0)WR`>fstB|bEqfY?6t3cWd1 z=DY8NTZT>MppI-I1EGusd-QDi>JO?PnentHgAwpN3t&oi<9wTNI z_eZqK<$Zlt1P$wBokBc_Eo#?;vgShLvw>T)XT`9CJS#^*O|=%LbqMB&(2hDh=*L(I#4k7-*ikYW6?FQEjkveDC)I9l(Z{)+dhP&jAxtijx;Fl8X}$Y#k^2 z5^7%)9OS=f>SioAr4d}Y>u@foZW4=nBimY}QSAp@3^McEXUQ#SsfjwGgj%QsrZE3h zQ#piyz>gN4%&R~lRzi4CLEbIrd#e&D5yia}@r;7}G+aSK{5(=lQv3iyVi-o~t!=`) z4Jw%|`{*_j9yK}*mJKz!7NSd~iHnuw;Aao0!aN5hPa5=_J;9p7yn$#XYK77gHEIdO z7pmki(r0ICS7XY;ysR%`D}1Fy{2P3spH8g06f>hlJ>Q+nxMO+zNj6KBDs+2Fl^l4s z8?$yAY0=x2sEHH8X61iZDh)&<_9lg)@F|pv73S%eQu1I`{h*HH|7urSVgjK+j8UpI zUn^LcC!B{Xb9puVI=AfNSD{25#JH_Q-G+ayF@3$76vji1jw~U3Qu3@qy*nJND=8dG zP^wg*AYZA3D~55XTX^;fHU?3zioi?s;Mfzx zsvtWSnLC;a^|K{}0~F*r%eoT79&d=2736nCsnJW7l)j=DLdra7s{3bcslrzv9~4va z=Oz}Rr`t&|?I2XnOo?LqzUn>qWPSe{7v99HIQmi~vaQW%qUU1C{cXJ>)s2CSk$hy& ziH(|t6J`qV{fm0#c@e}vfuqu+XZR6_j-)JfNCJ28qy4|}5|-mxgMWYronwsUImGQJMk-rISr zT+XBho5U^+p#{kOc&poPqdty3nTE8g64#1&uVS(!C&T5_^f*NP&mt!*-I}3Eo0T-g=dyIDba0jT0Z-dSy4OfOzF^o926zzsTfzWCS2{ zzd96t!g~FL1?2Sy7M1W{u%hGkdl5^Ybo--&@RS~Rn)?rKGycdMZn35NL)8xMVe$V7 zDvJMrDi-Z;RP5NFsIoy`YLoxritLqJ3V+xmYx{z=63Z)tMFrH0@UuR9E%~g^UQ?Q( z@V%aALv|3nY&s2r__k&0Ht%Ks#`N-K-lVxPw9HqQE7!HsW`w9gT_al{2k$6`|MumE z;Fm#q-P+Nu@#JUSQ-sqIN+ZE{HkszF3QL`k=Mi>kLjP0fVNPWSO6*p`dG}XHOb6m& z&uqbrMQqoD=F!pb;Vsyk+lmMM@nOeQB7R;7IGknXfq^IdU`Rqpw>V}$I7E2rsj|KA zozJpvLDM-&)HsJA&ZHqbI<2C9SPRN7K67(?CC>7*Csce&I`a#wJGya_JN{`Zct=kK z{0g(TX`AwrY>@8|zfvhq2uBF{7qB7b%@R!nPg*%TZohi)5Zgx)7ZNJKfKYCOD8qnM z9NT=BCj(bi1BwgHH-!Thf0q(A(>aH*f0q(@Jp#zb|58en%|KI~CWg*3+q)R)KaN=) zeicbGFDQkOBZnv}g-|2^Q$dA+IUD7bp`_3o=_~$Kl=XN+80oYBRg@`1Ba8_(|6fH} zVyKLme6>^pe(4YmYX4FTiwUDbHtBD1A0|e*-{L(?$R;2!s-!J6sLJ0^J*iOBnLvFq z)F9uPDN*}V$#lICN+fLL$lEf3bY<>H{lUJFmqS2tmVWE%P~+|W)=iw9Om0)=TBSl8LaXIJE7v%rr1L^u3@CF4+Naq z;%|T(oe&$mm4@2nw^q9c=LKID{`ZJMM{#f-D`(138S7a4v+>zUMp8J9Ms(uL_z8YJ z(|`D2NzSm`5|eK=b>{2iH;!((l?L#2P_C^AAXEjT9!FE_qEh^B`?z{+_e>!UCEg} zIO-M>zIb)}u%iFkgmcj_Jd<#LYodI_LM){aPG!=b1(cbDr!bmZ#))Wn9p<5cBB!BZ(!t0jqUx>808c}wqTPiF;x;Cd}CE-|{6aocZq?gQ?8OZnP zmCj!w6GFm)%UR6fXK2*31*`AgO1v&`D(5)J+%wLNr5v14w-*UW<6ta~i&CnZUu;sP zz4JJ`?547)!+B`hgpGSQ`M|ddwsDVOXdHMvcDdJm7XHP?&y@Sh3kpAd)D|zUFY_F; zKV?7GePvDdn>gRzdLS>~J+;CyRi`BaDO^X7PE88FbcLgjUOq#pDh zIyUXRO?R66oGIUvS^2{hPM|nuivNfi)mH{hIt)A>55L%SQ7_IuAOmkyWOI+yGk$Vc zjK`?YphBKM)T)4X9c`mBU+(-ggZ6SKMV!`RTf#G8C}O+5@|GRF>;XPeQ@8N~z5J%ekUJ9iq3AZJECJ7k+j~O#i*%PwOdA!p^`^@Kao5{rG zR0VKT^cW3f<%eZNS#brqL=+d6c#?-~Axc{&9yFnVWgEA~cUY9372nx#TFgfZT^>io z*HNx{-wkl#g=G*@f%Jnk1vgPvs1V5zT%(jk-*bAFnY8UX!ZCE@3OjDcC_36AMw0%7 zZs13*l+#5)$x)-$=_Z;P-4O=TQnxAN$zU&zhqTnr;5NxE`b8}8$JG_;x50AeX3P+E z2U%ifAZYlQdi}eEC~xRbmc}zq>ztVL;vKcv5!Uq}`Vq^}lhBVNBhA zvvVK20gJgKu$X!Eo9UxW+3TRFFt?}H;jdE5D~jBh_}^V_x|=VMbDspdAboUOylwa9 zNpGEnzvA0nD&VJJQtq2oGh8O8*?ZeO;aQ#4IlJKuzc&Gpdfme{o1&3_%pt7l1#W(Q zwmWiQb)*~7x!A!UIjiXn@V(zdeZTzVIO1E{uOpj_`|!-#HD4H?UKtX`g{?`-Y(xi& zrr>z?YD6;*(!{!Z;2bn9c%V6H{qFp5dfe^{<6-d;Y}y%9C#GKZn2slw-dZgxDN*3) zS;X_FpkzuQs~$%$-suuFU)w~rDuJqI7B!Ezv4NSz>?#XZJuue|w$HZZ$D81jTpVY| z#uo5v6J!l%v8R7FL&)*$u{}n`c&Dl5owZnCNU?eSJgXiUQXIdbkNDQokO|%59EAMk zOIEB=u#Y!Cmvv0m+TdhXmyD%TSdQL~R4l=eVq6YLZbY}~S3z5Ui!~DPsC-#N0Z9;g11+T0v5d^+iKf)1-=Yo^!_xh*f)rdy`obfm`{IA=EsT@ z9k$%WV&U7XuCog=qL62vCf|sc2a5b^n#d;xU+>tEJDDpHtsbIGe5IVv6JY+ts0@xp zlV-~rQ4q+r%lZJlWSPz?h?<=ThQ_Jbd71$t>1hkWY%`UUUma@zQ_%;6Ic+%@$)#e~ zsN)jgHG3~#!l7xa!Qd;E^GqF^01(7`wHo53y#bi>q~c?#0|tXzS_CL zeqohETOB>C-m5`=ZEZc+P^oeitAp^#l3v?mti1);yQg{nv*7>e!_?jY{F9)v_^Bhp cYN-ab;V&XYA2`Yoh#By52fknEaMioN0HeXP8UO$Q literal 0 HcmV?d00001 From e9446e4680ae509b2950c1611163bd09c6cae4bc Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:01:27 -0300 Subject: [PATCH 29/38] =?UTF-8?q?adiciona=20instru=C3=A7=C3=B5es=20para=20?= =?UTF-8?q?install=20de=20libreoffice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- compose/local/django/Dockerfile | 4 ++++ compose/production/django/Dockerfile | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index e668957..348a5a6 100755 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -72,12 +72,16 @@ ARG LLAMA_VERSION=0.3.14 WORKDIR ${APP_HOME} +RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list + # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ # psycopg2 dependencies libpq-dev \ # Translations dependencies gettext \ + # libreoffice for document conversions + default-jre libreoffice libreoffice-java-common ttf-mscorefonts-installer fonts-liberation fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea fonts-dejavu fonts-noto \ # cleaning up unused files && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && rm -rf /var/lib/apt/lists/* diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index a2189c1..8ad536d 100755 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -50,10 +50,14 @@ WORKDIR ${APP_HOME} RUN addgroup --system django \ && adduser --system --ingroup django django +RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list + # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ # psycopg2 dependencies libpq-dev \ + # libreoffice for document conversions + default-jre libreoffice libreoffice-java-common ttf-mscorefonts-installer fonts-liberation fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea fonts-dejavu fonts-noto \ # Translations dependencies gettext \ # cleaning up unused files From 27c071bcfd8bd1f1c97cdc707a8f2e6dc13d4040 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:01:59 -0300 Subject: [PATCH 30/38] =?UTF-8?q?Cria=20mais=20choices=20para=20rotular=20?= =?UTF-8?q?erros=20durante=20fluxo=20de=20gera=C3=A7=C3=A3o=20de=20pdf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tracker/choices.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tracker/choices.py b/tracker/choices.py index e70c3da..ec868ef 100644 --- a/tracker/choices.py +++ b/tracker/choices.py @@ -32,13 +32,17 @@ XML_DOCUMENT_PARSING_ERROR = "XPE" XML_DOCUMENT_VALIDATION_ERROR = "VDE" XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR = "CDE" +XML_DOCUMENT_CONVERSION_TO_HTML_ERROR = "CHE" XML_DOCUMENT_CONVERSION_TO_PDF_ERROR = "CPE" XML_DOCUMENT_CONVERSION_TO_TEX_ERROR = "CTE" +XML_DOCUMENT_UNKNOWN_ERROR = "UNE" XML_DOCUMENT_EVENT = [ (XML_DOCUMENT_PARSING_ERROR, _("XML Parsing Error")), (XML_DOCUMENT_VALIDATION_ERROR, _("XML Validation Error")), (XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, _("XML Conversion to DOCX Error")), + (XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, _("XML Conversion to HTML Error")), (XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, _("XML Conversion to PDF Error")), (XML_DOCUMENT_CONVERSION_TO_TEX_ERROR, _("XML Conversion to TEX Error")), + (XML_DOCUMENT_UNKNOWN_ERROR, _("Unknown Error")), ] From e68ebd737e75468c45c11d4939bf9d36db75798a Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:02:23 -0300 Subject: [PATCH 31/38] =?UTF-8?q?cria=20campo=20docx=20em=20XMLDocument=5F?= =?UTF-8?q?PDF=20para=20acomodar=20DOCX=20intermedi=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/models.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/xml_manager/models.py b/xml_manager/models.py index 2534978..4a1ad7b 100644 --- a/xml_manager/models.py +++ b/xml_manager/models.py @@ -45,6 +45,13 @@ class XMLDocumentPDF(models.Model): upload_to='xml_manager/pdf/', verbose_name=_("PDF File") ) + docx_file = models.FileField( + upload_to='xml_manager/docx/', + verbose_name=_("DOCX File"), + null=True, + blank=True, + help_text=_('Intermediate DOCX file generated during PDF creation') + ) language = models.CharField( max_length=32, default="pt", From 5d137f5257065203667475b7f5aa90931fde0db0 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:03:01 -0300 Subject: [PATCH 32/38] Passa a usar LinkColumn para permitir download em listagem de dados --- xml_manager/wagtail_hooks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/xml_manager/wagtail_hooks.py b/xml_manager/wagtail_hooks.py index d57720f..55e6719 100644 --- a/xml_manager/wagtail_hooks.py +++ b/xml_manager/wagtail_hooks.py @@ -84,7 +84,10 @@ class XMLDocumentPDFSnippetViewSet(SnippetViewSet): list_display = ( "xml_document", - "pdf_file", + LinkColumn("pdf_file", "PDF file"), + LinkColumn("docx_file", "DOCX file"), + "language", + "uploaded_at" ) search_fields = ("pdf_file",) @@ -102,13 +105,14 @@ class XMLDocumentHTMLSnippetViewSet(SnippetViewSet): list_display = ( "xml_document", - "html_file", + LinkColumn("html_file", "HTML file"), + "language", + "uploaded_at", ) search_fields = ("html_file",) - class XMLDocumentSnippetViewSetGroup(SnippetViewSetGroup): menu_name = 'xml_manager' menu_label = _("XML Manager") From 8e93c7b7f7e143343a44d5618ea6e1912c76ed19 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:03:26 -0300 Subject: [PATCH 33/38] =?UTF-8?q?Cria=20exce=C3=A7=C3=B5es=20para=20lidar?= =?UTF-8?q?=20com=20pipeline=20de=20convers=C3=A3o=20a=20pdf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/exceptions.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 xml_manager/exceptions.py diff --git a/xml_manager/exceptions.py b/xml_manager/exceptions.py new file mode 100644 index 0000000..89184ee --- /dev/null +++ b/xml_manager/exceptions.py @@ -0,0 +1,14 @@ +class XML_File_Parsing_Error(Exception): + pass + +class XML_File_Validation_Error(Exception): + pass + +class XML_File_DOCX_Generation_Error(Exception): + pass + +class XML_File_PDF_Generation_Error(Exception): + pass + +class XML_File_HTML_Generation_Error(Exception): + pass From d86cf3cbd119b16e5b6e0aedc91d8656e3304e77 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:03:50 -0300 Subject: [PATCH 34/38] =?UTF-8?q?cria=20utilit=C3=A1rio=20com=20fun=C3=A7?= =?UTF-8?q?=C3=B5es=20para=20validar=20xml=20e=20gerar=20pdf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/utils.py | 74 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 xml_manager/utils.py diff --git a/xml_manager/utils.py b/xml_manager/utils.py new file mode 100644 index 0000000..2cc5c92 --- /dev/null +++ b/xml_manager/utils.py @@ -0,0 +1,74 @@ +import os + +from packtools import data_checker +from packtools.sps.formats.pdf.pipeline import docx +from packtools.sps.formats.pdf.utils import file_utils +from packtools.sps.formats.pdf.pipeline.xml import extract_article_main_language +from packtools.sps.utils import xml_utils + +from xml_manager import exceptions + + +def validate_xml_document(xml_file_path, output_root_dir, params): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + base_fname, fext = os.path.splitext(os.path.basename(xml_file_path)) + path_csv = os.path.join(output_root_dir, f'{base_fname}.validation.csv') + path_exceptions = os.path.join(output_root_dir, f'{base_fname}.exceptions.json') + + try: + validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_file_path) + validator.validate(params=params, csv_per_xml=False) + except Exception as e: + raise exceptions.XML_File_Validation_Error(f"Error during XML validation: {e}") + + return path_csv, path_exceptions + + +def generate_pdf_for_xml_document(xml_file_path, output_root_dir, params): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + if not isinstance(params, dict): + params = { + 'base_layout': '/app/docx_layouts/layout.docx', + 'libreoffice_binary': 'libreoffice', + } + + if 'base_layout' not in params: + params['base_layout'] = '/app/docx_layouts/layout.docx' + + try: + xml_tree = xml_utils.get_xml_tree(xml_file_path) + except Exception as e: + raise exceptions.XML_File_Parsing_Error(f"Error parsing XML file: {e}") + + try: + docx_document = docx.pipeline_docx(xml_tree, data=params) + except Exception as e: + raise exceptions.XML_File_DOCX_Generation_Error(f"Error converting XML to DOCX: {e}") + + main_language = extract_article_main_language(xml_tree) or params.get('main_language', 'pt') + + base_name = os.path.basename(xml_file_path) + f_name, f_ext = os.path.splitext(base_name) + path_docx = os.path.join(output_root_dir, f'{f_name}.docx') + path_pdf = os.path.join(output_root_dir, f'{f_name}.pdf') + + docx_document.save(path_docx) + + try: + file_utils.convert_docx_to_pdf(path_docx, libreoffice_binary=params.get('libreoffice_binary', 'libreoffice')) + except Exception as e: + raise exceptions.XML_File_PDF_Generation_Error(f"Error generating PDF from DOCX: {e}") + + return path_pdf, path_docx, main_language + + +def generate_html_for_xml_document(xml_file_path, output_root_dir, config): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + # ToDo: Implement HTML generation logic here + return \ No newline at end of file From 4b4202a9021461f4d6ab81cdbf6df013ca80003b Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:04:10 -0300 Subject: [PATCH 35/38] corrige imports em tasks --- xml_manager/tasks.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 59af1a2..8d33b57 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -1,14 +1,7 @@ import logging import os -import tempfile -from packtools import data_checker -from packtools.sps.formats.pdf.pipeline import docx -from packtools.sps.formats.pdf.utils import file_utils -from packtools.sps.utils import xml_utils -from django.core.files import File from django.conf import settings - from django.contrib.auth import get_user_model from django.utils.translation import gettext as _ @@ -17,16 +10,18 @@ XML_DOCUMENT_PARSING_ERROR, XML_DOCUMENT_VALIDATION_ERROR, XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, + XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, - XML_DOCUMENT_CONVERSION_TO_TEX_ERROR, + XML_DOCUMENT_UNKNOWN_ERROR, ) from tracker.models import XMLDocumentEvent - -from .models import ( +from xml_manager.models import ( XMLDocument, XMLDocumentHTML, XMLDocumentPDF, ) +from xml_manager import exceptions +from xml_manager import utils User = get_user_model() From 764b54cba499075c77b52744f1279e3bb633ec0e Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:04:32 -0300 Subject: [PATCH 36/38] =?UTF-8?q?refatora=20tasks=20para=20usar=20utilit?= =?UTF-8?q?=C3=A1rio=20novo=20e=20tratar=20erros=20com=20detalhes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- xml_manager/tasks.py | 143 +++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 54 deletions(-) diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py index 8d33b57..0c5150f 100644 --- a/xml_manager/tasks.py +++ b/xml_manager/tasks.py @@ -66,20 +66,16 @@ def task_validate_xml_file(self, xml_id, user_id=None, username=None): logging.info(f'Starting XML validation for XML file {xml_document.xml_file.name}.') params = {} - base_filename = os.path.splitext(os.path.basename(xml_document.xml_file.name))[0] - - validation_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation') - exceptions_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation') - os.makedirs(validation_dir, exist_ok=True) - os.makedirs(exceptions_dir, exist_ok=True) - - path_csv = os.path.join(validation_dir, f"{base_filename}.validation.csv") - path_exceptions = os.path.join(exceptions_dir, f"{base_filename}.exceptions.json") - try: - validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_document.xml_file.path) - validator.validate(params=params, csv_per_xml=False) - except Exception as e: + path_csv, path_exceptions = utils.validate_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation'), + params=params, + ) + xml_document.validation_file.name = os.path.relpath(path_csv, settings.MEDIA_ROOT) + xml_document.exceptions_file.name = os.path.relpath(path_exceptions, settings.MEDIA_ROOT) + xml_document.save() + except exceptions.XML_File_Validation_Error as e: logging.error(f'Error during XML validation: {e}') XMLDocumentEvent.create( xml_document=xml_document, @@ -89,20 +85,24 @@ def task_validate_xml_file(self, xml_id, user_id=None, username=None): save=True, ) return False - - with open(path_csv, 'rb') as f_csv: - xml_document.validation_file.save(os.path.basename(path_csv), File(f_csv), save=False) - - with open(path_exceptions, 'rb') as f_exc: - xml_document.exceptions_file.save(os.path.basename(path_exceptions), File(f_exc), save=False) + except Exception as e: + logging.error(f'Unexpected error during XML validation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, + data={}, + message=str(e), + save=True, + ) + return False - xml_document.save() logging.info(f'XML validation completed successfully for {xml_document.xml_file.name}.') + return True @celery_app.task(bind=True, timelimit=-1) -def task_generate_pdf_file(self, xml_id, user_id=None, username=None): +def task_generate_pdf_file(self, xml_id, libreoffice_binary='libreoffice', user_id=None, username=None): try: xml_document = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: @@ -111,10 +111,23 @@ def task_generate_pdf_file(self, xml_id, user_id=None, username=None): user = _get_user(self.request, username=username, user_id=user_id) + params = {'libreoffice_binary': libreoffice_binary,} + + logging.info(f'Starting PDF generation for XML file {xml_document.xml_file.name}.') try: - xml_tree = xml_utils.get_xml_tree(xml_document.xml_file.path) - except Exception as e: - logging.error(f'Error parsing XML file: {e}') + path_pdf, path_docx, lang = utils.generate_pdf_for_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'pdf'), + params=params, + ) + + pdf_instance = XMLDocumentPDF(xml_document=xml_document, language=lang) + pdf_instance.pdf_file.name = os.path.relpath(path_pdf, settings.MEDIA_ROOT) + pdf_instance.docx_file.name = os.path.relpath(path_docx, settings.MEDIA_ROOT) + pdf_instance.save() + + except exceptions.XML_File_Parsing_Error as e: + logging.error(f'Error during XML parsing: {e}') XMLDocumentEvent.create( xml_document=xml_document, error_type=XML_DOCUMENT_PARSING_ERROR, @@ -122,16 +135,9 @@ def task_generate_pdf_file(self, xml_id, user_id=None, username=None): message=str(e), save=True, ) - return False - # TODO: Iterate over available languages - - logging.info(f'Starting PDF generation for XML file {xml_document.xml_file.name}.') - try: - base_docx_layout = os.path.join(settings.MEDIA_ROOT, 'layout.docx') - document = docx.pipeline_docx(xml_tree, data={'base_layout': base_docx_layout}) - except Exception as e: - logging.error(f'Error during PDF generation: {e}') + except exceptions.XML_File_DOCX_Generation_Error as e: + logging.error(f'Error during XML to DOCX conversion: {e}') XMLDocumentEvent.create( xml_document=xml_document, error_type=XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, @@ -139,47 +145,76 @@ def task_generate_pdf_file(self, xml_id, user_id=None, username=None): message=str(e), save=True, ) - return False - pdf_dir = os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'pdf') - os.makedirs(pdf_dir, exist_ok=True) - - # DOCX intermediário - temp_dir = tempfile.gettempdir() - base_filename = os.path.splitext(os.path.basename(xml_document.xml_file.name))[0] - path_intermediate_docx = os.path.join(temp_dir, f"{base_filename}.docx") - document.save(path_intermediate_docx) + except exceptions.XML_File_PDF_Generation_Error as e: + logging.error(f'Error during PDF generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + data={}, + message=str(e), + save=True, + ) - path_pdf = os.path.join(pdf_dir, f"{base_filename}.pdf") - try: - file_utils.convert_docx_to_pdf(path_intermediate_docx, path_pdf) except Exception as e: - logging.error(f'Error during DOCX to PDF conversion: {e}') + logging.error(f'Unexpected error during PDF generation: {e}') XMLDocumentEvent.create( xml_document=xml_document, - error_type=XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, data={}, message=str(e), save=True, ) return False - with open(path_pdf, 'rb') as f_pdf: - pdf_instance = XMLDocumentPDF(xml_document=xml_document, language="pt") - pdf_instance.pdf_file.save(os.path.basename(path_pdf), File(f_pdf), save=True) - return True @celery_app.task(bind=True, timelimit=-1) def task_generate_html_file(self, xml_id, user_id=None, username=None): try: - xml_file = XMLDocument.objects.get(id=xml_id) + xml_document = XMLDocument.objects.get(id=xml_id) except XMLDocument.DoesNotExist: logging.error(f'XML file with ID {xml_id} does not exist.') return False user = _get_user(self.request, username=username, user_id=user_id) - logging.info(f'Starting HTML generation for XML file {xml_file.xml_file.name}.') - # TODO: Implement HTML generation logic here + logging.info(f'Starting HTML generation for XML file {xml_document.xml_file.name}.') + try: + path_html, lang = utils.generate_html_for_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'html'), + config=settings.HTML_GENERATION_CONFIG, + ) + html_instance = XMLDocumentHTML(xml_document=xml_document, language=lang) + html_instance.html_file.name = os.path.relpath(path_html, settings.MEDIA_ROOT) + html_instance.save() + except exceptions.XML_File_Parsing_Error as e: + logging.error(f'Error during XML parsing: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_PARSING_ERROR, + data={}, + message=str(e), + save=True, + ) + except exceptions.XML_File_HTML_Generation_Error as e: + logging.error(f'Error during HTML generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, + data={}, + message=str(e), + save=True, + ) + except Exception as e: + logging.error(f'Unexpected error during HTML generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, + data={}, + message=str(e), + save=True, + ) + return False From 66bbe003227a60a7a6ae70b68aea9e5d6dccea84 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:04:41 -0300 Subject: [PATCH 37/38] =?UTF-8?q?gera=20migra=C3=A7=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../0005_alter_xmldocumentevent_error_type.py | 18 ++++++++++++++++++ .../0003_xmldocumentpdf_docx_file.py | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 tracker/migrations/0005_alter_xmldocumentevent_error_type.py create mode 100644 xml_manager/migrations/0003_xmldocumentpdf_docx_file.py diff --git a/tracker/migrations/0005_alter_xmldocumentevent_error_type.py b/tracker/migrations/0005_alter_xmldocumentevent_error_type.py new file mode 100644 index 0000000..09af602 --- /dev/null +++ b/tracker/migrations/0005_alter_xmldocumentevent_error_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2025-08-27 16:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0004_alter_xmldocumentevent_options_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='xmldocumentevent', + name='error_type', + field=models.CharField(blank=True, choices=[('XPE', 'XML Parsing Error'), ('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CHE', 'XML Conversion to HTML Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error'), ('UNE', 'Unknown Error')], max_length=3, null=True, verbose_name='Error Type'), + ), + ] diff --git a/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py b/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py new file mode 100644 index 0000000..76e062f --- /dev/null +++ b/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2025-08-27 16:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('xml_manager', '0002_alter_xmldocument_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='xmldocumentpdf', + name='docx_file', + field=models.FileField(blank=True, help_text='Intermediate DOCX file generated during PDF creation', null=True, upload_to='xml_manager/docx/', verbose_name='DOCX File'), + ), + ] From 6ac1c76e901b9cc06c62bbe0248d2ac9fe2d16d4 Mon Sep 17 00:00:00 2001 From: Pitanga Innovare Date: Wed, 27 Aug 2025 21:05:25 -0300 Subject: [PATCH 38/38] atualiza dep packtools para 4.11.22 --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index e12f8d1..930d606 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -44,7 +44,7 @@ llama-cpp-python==0.3.14 # Packtools # ------------------------------------------------------------------------------ -git+https://git@github.com/scieloorg/packtools@4.11.20#egg=packtools +git+https://git@github.com/scieloorg/packtools@4.11.22#egg=packtools # Langdetect # ------------------------------------------------------------------------------