diff --git a/compose/local/django/Dockerfile b/compose/local/django/Dockerfile index b37d495..348a5a6 100755 --- a/compose/local/django/Dockerfile +++ b/compose/local/django/Dockerfile @@ -44,6 +44,9 @@ RUN python -m pip install --upgrade pip setuptools wheel # Requirements are installed here to ensure they will be cached. COPY ./requirements . +# Create application directory to hold DOCX layouts +COPY ./docx_layouts . + # Update pip RUN python -m pip install --upgrade pip @@ -69,12 +72,16 @@ ARG LLAMA_VERSION=0.3.14 WORKDIR ${APP_HOME} +RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list + # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ # psycopg2 dependencies libpq-dev \ # Translations dependencies gettext \ + # libreoffice for document conversions + default-jre libreoffice libreoffice-java-common ttf-mscorefonts-installer fonts-liberation fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea fonts-dejavu fonts-noto \ # cleaning up unused files && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ && rm -rf /var/lib/apt/lists/* diff --git a/compose/production/django/Dockerfile b/compose/production/django/Dockerfile index 99e938c..8ad536d 100755 --- a/compose/production/django/Dockerfile +++ b/compose/production/django/Dockerfile @@ -19,6 +19,9 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ # Requirements are installed here to ensure they will be cached. COPY ./requirements . +# Create application directory to hold DOCX layouts +COPY ./docx_layouts . + # Update pip RUN python -m pip install --upgrade pip @@ -47,10 +50,14 @@ WORKDIR ${APP_HOME} RUN addgroup --system django \ && adduser --system --ingroup django django +RUN sed -i 's/main/main contrib non-free/' /etc/apt/sources.list + # Install required system dependencies RUN apt-get update && apt-get install --no-install-recommends -y \ # psycopg2 dependencies libpq-dev \ + # libreoffice for document conversions + default-jre libreoffice libreoffice-java-common ttf-mscorefonts-installer fonts-liberation fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea fonts-dejavu fonts-noto \ # Translations dependencies gettext \ # cleaning up unused files diff --git a/config/menu.py b/config/menu.py index c014971..8dc074e 100644 --- a/config/menu.py +++ b/config/menu.py @@ -1,4 +1,7 @@ WAGTAIL_MENU_APPS_ORDER = [ + "xml_manager", + "reference", + "tracker", "django_celery_beat", ] diff --git a/config/settings/base.py b/config/settings/base.py index 365a1d1..f3de397 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -80,7 +80,8 @@ "core_settings", "django_celery_beat", "tracker", - "reference" + "reference", + "xml_manager", ] INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS + WAGTAIL diff --git a/core/utils/__init__.py b/core/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/core/utils/requester.py b/core/utils/requester.py new file mode 100644 index 0000000..be2732d --- /dev/null +++ b/core/utils/requester.py @@ -0,0 +1,162 @@ +import logging +import requests + +from tenacity import ( + retry, + retry_if_exception_type, + stop_after_attempt, + wait_exponential, +) + + +logger = logging.getLogger(__name__) + + +class RetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +class NonRetryableError(Exception): + """Recoverable error without having to modify the data state on the client + side, e.g. timeouts, errors from network partitioning, etc. + """ + + +def _add_param(params, name, value): + if value: + params[name] = value + return params + + +@retry( + retry=retry_if_exception_type(RetryableError), + wait=wait_exponential(multiplier=1, min=1, max=5), + stop=stop_after_attempt(5), +) +def post_data( + url, + auth=None, + data=None, + files=None, + headers=None, + json=False, + timeout=2, + verify=True, +): + """ + Post data with HTTP + Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, + then up to 10 seconds, then 10 seconds afterwards + Args: + url: URL address + files: files + headers: HTTP headers + json: True|False + verify: Verify the SSL. + Returns: + Return a requests.response object. + Except: + Raise a RetryableError to retry. + """ + try: + params = dict( + headers=headers, + timeout=timeout, + verify=verify, + ) + params = _add_param(params, "auth", auth) + params = _add_param(params, "files", files) + params = _add_param(params, "data", data) + response = requests.post(url, **params) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + logger.error("Erro posting data (timeout=%s): %s, retry..., erro: %s" % (timeout, url, exc)) + raise RetryableError(exc) from exc + except ( + requests.exceptions.InvalidSchema, + requests.exceptions.MissingSchema, + requests.exceptions.InvalidURL, + ) as exc: + raise NonRetryableError(exc) from exc + try: + response.raise_for_status() + except requests.HTTPError as exc: + if response := is_http_error_json_response(json, response): + return response + if 400 <= exc.response.status_code < 500: + raise NonRetryableError(exc) from exc + elif 500 <= exc.response.status_code < 600: + logger.error( + "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) + ) + raise RetryableError(exc) from exc + else: + raise + + return response.content if not json else response.json() + + +def is_http_error_json_response(json, response): + """ + Algumas API, por exemplo, opac_5, retornam a mensagem de erro em formato + JSON + """ + if not json: + return + + try: + return response.json() + except Exception as json_error: + return + + +@retry( + retry=retry_if_exception_type(RetryableError), + wait=wait_exponential(multiplier=1, min=1, max=5), + stop=stop_after_attempt(5), +) +def fetch_data(url, params=None, headers=None, json=False, timeout=2, verify=True): + """ + Get the resource with HTTP + Retry: Wait 2^x * 1 second between each retry starting with 4 seconds, + then up to 10 seconds, then 10 seconds afterwards + Args: + url: URL address + headers: HTTP headers + json: True|False + verify: Verify the SSL. + Returns: + Return a requests.response object. + Except: + Raise a RetryableError to retry. + """ + + try: + logger.info("Fetching the URL: %s %s" % (url, params)) + response = requests.get( + url, params=params, headers=headers, timeout=timeout, verify=verify + ) + except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as exc: + logger.error("Erro fetching the content: %s, retry..., erro: %s" % (url, exc)) + raise RetryableError(exc) from exc + except ( + requests.exceptions.InvalidSchema, + requests.exceptions.MissingSchema, + requests.exceptions.InvalidURL, + ) as exc: + raise NonRetryableError(exc) from exc + try: + response.raise_for_status() + except requests.HTTPError as exc: + if 400 <= exc.response.status_code < 500: + raise NonRetryableError(exc) from exc + elif 500 <= exc.response.status_code < 600: + logger.error( + "Erro fetching the content: %s, retry..., erro: %s" % (url, exc) + ) + raise RetryableError(exc) from exc + else: + raise + + return response.content if not json else response.json() diff --git a/docx_layouts/layout.docx b/docx_layouts/layout.docx new file mode 100644 index 0000000..25b4aea Binary files /dev/null and b/docx_layouts/layout.docx differ diff --git a/requirements/base.txt b/requirements/base.txt index f548288..930d606 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -41,3 +41,11 @@ whitenoise==6.6.0 # https://github.com/evansd/whitenoise # ------------------------------------------------------------------------------ huggingface-hub==0.26.1 llama-cpp-python==0.3.14 + +# Packtools +# ------------------------------------------------------------------------------ +git+https://git@github.com/scieloorg/packtools@4.11.22#egg=packtools + +# Langdetect +# ------------------------------------------------------------------------------ +langdetect~=1.0.9 \ No newline at end of file diff --git a/tracker/choices.py b/tracker/choices.py index 268789c..ec868ef 100644 --- a/tracker/choices.py +++ b/tracker/choices.py @@ -28,3 +28,21 @@ (PROGRESS_STATUS_PENDING, _("Pending")), (PROGRESS_STATUS_IGNORED, _("ignored")), ) + +XML_DOCUMENT_PARSING_ERROR = "XPE" +XML_DOCUMENT_VALIDATION_ERROR = "VDE" +XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR = "CDE" +XML_DOCUMENT_CONVERSION_TO_HTML_ERROR = "CHE" +XML_DOCUMENT_CONVERSION_TO_PDF_ERROR = "CPE" +XML_DOCUMENT_CONVERSION_TO_TEX_ERROR = "CTE" +XML_DOCUMENT_UNKNOWN_ERROR = "UNE" + +XML_DOCUMENT_EVENT = [ + (XML_DOCUMENT_PARSING_ERROR, _("XML Parsing Error")), + (XML_DOCUMENT_VALIDATION_ERROR, _("XML Validation Error")), + (XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, _("XML Conversion to DOCX Error")), + (XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, _("XML Conversion to HTML Error")), + (XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, _("XML Conversion to PDF Error")), + (XML_DOCUMENT_CONVERSION_TO_TEX_ERROR, _("XML Conversion to TEX Error")), + (XML_DOCUMENT_UNKNOWN_ERROR, _("Unknown Error")), +] diff --git a/tracker/exceptions.py b/tracker/exceptions.py index 5d0f7dc..571c194 100644 --- a/tracker/exceptions.py +++ b/tracker/exceptions.py @@ -1,26 +1,6 @@ -class ProcEventCreateError(Exception): +class GeneralEventCreateError(Exception): ... -class UnexpectedEventCreateError(Exception): - ... - - -class EventCreateError(Exception): - ... - - -class EventReportCreateError(Exception): - ... - - -class EventReportSaveFileError(Exception): - ... - - -class EventReportCreateError(Exception): - ... - - -class EventReportDeleteEventsError(Exception): - ... +class XMLDocumentEventCreateError(Exception): + ... \ No newline at end of file diff --git a/tracker/migrations/0002_xmldocumentevent.py b/tracker/migrations/0002_xmldocumentevent.py new file mode 100644 index 0000000..4df8637 --- /dev/null +++ b/tracker/migrations/0002_xmldocumentevent.py @@ -0,0 +1,35 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:04 + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0001_initial'), + ('xml_manager', '0002_alter_xmldocument_options_and_more'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='XMLDocumentEvent', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True, verbose_name='Creation date')), + ('updated', models.DateTimeField(auto_now=True, verbose_name='Last update date')), + ('error_type', models.CharField(blank=True, choices=[('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error')], max_length=3, null=True, verbose_name='Error Type')), + ('data', models.JSONField(default=dict, verbose_name='Data')), + ('message', models.TextField(blank=True, null=True, verbose_name='Message')), + ('handled', models.BooleanField(default=False, verbose_name='Handled')), + ('creator', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_creator', to=settings.AUTH_USER_MODEL, verbose_name='Creator')), + ('updated_by', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='%(class)s_last_mod_user', to=settings.AUTH_USER_MODEL, verbose_name='Updater')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='xml_manager.xmldocument')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py b/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py new file mode 100644 index 0000000..5ee4d28 --- /dev/null +++ b/tracker/migrations/0003_rename_unexpectedevent_generalevent_and_more.py @@ -0,0 +1,32 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:17 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0002_xmldocumentevent'), + ] + + operations = [ + migrations.RenameModel( + old_name='UnexpectedEvent', + new_name='GeneralEvent', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_excepti_7389c9_idx', + old_name='tracker_une_excepti_47ede4_idx', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_item_77b66a_idx', + old_name='tracker_une_item_795336_idx', + ), + migrations.RenameIndex( + model_name='generalevent', + new_name='tracker_gen_action_1474c9_idx', + old_name='tracker_une_action_55cd4b_idx', + ), + ] diff --git a/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py b/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py new file mode 100644 index 0000000..9c9ec14 --- /dev/null +++ b/tracker/migrations/0004_alter_xmldocumentevent_options_and_more.py @@ -0,0 +1,22 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0003_rename_unexpectedevent_generalevent_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='xmldocumentevent', + options={'verbose_name': 'XML Document Event', 'verbose_name_plural': 'XML Document Events'}, + ), + migrations.AlterField( + model_name='xmldocumentevent', + name='error_type', + field=models.CharField(blank=True, choices=[('XPE', 'XML Parsing Error'), ('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error')], max_length=3, null=True, verbose_name='Error Type'), + ), + ] diff --git a/tracker/migrations/0005_alter_xmldocumentevent_error_type.py b/tracker/migrations/0005_alter_xmldocumentevent_error_type.py new file mode 100644 index 0000000..09af602 --- /dev/null +++ b/tracker/migrations/0005_alter_xmldocumentevent_error_type.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2025-08-27 16:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('tracker', '0004_alter_xmldocumentevent_options_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='xmldocumentevent', + name='error_type', + field=models.CharField(blank=True, choices=[('XPE', 'XML Parsing Error'), ('VDE', 'XML Validation Error'), ('CDE', 'XML Conversion to DOCX Error'), ('CHE', 'XML Conversion to HTML Error'), ('CPE', 'XML Conversion to PDF Error'), ('CTE', 'XML Conversion to TEX Error'), ('UNE', 'Unknown Error')], max_length=3, null=True, verbose_name='Error Type'), + ), + ] diff --git a/tracker/models.py b/tracker/models.py index 6786920..ae74c50 100644 --- a/tracker/models.py +++ b/tracker/models.py @@ -2,23 +2,76 @@ import logging import traceback import uuid -from datetime import datetime -from django.core.files.base import ContentFile from django.db import models from django.utils.translation import gettext_lazy as _ from core.models import CommonControlField +from xml_manager.models import XMLDocument from tracker import choices + from .exceptions import ( - EventCreateError, - EventReportCreateError, - EventReportSaveFileError, - UnexpectedEventCreateError + GeneralEventCreateError, + XMLDocumentEventCreateError, ) -class UnexpectedEvent(models.Model): +class XMLDocumentEvent(CommonControlField): + xml_document = models.ForeignKey( + XMLDocument, + on_delete=models.CASCADE, + null=False, + blank=False, + db_index=True, + ) + error_type = models.CharField( + _("Error Type"), + choices=choices.XML_DOCUMENT_EVENT, + max_length=3, + null=True, + blank=True, + ) + data = models.JSONField( + _("Data"), + default=dict, + ) + message = models.TextField( + _("Message"), + null=True, + blank=True, + ) + handled = models.BooleanField( + _("Handled"), + default=False + ) + + @classmethod + def create(cls, xml_document, error_type, data, message, save=False): + try: + obj = cls() + obj.xml_document = xml_document + obj.error_type = error_type + obj.data = data + obj.message = message + if save: + obj.save() + + return obj + + except Exception as exc: + raise XMLDocumentEventCreateError( + f"Unable to create XMLDocumentEvent. EXCEPTION {exc}" + ) + + def __str__(self): + return f"{self.data} - {self.message}" + + class Meta: + verbose_name = _("XML Document Event") + verbose_name_plural = _("XML Document Events") + + +class GeneralEvent(models.Model): id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) created = models.DateTimeField(verbose_name=_("Creation date"), auto_now_add=True) exception_type = models.TextField(_("Exception Type"), null=True, blank=True) @@ -45,6 +98,8 @@ class Meta: models.Index(fields=["action"]), ] ordering = ["-created"] + verbose_name = _("General Event") + verbose_name_plural = _("General Events") def __str__(self): if self.item or self.action: @@ -92,125 +147,6 @@ def create( obj.save() return obj except Exception as exc: - raise UnexpectedEventCreateError( - f"Unable to create unexpected event ({exception} {exc_traceback}). EXCEPTION {exc}" - ) - - -class Event(CommonControlField): - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - message = models.TextField(_("Message"), null=True, blank=True) - message_type = models.CharField( - _("Message type"), - choices=choices.EVENT_MSG_TYPE, - max_length=16, - null=True, - blank=True, - ) - detail = models.JSONField(null=True, blank=True) - unexpected_event = models.ForeignKey( - UnexpectedEvent, on_delete=models.SET_NULL, null=True, blank=True - ) - - class Meta: - abstract = True - indexes = [ - models.Index(fields=["message_type"]), - ] - - @property - def data(self): - d = {} - d["created"] = self.created.isoformat() - d["user"] = self.user.username - d.update( - dict( - message=self.message, message_type=self.message_type, detail=self.detail - ) - ) - if self.unexpected_event: - d.update(self.unexpected_event.data) - return d - - @classmethod - def create( - cls, - user=None, - message_type=None, - message=None, - e=None, - exc_traceback=None, - detail=None, - ): - try: - obj = cls() - obj.creator = user - obj.message = message - obj.message_type = message_type - obj.detail = detail - obj.save() - - if e: - logging.exception(f"{message}: {e}") - obj.unexpected_event = UnexpectedEvent.create( - exception=e, - exc_traceback=exc_traceback, - ) - obj.save() - except Exception as exc: - raise EventCreateError( - f"Unable to create Event ({message} {e}). EXCEPTION: {exc}" - ) - return obj - - -def tracker_file_directory_path(instance, filename): - # file will be uploaded to MEDIA_ROOT/user_/ - - d = datetime.utcnow() - return f"tracker/{d.year}/{d.month}/{d.day}/{filename}" - - -class EventReport(CommonControlField): - file = models.FileField( - upload_to=tracker_file_directory_path, null=True, blank=True - ) - - class Meta: - abstract = True - - def save_file(self, events, ext=None): - if not events: - return - try: - ext = ".json" - content = json.dumps(list([item.data for item in events])) - name = datetime.utcnow().isoformat() + ext - self.file.save(name, ContentFile(content)) - self.delete_events(events) - except Exception as e: - raise EventReportSaveFileError( - f"Unable to save EventReport.file ({name}). Exception: {e}" - ) - - def delete_events(self, events): - for item in events: - try: - item.unexpected_event.delete() - except: - pass - try: - item.delete() - except: - pass - - @classmethod - def create(cls, user): - try: - obj = cls() - obj.creator = user - obj.save() - except Exception as e: - raise EventReportCreateError( - f"Unable to create EventReport. Exception: {e}" + raise GeneralEventCreateError( + f"Unable to create general event ({exception} {exc_traceback}). EXCEPTION {exc}" ) diff --git a/tracker/tasks.py b/tracker/tasks.py index af422b8..9f9154f 100644 --- a/tracker/tasks.py +++ b/tracker/tasks.py @@ -1,16 +1,13 @@ -# tasks.py -import logging -import sys from datetime import datetime from config import celery_app -from .models import UnexpectedEvent +from .models import GeneralEvent -@celery_app.task(bind=True, name="cleanup_unexpected_events") -def delete_unexpected_events(self, exception_type, start_date=None, end_date=None, user_id=None, username=None): +@celery_app.task(bind=True) +def delete_general_events(self, exception_type, start_date=None, end_date=None, user_id=None, username=None): """ - Delete UnexpectedEvent records based on exception type and optional date range. + Delete GeneralEvent records based on exception type and optional date range. """ filters = {} @@ -24,4 +21,4 @@ def delete_unexpected_events(self, exception_type, start_date=None, end_date=Non end_date = datetime.fromisoformat(end_date) filters['created__lte'] = end_date - UnexpectedEvent.objects.filter(**filters).delete() + GeneralEvent.objects.filter(**filters).delete() diff --git a/tracker/wagtail_hooks.py b/tracker/wagtail_hooks.py index e1b82e0..8c87445 100644 --- a/tracker/wagtail_hooks.py +++ b/tracker/wagtail_hooks.py @@ -7,14 +7,45 @@ from config.menu import get_menu_order -from .models import UnexpectedEvent +from .models import XMLDocumentEvent, GeneralEvent -class UnexpectedEventModelAdmin(ModelAdmin): - model = UnexpectedEvent +class XMLDocumentEventModelAdmin(ModelAdmin): + model = XMLDocumentEvent inspect_view_enabled = True - menu_label = _("Unexpected Events") - menu_icon = "folder" + menu_label = _("XML Document Events") + menu_icon = "warning" + menu_order = 100 + add_to_settings_menu = False + exclude_from_explorer = False + list_per_page = 10 + + list_display = ( + "xml_document", + "error_type", + "data", + "message", + "created", + ) + list_filter = ("error_type", ) + search_fields = ( + "message", + "data", + ) + inspect_view_fields = ( + "xml_document", + "error_type", + "data", + "message", + "created", + ) + + +class GeneralEventModelAdmin(ModelAdmin): + model = GeneralEvent + inspect_view_enabled = True + menu_label = _("General Events") + menu_icon = "warning" menu_order = 200 add_to_settings_menu = False exclude_from_explorer = False @@ -44,11 +75,12 @@ class UnexpectedEventModelAdmin(ModelAdmin): "created", ) -class UnexpectedEventModelAdminGroup(ModelAdminGroup): - menu_icon = "folder" - menu_label = _("Unexpected errors") + +class EventModelAdminGroup(ModelAdminGroup): + menu_icon = "warning" + menu_label = _("Unexpected Events") menu_order = get_menu_order("tracker") - items = (UnexpectedEventModelAdmin) + items = (GeneralEventModelAdmin, XMLDocumentEventModelAdmin) -modeladmin_register(UnexpectedEventModelAdmin) +modeladmin_register(EventModelAdminGroup) diff --git a/xml_manager/__init__.py b/xml_manager/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xml_manager/admin.py b/xml_manager/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/xml_manager/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/xml_manager/apps.py b/xml_manager/apps.py new file mode 100644 index 0000000..1292960 --- /dev/null +++ b/xml_manager/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class XmlManagerConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'xml_manager' diff --git a/xml_manager/exceptions.py b/xml_manager/exceptions.py new file mode 100644 index 0000000..89184ee --- /dev/null +++ b/xml_manager/exceptions.py @@ -0,0 +1,14 @@ +class XML_File_Parsing_Error(Exception): + pass + +class XML_File_Validation_Error(Exception): + pass + +class XML_File_DOCX_Generation_Error(Exception): + pass + +class XML_File_PDF_Generation_Error(Exception): + pass + +class XML_File_HTML_Generation_Error(Exception): + pass diff --git a/xml_manager/migrations/0001_initial.py b/xml_manager/migrations/0001_initial.py new file mode 100644 index 0000000..fd2b460 --- /dev/null +++ b/xml_manager/migrations/0001_initial.py @@ -0,0 +1,45 @@ +# Generated by Django 5.0.8 on 2025-08-25 02:26 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='XMLDocument', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('xml_file', models.FileField(help_text='Upload an XML file for processing.', upload_to='xml_manager/xml/', verbose_name='XML File')), + ('validation_file', models.FileField(blank=True, null=True, upload_to='xml_manager/csv/', verbose_name='Validation File')), + ('exceptions_file', models.FileField(blank=True, null=True, upload_to='xml_manager/exceptions/', verbose_name='Exceptions File')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, help_text='The date and time when the file was uploaded.', verbose_name='Uploaded At')), + ], + ), + migrations.CreateModel( + name='XMLDocumentHTML', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('html_file', models.FileField(upload_to='xml_manager/html/', verbose_name='HTML File')), + ('language', models.CharField(default='pt', help_text='Language code or name', max_length=32, verbose_name='Language')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='Uploaded At')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='htmls', to='xml_manager.xmldocument', verbose_name='XML Document')), + ], + ), + migrations.CreateModel( + name='XMLDocumentPDF', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pdf_file', models.FileField(upload_to='xml_manager/pdf/', verbose_name='PDF File')), + ('language', models.CharField(default='pt', help_text='Language code or name', max_length=32, verbose_name='Language')), + ('uploaded_at', models.DateTimeField(auto_now_add=True, verbose_name='Uploaded At')), + ('xml_document', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pdfs', to='xml_manager.xmldocument', verbose_name='XML Document')), + ], + ), + ] diff --git a/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py b/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py new file mode 100644 index 0000000..3eb3bf9 --- /dev/null +++ b/xml_manager/migrations/0002_alter_xmldocument_options_and_more.py @@ -0,0 +1,35 @@ +# Generated by Django 5.0.8 on 2025-08-25 17:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('xml_manager', '0001_initial'), + ] + + operations = [ + migrations.AlterModelOptions( + name='xmldocument', + options={'verbose_name': 'XML Document', 'verbose_name_plural': 'XML Documents'}, + ), + migrations.AlterModelOptions( + name='xmldocumenthtml', + options={'verbose_name': 'XML Document HTML', 'verbose_name_plural': 'XML Document HTMLs'}, + ), + migrations.AlterModelOptions( + name='xmldocumentpdf', + options={'verbose_name': 'XML Document PDF', 'verbose_name_plural': 'XML Document PDFs'}, + ), + migrations.AlterField( + model_name='xmldocument', + name='exceptions_file', + field=models.FileField(blank=True, null=True, upload_to='xml_manager/validation/', verbose_name='Exceptions File'), + ), + migrations.AlterField( + model_name='xmldocument', + name='validation_file', + field=models.FileField(blank=True, null=True, upload_to='xml_manager/validation/', verbose_name='Validation File'), + ), + ] diff --git a/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py b/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py new file mode 100644 index 0000000..76e062f --- /dev/null +++ b/xml_manager/migrations/0003_xmldocumentpdf_docx_file.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2025-08-27 16:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('xml_manager', '0002_alter_xmldocument_options_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='xmldocumentpdf', + name='docx_file', + field=models.FileField(blank=True, help_text='Intermediate DOCX file generated during PDF creation', null=True, upload_to='xml_manager/docx/', verbose_name='DOCX File'), + ), + ] diff --git a/xml_manager/migrations/__init__.py b/xml_manager/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/xml_manager/models.py b/xml_manager/models.py new file mode 100644 index 0000000..4a1ad7b --- /dev/null +++ b/xml_manager/models.py @@ -0,0 +1,110 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ +from wagtail.admin.panels import FieldPanel + + +class XMLDocument(models.Model): + xml_file = models.FileField( + upload_to='xml_manager/xml/', + verbose_name=_("XML File"), + help_text=_("Upload an XML file for processing.") + ) + validation_file = models.FileField( + upload_to='xml_manager/validation/', + blank=True, + null=True, + verbose_name=_("Validation File") + ) + exceptions_file = models.FileField( + upload_to='xml_manager/validation/', + blank=True, + null=True, + verbose_name=_("Exceptions File") + ) + uploaded_at = models.DateTimeField( + auto_now_add=True, + verbose_name=_("Uploaded At"), + help_text=_("The date and time when the file was uploaded.") + ) + + panels = [ + FieldPanel("xml_file"), + ] + + def __str__(self): + return f"{self.xml_file.name}" + + class Meta: + verbose_name = _("XML Document") + verbose_name_plural = _("XML Documents") + + +class XMLDocumentPDF(models.Model): + xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="pdfs", verbose_name=_("XML Document")) + pdf_file = models.FileField( + upload_to='xml_manager/pdf/', + verbose_name=_("PDF File") + ) + docx_file = models.FileField( + upload_to='xml_manager/docx/', + verbose_name=_("DOCX File"), + null=True, + blank=True, + help_text=_('Intermediate DOCX file generated during PDF creation') + ) + language = models.CharField( + max_length=32, + default="pt", + verbose_name=_("Language"), + help_text=_("Language code or name") + ) + uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) + + def __str__(self): + return f"PDF for {self.xml_document.xml_file.name} ({self.language})" + + class Meta: + verbose_name = _("XML Document PDF") + verbose_name_plural = _("XML Document PDFs") + + @classmethod + def create(cls, xml_document, pdf_file, language="pt"): + pdf_instance = cls( + xml_document=xml_document, + pdf_file=pdf_file, + language=language + ) + pdf_instance.save() + return pdf_instance + + +class XMLDocumentHTML(models.Model): + xml_document = models.ForeignKey(XMLDocument, on_delete=models.CASCADE, related_name="htmls", verbose_name=_("XML Document")) + html_file = models.FileField( + upload_to='xml_manager/html/', + verbose_name=_("HTML File") + ) + language = models.CharField( + max_length=32, + default="pt", + verbose_name=_("Language"), + help_text=_("Language code or name") + ) + uploaded_at = models.DateTimeField(auto_now_add=True, verbose_name=_("Uploaded At")) + + def __str__(self): + return f"HTML for {self.xml_document.xml_file.name} ({self.language})" + + class Meta: + verbose_name = _("XML Document HTML") + verbose_name_plural = _("XML Document HTMLs") + + @classmethod + def create(cls, xml_document, html_file, language="pt"): + html_instance = cls( + xml_document=xml_document, + html_file=html_file, + language=language + ) + html_instance.save() + return html_instance diff --git a/xml_manager/tasks.py b/xml_manager/tasks.py new file mode 100644 index 0000000..0c5150f --- /dev/null +++ b/xml_manager/tasks.py @@ -0,0 +1,220 @@ +import logging +import os + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.utils.translation import gettext as _ + +from config import celery_app +from tracker.choices import ( + XML_DOCUMENT_PARSING_ERROR, + XML_DOCUMENT_VALIDATION_ERROR, + XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, + XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, + XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + XML_DOCUMENT_UNKNOWN_ERROR, +) +from tracker.models import XMLDocumentEvent +from xml_manager.models import ( + XMLDocument, + XMLDocumentHTML, + XMLDocumentPDF, +) +from xml_manager import exceptions +from xml_manager import utils + + +User = get_user_model() + + +def _get_user(request, username=None, user_id=None): + try: + return User.objects.get(pk=request.user_id) + except AttributeError: + if user_id: + return User.objects.get(pk=user_id) + if username: + return User.objects.get(username=username) + + +@celery_app.task(bind=True, timelimit=-1) +def task_process_xml_document(self, xml_id, user_id=None, username=None): + try: + xml_document = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML document with ID {xml_id} does not exist.') + return False + + logging.info(f'Processing XML file {xml_document.xml_file.name}.') + task_validate_xml_file.delay(xml_id, user_id=user_id, username=username) + task_generate_pdf_file.delay(xml_id, user_id=user_id, username=username) + task_generate_html_file.delay(xml_id, user_id=user_id, username=username) + + return True + + +@celery_app.task(bind=True, timelimit=-1) +def task_validate_xml_file(self, xml_id, user_id=None, username=None): + try: + xml_document = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + user = _get_user(self.request, username=username, user_id=user_id) + + logging.info(f'Starting XML validation for XML file {xml_document.xml_file.name}.') + params = {} + + try: + path_csv, path_exceptions = utils.validate_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'validation'), + params=params, + ) + xml_document.validation_file.name = os.path.relpath(path_csv, settings.MEDIA_ROOT) + xml_document.exceptions_file.name = os.path.relpath(path_exceptions, settings.MEDIA_ROOT) + xml_document.save() + except exceptions.XML_File_Validation_Error as e: + logging.error(f'Error during XML validation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_VALIDATION_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + except Exception as e: + logging.error(f'Unexpected error during XML validation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + + logging.info(f'XML validation completed successfully for {xml_document.xml_file.name}.') + + return True + + +@celery_app.task(bind=True, timelimit=-1) +def task_generate_pdf_file(self, xml_id, libreoffice_binary='libreoffice', user_id=None, username=None): + try: + xml_document = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + user = _get_user(self.request, username=username, user_id=user_id) + + params = {'libreoffice_binary': libreoffice_binary,} + + logging.info(f'Starting PDF generation for XML file {xml_document.xml_file.name}.') + try: + path_pdf, path_docx, lang = utils.generate_pdf_for_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'pdf'), + params=params, + ) + + pdf_instance = XMLDocumentPDF(xml_document=xml_document, language=lang) + pdf_instance.pdf_file.name = os.path.relpath(path_pdf, settings.MEDIA_ROOT) + pdf_instance.docx_file.name = os.path.relpath(path_docx, settings.MEDIA_ROOT) + pdf_instance.save() + + except exceptions.XML_File_Parsing_Error as e: + logging.error(f'Error during XML parsing: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_PARSING_ERROR, + data={}, + message=str(e), + save=True, + ) + + except exceptions.XML_File_DOCX_Generation_Error as e: + logging.error(f'Error during XML to DOCX conversion: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_DOCX_ERROR, + data={}, + message=str(e), + save=True, + ) + + except exceptions.XML_File_PDF_Generation_Error as e: + logging.error(f'Error during PDF generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_PDF_ERROR, + data={}, + message=str(e), + save=True, + ) + + except Exception as e: + logging.error(f'Unexpected error during PDF generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, + data={}, + message=str(e), + save=True, + ) + return False + + return True + + +@celery_app.task(bind=True, timelimit=-1) +def task_generate_html_file(self, xml_id, user_id=None, username=None): + try: + xml_document = XMLDocument.objects.get(id=xml_id) + except XMLDocument.DoesNotExist: + logging.error(f'XML file with ID {xml_id} does not exist.') + return False + + user = _get_user(self.request, username=username, user_id=user_id) + + logging.info(f'Starting HTML generation for XML file {xml_document.xml_file.name}.') + try: + path_html, lang = utils.generate_html_for_xml_document( + xml_document.xml_file.path, + output_root_dir=os.path.join(settings.MEDIA_ROOT, 'xml_manager', 'html'), + config=settings.HTML_GENERATION_CONFIG, + ) + html_instance = XMLDocumentHTML(xml_document=xml_document, language=lang) + html_instance.html_file.name = os.path.relpath(path_html, settings.MEDIA_ROOT) + html_instance.save() + except exceptions.XML_File_Parsing_Error as e: + logging.error(f'Error during XML parsing: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_PARSING_ERROR, + data={}, + message=str(e), + save=True, + ) + except exceptions.XML_File_HTML_Generation_Error as e: + logging.error(f'Error during HTML generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_CONVERSION_TO_HTML_ERROR, + data={}, + message=str(e), + save=True, + ) + except Exception as e: + logging.error(f'Unexpected error during HTML generation: {e}') + XMLDocumentEvent.create( + xml_document=xml_document, + error_type=XML_DOCUMENT_UNKNOWN_ERROR, + data={}, + message=str(e), + save=True, + ) + return False diff --git a/xml_manager/tests.py b/xml_manager/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/xml_manager/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/xml_manager/urls.py b/xml_manager/urls.py new file mode 100644 index 0000000..226389b --- /dev/null +++ b/xml_manager/urls.py @@ -0,0 +1,7 @@ +from django.urls import path +from . import views + + +urlpatterns = [ + path("process//", views.process_xml_pk, name="process_xml_pk"), +] \ No newline at end of file diff --git a/xml_manager/utils.py b/xml_manager/utils.py new file mode 100644 index 0000000..2cc5c92 --- /dev/null +++ b/xml_manager/utils.py @@ -0,0 +1,74 @@ +import os + +from packtools import data_checker +from packtools.sps.formats.pdf.pipeline import docx +from packtools.sps.formats.pdf.utils import file_utils +from packtools.sps.formats.pdf.pipeline.xml import extract_article_main_language +from packtools.sps.utils import xml_utils + +from xml_manager import exceptions + + +def validate_xml_document(xml_file_path, output_root_dir, params): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + base_fname, fext = os.path.splitext(os.path.basename(xml_file_path)) + path_csv = os.path.join(output_root_dir, f'{base_fname}.validation.csv') + path_exceptions = os.path.join(output_root_dir, f'{base_fname}.exceptions.json') + + try: + validator = data_checker.XMLDataChecker(path_csv, path_exceptions, xml_file_path) + validator.validate(params=params, csv_per_xml=False) + except Exception as e: + raise exceptions.XML_File_Validation_Error(f"Error during XML validation: {e}") + + return path_csv, path_exceptions + + +def generate_pdf_for_xml_document(xml_file_path, output_root_dir, params): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + if not isinstance(params, dict): + params = { + 'base_layout': '/app/docx_layouts/layout.docx', + 'libreoffice_binary': 'libreoffice', + } + + if 'base_layout' not in params: + params['base_layout'] = '/app/docx_layouts/layout.docx' + + try: + xml_tree = xml_utils.get_xml_tree(xml_file_path) + except Exception as e: + raise exceptions.XML_File_Parsing_Error(f"Error parsing XML file: {e}") + + try: + docx_document = docx.pipeline_docx(xml_tree, data=params) + except Exception as e: + raise exceptions.XML_File_DOCX_Generation_Error(f"Error converting XML to DOCX: {e}") + + main_language = extract_article_main_language(xml_tree) or params.get('main_language', 'pt') + + base_name = os.path.basename(xml_file_path) + f_name, f_ext = os.path.splitext(base_name) + path_docx = os.path.join(output_root_dir, f'{f_name}.docx') + path_pdf = os.path.join(output_root_dir, f'{f_name}.pdf') + + docx_document.save(path_docx) + + try: + file_utils.convert_docx_to_pdf(path_docx, libreoffice_binary=params.get('libreoffice_binary', 'libreoffice')) + except Exception as e: + raise exceptions.XML_File_PDF_Generation_Error(f"Error generating PDF from DOCX: {e}") + + return path_pdf, path_docx, main_language + + +def generate_html_for_xml_document(xml_file_path, output_root_dir, config): + if not os.path.exists(output_root_dir): + os.makedirs(output_root_dir) + + # ToDo: Implement HTML generation logic here + return \ No newline at end of file diff --git a/xml_manager/views.py b/xml_manager/views.py new file mode 100644 index 0000000..8f67642 --- /dev/null +++ b/xml_manager/views.py @@ -0,0 +1,14 @@ +from django.contrib import messages +from django.contrib.admin.views.decorators import staff_member_required +from django.shortcuts import redirect, get_object_or_404 + +from .tasks import task_process_xml_document +from .models import XMLDocument + + +@staff_member_required +def process_xml_pk(request, pk): + obj = get_object_or_404(XMLDocument, pk=pk) + task_process_xml_document.delay(obj.id) + messages.success(request, f"XML {obj.id} enviado para processamento") + return redirect("/admin/snippets/xml_manager/xmldocument/") diff --git a/xml_manager/wagtail_hooks.py b/xml_manager/wagtail_hooks.py new file mode 100644 index 0000000..55e6719 --- /dev/null +++ b/xml_manager/wagtail_hooks.py @@ -0,0 +1,135 @@ +import os + +from django.urls import path, include +from django.urls import reverse +from django.utils.html import format_html +from django.utils.translation import gettext_lazy as _ +from wagtail import hooks +from wagtail.admin.ui.tables import Column +from wagtail.snippets.models import register_snippet +from wagtail.snippets.views.snippets import SnippetViewSet, SnippetViewSetGroup + +from config.menu import get_menu_order + +from .models import XMLDocument, XMLDocumentPDF, XMLDocumentHTML +from . import urls + + +class FileNameColumn(Column): + """ + Coluna que mostra apenas o nome do arquivo de um FileField. + """ + def get_value(self, instance): + val = super().get_value(instance) + if not val: + return "-" + return os.path.basename(getattr(val, "name", str(val))) + + +class LinkColumn(Column): + """ + Column que recebe um FileField (FieldFile) e renderiza um . + Cria '-' se não houver arquivo. + """ + def get_value(self, instance): + val = super().get_value(instance) + if not val: + return "-" + name = os.path.basename(getattr(val, "name", str(val))) + try: + url = val.url + except Exception: + return name + return format_html('{}', url, name) + + +class ActionColumn(Column): + def get_value(self, instance): + url = reverse("process_xml_pk", args=[instance.pk]) + return format_html( + 'Processar', url + ) + + +class XMLDocumentSnippetViewSet(SnippetViewSet): + model = XMLDocument + verbose_name = _("XML Document") + verbose_name_plural = _("XML Documents") + icon = "folder-open-inverse" + menu_name = "xml_manager" + menu_label = _("XML Document") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = False + + list_display = ( + "xml_file", + LinkColumn("validation_file", label=_("Validation file")), + LinkColumn("exceptions_file", label=_("Exceptions file")), + "uploaded_at", + ActionColumn("actions", label=_("Ações")), + ) + + search_fields = ("xml_file",) + + +class XMLDocumentPDFSnippetViewSet(SnippetViewSet): + model = XMLDocumentPDF + verbose_name = _("XML Document PDF") + verbose_name_plural = _("XML Document PDFs") + icon = "doc-full" + menu_name = "xml_manager" + menu_label = _("XML Document PDF") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = False + + list_display = ( + "xml_document", + LinkColumn("pdf_file", "PDF file"), + LinkColumn("docx_file", "DOCX file"), + "language", + "uploaded_at" + ) + + search_fields = ("pdf_file",) + + +class XMLDocumentHTMLSnippetViewSet(SnippetViewSet): + model = XMLDocumentHTML + verbose_name = _("XML Document HTML") + verbose_name_plural = _("XML Document HTMLs") + icon = "doc-full" + menu_name = "xml_manager" + menu_label = _("XML Document HTML") + menu_order = get_menu_order("xml_manager") + add_to_admin_menu = False + + list_display = ( + "xml_document", + LinkColumn("html_file", "HTML file"), + "language", + "uploaded_at", + ) + + search_fields = ("html_file",) + + +class XMLDocumentSnippetViewSetGroup(SnippetViewSetGroup): + menu_name = 'xml_manager' + menu_label = _("XML Manager") + menu_icon = "folder-open-inverse" + menu_order = get_menu_order("xml_manager") + items = ( + XMLDocumentSnippetViewSet, + XMLDocumentHTMLSnippetViewSet, + XMLDocumentPDFSnippetViewSet, + ) + + +register_snippet(XMLDocumentSnippetViewSetGroup) + + +@hooks.register('register_admin_urls') +def register_admin_urls(): + return [ + path('xml-manager/', include(urls)), + ]