diff --git a/.cursor/rules/tests.mdc b/.cursor/rules/tests.mdc new file mode 100644 index 0000000..3dca909 --- /dev/null +++ b/.cursor/rules/tests.mdc @@ -0,0 +1,3 @@ +--- +alwaysApply: true +--- diff --git a/compose/local/django/Dockerfile.llama b/compose/local/django/Dockerfile.llama new file mode 100644 index 0000000..75514b2 --- /dev/null +++ b/compose/local/django/Dockerfile.llama @@ -0,0 +1,110 @@ +ARG PYTHON_VERSION=3.12-bookworm + +FROM python:${PYTHON_VERSION} AS python + +FROM python AS python-build-stage + +ARG BUILD_ENVIRONMENT=local + +RUN apt-get update && apt-get install --no-install-recommends -y \ + build-essential \ + git \ + libpq-dev \ + software-properties-common \ + libopenblas-dev \ + libomp-dev + +RUN apt-get update && \ + apt-get install -y ninja-build cmake && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + +RUN python -m pip install --upgrade pip setuptools wheel + +COPY ./requirements . + +COPY ./docx_layouts . + +RUN python -m pip install --upgrade pip + +RUN pip wheel --wheel-dir /usr/src/app/wheels \ + -r ${BUILD_ENVIRONMENT}.txt + +FROM python AS python-run-stage + +ARG BUILD_ENVIRONMENT=local +ARG APP_HOME=/app +ARG DISABLE_AVX=true +ARG LLAMA_VERSION=0.3.14 + +ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV BUILD_ENV ${BUILD_ENVIRONMENT} + +WORKDIR ${APP_HOME} + +RUN if [ -f /etc/apt/sources.list ]; then \ + sed -i 's/main/main contrib non-free/' /etc/apt/sources.list; \ + elif [ -f /etc/apt/sources.list.d/debian.sources ]; then \ + sed -i 's/Components: main/Components: main contrib non-free non-free-firmware/' /etc/apt/sources.list.d/debian.sources; \ + fi + +RUN apt-get update && apt-get install --no-install-recommends -y \ + libpq-dev \ + gettext \ + default-jre libreoffice libreoffice-java-common ttf-mscorefonts-installer fonts-liberation fonts-liberation2 fonts-crosextra-carlito fonts-crosextra-caladea fonts-dejavu fonts-noto \ + build-essential cmake ninja-build \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && rm -rf /var/lib/apt/lists/* + +COPY --from=python-build-stage /usr/src/app/wheels /wheels/ + +RUN pip install --no-cache-dir --no-index --find-links=/wheels/ $(find /wheels/ -name "*.whl" ! -name "llama_cpp_python*") && rm -rf /wheels/ + +ARG TARGETARCH +RUN set -eux; \ + ARCH="${TARGETARCH:-}"; \ + if [ -z "${ARCH}" ]; then ARCH="$(uname -m)"; fi; \ + if [ "${ARCH}" = "arm64" ] || [ "${ARCH}" = "aarch64" ]; then \ + pip install "llama-cpp-python==${LLAMA_VERSION}" \ + --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu \ + --only-binary=:all: \ + --no-cache-dir \ + || { \ + export FORCE_CMAKE=1; \ + export CMAKE_ARGS="-DGGML_NATIVE=OFF -DGGML_CPU_ALL_VARIANTS=OFF -DGGML_CPU_ARM_ARCH=armv8-a -DGGML_LLAMAFILE=OFF -DGGML_CPU_REPACK=OFF"; \ + pip install "llama-cpp-python==${LLAMA_VERSION}" --force-reinstall --no-cache-dir; \ + }; \ + else \ + pip install "llama-cpp-python==${LLAMA_VERSION}" --prefer-binary --no-cache-dir \ + || { \ + export FORCE_CMAKE=1; \ + if [ "${DISABLE_AVX}" = "true" ]; then \ + export CMAKE_ARGS="-DLLAMA_AVX=OFF -DLLAMA_AVX2=OFF -DLLAMA_FMA=OFF -DLLAMA_F16C=OFF -DLLAMA_OPENMP=ON"; \ + fi; \ + pip install "llama-cpp-python==${LLAMA_VERSION}" --force-reinstall --no-cache-dir; \ + }; \ + fi + +COPY ./compose/production/django/entrypoint /entrypoint +RUN sed -i 's/\r$//g' /entrypoint +RUN chmod +x /entrypoint + +COPY ./compose/local/django/start /start +RUN sed -i 's/\r$//g' /start +RUN chmod +x /start + +COPY ./compose/local/django/celery/worker/start /start-celeryworker +RUN sed -i 's/\r$//g' /start-celeryworker +RUN chmod +x /start-celeryworker + +COPY ./compose/local/django/celery/beat/start /start-celerybeat +RUN sed -i 's/\r$//g' /start-celerybeat +RUN chmod +x /start-celerybeat + +COPY ./compose/local/django/celery/flower/start /start-flower +RUN sed -i 's/\r$//g' /start-flower +RUN chmod +x /start-flower + +COPY . ${APP_HOME} + +ENTRYPOINT ["/entrypoint"] diff --git a/markup_doc/labeling_utils.py b/markup_doc/labeling_utils.py index d0247f1..f66d3d6 100644 --- a/markup_doc/labeling_utils.py +++ b/markup_doc/labeling_utils.py @@ -1,21 +1,22 @@ # Standard library imports +import html import json import re -import requests -from lxml import etree +import requests # Third-party imports from django.contrib.auth import get_user_model +from lxml import etree from rest_framework_simplejwt.tokens import RefreshToken # Local application imports from model_ai.models import LlamaModel -from .choices import order_labels +from .choices import order_labels -MODEL_NAME_GEMINI = 'GEMINI' -MODEL_NAME_LLAMA = 'LLAMA' +MODEL_NAME_GEMINI = "GEMINI" +MODEL_NAME_LLAMA = "LLAMA" def get_llm_model_name(): @@ -26,22 +27,22 @@ def get_llm_model_name(): return MODEL_NAME_GEMINI else: return MODEL_NAME_LLAMA - + def split_in_three(obj_reference, chunk_size=5): if not obj_reference: return [] - return [obj_reference[i:i + chunk_size] - for i in range(0, len(obj_reference), chunk_size)] + return [ + obj_reference[i : i + chunk_size] + for i in range(0, len(obj_reference), chunk_size) + ] User = get_user_model() def process_reference(num_ref, obj, user_id): - payload = { - 'reference': obj['value']['paragraph'] - } + payload = {"reference": obj["value"]["paragraph"]} # FIXME: This function always fetches the first LlamaModel instance. model = LlamaModel.objects.first() @@ -51,65 +52,65 @@ def process_reference(num_ref, obj, user_id): refresh = RefreshToken.for_user(user) access_token = refresh.access_token - #url = "http://172.17.0.1:8400/api/v1/mix_citation/reference/" - #url = "http://172.17.0.1:8009/api/v1/mix_citation/reference/" + # url = "http://172.17.0.1:8400/api/v1/mix_citation/reference/" + # url = "http://172.17.0.1:8009/api/v1/mix_citation/reference/" # FIXME: Hardcoded URL - url = "http://django:8000/api/v1/reference/" + url = "http://django:8000/api/v1/reference/" headers = { - 'Authorization': f'Bearer {access_token}', - 'Content-Type': 'application/json' + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", } response = requests.post(url, json=payload, headers=headers) if response.status_code == 200: response_json = response.json() - message_str = response_json['message'] + message_str = response_json["message"] - json_str = message_str.replace('reference: ', '', 1) + json_str = message_str.replace("reference: ", "", 1) ref_json = json.loads(json_str) - #ref_json = json.loads(response) - obj['type'] = 'ref_paragraph' - obj['value'] = { - 'paragraph': ref_json.get('full_text', None), - 'label': '

', - 'reftype': ref_json.get('reftype', None), - 'refid': 'B'+str(num_ref), - 'date': ref_json.get('date', None), - 'title': ref_json.get('title', None), - 'chapter': ref_json.get('chapter', None), - 'edition': ref_json.get('edition', None), - 'source': ref_json.get('source', None), - 'vol': ref_json.get('vol', None), - 'issue': ref_json.get('num', None), - 'pages': ref_json.get('pages', None), - 'lpage': ref_json.get('lpage', None), - 'fpage': ref_json.get('fpage', None), - 'doi': ref_json.get('doi', None), - 'access_id': ref_json.get('access_id', None), - 'degree': ref_json.get('degree', None), - 'organization': ref_json.get('organization', None), - 'location': ref_json.get('location', None), - 'org_location': ref_json.get('org_location', None), - 'num_pages': ref_json.get('num_pages', None), - 'uri': ref_json.get('uri', None), - 'version': ref_json.get('version', None), - 'access_date': ref_json.get('access_date', None), - 'authors': [] + # ref_json = json.loads(response) + obj["type"] = "ref_paragraph" + obj["value"] = { + "paragraph": ref_json.get("full_text", None), + "label": "

", + "reftype": ref_json.get("reftype", None), + "refid": "B" + str(num_ref), + "date": ref_json.get("date", None), + "title": ref_json.get("title", None), + "chapter": ref_json.get("chapter", None), + "edition": ref_json.get("edition", None), + "source": ref_json.get("source", None), + "vol": ref_json.get("vol", None), + "issue": ref_json.get("num", None), + "pages": ref_json.get("pages", None), + "lpage": ref_json.get("lpage", None), + "fpage": ref_json.get("fpage", None), + "doi": ref_json.get("doi", None), + "access_id": ref_json.get("access_id", None), + "degree": ref_json.get("degree", None), + "organization": ref_json.get("organization", None), + "location": ref_json.get("location", None), + "org_location": ref_json.get("org_location", None), + "num_pages": ref_json.get("num_pages", None), + "uri": ref_json.get("uri", None), + "version": ref_json.get("version", None), + "access_date": ref_json.get("access_date", None), + "authors": [], } - authors = ref_json.get('authors', []) + authors = ref_json.get("authors", []) for author in authors: obj_auth = {} - obj_auth['type'] = 'Author' - obj_auth['value'] = {} - obj_auth['value']['surname'] = author.get('surname', None) - obj_auth['value']['given_names'] = author.get('fname', None) - obj['value']['authors'].append(obj_auth) - + obj_auth["type"] = "Author" + obj_auth["value"] = {} + obj_auth["value"]["surname"] = author.get("surname", None) + obj_auth["value"]["given_names"] = author.get("fname", None) + obj["value"]["authors"].append(obj_auth) + return obj @@ -118,44 +119,44 @@ def process_references(num_refs, references): for i, ref_json in enumerate(references): obj = {} - obj['type'] = 'ref_paragraph' - obj['value'] = { - 'paragraph': ref_json.get('full_text', None), - 'label': '

', - 'reftype': ref_json.get('reftype', None), - 'refid': 'B'+str(num_refs[i] if i < len(num_refs) else ''), - 'date': ref_json.get('date', None), - 'title': ref_json.get('title', None), - 'chapter': ref_json.get('chapter', None), - 'edition': ref_json.get('edition', None), - 'source': ref_json.get('source', None), - 'vol': ref_json.get('vol', None), - 'issue': ref_json.get('num', None), - 'pages': ref_json.get('pages', None), - 'lpage': ref_json.get('lpage', None), - 'fpage': ref_json.get('fpage', None), - 'doi': ref_json.get('doi', None), - 'access_id': ref_json.get('access_id', None), - 'degree': ref_json.get('degree', None), - 'organization': ref_json.get('organization', None), - 'location': ref_json.get('location', None), - 'org_location': ref_json.get('org_location', None), - 'num_pages': ref_json.get('num_pages', None), - 'uri': ref_json.get('uri', None), - 'version': ref_json.get('version', None), - 'access_date': ref_json.get('access_date', None), - 'authors': [] + obj["type"] = "ref_paragraph" + obj["value"] = { + "paragraph": ref_json.get("full_text", None), + "label": "

", + "reftype": ref_json.get("reftype", None), + "refid": "B" + str(num_refs[i] if i < len(num_refs) else ""), + "date": ref_json.get("date", None), + "title": ref_json.get("title", None), + "chapter": ref_json.get("chapter", None), + "edition": ref_json.get("edition", None), + "source": ref_json.get("source", None), + "vol": ref_json.get("vol", None), + "issue": ref_json.get("num", None), + "pages": ref_json.get("pages", None), + "lpage": ref_json.get("lpage", None), + "fpage": ref_json.get("fpage", None), + "doi": ref_json.get("doi", None), + "access_id": ref_json.get("access_id", None), + "degree": ref_json.get("degree", None), + "organization": ref_json.get("organization", None), + "location": ref_json.get("location", None), + "org_location": ref_json.get("org_location", None), + "num_pages": ref_json.get("num_pages", None), + "uri": ref_json.get("uri", None), + "version": ref_json.get("version", None), + "access_date": ref_json.get("access_date", None), + "authors": [], } - authors = ref_json.get('authors', []) + authors = ref_json.get("authors", []) for author in authors: obj_auth = {} - obj_auth['type'] = 'Author' - obj_auth['value'] = {} - obj_auth['value']['surname'] = author.get('surname', None) - obj_auth['value']['given_names'] = author.get('fname', None) - obj['value']['authors'].append(obj_auth) + obj_auth["type"] = "Author" + obj_auth["value"] = {} + obj_auth["value"]["surname"] = author.get("surname", None) + obj_auth["value"]["given_names"] = author.get("fname", None) + obj["value"]["authors"].append(obj_auth) arr_references.append(obj) - + return arr_references @@ -165,55 +166,81 @@ def buscar_refid_por_surname_y_date(data_back, surname_buscado, date_buscado): y que coincida con la fecha dada. Retorna el refid si encuentra una coincidencia. """ for bloque in data_back: # Reemplaza 'contenido' con el nombre de tu StreamField - if bloque['type'] == 'ref_paragraph': # o el nombre que usaste en el StreamField - data = bloque['value'] + if ( + bloque["type"] == "ref_paragraph" + ): # o el nombre que usaste en el StreamField + data = bloque["value"] # Verificar la fecha - if str(data.get('date')) != str(date_buscado[:4]): + if str(data.get("date")) != str(date_buscado[:4]): continue # Revisar autores - authors = data.get('authors', []) + authors = data.get("authors", []) - surname_buscados = surname_buscado.split(',') + surname_buscados = surname_buscado.split(",") for surname_buscado in surname_buscados: - if ' y ' in surname_buscado or ' and ' in surname_buscado or ' e ' in surname_buscado or ' & ' in surname_buscado: - if ' y ' in surname_buscado: - surname1 = surname_buscado.split(' y ')[0].strip().lower() - surname2 = surname_buscado.split(' y ')[1].strip().lower() - - if ' and ' in surname_buscado: - surname1 = surname_buscado.split(' and ')[0].strip().lower() - surname2 = surname_buscado.split(' and ')[1].strip().lower() - - if ' & ' in surname_buscado: - surname1 = surname_buscado.split(' & ')[0].strip().lower() - surname2 = surname_buscado.split(' & ')[1].strip().lower() - - if ' e ' in surname_buscado: - surname1 = surname_buscado.split(' e ')[0].strip().lower() - surname2 = surname_buscado.split(' e ')[1].strip().lower() + if ( + " y " in surname_buscado + or " and " in surname_buscado + or " e " in surname_buscado + or " & " in surname_buscado + ): + if " y " in surname_buscado: + surname1 = surname_buscado.split(" y ")[0].strip().lower() + surname2 = surname_buscado.split(" y ")[1].strip().lower() + + if " and " in surname_buscado: + surname1 = surname_buscado.split(" and ")[0].strip().lower() + surname2 = surname_buscado.split(" and ")[1].strip().lower() + + if " & " in surname_buscado: + surname1 = surname_buscado.split(" & ")[0].strip().lower() + surname2 = surname_buscado.split(" & ")[1].strip().lower() + + if " e " in surname_buscado: + surname1 = surname_buscado.split(" e ")[0].strip().lower() + surname2 = surname_buscado.split(" e ")[1].strip().lower() for author_bloque in authors: - if author_bloque['type'] == 'Author': - author_data = author_bloque['value'] - if surname1 in (author_data.get('surname') or '').lower() + ' ' + (author_data.get('given_names') or '').lower(): + if author_bloque["type"] == "Author": + author_data = author_bloque["value"] + if ( + surname1 + in (author_data.get("surname") or "").lower() + + " " + + (author_data.get("given_names") or "").lower() + ): for author_bloque2 in authors: - if author_bloque2['type'] == 'Author': - author_data = author_bloque2['value'] - if surname2 in (author_data.get('surname') or '').lower() + ' ' + (author_data.get('given_names') or '').lower(): - return data.get('refid') + if author_bloque2["type"] == "Author": + author_data = author_bloque2["value"] + if ( + surname2 + in ( + author_data.get("surname") or "" + ).lower() + + " " + + ( + author_data.get("given_names") or "" + ).lower() + ): + return data.get("refid") for author_bloque in authors: - if author_bloque['type'] == 'Author': - author_data = author_bloque['value'] + if author_bloque["type"] == "Author": + author_data = author_bloque["value"] + + if ( + surname_buscado.strip().lower() + in (author_data.get("surname") or "").lower() + + " " + + (author_data.get("given_names") or "").lower() + ): + return data.get("refid") - if surname_buscado.strip().lower() in (author_data.get('surname') or '').lower() + ' ' + (author_data.get('given_names') or '').lower(): - return data.get('refid') - - if surname_buscado.strip().lower() in (data.get('paragraph') or '').lower(): - return data.get('refid') + if surname_buscado.strip().lower() in (data.get("paragraph") or "").lower(): + return data.get("refid") return None @@ -226,242 +253,265 @@ def extract_citation_apa(texto, data_back): - el año. Acepta múltiples espacios entre palabras y símbolos. """ - + # Preposiciones comunes en apellidos - preposiciones = r'(?:de|del|la|los|las|da|do|dos|das|van|von)' + preposiciones = r"(?:de|del|la|los|las|da|do|dos|das|van|von)" # Apellido compuesto o con preposición (incluye caracteres portugueses: ç, ã, õ, etc.) - apellido = rf'[A-ZÁÉÍÓÚÑÇÃÕÂÊÎÔÛ][a-záéíóúñçãõâêîôû]+(?:[-‐\s]+(?:{preposiciones})?\s*[A-ZÁÉÍÓÚÑÇÃÕÂÊÎÔÛ]?[a-záéíóúñçãõâêîôû]+)*' + apellido = rf"[A-ZÁÉÍÓÚÑÇÃÕÂÊÎÔÛ][a-záéíóúñçãõâêîôû]+(?:[-‐\s]+(?:{preposiciones})?\s*[A-ZÁÉÍÓÚÑÇÃÕÂÊÎÔÛ]?[a-záéíóúñçãõâêîôû]+)*" resultados = [] - + # 1. Buscar todas las citas dentro de paréntesis - for paren in re.finditer(r'\(([^)]+)\)', texto): + for paren in re.finditer(r"\(([^)]+)\)", texto): contenido_completo = paren.group(1) - + # Si hay punto y coma, dividir las citas - if ';' in contenido_completo: - partes = [parte for parte in contenido_completo.split(';')] + if ";" in contenido_completo: + partes = [parte for parte in contenido_completo.split(";")] else: partes = [contenido_completo] - + # Variable para rastrear el contexto de autores dentro del mismo paréntesis autores_en_parentesis = [] - + for i, parte in enumerate(partes): parte = parte # Agregar strip aquí para limpiar espacios if not parte: continue - + # Caso especial: solo año (para citas como "2017" después de "2014;") # MEJORADO: Solo aplicar si hay citas previas EN EL MISMO PARÉNTESIS - if re.match(r'^\s*\d{4}[a-z]?\s*$', parte): + if re.match(r"^\s*\d{4}[a-z]?\s*$", parte): # Solo usar el último autor si hay autores en el mismo paréntesis if autores_en_parentesis: ultimo_autor = autores_en_parentesis[-1] - refid = buscar_refid_por_surname_y_date(data_back, ultimo_autor, parte) - resultados.append({ - "cita": parte, - "autor": ultimo_autor, - "anio": parte, - "refid": refid - }) + refid = buscar_refid_por_surname_y_date( + data_back, ultimo_autor, parte + ) + resultados.append( + { + "cita": parte, + "autor": ultimo_autor, + "anio": parte, + "refid": refid, + } + ) # Si no hay autores previos en el paréntesis, ignorar (posible error) continue - + # Patrones para diferentes tipos de citas cita_encontrada = False - + # Patrón 1: Múltiples autores con & y coma antes del año # Ejemplo: "Porta, Lopez-De-Silanes, & Shleifer, 1999" - pattern1 = rf'(?P{apellido}(?:\s*,\s*{apellido})*\s*,?\s*&\s*{apellido})\s*,\s*(?P\d{{4}}[a-z]?)' + pattern1 = rf"(?P{apellido}(?:\s*,\s*{apellido})*\s*,?\s*&\s*{apellido})\s*,\s*(?P\d{{4}}[a-z]?)" match = re.search(pattern1, parte) if match: autores_completos = match.group("autores") anio = match.group("anio") - primer_autor = re.split(r'\s*,\s*', autores_completos)[0] + primer_autor = re.split(r"\s*,\s*", autores_completos)[0] autores_en_parentesis.append(primer_autor) refid = buscar_refid_por_surname_y_date(data_back, primer_autor, anio) - resultados.append({ - "cita": parte, - "autor": primer_autor, - "anio": anio, - "refid": refid - }) + resultados.append( + {"cita": parte, "autor": primer_autor, "anio": anio, "refid": refid} + ) cita_encontrada = True - + # Patrón 2: Múltiples autores con & SIN coma antes del año # Ejemplo: "Silva, Peixoto, & Tizziotti 2021" if not cita_encontrada: - pattern2 = rf'(?P{apellido}(?:\s*,\s*{apellido})*\s*,?\s*&\s*{apellido})\s+(?P\d{{4}}[a-z]?)' + pattern2 = rf"(?P{apellido}(?:\s*,\s*{apellido})*\s*,?\s*&\s*{apellido})\s+(?P\d{{4}}[a-z]?)" match = re.search(pattern2, parte) if match: autores_completos = match.group("autores") anio = match.group("anio") - primer_autor = re.split(r'\s*,\s*', autores_completos)[0] + primer_autor = re.split(r"\s*,\s*", autores_completos)[0] autores_en_parentesis.append(primer_autor) - refid = buscar_refid_por_surname_y_date(data_back, primer_autor, anio) - resultados.append({ - "cita": parte, - "autor": primer_autor, - "anio": anio, - "refid": refid - }) + refid = buscar_refid_por_surname_y_date( + data_back, primer_autor, anio + ) + resultados.append( + { + "cita": parte, + "autor": primer_autor, + "anio": anio, + "refid": refid, + } + ) cita_encontrada = True - + # Patrón 3: Dos autores con & (simple) # Ejemplo: "Crisóstomo & Brandão, 2019" if not cita_encontrada: - pattern3 = rf'(?P{apellido})\s*&\s*(?P{apellido})\s*,\s*(?P\d{{4}}[a-z]?)' + pattern3 = rf"(?P{apellido})\s*&\s*(?P{apellido})\s*,\s*(?P\d{{4}}[a-z]?)" match = re.search(pattern3, parte) if match: primer_autor = match.group("autor1") anio = match.group("anio") autores_en_parentesis.append(primer_autor) - refid = buscar_refid_por_surname_y_date(data_back, primer_autor, anio) - resultados.append({ - "cita": parte, - "autor": primer_autor, - "anio": anio, - "refid": refid - }) + refid = buscar_refid_por_surname_y_date( + data_back, primer_autor, anio + ) + resultados.append( + { + "cita": parte, + "autor": primer_autor, + "anio": anio, + "refid": refid, + } + ) cita_encontrada = True - + # Patrón 4: Autor con "et al." con coma # Ejemplo: "Brandão et al., 2019" if not cita_encontrada: - pattern4 = rf'(?P{apellido})\s+et\s+al\s*\.?\s*,\s*(?P\d{{4}}[a-z]?)' + pattern4 = rf"(?P{apellido})\s+et\s+al\s*\.?\s*,\s*(?P\d{{4}}[a-z]?)" match = re.search(pattern4, parte) if match: autor = match.group("autor") anio = match.group("anio") autores_en_parentesis.append(autor) refid = buscar_refid_por_surname_y_date(data_back, autor, anio) - resultados.append({ - "cita": parte, - "autor": autor, - "anio": anio, - "refid": refid - }) + resultados.append( + {"cita": parte, "autor": autor, "anio": anio, "refid": refid} + ) cita_encontrada = True - + # Patrón 5: Autor con "et al." sin coma # Ejemplo: "Brandão et al. 2019" if not cita_encontrada: - pattern5 = rf'(?P{apellido})\s+et\s+al\s*\.?\s+(?P\d{{4}}[a-z]?)' + pattern5 = ( + rf"(?P{apellido})\s+et\s+al\s*\.?\s+(?P\d{{4}}[a-z]?)" + ) match = re.search(pattern5, parte) if match: autor = match.group("autor") anio = match.group("anio") autores_en_parentesis.append(autor) refid = buscar_refid_por_surname_y_date(data_back, autor, anio) - resultados.append({ - "cita": parte, - "autor": autor, - "anio": anio, - "refid": refid - }) + resultados.append( + {"cita": parte, "autor": autor, "anio": anio, "refid": refid} + ) cita_encontrada = True - + # Patrón 6: Múltiples autores solo con comas (sin &) # Ejemplo: "Adam, Tene, Mucci, Beck, 2020" o "Correia, Amaral, Louvet, 2014a" if not cita_encontrada: - pattern6 = rf'(?P{apellido}(?:\s*,\s*{apellido}){{2,}})\s*,\s*(?P\d{{4}}[a-z]?)' + pattern6 = rf"(?P{apellido}(?:\s*,\s*{apellido}){{2,}})\s*,\s*(?P\d{{4}}[a-z]?)" match = re.search(pattern6, parte) if match: autores_completos = match.group("autores") anio = match.group("anio") - primer_autor = re.split(r'\s*,\s*', autores_completos)[0] + primer_autor = re.split(r"\s*,\s*", autores_completos)[0] autores_en_parentesis.append(primer_autor) - refid = buscar_refid_por_surname_y_date(data_back, primer_autor, anio) - resultados.append({ - "cita": parte, - "autor": primer_autor, - "anio": anio, - "refid": refid - }) + refid = buscar_refid_por_surname_y_date( + data_back, primer_autor, anio + ) + resultados.append( + { + "cita": parte, + "autor": primer_autor, + "anio": anio, + "refid": refid, + } + ) cita_encontrada = True - + # Patrón 7: Autor simple con coma # Ejemplo: "Smith, 2020" if not cita_encontrada: - pattern7 = rf'(?P{apellido})\s*,\s*(?P\d{{4}}[a-z]?)' + pattern7 = rf"(?P{apellido})\s*,\s*(?P\d{{4}}[a-z]?)" match = re.search(pattern7, parte) if match: autor = match.group("autor") anio = match.group("anio") autores_en_parentesis.append(autor) refid = buscar_refid_por_surname_y_date(data_back, autor, anio) - resultados.append({ - "cita": parte, - "autor": autor, - "anio": anio, - "refid": refid - }) + resultados.append( + {"cita": parte, "autor": autor, "anio": anio, "refid": refid} + ) cita_encontrada = True - + # 2. Citas fuera del paréntesis: Nombre (2000) o Nombre et al. (2000) # MEJORADO: Filtrar citas que están precedidas por preposiciones - + # Lista de preposiciones a evitar - preposiciones_evitar = ['de', 'del', 'la', 'los', 'las', 'da', 'do', 'dos', 'das', 'van', 'von'] - + preposiciones_evitar = [ + "de", + "del", + "la", + "los", + "las", + "da", + "do", + "dos", + "das", + "van", + "von", + ] + # Patrón para citas con múltiples años: Autor (2018, 2019) - patron_multiples_años = rf'(?P{apellido})(?:\s*[-‐]\s*{apellido})*(?:\s+et\s+al\s*\.?|\s+(?:y|and|&)\s+{apellido})?\s*\(\s*(?P\d{{4}}[a-z]?(?:\s*,\s*\d{{4}}[a-z]?)+)\s*\)' - + patron_multiples_años = rf"(?P{apellido})(?:\s*[-‐]\s*{apellido})*(?:\s+et\s+al\s*\.?|\s+(?:y|and|&)\s+{apellido})?\s*\(\s*(?P\d{{4}}[a-z]?(?:\s*,\s*\d{{4}}[a-z]?)+)\s*\)" + for match in re.finditer(patron_multiples_años, texto): # Verificar que no haya preposición antes de la cita inicio_match = match.start() texto_anterior = texto[:inicio_match].split() - + # Si hay palabras antes y la última es una preposición, saltarse esta cita if texto_anterior and texto_anterior[-1].lower() in preposiciones_evitar: continue - + autor = match.group("autor") años_str = match.group("años") # Separar los años y crear una cita para cada uno - años = [año for año in años_str.split(',')] - + años = [año for año in años_str.split(",")] + for año in años: refid = buscar_refid_por_surname_y_date(data_back, autor, año) - resultados.append({ - "cita": f"{autor} et al. ({año})" if "et al" in match.group(0) else f"{autor} ({año})", - "autor": autor, - "anio": año, - "refid": refid - }) - + resultados.append( + { + "cita": f"{autor} et al. ({año})" + if "et al" in match.group(0) + else f"{autor} ({año})", + "autor": autor, + "anio": año, + "refid": refid, + } + ) + # Patrón para citas simples: Nombre (2000) o Nombre et al. (2000) - patron_afuera = rf'(?P{apellido})(?:\s*[-‐]\s*{apellido})*(?:\s+et\s+al\s*\.?|\s+(?:y|and|&)\s+{apellido})?\s*\(\s*(?P\d{{4}}[a-z]?)\s*\)' - + patron_afuera = rf"(?P{apellido})(?:\s*[-‐]\s*{apellido})*(?:\s+et\s+al\s*\.?|\s+(?:y|and|&)\s+{apellido})?\s*\(\s*(?P\d{{4}}[a-z]?)\s*\)" + for match in re.finditer(patron_afuera, texto): # Verificar que no haya preposición antes de la cita inicio_match = match.start() texto_anterior = texto[:inicio_match].split() - + # Si hay palabras antes y la última es una preposición, saltarse esta cita if texto_anterior and texto_anterior[-1].lower() in preposiciones_evitar: continue - + autor = match.group("autor") anio = match.group("anio") - + # Verificar que no sea parte de una cita con múltiples años ya procesada cita_completa = match.group(0) es_multiple = False for resultado in resultados: - if resultado["autor"] == autor and resultado["anio"] == anio and "," in cita_completa: + if ( + resultado["autor"] == autor + and resultado["anio"] == anio + and "," in cita_completa + ): es_multiple = True break - + if not es_multiple: refid = buscar_refid_por_surname_y_date(data_back, autor, anio) - resultados.append({ - "cita": cita_completa, - "autor": autor, - "anio": anio, - "refid": refid - }) - + resultados.append( + {"cita": cita_completa, "autor": autor, "anio": anio, "refid": refid} + ) + return resultados @@ -470,13 +520,13 @@ def clean_labels(texto): Elimina todas las etiquetas XML del texto. """ # Patrón para encontrar etiquetas XML (apertura y cierre) - patron_etiquetas = r'<[^>]+>' - texto_limpio = re.sub(patron_etiquetas, '', texto) - + patron_etiquetas = r"<[^>]+>" + texto_limpio = re.sub(patron_etiquetas, "", texto) + # Limpiar espacios múltiples que puedan haber quedado - #texto_limpio = re.sub(r'\s+', ' ', texto_limpio) - - return texto_limpio#.strip() + # texto_limpio = re.sub(r'\s+', ' ', texto_limpio) + + return texto_limpio # .strip() def map_text(texto): @@ -485,16 +535,16 @@ def map_text(texto): Clave: texto sin etiquetas, Valor: texto con etiquetas """ mapa = {} - + # Buscar TODAS las etiquetas y su contenido - patron = r'<[^>]+>.*?]+>|<[^/>]+/>' + patron = r"<[^>]+>.*?]+>|<[^/>]+/>" matches = re.findall(patron, texto, re.DOTALL) - + for match in matches: - contenido_limpio = clean_labels(match)#.strip() + contenido_limpio = clean_labels(match) # .strip() if contenido_limpio: # Solo si hay contenido real - mapa[contenido_limpio] = match#.strip() - + mapa[contenido_limpio] = match # .strip() + return mapa @@ -520,38 +570,38 @@ def extract_labels(texto_original, texto_limpio, pos_inicio, pos_fin): contador_chars_limpios = 0 resultado = "" dentro_del_rango = False - + i = 0 while i < len(texto_original) and contador_chars_limpios <= pos_fin: char = texto_original[i] - - if char == '<': + + if char == "<": # Encontrar el final de la etiqueta - fin_etiqueta = texto_original.find('>', i) + fin_etiqueta = texto_original.find(">", i) if fin_etiqueta != -1: - etiqueta = texto_original[i:fin_etiqueta + 1] - + etiqueta = texto_original[i : fin_etiqueta + 1] + # Si estamos dentro del rango, incluir la etiqueta if dentro_del_rango: resultado += etiqueta - + i = fin_etiqueta + 1 continue - + # Verificar si entramos o salimos del rango if contador_chars_limpios == pos_inicio: dentro_del_rango = True elif contador_chars_limpios == pos_fin: dentro_del_rango = False break - + # Si estamos dentro del rango, incluir el caracter if dentro_del_rango: resultado += char - + contador_chars_limpios += 1 i += 1 - + return resultado @@ -562,26 +612,26 @@ def restore_labels_ref(ref, mapa_etiquetado, texto_original, texto_limpio): """ # Encontrar todas las posiciones donde aparece esta ref en el texto limpio posiciones_ref = search_position(texto_limpio, ref) - + if not posiciones_ref: return ref - + # Para cada posición, extraer el fragmento original y ver si contiene etiquetas mejores_candidatos = [] - + for pos_inicio, pos_fin in posiciones_ref: fragmento_original = extract_labels( texto_original, texto_limpio, pos_inicio, pos_fin ) - + # Si el fragmento original es diferente al ref, significa que tenía etiquetas if fragmento_original != ref: mejores_candidatos.append(fragmento_original) - + # Si encontramos candidatos con etiquetas, devolver el primero if mejores_candidatos: return mejores_candidatos[0] - + # Si no hay candidatos con etiquetas, devolver el ref original sin modificar return ref @@ -590,88 +640,112 @@ def proccess_labeled_text(texto, data_back): """ Procesa un texto eliminando etiquetas XML, extrae citas APA y las devuelve con sus etiquetas originales restauradas. - + Args: texto (str): Texto original con etiquetas XML extraer_citas_apa (function): Función que extrae citas del texto limpio - + Returns: list: Lista de citas con etiquetas XML restauradas """ - + # Crear mapa de transformaciones mapa_transformaciones = map_text(texto) - #print(f"mapa: {mapa_transformaciones}") - + # print(f"mapa: {mapa_transformaciones}") + # Limpiar texto eliminando etiquetas texto_limpio = clean_labels(texto) - + # Extraer citas del texto limpio refs = extract_citation_apa(texto_limpio, data_back) - #print(f"refs: {refs}") + # print(f"refs: {refs}") - # 4. Para cada ref, usar posición para restaurar solo lo que realmente estaba etiquetado + # 4. Para cada ref, usar posición para restaurar solo lo que realmente estaba etiquetado refs_con_etiquetas = [] for ref in refs: ref_restaurada = ref - ref_restaurada['cita'] = restore_labels_ref(ref['cita'], mapa_transformaciones, texto, texto_limpio) + ref_restaurada["cita"] = restore_labels_ref( + ref["cita"], mapa_transformaciones, texto, texto_limpio + ) refs_con_etiquetas.append(ref_restaurada) - + return refs_con_etiquetas def match_by_regex(text, order_labels): return next( - (key_obj for key_obj in order_labels.items() - if "regex" in key_obj[1] and re.search(key_obj[1]["regex"], text)), - None + ( + key_obj + for key_obj in order_labels.items() + if "regex" in key_obj[1] and re.search(key_obj[1]["regex"], text) + ), + None, ) -def match_by_style_and_size(item, order_labels, style='bold'): +def match_by_style_and_size(item, order_labels, style="bold"): return next( - (key_obj for key_obj in order_labels.items() - if "size" in key_obj[1] and style in key_obj[1] and - key_obj[1]["size"] == item.get('font_size') and - key_obj[1][style] == item.get(style)), - None + ( + key_obj + for key_obj in order_labels.items() + if "size" in key_obj[1] + and style in key_obj[1] + and key_obj[1]["size"] == item.get("font_size") + and key_obj[1][style] == item.get(style) + ), + None, ) def match_next_label(item, label_next, order_labels): return next( - (key_obj for key_obj in order_labels.items() - if "size" in key_obj[1] and key_obj[1]["size"] == item.get('font_size') - and key_obj[0] == label_next), - None + ( + key_obj + for key_obj in order_labels.items() + if "size" in key_obj[1] + and key_obj[1]["size"] == item.get("font_size") + and key_obj[0] == label_next + ), + None, ) def match_paragraph(item, order_labels): return next( - (key_obj for key_obj in order_labels.items() - if "size" in key_obj[1] and - "next" in key_obj[1] and - key_obj[1]["size"] == item.get('font_size') and - key_obj[1]["next"] == "

"), - None + ( + key_obj + for key_obj in order_labels.items() + if "size" in key_obj[1] + and "next" in key_obj[1] + and key_obj[1]["size"] == item.get("font_size") + and key_obj[1]["next"] == "

" + ), + None, ) def match_section(item, sections): - return {'label': '', 'body': True} if ( - item.get('font_size') == sections[0].get('size') and - item.get('bold') == sections[0].get('bold') and - item.get('text', '').isupper() == sections[0].get('isupper') - ) else None + return ( + {"label": "", "body": True} + if ( + item.get("font_size") == sections[0].get("size") + and item.get("bold") == sections[0].get("bold") + and item.get("text", "").isupper() == sections[0].get("isupper") + ) + else None + ) def match_subsection(item, sections): - return {'label': '', 'body': True} if ( - item.get('font_size') == sections[1].get('size') and - item.get('bold') == sections[1].get('bold') and - item.get('text', '').isupper() == sections[1].get('isupper') - ) else None + return ( + {"label": "", "body": True} + if ( + item.get("font_size") == sections[1].get("size") + and item.get("bold") == sections[1].get("bold") + and item.get("text", "").isupper() == sections[1].get("isupper") + ) + else None + ) def create_labeled_object2(i, item, state, sections): @@ -680,114 +754,110 @@ def create_labeled_object2(i, item, state, sections): if match_section(item, sections): result = match_section(item, sections) - state['label'] = result.get('label') - state['body'] = result.get('body') - + state["label"] = result.get("label") + state["body"] = result.get("body") + if match_subsection(item, sections): result = match_subsection(item, sections) - state['label'] = result.get('label') - state['body'] = result.get('body') - - if state.get('body') and re.search(r"^(refer)", item.get('text').lower()) and match_section(item, sections): - state['label'] = '' - state['body'] = False - state['back'] = True - obj['type'] = 'paragraph' - obj['value'] = { - 'label': state['label'], - 'paragraph': item.get('text') - } - + state["label"] = result.get("label") + state["body"] = result.get("body") + + if ( + state.get("body") + and re.search(r"^(refer)", item.get("text").lower()) + and match_section(item, sections) + ): + state["label"] = "" + state["body"] = False + state["back"] = True + obj["type"] = "paragraph" + obj["value"] = {"label": state["label"], "paragraph": item.get("text")} + if not result: - result = {'label': '

', 'body': state['body'], 'back': state['back']} - state['label'] = result.get('label') - state['body'] = result.get('body') - state['back'] = result.get('back') + result = {"label": "

", "body": state["body"], "back": state["back"]} + state["label"] = result.get("label") + state["body"] = result.get("body") + state["back"] = result.get("back") if result: pass else: - if state.get('label_next'): - if state.get('repeat'): - result = match_by_regex(item.get('text'), order_labels) + if state.get("label_next"): + if state.get("repeat"): + result = match_by_regex(item.get("text"), order_labels) if result: - state['label'] = result[0] + state["label"] = result[0] else: - result = match_by_style_and_size(item, order_labels, style='bold') + result = match_by_style_and_size(item, order_labels, style="bold") if result: - state['label'] = result[0] - state['repeat'] = None - state['reset'] = None - state['label_next'] = result[1].get("next") - state['body'] = result[1].get("size") == 16 - if state['body'] and re.search(r"^(refer)", item.get('text').lower()): - state['body'] = False - state['back'] = True + state["label"] = result[0] + state["repeat"] = None + state["reset"] = None + state["label_next"] = result[1].get("next") + state["body"] = result[1].get("size") == 16 + if state["body"] and re.search( + r"^(refer)", item.get("text").lower() + ): + state["body"] = False + state["back"] = True if not result: - result = match_next_label(item, state['label_next'], order_labels) + result = match_next_label(item, state["label_next"], order_labels) if result: - state['label'] = result[0] - state['label_next_reset'] = result[1].get("next") - state['reset'] = result[1].get("reset", False) - state['repeat'] = result[1].get("repeat", False) + state["label"] = result[0] + state["label_next_reset"] = result[1].get("next") + state["reset"] = result[1].get("reset", False) + state["repeat"] = result[1].get("repeat", False) else: - result = match_by_style_and_size(item, order_labels, style='bold') + result = match_by_style_and_size(item, order_labels, style="bold") if result: - state['label'] = result[0] - state['label_next'] = result[1].get("next") - if state.get('body') and re.search(r"^(refer)", item.get('text').lower()): - state['body'] = False - state['back'] = True + state["label"] = result[0] + state["label_next"] = result[1].get("next") + if state.get("body") and re.search( + r"^(refer)", item.get("text").lower() + ): + state["body"] = False + state["back"] = True else: - result = match_by_style_and_size(item, order_labels, style='italic') + result = match_by_style_and_size(item, order_labels, style="italic") if result: - state['label'] = re.sub(r"-\d+", "", result[0]) - state['label_next'] = result[1].get("next") + state["label"] = re.sub(r"-\d+", "", result[0]) + state["label_next"] = result[1].get("next") else: - result = match_by_regex(item.get('text'), order_labels) + result = match_by_regex(item.get("text"), order_labels) if result: - state['label'] = result[0] + state["label"] = result[0] else: result = match_paragraph(item, order_labels) if result: - state['label'] = result[0] + state["label"] = result[0] if result: - if state['label'] in ['']: - order_labels.pop(state['label'], None) + if state["label"] in [""]: + order_labels.pop(state["label"], None) - #label_info = result[1] - #obj['type'] = 'paragraph_with_language' if label_info.get("lan") else 'paragraph' - obj['type'] = 'paragraph' + # label_info = result[1] + # obj['type'] = 'paragraph_with_language' if label_info.get("lan") else 'paragraph' + obj["type"] = "paragraph" - obj['value'] = { - 'label': state['label'], - 'paragraph': item.get('text') - } + obj["value"] = {"label": state["label"], "paragraph": item.get("text")} - if state['label'] == '': - obj['type'] = 'author_paragraph' - elif state['label'] == '': - obj['type'] = 'aff_paragraph' - - if re.search(r"^(translation)", item.get('text').lower()): - state['label'] = '' - state['body'] = False - state['back'] = False - obj['type'] = 'paragraph_with_language' - obj['value'] = { - 'label': state['label'], - 'paragraph': item.get('text') - } + if state["label"] == "": + obj["type"] = "author_paragraph" + elif state["label"] == "": + obj["type"] = "aff_paragraph" + + if re.search(r"^(translation)", item.get("text").lower()): + state["label"] = "" + state["body"] = False + state["back"] = False + obj["type"] = "paragraph_with_language" + obj["value"] = {"label": state["label"], "paragraph": item.get("text")} return obj, result, state def get_data_first_block(text, metadata, user_id): - payload = { - 'text': text, - 'metadata': metadata - } + payload = {"text": text, "metadata": metadata} model = LlamaModel.objects.first() @@ -797,33 +867,33 @@ def get_data_first_block(text, metadata, user_id): access_token = refresh.access_token # FIXME: Hardcoded URL - url = "http://django:8000/api/v1/first_block/" + url = "http://django:8000/api/v1/first_block/" headers = { - 'Authorization': f'Bearer {access_token}', - 'Content-Type': 'application/json' + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", } response = requests.post(url, json=payload, headers=headers) if response.status_code == 200: response_json = response.json() - message_str = response_json['message'] + message_str = response_json["message"] resp_json = json.loads(message_str) - + return resp_json def extract_keywords(text): # Quitar punto final si existe text = text.strip() - if text.endswith('.'): + if text.endswith("."): text = text[:-1].strip() # Ver si contiene una etiqueta con dos puntos - match = re.match(r'(?i)\s*(.+?)\s*:\s*(.+)', text) - + match = re.match(r"(?i)\s*(.+?)\s*:\s*(.+)", text) + if match: label = match.group(1).strip() content = match.group(2).strip() @@ -832,7 +902,7 @@ def extract_keywords(text): content = text # Separar por punto y coma o coma - keywords = re.split(r'\s*[;,]\s*', content) + keywords = re.split(r"\s*[;,]\s*", content) clean_keywords = [p.strip() for p in keywords if p.strip()] clean_keywords = ", ".join(keywords) @@ -843,92 +913,82 @@ def create_special_content_object(item, stream_data_body, counts): """Create objects for special content types (image, table, list, compound)""" obj = {} - if item.get('type') == 'image': + if item.get("type") == "image": obj = {} - counts['numfig'] += 1 - obj['type'] = 'image' - obj['value'] = { - 'figid' : f"f{counts['numfig']}", - 'label' : '', - 'image' : item.get('image') - } - - #Obitiene el elemento aterior + counts["numfig"] += 1 + obj["type"] = "image" + obj["value"] = { + "figid": f"f{counts['numfig']}", + "label": "", + "image": item.get("image"), + } + + # Obitiene el elemento aterior try: prev_element = stream_data_body[-1] - label_title = extract_label_and_title(prev_element['value']['paragraph']) - obj['value']['figlabel'] = label_title['label'] - obj['value']['title'] = label_title['title'] + label_title = extract_label_and_title(prev_element["value"]["paragraph"]) + obj["value"]["figlabel"] = label_title["label"] + obj["value"]["title"] = label_title["title"] stream_data_body.pop(-1) except: pass - elif item.get('type') == 'table': + elif item.get("type") == "table": obj = {} - counts['numtab'] += 1 - obj['type'] = 'table' - obj['value'] = { - 'tabid' : f"t{counts['numtab']}", - 'label' : '', - 'content' : item.get('table') + counts["numtab"] += 1 + obj["type"] = "table" + obj["value"] = { + "tabid": f"t{counts['numtab']}", + "label": "
", + "content": item.get("table"), } - #Obitiene el elemento aterior + # Obitiene el elemento aterior try: prev_element = stream_data_body[-1] - label_title = extract_label_and_title(prev_element['value']['paragraph']) - obj['value']['tablabel'] = label_title['label'] - obj['value']['title'] = label_title['title'] + label_title = extract_label_and_title(prev_element["value"]["paragraph"]) + obj["value"]["tablabel"] = label_title["label"] + obj["value"]["title"] = label_title["title"] stream_data_body.pop(-1) except: - #No hay elemento anterior + # No hay elemento anterior pass - elif item.get('type') == 'list': + elif item.get("type") == "list": obj = {} - obj['type'] = 'paragraph' - obj['value'] = { - 'label' : '', - 'paragraph' : item.get('list') - } - - elif item.get('type') == 'compound': + obj["type"] = "paragraph" + obj["value"] = {"label": "", "paragraph": item.get("list")} + + elif item.get("type") == "compound": obj = {} - counts['numeq'] += 1 - obj['type'] = 'compound_paragraph' - obj['value'] = { - 'eid' : f"e{counts['numeq']}", + counts["numeq"] += 1 + obj["type"] = "compound_paragraph" + obj["value"] = { + "eid": f"e{counts['numeq']}", #'label' : '', - 'content': item.get('text') + "content": item.get("text"), } - text_count = sum( - 1 for c in obj['value']['content'] - if c['type'] == 'text' - ) - + text_count = sum(1 for c in obj["value"]["content"] if c["type"] == "text") + if text_count > 1: - obj['value']['label'] = '' + obj["value"]["label"] = "" return obj, counts if text_count == 0: - obj['value']['label'] = '' + obj["value"]["label"] = "" return obj, counts - + text_value = next( - item['value'] - for item in obj['value']['content'] - if item['type'] == 'text' + item["value"] for item in obj["value"]["content"] if item["type"] == "text" ) text = is_number_parenthesis(text_value) if text: - obj['value']['label'] = '' - next( - item - for item in obj['value']['content'] - if item['type'] == 'text' - )['value'] = text + obj["value"]["label"] = "" + next(item for item in obj["value"]["content"] if item["type"] == "text")[ + "value" + ] = text else: - obj['value']['label'] = '' + obj["value"]["label"] = "" return obj, counts @@ -938,8 +998,8 @@ def extract_subsection(text): text = text.strip() # Ver si contiene una etiqueta con dos puntos - match = re.match(r'(?i)\s*(.+?)\s*:\s*(.+)', text) - + match = re.match(r"(?i)\s*(.+?)\s*:\s*(.+)", text) + if match: label = match.group(1).strip() content = match.group(2).strip() @@ -951,37 +1011,43 @@ def extract_subsection(text): def search_special_id(data_body, label): - for d in data_body: - if d['type'] in ['image', 'table']: - data = d['value'] - clean_label = re.sub(r'^[\s\.,;:–—-]+', '', label).capitalize() - - if d['type'] == 'image': - if clean_label == data['figlabel']: - return data.get('figid') - if data['figid'][0] == clean_label.lower()[0] and data['figid'][1] in clean_label.lower(): - return data.get('figid') - - if d['type'] == 'table': - if clean_label == data['tablabel']: - return data.get('tabid') - if data['tabid'][0] == clean_label.lower()[0] and data['tabid'][1] in clean_label.lower(): - return data.get('tabid') - - for d in data_body: - if d['type'] in ['compound_paragraph']: - data = d['value'] - clean_label = re.sub(r'^[\s\.,;:–—-]+', '', label).lower() - - if d['type'] == 'compound_paragraph': - if data['eid'][0] in clean_label[0] and data['eid'][1] in clean_label: - return data.get('eid') + for d in data_body: + if d["type"] in ["image", "table"]: + data = d["value"] + clean_label = re.sub(r"^[\s\.,;:–—-]+", "", label).capitalize() + + if d["type"] == "image": + if clean_label == data["figlabel"]: + return data.get("figid") + if ( + data["figid"][0] == clean_label.lower()[0] + and data["figid"][1] in clean_label.lower() + ): + return data.get("figid") + + if d["type"] == "table": + if clean_label == data["tablabel"]: + return data.get("tabid") + if ( + data["tabid"][0] == clean_label.lower()[0] + and data["tabid"][1] in clean_label.lower() + ): + return data.get("tabid") + + for d in data_body: + if d["type"] in ["compound_paragraph"]: + data = d["value"] + clean_label = re.sub(r"^[\s\.,;:–—-]+", "", label).lower() + + if d["type"] == "compound_paragraph": + if data["eid"][0] in clean_label[0] and data["eid"][1] in clean_label: + return data.get("eid") return None def is_number_parenthesis(text): - pattern = re.compile(r'^\s*\(\s*(\d+)\s*\)\s*$') + pattern = re.compile(r"^\s*\(\s*(\d+)\s*\)\s*$") match = pattern.fullmatch(text) if match: return f"({match.group(1)})" @@ -990,22 +1056,22 @@ def is_number_parenthesis(text): def remove_unpaired_tags(text): # Match opening/closing tags, capturing only the tag name (before any space or >) - pattern = re.compile(r'<(/?)([a-zA-Z0-9]+)(?:\s[^>]*)?>') - + pattern = re.compile(r"<(/?)([a-zA-Z0-9]+)(?:\s[^>]*)?>") + result = [] stack = [] # Stores (tag_name, position_in_result) - + i = 0 for match in pattern.finditer(text): is_closing, tag_name = match.groups() is_closing = bool(is_closing) - + # Text between tags if match.start() > i: - result.append(text[i:match.start()]) - - tag_text = text[match.start():match.end()] - + result.append(text[i : match.start()]) + + tag_text = text[match.start() : match.end()] + if not is_closing: # Opening tag stack.append((tag_name, len(result))) @@ -1018,18 +1084,180 @@ def remove_unpaired_tags(text): else: # Orphan closing tag - skip pass - + i = match.end() - + # Append remaining text if i < len(text): result.append(text[i:]) - + # Remove unclosed opening tags for tag_name, pos in sorted(stack, reverse=True, key=lambda x: x[1]): result.pop(pos) - - return ''.join(result) + + return "".join(result) + + +INLINE_XML_TAG_NAMES = ( + "italic", + "xref", + "bold", + "sub", + "sup", + "underline", + "sc", + "strike", +) + +_INLINE_TAG_PATTERN = re.compile( + rf"]*)?>", + re.IGNORECASE, +) + + +def escape_angle_brackets_outside_tags(text): + if not text or "<" not in text: + return text + + parts = [] + pos = 0 + for match in _INLINE_TAG_PATTERN.finditer(text): + if match.start() > pos: + parts.append(text[pos : match.start()].replace("<", "<")) + parts.append(match.group(0)) + pos = match.end() + if pos < len(text): + parts.append(text[pos:].replace("<", "<")) + return "".join(parts) + + +class StreamBlockAdapter: + __slots__ = ("block_type", "value") + + def __init__(self, block_type, value): + self.block_type = block_type + self.value = value + + +def iter_front_blocks(article_docx, data_front=None): + if data_front: + first = data_front[0] + if isinstance(first, dict) and "type" in first and "value" in first: + for item in data_front: + yield StreamBlockAdapter(item["type"], item["value"]) + return + for block in article_docx.content: + yield block + + +def plain_paragraph_text(paragraph): + if not paragraph: + return "" + text = str(paragraph) + text = re.sub(r"(?i)", "", text) + text = re.sub(r"\[\s*/?\s*\w+(?:\s+[^\]]+)?\s*\]", "", text) + return re.sub(r"\s+", " ", text).strip() + + +def article_title_from_front_stream(stream_data): + if not stream_data: + return None + for item in stream_data: + if not isinstance(item, dict): + continue + if item.get("type") not in ("paragraph", "paragraph_with_language"): + continue + value = item.get("value") or {} + if value.get("label") != "": + continue + text = plain_paragraph_text(value.get("paragraph")) + if text: + return text + return None + + +def article_title_from_front_content(article): + for block in article.content: + if block.block_type not in ("paragraph", "paragraph_with_language"): + continue + if block.value.get("label") != "": + continue + text = plain_paragraph_text(block.value.get("paragraph")) + if text: + return text + return None + + +def apply_document_title_from_article_title(article, stream_data=None): + if (article.title or "").strip(): + return + title = article_title_from_front_stream(stream_data) + if not title: + title = article_title_from_front_content(article) + if title: + article.title = title + + +def normalize_aff_ids(affid): + if affid is None or affid == "": + return [] + + items = affid if isinstance(affid, list) else [affid] + result = [] + for item in items: + if item is None or item == "": + continue + if isinstance(item, int): + result.append(item) + continue + if isinstance(item, str) and item.isdigit(): + result.append(int(item)) + continue + digits = re.sub(r"\D", "", str(item)) + if digits: + result.append(int(digits)) + return result + + +_TABLE_CELL_PATTERN = re.compile( + r"(]*)?>)(.*?)()", + re.DOTALL | re.IGNORECASE, +) + + +def _escape_table_cell_content(inner): + if not inner: + return inner + if "&" in inner or "<" in inner or ">" in inner: + inner = re.sub(r"&(?!\w+;|#\d+;)", "&", inner) + if "<" in inner: + inner = escape_angle_brackets_outside_tags(inner) + return inner + return html.escape(inner, quote=False) + + +def sanitize_table_html_fragment(table_html): + if not table_html: + return table_html + table_html = re.sub(r"&(?!\w+;|#\d+;)", "&", table_html) + + def fix_cell(match): + return ( + match.group(1) + _escape_table_cell_content(match.group(2)) + match.group(3) + ) + + return _TABLE_CELL_PATTERN.sub(fix_cell, table_html) + + +def sanitize_inline_xml_fragment(fragment): + if not fragment: + return fragment + fragment = re.sub(r"&(?!\w+;|#\d+;)", "&", fragment) + return escape_angle_brackets_outside_tags(fragment) + + +def parse_xml_fragment(fragment): + return etree.XML(sanitize_table_html_fragment(fragment)) def append_fragment(node_dest, val): @@ -1047,8 +1275,9 @@ def append_fragment(node_dest, val): # normaliza entidades problemáticas clean = clean.replace(" ", " ") - clean = re.sub(r'&(?!\w+;|#\d+;)', '&', clean) + clean = re.sub(r"&(?!\w+;|#\d+;)", "&", clean) + clean = escape_angle_brackets_outside_tags(clean) clean = remove_unpaired_tags(clean) if clean == "": @@ -1080,19 +1309,21 @@ def extract_label_and_title(text): Ignora mayúsculas y minúsculas y limpia puntuación/espacios entre el número y el título. """ # Acepta Figura/Figure y Tabla/Table/Tabela - pattern = r'\b(Imagen|Imágen|Image|Imagem|Figura|Figure|Tabla|Table|Tabela)\s+(\d+)\b' + pattern = ( + r"\b(Imagen|Imágen|Image|Imagem|Figura|Figure|Tabla|Table|Tabela)\s+(\d+)\b" + ) match = re.search(pattern, text, re.IGNORECASE) if match: - word = match.group(1).capitalize() # Normaliza capitalización + word = match.group(1).capitalize() # Normaliza capitalización number = match.group(2) label = f"{word} {number}" # Texto después del número - rest = text[match.end():] + rest = text[match.end() :] # Quita puntuación/espacios iniciales (.,;: guiones, etc.) - rest_clean = re.sub(r'^[\s\.,;:–—-]+', '', rest) + rest_clean = re.sub(r"^[\s\.,;:–—-]+", "", rest) return {"label": label, "title": rest_clean.strip()} else: @@ -1101,7 +1332,7 @@ def extract_label_and_title(text): def proccess_special_content(text, data_body): # normaliza espacios no separables por si acaso - text = re.sub(r'[\u00A0\u2007\u202F]', ' ', text) + text = re.sub(r"[\u00A0\u2007\u202F]", " ", text) pattern = r""" (?', - '', - '', - '' - ]: - if item.get('type') == '': + if item.get("type") in [ + "", + "", + "", + "", + ]: + if item.get("type") == "": if i + 1 < len(content): - obj['type'] = 'paragraph' - obj['value'] = { - 'label': '', - 'paragraph': item.get('text') + obj["type"] = "paragraph" + obj["value"] = { + "label": "", + "paragraph": item.get("text"), } stream_data.append(obj.copy()) next_item = content[i + 1] - obj['type'] = 'paragraph_with_language' - obj['value'] = { - 'label': '', - 'paragraph': next_item.get('text'), - 'language': langid.classify(next_item.get('text'))[0] or None + obj["type"] = "paragraph_with_language" + obj["value"] = { + "label": "", + "paragraph": next_item.get("text"), + "language": langid.classify(next_item.get("text"))[0] or None, } stream_data.append(obj.copy()) - - elif item.get('type') == '': - keywords = extract_keywords(item.get('text')) - obj['type'] = 'paragraph' - obj['value'] = { - 'label': '', - 'paragraph': keywords['title'] - } + + elif item.get("type") == "": + keywords = extract_keywords(item.get("text")) + obj["type"] = "paragraph" + obj["value"] = {"label": "", "paragraph": keywords["title"]} stream_data.append(obj.copy()) - obj['type'] = 'paragraph_with_language' - obj['value'] = { - 'label': '', - 'paragraph': keywords['keywords'], - 'language': langid.classify(keywords['title'].replace('', '').replace('', ''))[0] or None - } + obj["type"] = "paragraph_with_language" + obj["value"] = { + "label": "", + "paragraph": keywords["keywords"], + "language": langid.classify( + keywords["title"] + .replace("", "") + .replace("", "") + )[0] + or None, + } stream_data.append(obj.copy()) - else: - obj['type'] = 'paragraph' - obj['value'] = { - 'label': item.get('type') , - 'paragraph': item.get('text') + else: + obj["type"] = "paragraph" + obj["value"] = { + "label": item.get("type"), + "paragraph": item.get("text"), } stream_data.append(obj.copy()) continue - if item.get('type') == 'first_block': - llm_first_block = LlamaService(mode='prompt', temperature=0.1) + if item.get("type") == "first_block": + llm_first_block = LlamaService(mode="prompt", temperature=0.1) + output = None if get_llm_model_name() == MODEL_NAME_GEMINI: - output = llm_first_block.run(LlamaInputSettings.get_first_metadata(clean_labels(item.get('text')))) - match = re.search(r'\{.*\}', output, re.DOTALL) + logger.info("get_labels: processando first_block com Gemini") + raw_output = llm_first_block.run( + LlamaInputSettings.get_first_metadata( + clean_labels(item.get("text")) + ) + ) + match = re.search(r"\{.*\}", raw_output, re.DOTALL) if match: - output = match.group(0) - output = json.loads(output) + output = json.loads(match.group(0)) + else: + logger.warning( + "get_labels: Gemini não devolveu JSON válido no first_block" + ) if get_llm_model_name() == MODEL_NAME_LLAMA: + output_author = get_data_first_block( + clean_labels(item.get("text")), "author", user_id + ) + + output_affiliation = get_data_first_block( + clean_labels(item.get("text")), "affiliation", user_id + ) - output_author = get_data_first_block(clean_labels(item.get('text')), 'author', user_id) - - output_affiliation = get_data_first_block(clean_labels(item.get('text')), 'affiliation', user_id) - - output_doi = get_data_first_block(clean_labels(item.get('text')), 'doi', user_id) - - output_title = get_data_first_block(clean_labels(item.get('text')), 'title', user_id) + output_doi = get_data_first_block( + clean_labels(item.get("text")), "doi", user_id + ) + + output_title = get_data_first_block( + clean_labels(item.get("text")), "title", user_id + ) # 1. Parsear cada salida doi_section = output_doi @@ -173,158 +218,171 @@ def get_labels(title, user_id): "section": doi_section.get("section", ""), "titles": titles, "authors": authors, - "affiliations": affiliations + "affiliations": affiliations, } - obj['type'] = 'paragraph' - obj['value'] = { - 'label': '', - 'paragraph': output['doi'] - } + if not output: + continue + + obj["type"] = "paragraph" + obj["value"] = {"label": "", "paragraph": output["doi"]} stream_data.append(obj.copy()) - obj['value'] = { - 'label': '', - 'paragraph': output['section'] - } + obj["value"] = {"label": "", "paragraph": output["section"]} stream_data.append(obj.copy()) - for i, tit in enumerate(output['titles']): - obj['type'] = 'paragraph_with_language' - obj['value'] = { - 'label': '' if i == 0 else '', - 'paragraph': tit['title'], - 'language': tit['language'] + for i, tit in enumerate(output["titles"]): + obj["type"] = "paragraph_with_language" + obj["value"] = { + "label": "" if i == 0 else "", + "paragraph": tit["title"], + "language": tit["language"], } stream_data.append(obj.copy()) - for i, auth in enumerate(output['authors']): - obj['type'] = 'author_paragraph' - obj['value'] = { - 'label': '', - 'surname': auth['surname'], - 'given_names': auth['name'], - 'orcid': auth['orcid'], - 'affid': auth['aff'], - 'char': auth['char'] + for i, auth in enumerate(output["authors"]): + obj["type"] = "author_paragraph" + obj["value"] = { + "label": "", + "surname": auth["surname"], + "given_names": auth["name"], + "orcid": auth["orcid"], + "affid": auth["aff"], + "char": auth["char"], } stream_data.append(obj.copy()) - for i, aff in enumerate(output['affiliations']): - obj['type'] = 'aff_paragraph' - obj['value'] = { - 'label': '', - 'affid': aff['aff'], - 'char': aff['char'], - 'orgname': aff['orgname'], - 'orgdiv2': aff['orgdiv2'], - 'orgdiv1': aff['orgdiv1'], - 'zipcode': aff['postal'], - 'city': aff['city'], - 'country': aff['name_country'], - 'code_country': aff['code_country'], - 'state': aff['state'], - 'text_aff': aff['text_aff'], + for i, aff in enumerate(output["affiliations"]): + obj["type"] = "aff_paragraph" + obj["value"] = { + "label": "", + "affid": aff["aff"], + "char": aff["char"], + "orgname": aff["orgname"], + "orgdiv2": aff["orgdiv2"], + "orgdiv1": aff["orgdiv1"], + "zipcode": aff["postal"], + "city": aff["city"], + "country": aff["name_country"], + "code_country": aff["code_country"], + "state": aff["state"], + "text_aff": aff["text_aff"], #'original': aff['original'] } stream_data.append(obj.copy()) - - if item.get('type') in ['image', 'table', 'list', 'compound']: + + if item.get("type") in ["image", "table", "list", "compound"]: obj, counts = create_special_content_object(item, stream_data_body, counts) stream_data_body.append(obj) continue - if item.get('text') is None or item.get('text') == '': - state['label_next'] = state['label_next_reset'] if state['reset'] else state['label_next'] - if state['back']: - state['back'] = False - state['body'] = False - state['references'] = True + if item.get("text") is None or item.get("text") == "": + state["label_next"] = ( + state["label_next_reset"] if state["reset"] else state["label_next"] + ) + if state["back"]: + state["back"] = False + state["body"] = False + state["references"] = True else: - obj, result, state = create_labeled_object2(i, item, state, sections) - - if result: - if item.get('text').lower() in ['introducción', 'introduction', 'introdução'] and state['references']: - state['body_trans'] = True + + if result: + if ( + item.get("text").lower() + in ["introducción", "introduction", "introdução"] + and state["references"] + ): + state["body_trans"] = True obj_trans = { - 'type': 'paragraph_with_language', - 'value': { - 'label': '', - 'paragraph': 'Translate' - } - } - stream_data_body.append(obj_trans) - if state['body']: - if state['references']: - if state['body_trans']: + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Translate", + }, + } + stream_data_body.append(obj_trans) + if state["body"]: + if state["references"]: + if state["body_trans"]: stream_data_body.append(obj) else: stream_data.append(obj) else: stream_data_body.append(obj) - elif state['back']: - if state['label'] == '': + elif state["back"]: + if state["label"] == "": stream_data_back.append(obj) - if state['label'] == '

': + if state["label"] == "

": num_ref = num_ref + 1 - #obj = {}#process_reference(num_ref, obj, user_id) - obj_reference.append({"num_ref": num_ref, "obj": obj, "text": obj['value']['paragraph'],}) - #stream_data_back.append(obj) + # obj = {}#process_reference(num_ref, obj, user_id) + obj_reference.append( + { + "num_ref": num_ref, + "obj": obj, + "text": obj["value"]["paragraph"], + } + ) + # stream_data_back.append(obj) else: stream_data.append(obj) - + num_refs = [item["num_ref"] for item in obj_reference] - if get_llm_model_name() == 'LLAMA': + if get_llm_model_name() == "LLAMA": for obj_ref in obj_reference: - obj = process_reference(obj_ref['num_ref'], obj_ref['obj'], user_id) + obj = process_reference(obj_ref["num_ref"], obj_ref["obj"], user_id) stream_data_back.append(obj) else: + if llm_first_block is None: + llm_first_block = LlamaService(mode="prompt", temperature=0.1) chunks = split_in_three(obj_reference) - output=[] + output = [] + logger.info( + "get_labels: processando %d referências com Gemini (%d chunks)", + len(obj_reference), + len(chunks), + ) for chunk in chunks: if len(chunk) > 0: - text_references = "\n".join([item["text"] for item in chunk]).replace('', '').replace('', '') + text_references = ( + "\n".join([item["text"] for item in chunk]) + .replace("", "") + .replace("", "") + ) prompt_reference = create_prompt_reference(text_references) - result = llm_first_block.run(prompt_reference) + result = llm_first_block.run(prompt_reference) - match = re.search(r'\[.*\]', result, re.DOTALL) + match = re.search(r"\[.*\]", result, re.DOTALL) if match: parsed = json.loads(match.group(0)) output.extend(parsed) # Agrega a la lista de salida - + stream_data_back.extend(process_references(num_refs, output)) article_docx_markup.content = stream_data article_docx_markup.content_body = stream_data_body article_docx_markup.content_back = stream_data_back + apply_document_title_from_article_title(article_docx_markup, stream_data) article_docx_markup.save() - article_docx.estatus = ProcessStatus.PROCESSED - article_docx.save() - - xml, stream_data_body = get_xml(article_docx, stream_data, stream_data_body, stream_data_back) - article_docx_markup.content_body = stream_data_body - - # Guardar el XML - article_docx_markup.text_xml = xml - article_docx.save() + xml, stream_data_body = get_xml( + article_docx, stream_data, stream_data_body, stream_data_back + ) + persist_article_xml(article_docx_markup, xml, stream_data_body) @celery_app.task() -def update_xml(instance_id, instance_content, instance_content_body, instance_content_back): +def update_xml( + instance_id, instance_content, instance_content_body, instance_content_back +): instance = MarkupXML.objects.get(id=instance_id) content_head = instance_content content_body_dict = instance_content_body - xml, stream_data_body = get_xml(instance, content_head, content_body_dict, instance_content_back) - - instance.content_body = stream_data_body - # Guardar el XML en el campo `file_xml` - #archive_xml = ContentFile(xml) # Crea un archivo temporal en memoria - instance.estatus = ProcessStatus.PROCESSED - #instance.file_xml.save("archivo.xml", archive_xml) # Guarda en el campo `file_xml` - instance.text_xml = xml + apply_document_title_from_article_title(instance, content_head) + xml, stream_data_body = get_xml( + instance, content_head, content_body_dict, instance_content_back + ) - instance.save() + persist_article_xml(instance, xml, stream_data_body) diff --git a/markup_doc/tests.py b/markup_doc/tests.py index 7ce503c..73eb811 100644 --- a/markup_doc/tests.py +++ b/markup_doc/tests.py @@ -1,3 +1,343 @@ -from django.test import TestCase +from types import SimpleNamespace -# Create your tests here. +from django.test import SimpleTestCase, TestCase +from lxml import etree + +from markup_doc.labeling_utils import ( + StreamBlockAdapter, + append_fragment, + apply_document_title_from_article_title, + article_title_from_front_content, + article_title_from_front_stream, + escape_angle_brackets_outside_tags, + iter_front_blocks, + normalize_aff_ids, + parse_xml_fragment, + plain_paragraph_text, + sanitize_inline_xml_fragment, + sanitize_table_html_fragment, +) +from markup_doc.models import ProcessStatus, UploadDocx +from markup_doc.tasks import persist_article_xml +from markup_doc.xml import get_xml + + +def minimal_article_stub(**overrides): + defaults = { + "language": "en", + "acronym": None, + "title_nlm": None, + "journal_title": None, + "short_title": None, + "pissn": None, + "eissn": None, + "pubname": None, + "artdate": None, + "dateiso": None, + "vol": None, + "issue": None, + "elocatid": None, + "license": None, + "content": [], + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +class EscapeAngleBracketsTests(SimpleTestCase): + def test_escapes_comparison_operator(self): + result = escape_angle_brackets_outside_tags( + "Values with p < 0.05 were significant" + ) + self.assertIn("<", result) + self.assertNotIn("p < 0.05", result) + + def test_preserves_inline_tags(self): + text = "α with p < 0.05" + result = escape_angle_brackets_outside_tags(text) + self.assertIn("α", result) + self.assertIn("< 0.05", result) + + def test_preserves_xref_tags(self): + text = 'See Smith and p < 0.01' + result = escape_angle_brackets_outside_tags(text) + self.assertIn('', result) + self.assertIn("< 0.01", result) + + +class AppendFragmentTests(SimpleTestCase): + def test_parses_paragraph_with_less_than(self): + node = etree.Element("p") + append_fragment(node, "Values with p < 0.05 were significant") + xml = etree.tostring(node, encoding="unicode") + self.assertIn("0.05", xml) + self.assertNotIn("p < 0.05", xml) + + def test_parses_italic_inline(self): + node = etree.Element("p") + append_fragment(node, "significant result") + xml = etree.tostring(node, encoding="unicode") + self.assertIn("significant", xml) + + def test_empty_value_removes_node(self): + parent = etree.Element("body") + node = etree.SubElement(parent, "p") + append_fragment(node, "") + self.assertEqual(len(parent), 0) + + +class SanitizeTableHtmlTests(SimpleTestCase): + def test_ampersand_in_cell_parses(self): + table_html = "

AT&T Corp
" + result = sanitize_table_html_fragment(table_html) + parse_xml_fragment(result) + self.assertIn("AT&T", result) + + def test_less_than_in_cell_parses(self): + table_html = "
p < 0.05
" + result = sanitize_table_html_fragment(table_html) + parse_xml_fragment(result) + self.assertIn("<", result) + + def test_th_cell_content_sanitized(self): + table_html = "
A & B
" + result = sanitize_table_html_fragment(table_html) + root = parse_xml_fragment(result) + self.assertEqual(root.tag, "table") + + +class SanitizeInlineXmlTests(SimpleTestCase): + def test_list_item_with_ampersand(self): + fragment = "

A & B

" + result = sanitize_inline_xml_fragment(fragment) + etree.fromstring(f"{result}") + self.assertIn("&", result) + + +class IterFrontBlocksTests(SimpleTestCase): + def test_yields_from_stream_dicts(self): + stream = [ + { + "type": "paragraph_with_language", + "value": {"label": "", "paragraph": "T"}, + }, + ] + blocks = list(iter_front_blocks(minimal_article_stub(), stream)) + self.assertEqual(len(blocks), 1) + self.assertIsInstance(blocks[0], StreamBlockAdapter) + self.assertEqual(blocks[0].block_type, "paragraph_with_language") + self.assertEqual(blocks[0].value["label"], "") + + def test_falls_back_to_article_content(self): + block = StreamBlockAdapter( + "paragraph", + {"label": "", "paragraph": "Research"}, + ) + article = minimal_article_stub(content=[block]) + blocks = list(iter_front_blocks(article, None)) + self.assertEqual(len(blocks), 1) + self.assertEqual(blocks[0].value["paragraph"], "Research") + + +class PlainParagraphTextTests(SimpleTestCase): + def test_strips_italic_and_brackets(self): + text = "[doctitle] My title" + self.assertEqual(plain_paragraph_text(text), "My title") + + +class ArticleTitleFromFrontTests(SimpleTestCase): + def test_extracts_article_title_block(self): + stream_data = [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "My Study", + "language": "en", + }, + }, + ] + self.assertEqual(article_title_from_front_stream(stream_data), "My Study") + + def test_skips_trans_title(self): + stream_data = [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Translated", + "language": "es", + }, + }, + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Main title", + "language": "en", + }, + }, + ] + self.assertEqual(article_title_from_front_stream(stream_data), "Main title") + + def test_from_saved_content_blocks(self): + article = minimal_article_stub( + content=[ + StreamBlockAdapter( + "paragraph_with_language", + { + "label": "", + "paragraph": "From content", + "language": "pt", + }, + ), + ], + ) + self.assertEqual(article_title_from_front_content(article), "From content") + + def test_apply_sets_title_when_blank(self): + article = minimal_article_stub(title="") + apply_document_title_from_article_title( + article, + [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Título do artigo", + "language": "pt", + }, + }, + ], + ) + self.assertEqual(article.title, "Título do artigo") + + def test_apply_keeps_existing_title(self): + article = minimal_article_stub(title="Já definido") + apply_document_title_from_article_title( + article, + [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Outro", + "language": "pt", + }, + }, + ], + ) + self.assertEqual(article.title, "Já definido") + + +class NormalizeAffIdsTests(SimpleTestCase): + def test_single_int(self): + self.assertEqual(normalize_aff_ids(2), [2]) + + def test_list_from_gemini(self): + self.assertEqual(normalize_aff_ids([1, 2]), [1, 2]) + + def test_string_digit(self): + self.assertEqual(normalize_aff_ids("3"), [3]) + + def test_empty_values(self): + self.assertEqual(normalize_aff_ids(None), []) + self.assertEqual(normalize_aff_ids([]), []) + self.assertEqual(normalize_aff_ids(""), []) + + +class GetXmlFrontTests(SimpleTestCase): + def test_includes_article_title_from_data_front(self): + article = minimal_article_stub() + data_front = [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Effects of intervention", + "language": "en", + }, + }, + ] + xml, _ = get_xml(article, data_front, [], []) + self.assertIn("", xml) + self.assertIn("Effects of intervention", xml) + + def test_body_paragraph_with_less_than(self): + article = minimal_article_stub() + data_body = [ + { + "value": { + "label": "

", + "paragraph": "Result p < 0.05 was significant", + }, + }, + ] + xml, _ = get_xml(article, [], data_body, []) + self.assertIn("0.05", xml) + self.assertNotIn("p < 0.05", xml) + + def test_author_with_multiple_aff_ids(self): + article = minimal_article_stub() + data_front = [ + { + "type": "paragraph_with_language", + "value": { + "label": "", + "paragraph": "Title", + "language": "en", + }, + }, + { + "type": "author_paragraph", + "value": { + "label": "", + "surname": "Silva", + "given_names": "Ana", + "orcid": None, + "affid": [1, 2], + "char": "*", + }, + }, + { + "type": "aff_paragraph", + "value": { + "label": "", + "affid": 1, + "char": "*", + "orgname": "Universidade", + "orgdiv1": None, + "orgdiv2": None, + "city": None, + "state": None, + "code_country": "BR", + "text_aff": None, + }, + }, + ] + xml, _ = get_xml(article, data_front, [], []) + self.assertIn('ref-type="aff"', xml) + self.assertIn('rid="aff1"', xml) + self.assertIn('rid="aff2"', xml) + + +class PersistArticleXmlTests(TestCase): + def test_persists_text_xml_file_xml_and_processed_status(self): + article = UploadDocx.objects.create(title="Artigo de teste") + xml = ( + '' + "

" + ) + persist_article_xml(article, xml) + article.refresh_from_db() + self.assertEqual(article.estatus, ProcessStatus.PROCESSED) + self.assertIn("
", article.text_xml) + self.assertTrue(article.file_xml.name.endswith(".xml")) + self.assertTrue(article.file_xml.storage.exists(article.file_xml.name)) + + def test_uses_slugified_title_for_xml_filename(self): + article = UploadDocx.objects.create(title="Meu Artigo 2026") + xml = "
" + persist_article_xml(article, xml) + article.refresh_from_db() + self.assertIn("meu-artigo-2026", article.file_xml.name) diff --git a/markup_doc/wagtail_hooks.py b/markup_doc/wagtail_hooks.py index fc3abe7..6ecb304 100644 --- a/markup_doc/wagtail_hooks.py +++ b/markup_doc/wagtail_hooks.py @@ -69,9 +69,9 @@ def form_valid(self, form): self.object = form.save_all(self.request.user) self.object.estatus = ProcessStatus.PROCESSING self.object.save() - transaction.on_commit( - lambda: get_labels.delay(self.object.title, self.request.user.id) - ) + article_pk = self.object.pk + user_id = self.request.user.id + transaction.on_commit(lambda: get_labels.delay(article_pk, user_id)) return HttpResponseRedirect(self.get_success_url()) diff --git a/markup_doc/xml.py b/markup_doc/xml.py index d200675..3698ca9 100644 --- a/markup_doc/xml.py +++ b/markup_doc/xml.py @@ -1,20 +1,30 @@ +import html +import os +import re +from urllib.parse import urlparse + from lxml import etree -import re, html, os +from wagtail.images import get_image_model + from markup_doc.labeling_utils import ( + append_fragment, extract_subsection, + iter_front_blocks, + normalize_aff_ids, + parse_xml_fragment, proccess_labeled_text, proccess_special_content, - append_fragment + sanitize_inline_xml_fragment, ) -from wagtail.images import get_image_model -from urllib.parse import urlparse def extract_date(texto): try: # Patrón para detectar YYYY-MM-DD, YYYY/MM/DD, DD-MM-YYYY, DD/MM/YYYY - patron_fecha = r'\b(\d{4})[-/](\d{2})[-/](\d{2})\b|\b(\d{2})[-/](\d{2})[-/](\d{4})\b' - + patron_fecha = ( + r"\b(\d{4})[-/](\d{2})[-/](\d{2})\b|\b(\d{2})[-/](\d{2})[-/](\d{4})\b" + ) + match = re.search(patron_fecha, texto) if match: if match.group(1): # Formato YYYY-MM-DD o YYYY/MM/DD @@ -28,75 +38,77 @@ def extract_date(texto): return (dia, mes, año) except: pass - + return None # No se encontró def get_xml(article_docx, data_front, data, data_back): # Crear el elemento raíz nsmap = { - 'mml': 'http://www.w3.org/1998/Math/MathML', - 'xlink': 'http://www.w3.org/1999/xlink' + "mml": "http://www.w3.org/1998/Math/MathML", + "xlink": "http://www.w3.org/1999/xlink", } - root = etree.Element('article', - nsmap=nsmap, - attrib={ - 'article-type': 'research-article', - 'dtd-version': '1.1', - 'specific-use': 'sps-1.9', - '{http://www.w3.org/XML/1998/namespace}lang': article_docx.language or 'en'} - #'{http://www.w3.org/1998/Math/MathML}mml': 'http://www.w3.org/1998/Math/MathML', - #'{http://www.w3.org/1999/xlink}xlink': 'http://www.w3.org/1999/xlink'} - ) + root = etree.Element( + "article", + nsmap=nsmap, + attrib={ + "article-type": "research-article", + "dtd-version": "1.1", + "specific-use": "sps-1.9", + "{http://www.w3.org/XML/1998/namespace}lang": article_docx.language or "en", + } + #'{http://www.w3.org/1998/Math/MathML}mml': 'http://www.w3.org/1998/Math/MathML', + #'{http://www.w3.org/1999/xlink}xlink': 'http://www.w3.org/1999/xlink'} + ) # Añadir un elemento hijo front = etree.SubElement(root, "front") body = etree.SubElement(root, "body") back = etree.SubElement(root, "back") - node_reflist = etree.SubElement(back, 'ref-list') + node_reflist = etree.SubElement(back, "ref-list") subsec = None num_table = 1 continue_t = False arr_subarticle = [] - node = etree.SubElement(front, 'journal-meta') - + node = etree.SubElement(front, "journal-meta") + if article_docx.acronym: - node_tmp = etree.SubElement(node, 'journal-id') - node_tmp.set('journal-id-type', 'publisher-id') + node_tmp = etree.SubElement(node, "journal-id") + node_tmp.set("journal-id-type", "publisher-id") node_tmp.text = article_docx.acronym if article_docx.title_nlm: - node_tmp = etree.SubElement(node, 'journal-id') - node_tmp.set('journal-id-type', 'nlm-ta') + node_tmp = etree.SubElement(node, "journal-id") + node_tmp.set("journal-id-type", "nlm-ta") node_tmp.text = article_docx.title_nlm - node_tmp = etree.SubElement(node, 'journal-title-group') + node_tmp = etree.SubElement(node, "journal-title-group") if article_docx.journal_title: - node_tmp2 = etree.SubElement(node_tmp, 'journal-title') + node_tmp2 = etree.SubElement(node_tmp, "journal-title") node_tmp2.text = article_docx.journal_title if article_docx.short_title: - node_tmp2 = etree.SubElement(node_tmp, 'abbrev-journal-title') - node_tmp2.set('abbrev-type', 'publisher') + node_tmp2 = etree.SubElement(node_tmp, "abbrev-journal-title") + node_tmp2.set("abbrev-type", "publisher") node_tmp2.text = article_docx.short_title if article_docx.pissn: - node_tmp = etree.SubElement(node, 'issn') - node_tmp.set('pub-type', 'ppub') + node_tmp = etree.SubElement(node, "issn") + node_tmp.set("pub-type", "ppub") node_tmp.text = article_docx.pissn if article_docx.eissn: - node_tmp = etree.SubElement(node, 'issn') - node_tmp.set('pub-type', 'epub') + node_tmp = etree.SubElement(node, "issn") + node_tmp.set("pub-type", "epub") node_tmp.text = article_docx.eissn - node_tmp = etree.SubElement(node, 'publisher') + node_tmp = etree.SubElement(node, "publisher") if article_docx.pubname: - node_tmp2 = etree.SubElement(node_tmp, 'publisher-name') + node_tmp2 = etree.SubElement(node_tmp, "publisher-name") node_tmp2.text = article_docx.pubname ##### Article Meta @@ -104,9 +116,9 @@ def get_xml(article_docx, data_front, data, data_back): translates = [] current_trans = [] - for block in article_docx.content: + for block in iter_front_blocks(article_docx, data_front): if ( - block.block_type == "paragraph_with_language" + block.block_type == "paragraph_with_language" and block.value.get("label") == "" ): # Si ya tenemos contenido acumulado, lo guardamos como parte @@ -120,344 +132,392 @@ def get_xml(article_docx, data_front, data, data_back): for i, data_t in enumerate(translates): if i == 0: - node = etree.SubElement(front, 'article-meta') + node = etree.SubElement(front, "article-meta") else: subarticle = etree.SubElement(root, "sub-article") arr_subarticle.append(subarticle) - subarticle.attrib['article-type'] = "translation" - subarticle.attrib['id'] = f"S{len(arr_subarticle)}" - subarticle.attrib['{http://www.w3.org/XML/1998/namespace}lang'] = data_t[0].value['language'] + subarticle.attrib["article-type"] = "translation" + subarticle.attrib["id"] = f"S{len(arr_subarticle)}" + subarticle.attrib["{http://www.w3.org/XML/1998/namespace}lang"] = data_t[ + 0 + ].value["language"] - node = etree.SubElement(subarticle, 'front-stub') + node = etree.SubElement(subarticle, "front-stub") val = next( - (b.value['paragraph'] for b in data_t - if b.block_type == 'paragraph' and b.value.get('label') == ''), - None + ( + b.value["paragraph"] + for b in data_t + if b.block_type == "paragraph" + and b.value.get("label") == "" + ), + None, ) if val: - node_tmp = etree.SubElement(node, 'article-id') - node_tmp.set('pub-id-type', 'doi') + node_tmp = etree.SubElement(node, "article-id") + node_tmp.set("pub-id-type", "doi") node_tmp.text = val - + val = next( - (b.value['paragraph'] for b in data_t - if b.block_type == 'paragraph' and b.value.get('label') == ''), - None + ( + b.value["paragraph"] + for b in data_t + if b.block_type == "paragraph" and b.value.get("label") == "" + ), + None, ) if val: - node_tmp = etree.SubElement(node, 'article-categories') - node_tmp2 = etree.SubElement(node_tmp, 'subj-group') - node_tmp2.set('subj-group-type', 'heading') - node_tmp3 = etree.SubElement(node_tmp2, 'subject') + node_tmp = etree.SubElement(node, "article-categories") + node_tmp2 = etree.SubElement(node_tmp, "subj-group") + node_tmp2.set("subj-group-type", "heading") + node_tmp3 = etree.SubElement(node_tmp2, "subject") node_tmp3.text = val - + val = next( - (b.value['paragraph'] for b in data_t - if b.block_type == 'paragraph_with_language' and b.value.get('label') == ''), - None + ( + b.value["paragraph"] + for b in data_t + if b.block_type == "paragraph_with_language" + and b.value.get("label") == "" + ), + None, ) if val: - node_tmp = etree.SubElement(node, 'title-group') - node_tmp2 = etree.SubElement(node_tmp, 'article-title') + node_tmp = etree.SubElement(node, "title-group") + node_tmp2 = etree.SubElement(node_tmp, "article-title") append_fragment(node_tmp2, val) - + vals = [ b for b in data_t - if b.block_type == 'paragraph_with_language' and b.value.get('label') == '' + if b.block_type == "paragraph_with_language" + and b.value.get("label") == "" ] for val in vals: - node_tmp2 = etree.SubElement(node_tmp, 'trans-title-group') - node_tmp2.set('{http://www.w3.org/XML/1998/namespace}lang', val.value.get('language')) - node_tmp3 = etree.SubElement(node_tmp2, 'trans-title') - append_fragment(node_tmp3, val.value.get('paragraph')) + node_tmp2 = etree.SubElement(node_tmp, "trans-title-group") + node_tmp2.set( + "{http://www.w3.org/XML/1998/namespace}lang", + val.value.get("language"), + ) + node_tmp3 = etree.SubElement(node_tmp2, "trans-title") + append_fragment(node_tmp3, val.value.get("paragraph")) - node_tmp = etree.SubElement(node, 'contrib-group') + node_tmp = etree.SubElement(node, "contrib-group") - vals = [ - b - for b in data_t - if b.block_type == 'author_paragraph' - ] - - for val in vals: - node_tmp2 = etree.SubElement(node_tmp, 'contrib') - node_tmp2.set('contrib-type', 'author') - if val.value.get('orcid'): - node_tmp3 = etree.SubElement(node_tmp2, 'contrib-id') - node_tmp3.set('contrib-id-type', 'orcid') - node_tmp3.text = val.value.get('orcid') - node_tmp3 = etree.SubElement(node_tmp2, 'name') - if val.value.get('surname'): - node_tmp4 = etree.SubElement(node_tmp3, 'surname') - append_fragment(node_tmp4, val.value.get('surname')) - - if val.value.get('given_names'): - node_tmp4 = etree.SubElement(node_tmp3, 'given-names') - append_fragment(node_tmp4, val.value.get('given_names')) - - if val.value.get('affid'): - node_tmp3 = etree.SubElement(node_tmp2, 'xref') - node_tmp3.set('ref-type', 'aff') - node_tmp3.set('rid', f"aff{val.value.get('affid')}") - node_tmp3.text = val.value.get('char') or ('*' * int(val.value.get('affid'))) + vals = [b for b in data_t if b.block_type == "author_paragraph"] - vals = [ - b - for b in data_t - if b.block_type == 'aff_paragraph' - ] + for val in vals: + node_tmp2 = etree.SubElement(node_tmp, "contrib") + node_tmp2.set("contrib-type", "author") + if val.value.get("orcid"): + node_tmp3 = etree.SubElement(node_tmp2, "contrib-id") + node_tmp3.set("contrib-id-type", "orcid") + node_tmp3.text = val.value.get("orcid") + node_tmp3 = etree.SubElement(node_tmp2, "name") + if val.value.get("surname"): + node_tmp4 = etree.SubElement(node_tmp3, "surname") + append_fragment(node_tmp4, val.value.get("surname")) + + if val.value.get("given_names"): + node_tmp4 = etree.SubElement(node_tmp3, "given-names") + append_fragment(node_tmp4, val.value.get("given_names")) + + for aff_id in normalize_aff_ids(val.value.get("affid")): + node_tmp3 = etree.SubElement(node_tmp2, "xref") + node_tmp3.set("ref-type", "aff") + node_tmp3.set("rid", f"aff{aff_id}") + node_tmp3.text = val.value.get("char") or ("*" * int(aff_id)) + + vals = [b for b in data_t if b.block_type == "aff_paragraph"] for val in vals: - node_tmp = etree.SubElement(node, 'aff') - node_tmp.set('id', f"aff{val.value.get('affid')}") - - node_tmp2 = etree.SubElement(node_tmp, 'label') - node_tmp2.text = val.value.get('char') or ('*' * int(val.value.get('affid'))) - - if val.value.get('orgname'): - node_tmp2 = etree.SubElement(node_tmp, 'institution') - node_tmp2.set('content-type', 'orgname') - append_fragment(node_tmp2, val.value.get('orgname')) - - if val.value.get('orgdiv1'): - node_tmp2 = etree.SubElement(node_tmp, 'institution') - node_tmp2.set('content-type', 'orgdiv1') - append_fragment(node_tmp2, val.value.get('orgdiv1')) - - if val.value.get('orgdiv2'): - node_tmp2 = etree.SubElement(node_tmp, 'institution') - node_tmp2.set('content-type', 'orgdiv2') - append_fragment(node_tmp2, val.value.get('orgdiv2')) - - node_tmp2 = etree.SubElement(node_tmp, 'addr-line') - - if val.value.get('city'): - node_tmp3 = etree.SubElement(node_tmp2, 'city') - append_fragment(node_tmp3, val.value.get('city')) - - if val.value.get('state'): - node_tmp3 = etree.SubElement(node_tmp2, 'state') - append_fragment(node_tmp3, val.value.get('state')) - - if val.value.get('country'): - node_tmp2 = etree.SubElement(node_tmp, 'country') - node_tmp2.set('country', val.value.get('code_country')) - append_fragment(node_tmp2, val.value.get('code_country')) - - node_tmp = etree.SubElement(node, 'author-notes') + aff_ids = normalize_aff_ids(val.value.get("affid")) + if not aff_ids: + continue + aff_id = aff_ids[0] + node_tmp = etree.SubElement(node, "aff") + node_tmp.set("id", f"aff{aff_id}") + + node_tmp2 = etree.SubElement(node_tmp, "label") + node_tmp2.text = val.value.get("char") or ("*" * int(aff_id)) + + if val.value.get("orgname"): + node_tmp2 = etree.SubElement(node_tmp, "institution") + node_tmp2.set("content-type", "orgname") + append_fragment(node_tmp2, val.value.get("orgname")) + + if val.value.get("orgdiv1"): + node_tmp2 = etree.SubElement(node_tmp, "institution") + node_tmp2.set("content-type", "orgdiv1") + append_fragment(node_tmp2, val.value.get("orgdiv1")) + + if val.value.get("orgdiv2"): + node_tmp2 = etree.SubElement(node_tmp, "institution") + node_tmp2.set("content-type", "orgdiv2") + append_fragment(node_tmp2, val.value.get("orgdiv2")) + + node_tmp2 = etree.SubElement(node_tmp, "addr-line") + + if val.value.get("city"): + node_tmp3 = etree.SubElement(node_tmp2, "city") + append_fragment(node_tmp3, val.value.get("city")) + + if val.value.get("state"): + node_tmp3 = etree.SubElement(node_tmp2, "state") + append_fragment(node_tmp3, val.value.get("state")) + + if val.value.get("country"): + node_tmp2 = etree.SubElement(node_tmp, "country") + node_tmp2.set("country", val.value.get("code_country")) + append_fragment(node_tmp2, val.value.get("code_country")) + + node_tmp = etree.SubElement(node, "author-notes") for val in vals: + if val.value.get("text_aff"): + node_tmp2 = etree.SubElement(node_tmp, "fn") + node_tmp2.set("fn-type", "other") + aff_ids = normalize_aff_ids(val.value.get("affid")) + if not aff_ids: + continue + aff_id = aff_ids[0] + node_tmp2.set("id", f"fn{aff_id}") + + node_tmp3 = etree.SubElement(node_tmp2, "label") + node_tmp3.text = val.value.get("char") or ("*" * int(aff_id)) + + node_tmp3 = etree.SubElement(node_tmp2, "p") + append_fragment(node_tmp3, val.value.get("text_aff")) - if val.value.get('text_aff'): - node_tmp2 = etree.SubElement(node_tmp, 'fn') - node_tmp2.set('fn-type', 'other') - node_tmp2.set('id', f"fn{val.value.get('affid')}") - - node_tmp3 = etree.SubElement(node_tmp2, 'label') - node_tmp3.text = val.value.get('char') or ('*' * int(val.value.get('affid'))) - - node_tmp3 = etree.SubElement(node_tmp2, 'p') - append_fragment(node_tmp3, val.value.get('text_aff')) - if article_docx.artdate: - node_tmp = etree.SubElement(node, 'pub-date') - node_tmp.set('date-type', 'pub') - node_tmp.set('publication-format', 'electronic') + node_tmp = etree.SubElement(node, "pub-date") + node_tmp.set("date-type", "pub") + node_tmp.set("publication-format", "electronic") - node_tmp2 = etree.SubElement(node_tmp, 'day') + node_tmp2 = etree.SubElement(node_tmp, "day") node_tmp2.text = article_docx.artdate.strftime("%d") - node_tmp2 = etree.SubElement(node_tmp, 'month') + node_tmp2 = etree.SubElement(node_tmp, "month") node_tmp2.text = article_docx.artdate.strftime("%m") - node_tmp2 = etree.SubElement(node_tmp, 'year') + node_tmp2 = etree.SubElement(node_tmp, "year") node_tmp2.text = article_docx.artdate.strftime("%Y") - - if article_docx.dateiso: - node_tmp = etree.SubElement(node, 'pub-date') - node_tmp.set('date-type', 'collection') - node_tmp.set('publication-format', 'electronic') - - if article_docx.dateiso.split('-')[2] and article_docx.dateiso.split('-')[2] != '00': - node_tmp2 = etree.SubElement(node_tmp, 'day') - node_tmp2.text = article_docx.dateiso.split('-')[2] - if article_docx.dateiso.split('-')[1] and article_docx.dateiso.split('-')[1] != '00': - node_tmp2 = etree.SubElement(node_tmp, 'month') - node_tmp2.text = article_docx.dateiso.split('-')[1] - - node_tmp2 = etree.SubElement(node_tmp, 'year') - node_tmp2.text = article_docx.dateiso.split('-')[0] + if article_docx.dateiso: + node_tmp = etree.SubElement(node, "pub-date") + node_tmp.set("date-type", "collection") + node_tmp.set("publication-format", "electronic") + + if ( + article_docx.dateiso.split("-")[2] + and article_docx.dateiso.split("-")[2] != "00" + ): + node_tmp2 = etree.SubElement(node_tmp, "day") + node_tmp2.text = article_docx.dateiso.split("-")[2] + + if ( + article_docx.dateiso.split("-")[1] + and article_docx.dateiso.split("-")[1] != "00" + ): + node_tmp2 = etree.SubElement(node_tmp, "month") + node_tmp2.text = article_docx.dateiso.split("-")[1] + + node_tmp2 = etree.SubElement(node_tmp, "year") + node_tmp2.text = article_docx.dateiso.split("-")[0] if article_docx.vol: - node_tmp = etree.SubElement(node, 'volume') + node_tmp = etree.SubElement(node, "volume") node_tmp.text = str(article_docx.vol) if article_docx.issue: - node_tmp = etree.SubElement(node, 'issue') + node_tmp = etree.SubElement(node, "issue") node_tmp.text = str(article_docx.issue) - + if article_docx.elocatid: - node_tmp = etree.SubElement(node, 'elocation-id') + node_tmp = etree.SubElement(node, "elocation-id") node_tmp.text = article_docx.elocatid - node_tmp = etree.SubElement(node, 'history') + node_tmp = etree.SubElement(node, "history") val = next( - (b.value['paragraph'] for b in data_t - if b.block_type == 'paragraph' and b.value.get('label') == ''), - None + ( + b.value["paragraph"] + for b in data_t + if b.block_type == "paragraph" + and b.value.get("label") == "" + ), + None, ) date = extract_date(val) if date: - node_tmp2 = etree.SubElement(node_tmp, 'date') - node_tmp2.set('date-type', 'received') - - node_tmp3 = etree.SubElement(node_tmp2, 'day') + node_tmp2 = etree.SubElement(node_tmp, "date") + node_tmp2.set("date-type", "received") + + node_tmp3 = etree.SubElement(node_tmp2, "day") node_tmp3.text = date[0] - node_tmp3 = etree.SubElement(node_tmp2, 'month') + node_tmp3 = etree.SubElement(node_tmp2, "month") node_tmp3.text = date[1] - node_tmp3 = etree.SubElement(node_tmp2, 'year') + node_tmp3 = etree.SubElement(node_tmp2, "year") node_tmp3.text = date[2] val = next( - (b.value['paragraph'] for b in data_t - if b.block_type == 'paragraph' and b.value.get('label') == ''), - None + ( + b.value["paragraph"] + for b in data_t + if b.block_type == "paragraph" + and b.value.get("label") == "" + ), + None, ) date = extract_date(val) if date: - node_tmp2 = etree.SubElement(node_tmp, 'date') - node_tmp2.set('date-type', 'accepted') - - node_tmp3 = etree.SubElement(node_tmp2, 'day') + node_tmp2 = etree.SubElement(node_tmp, "date") + node_tmp2.set("date-type", "accepted") + + node_tmp3 = etree.SubElement(node_tmp2, "day") node_tmp3.text = date[0] - node_tmp3 = etree.SubElement(node_tmp2, 'month') + node_tmp3 = etree.SubElement(node_tmp2, "month") node_tmp3.text = date[1] - node_tmp3 = etree.SubElement(node_tmp2, 'year') + node_tmp3 = etree.SubElement(node_tmp2, "year") node_tmp3.text = date[2] - - node_tmp = etree.SubElement(node, 'permissions') + + node_tmp = etree.SubElement(node, "permissions") if article_docx.license: - node_tmp2 = etree.SubElement(node_tmp, 'license') - node_tmp2.set('license-type', 'open-access') + node_tmp2 = etree.SubElement(node_tmp, "license") + node_tmp2.set("license-type", "open-access") node_tmp2.set("{http://www.w3.org/1999/xlink}href", article_docx.license) - node_tmp2.set("{http://www.w3.org/XML/1998/namespace}lang", article_docx.language) + node_tmp2.set( + "{http://www.w3.org/XML/1998/namespace}lang", article_docx.language + ) - node_tmp3 = etree.SubElement(node_tmp2, 'license-p') + node_tmp3 = etree.SubElement(node_tmp2, "license-p") node_tmp3.text = "Este es un artículo con licencia..." - + vals = [ b for b in data_t - if b.block_type == 'paragraph' - and b.value.get('label') == '' - ] + if b.block_type == "paragraph" + and b.value.get("label") == "" + ] vals2 = [ b for b in data_t - if b.block_type == 'paragraph_with_language' - and b.value.get('label') == '' - ] - - node_tmp = etree.SubElement(node, 'abstract') + if b.block_type == "paragraph_with_language" + and b.value.get("label") == "" + ] - if vals[0]: - node_tmp2 = etree.SubElement(node_tmp, 'title') - append_fragment(node_tmp2, vals[0].value.get('paragraph')) + node_tmp = etree.SubElement(node, "abstract") - if vals2[0]: + if vals: + node_tmp2 = etree.SubElement(node_tmp, "title") + append_fragment(node_tmp2, vals[0].value.get("paragraph")) + if vals2: # Encuentra su índice original en article_docx.content last_index = data_t.index(vals2[0]) # Recorre los bloques siguientes following_paragraphs = [] for block in data_t[last_index:]: - if (block.block_type == 'paragraph' and block.value.get('label') == '

') or (block.block_type == 'paragraph_with_language' and block.value.get('label') == ''): - subsection = extract_subsection(block.value.get('paragraph')) - - if subsection['title']: - node_tmp2 = etree.SubElement(node_tmp, 'sec') - node_tmp3 = etree.SubElement(node_tmp2, 'title') - append_fragment(node_tmp3, subsection['title']) - node_tmp3 = etree.SubElement(node_tmp2, 'p') - append_fragment(node_tmp3, subsection['content']) + if ( + block.block_type == "paragraph" + and block.value.get("label") == "

" + ) or ( + block.block_type == "paragraph_with_language" + and block.value.get("label") == "" + ): + subsection = extract_subsection(block.value.get("paragraph")) + + if subsection["title"]: + node_tmp2 = etree.SubElement(node_tmp, "sec") + node_tmp3 = etree.SubElement(node_tmp2, "title") + append_fragment(node_tmp3, subsection["title"]) + node_tmp3 = etree.SubElement(node_tmp2, "p") + append_fragment(node_tmp3, subsection["content"]) else: - node_tmp2 = etree.SubElement(node_tmp, 'p') - append_fragment(node_tmp2, subsection['content']) + node_tmp2 = etree.SubElement(node_tmp, "p") + append_fragment(node_tmp2, subsection["content"]) else: break for i, val in enumerate(vals[1:], start=1): - node_tmp = etree.SubElement(node, 'trans-abstract') - node_tmp.set("{http://www.w3.org/XML/1998/namespace}lang", vals2[i].value.get('language')) + node_tmp = etree.SubElement(node, "trans-abstract") + node_tmp.set( + "{http://www.w3.org/XML/1998/namespace}lang", + vals2[i].value.get("language"), + ) - node_tmp2 = etree.SubElement(node_tmp, 'title') - append_fragment(node_tmp2, val.value.get('paragraph')) + node_tmp2 = etree.SubElement(node_tmp, "title") + append_fragment(node_tmp2, val.value.get("paragraph")) last_index = data_t.index(vals2[i]) # Recorre los bloques siguientes following_paragraphs = [] for block in data_t[last_index:]: - if (block.block_type == 'paragraph' and block.value.get('label') == '

') or (block.block_type == 'paragraph_with_language' and block.value.get('label') == ''): - subsection = extract_subsection(block.value.get('paragraph')) - - if subsection['title']: - node_tmp2 = etree.SubElement(node_tmp, 'sec') - node_tmp3 = etree.SubElement(node_tmp2, 'title') - append_fragment(node_tmp3, subsection['title']) - node_tmp3 = etree.SubElement(node_tmp2, 'p') - append_fragment(node_tmp3, subsection['content']) + if ( + block.block_type == "paragraph" + and block.value.get("label") == "

" + ) or ( + block.block_type == "paragraph_with_language" + and block.value.get("label") == "" + ): + subsection = extract_subsection(block.value.get("paragraph")) + + if subsection["title"]: + node_tmp2 = etree.SubElement(node_tmp, "sec") + node_tmp3 = etree.SubElement(node_tmp2, "title") + append_fragment(node_tmp3, subsection["title"]) + node_tmp3 = etree.SubElement(node_tmp2, "p") + append_fragment(node_tmp3, subsection["content"]) else: - node_tmp2 = etree.SubElement(node_tmp, 'p') - append_fragment(node_tmp2, subsection['content']) + node_tmp2 = etree.SubElement(node_tmp, "p") + append_fragment(node_tmp2, subsection["content"]) else: break vals = [ b for b in data_t - if b.block_type == 'paragraph' - and b.value.get('label') == '' - ] + if b.block_type == "paragraph" and b.value.get("label") == "" + ] vals2 = [ b for b in data_t - if b.block_type == 'paragraph_with_language' - and b.value.get('label') == '' - ] - - for i, val in enumerate(vals): - node_tmp = etree.SubElement(node, 'kwd-group') - node_tmp.set("{http://www.w3.org/XML/1998/namespace}lang", vals2[i].value.get('language')) - - node_tmp2 = etree.SubElement(node_tmp, 'title') - append_fragment(node_tmp2, val.value.get('paragraph')) - #node_tmp2.text = val.value.get('paragraph') + if b.block_type == "paragraph_with_language" + and b.value.get("label") == "" + ] - for kw in vals2[i].value.get('paragraph').split(', '): - node_tmp2 = etree.SubElement(node_tmp, 'kwd') + for i, val in enumerate(vals): + node_tmp = etree.SubElement(node, "kwd-group") + node_tmp.set( + "{http://www.w3.org/XML/1998/namespace}lang", + vals2[i].value.get("language"), + ) + + node_tmp2 = etree.SubElement(node_tmp, "title") + append_fragment(node_tmp2, val.value.get("paragraph")) + # node_tmp2.text = val.value.get('paragraph') + + for kw in vals2[i].value.get("paragraph").split(", "): + node_tmp2 = etree.SubElement(node_tmp, "kwd") append_fragment(node_tmp2, kw) - + countFN = 0 for i, d in enumerate(data): node = body @@ -466,190 +526,206 @@ def get_xml(article_docx, data_front, data, data_back): continue_t = False continue - if d['value']['label'] == '': - val_p = d['value']['paragraph'].lower() - attrib={} + if d["value"]["label"] == "": + val_p = d["value"]["paragraph"].lower() + attrib = {} if re.search(r"^(intro|sinops|synops)", val_p): - attrib={'sec-type': 'intro'} + attrib = {"sec-type": "intro"} elif re.search(r"^(caso|case)", val_p): - attrib={'sec-type': 'cases'} + attrib = {"sec-type": "cases"} elif re.search(r"^(conclus|comment|coment)", val_p): - attrib={'sec-type': 'conclusions'} + attrib = {"sec-type": "conclusions"} elif re.search(r"^(discus)", val_p): - attrib={'sec-type': 'discussion'} + attrib = {"sec-type": "discussion"} elif re.search(r"^(materia)", val_p): - attrib={'sec-type': 'materials'} + attrib = {"sec-type": "materials"} elif re.search(r"^(proced|method|métod|metod)", val_p): - attrib={'sec-type': 'methods'} + attrib = {"sec-type": "methods"} elif re.search(r"^(result|statement|finding|declara|hallaz)", val_p): - attrib={'sec-type': 'results'} - elif re.search(r"^(subject|participant|patient|pacient|assunt|sujeto)", val_p): - attrib={'sec-type': 'subjects'} + attrib = {"sec-type": "results"} + elif re.search( + r"^(subject|participant|patient|pacient|assunt|sujeto)", val_p + ): + attrib = {"sec-type": "subjects"} elif re.search(r"^(suplement|material)", val_p): - attrib={'sec-type': 'supplementary-material'} + attrib = {"sec-type": "supplementary-material"} + + node = etree.SubElement(body, "sec", attrib=attrib) + node_title = etree.SubElement(node, "title") + append_fragment(node_title, d["value"]["paragraph"]) - node = etree.SubElement(body, 'sec', attrib=attrib) - node_title = etree.SubElement(node, 'title') - append_fragment(node_title, d['value']['paragraph']) - subsec = False - if d['value']['label'] == '': + if d["value"]["label"] == "": subsec = True - node_sec = etree.SubElement(node, 'sec') - node_title = etree.SubElement(node_sec, 'title') - #if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', d['value']['paragraph']): - if re.search(r'^(.*?)$', d['value']['paragraph']): - #sech = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') - sech = d['value']['paragraph'] - node_subtitle = etree.SubElement(node_title, 'italic') + node_sec = etree.SubElement(node, "sec") + node_title = etree.SubElement(node_sec, "title") + # if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', d['value']['paragraph']): + if re.search(r"^(.*?)$", d["value"]["paragraph"]): + # sech = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') + sech = d["value"]["paragraph"] + node_subtitle = etree.SubElement(node_title, "italic") append_fragment(node_subtitle, sech) - #node_subtitle.text = sech + # node_subtitle.text = sech else: - #node_title.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') + # node_title.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') pass - if d['value']['label'] == '': - re_search = re.search(r'list list-type="(.*?)"\]', d['value']['paragraph']) + if d["value"]["label"] == "": + re_search = re.search(r'list list-type="(.*?)"\]', d["value"]["paragraph"]) list_type = re_search.group(1) - attrib={'list-type': list_type} + attrib = {"list-type": list_type} if subsec: - node_p = etree.SubElement(node_sec, 'p') - node_list = etree.SubElement(node_p, 'list', attrib=attrib) + node_p = etree.SubElement(node_sec, "p") + node_list = etree.SubElement(node_p, "list", attrib=attrib) else: - node_p = etree.SubElement(node, 'p') - node_list = etree.SubElement(node_p, 'list', attrib=attrib) - - content_list = re.search(r'\[list list-type="[^"]*"\](.*?)\[/list\]', d['value']['paragraph'], re.DOTALL) + node_p = etree.SubElement(node, "p") + node_list = etree.SubElement(node_p, "list", attrib=attrib) + + content_list = re.search( + r'\[list list-type="[^"]*"\](.*?)\[/list\]', + d["value"]["paragraph"], + re.DOTALL, + ) content_list = content_list.group(1) - node_list_text = content_list \ - .replace('[list-item]', '

') \ - .replace('[/list-item]', '

') - #.replace('[style name="italic"]', '').replace('[/style]', '') \ + node_list_text = content_list.replace( + "[list-item]", "

" + ).replace("[/list-item]", "

") + node_list_text = sanitize_inline_xml_fragment(node_list_text) node_list_text = etree.fromstring(f"{node_list_text}") for child in node_list_text: node_list.append(child) - - if d['value']['label'] == '' or d['value']['label'] == '': - - attrib={'id': d['value']['tabid']} - + + if d["value"]["label"] == "
" or d["value"]["label"] == "": + attrib = {"id": d["value"]["tabid"]} + if subsec: - node_p = etree.SubElement(node_sec, 'p') - node_table = etree.SubElement(node_p, 'table-wrap', attrib=attrib) + node_p = etree.SubElement(node_sec, "p") + node_table = etree.SubElement(node_p, "table-wrap", attrib=attrib) else: - node_p = etree.SubElement(node, 'p') - node_table = etree.SubElement(node_p, 'table-wrap', attrib=attrib) - - node_label = etree.SubElement(node_table, 'label') - append_fragment(node_label, d.get('value', {}).get('tablabel')) - - node_caption = etree.SubElement(node_table, 'caption') - node_title = etree.SubElement(node_caption, 'title') - append_fragment(node_title, d.get('value', {}).get('title')) - - node_table_text = d['value']['content'] + node_p = etree.SubElement(node, "p") + node_table = etree.SubElement(node_p, "table-wrap", attrib=attrib) + + node_label = etree.SubElement(node_table, "label") + append_fragment(node_label, d.get("value", {}).get("tablabel")) + + node_caption = etree.SubElement(node_table, "caption") + node_title = etree.SubElement(node_caption, "title") + append_fragment(node_title, d.get("value", {}).get("title")) + + node_table_text = d["value"]["content"] # Quitar saltos de línea y espacios extra - node_table_text = re.sub(r"\s*\n\s*", "", node_table_text).replace('
','') + node_table_text = re.sub(r"\s*\n\s*", "", node_table_text).replace( + "
", "" + ) - # Parsear la tabla como fragmento XML/HTML - tabla_element = etree.XML(node_table_text) + tabla_element = parse_xml_fragment(node_table_text) # Insertar en el XML principal node_table.append(tabla_element) - node_foot = etree.SubElement(node_p, 'table-wrap-foot') + node_foot = etree.SubElement(node_p, "table-wrap-foot") - if d['value']['label'] == '': + if d["value"]["label"] == "": countFN += 1 - node_fn = etree.SubElement(node_foot, 'fn', attrib={"id":f"TFN{str(countFN)}"}) - node_fnp = etree.SubElement(node_fn, 'p') - append_fragment(node_fnp, d['value']['paragraph']) - - if d['value']['label'] == '': - - attrib={'id': d['value']['figid']} - + node_fn = etree.SubElement( + node_foot, "fn", attrib={"id": f"TFN{str(countFN)}"} + ) + node_fnp = etree.SubElement(node_fn, "p") + append_fragment(node_fnp, d["value"]["paragraph"]) + + if d["value"]["label"] == "": + attrib = {"id": d["value"]["figid"]} + if subsec: - node_p = etree.SubElement(node_sec, 'p') - node_fig = etree.SubElement(node_p, 'fig', attrib=attrib) + node_p = etree.SubElement(node_sec, "p") + node_fig = etree.SubElement(node_p, "fig", attrib=attrib) else: - node_p = etree.SubElement(node, 'p') - node_fig = etree.SubElement(node_p, 'fig', attrib=attrib) + node_p = etree.SubElement(node, "p") + node_fig = etree.SubElement(node_p, "fig", attrib=attrib) - etree.SubElement(node_fig, 'label').text = d['value']['figlabel'] - node_caption = etree.SubElement(node_fig, 'caption') - etree.SubElement(node_caption, 'title').text = d['value']['title'] if 'title' in d['value'] else None + etree.SubElement(node_fig, "label").text = d["value"]["figlabel"] + node_caption = etree.SubElement(node_fig, "caption") + etree.SubElement(node_caption, "title").text = ( + d["value"]["title"] if "title" in d["value"] else None + ) Image = get_image_model() - image_id = d['value']['image'] + image_id = d["value"]["image"] image_obj = Image.objects.get(pk=image_id) file_name = os.path.basename(image_obj.file.name) - original_url = image_obj.get_rendition('original').url + original_url = image_obj.get_rendition("original").url original_filename = os.path.basename(urlparse(original_url).path) - #node_caption = etree.SubElement(node_fig, 'graphic', attrib={'{http://www.w3.org/1999/xlink}ref': f"{d['value']['figid']}.jpeg"}) - node_caption = etree.SubElement(node_fig, 'graphic', attrib={'{http://www.w3.org/1999/xlink}href': original_filename}) - - if d['value']['label'] == '': + # node_caption = etree.SubElement(node_fig, 'graphic', attrib={'{http://www.w3.org/1999/xlink}ref': f"{d['value']['figid']}.jpeg"}) + node_caption = etree.SubElement( + node_fig, + "graphic", + attrib={"{http://www.w3.org/1999/xlink}href": original_filename}, + ) + + if d["value"]["label"] == "": + node_attrib = etree.SubElement(node_fig, "attrib") + append_fragment(node_attrib, d["value"]["paragraph"]) - node_attrib = etree.SubElement(node_fig, 'attrib') - append_fragment(node_attrib, d['value']['paragraph']) + if d["value"]["label"] == "": + attrib = {"id": d["value"]["eid"]} - if d['value']['label'] == '': - attrib={'id': d['value']['eid']} - if subsec: - node_p = etree.SubElement(node_sec, 'p') - node_f = etree.SubElement(node_p, 'disp-formula', attrib=attrib) + node_p = etree.SubElement(node_sec, "p") + node_f = etree.SubElement(node_p, "disp-formula", attrib=attrib) else: - node_p = etree.SubElement(node, 'p') - node_f = etree.SubElement(node_p, 'disp-formula', attrib=attrib) - - - for c in d['value']['content']: - if c['type'] == 'text': - node_t = etree.SubElement(node_f, 'label') - append_fragment(node_t, c['value']) - if c['type'] == 'formula': - append_fragment(node_f, c['value']) - - if d['value']['label'] == '': - attrib={'id': d['value']['eid']} - + node_p = etree.SubElement(node, "p") + node_f = etree.SubElement(node_p, "disp-formula", attrib=attrib) + + for c in d["value"]["content"]: + if c["type"] == "text": + node_t = etree.SubElement(node_f, "label") + append_fragment(node_t, c["value"]) + if c["type"] == "formula": + append_fragment(node_f, c["value"]) + + if d["value"]["label"] == "": + attrib = {"id": d["value"]["eid"]} + if subsec: - node_p = etree.SubElement(node_sec, 'p') + node_p = etree.SubElement(node_sec, "p") else: - node_p = etree.SubElement(node, 'p') - - content = '' - for c in d['value']['content']: - if c['type'] == 'text': - content += c['value'] - if c['type'] == 'formula': - node_f = etree.Element('inline-formula', attrib=attrib) - append_fragment(node_f, c['value']) - content += etree.tostring(node_f, pretty_print=True, encoding="unicode") - + node_p = etree.SubElement(node, "p") + + content = "" + for c in d["value"]["content"]: + if c["type"] == "text": + content += c["value"] + if c["type"] == "formula": + node_f = etree.Element("inline-formula", attrib=attrib) + append_fragment(node_f, c["value"]) + content += etree.tostring( + node_f, pretty_print=True, encoding="unicode" + ) + append_fragment(node_p, content) - if d['value']['label'] == '

': + if d["value"]["label"] == "

": if subsec: - node_p = etree.SubElement(node_sec, 'p') + node_p = etree.SubElement(node_sec, "p") else: - node_p = etree.SubElement(node, 'p') + node_p = etree.SubElement(node, "p") - #refs = extraer_citas_apa(d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', ''), data_back) - #refs = extraer_citas_apa(d['value']['paragraph'].replace('', '').replace('', ''), data_back) - if 'xref' not in d['value']['paragraph']: - refs = proccess_labeled_text(d['value']['paragraph'], data_back) + # refs = extraer_citas_apa(d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', ''), data_back) + # refs = extraer_citas_apa(d['value']['paragraph'].replace('', '').replace('', ''), data_back) + if "xref" not in d["value"]["paragraph"]: + refs = proccess_labeled_text(d["value"]["paragraph"], data_back) for r in refs: - #print(f"r in refs: {r}") - d['value']['paragraph'] = d['value']['paragraph'].replace(r['cita'], f"{r['cita']}") + # print(f"r in refs: {r}") + d["value"]["paragraph"] = d["value"]["paragraph"].replace( + r["cita"], + f"{r['cita']}", + ) """ if 'et al' in r['cita']: et_al_replace = r['cita'].replace('et al', 'et al') @@ -659,208 +735,320 @@ def get_xml(article_docx, data_front, data, data_back): d['value']['paragraph'] = d['value']['paragraph'].replace(r['cita'], f"{r['cita']}") """ - elements = proccess_special_content(d['value']['paragraph'], data) + elements = proccess_special_content(d["value"]["paragraph"], data) for e in elements: - d['value']['paragraph'] = d['value']['paragraph'].replace(e['label'], f"{e['label']}") + d["value"]["paragraph"] = d["value"]["paragraph"].replace( + e["label"], + f"{e['label']}", + ) - append_fragment(node_p, d['value']['paragraph']) + append_fragment(node_p, d["value"]["paragraph"]) - #if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', d['value']['paragraph']): - #if re.search(r'^(.*?)$', d['value']['paragraph']): + # if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', d['value']['paragraph']): + # if re.search(r'^(.*?)$', d['value']['paragraph']): # node_title.text = '' - #ph = d['value']['paragraph'].replace('', '').replace('', '') - #node_subtitle = etree.SubElement(node_title, 'italic') - #node_subtitle.text = ph - #node_subtitle = etree.fromstring(f"{d['value']['paragraph']}") - #for child in node_subtitle: - # node_title.append(child) - #else: - #node_p.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') - #p_text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') - # append_fragment(node_p, d['value']['paragraph']) - #node_p.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') - #p_text = d['value']['paragraph'] - #try: - #node_text = etree.fromstring(f"{p_text}") - #for child in node_text: - #node_p.append(child) - #except Exception as e: - #print(p_text) - #print(e) - - if d['value']['label'] == '': + # ph = d['value']['paragraph'].replace('', '').replace('', '') + # node_subtitle = etree.SubElement(node_title, 'italic') + # node_subtitle.text = ph + # node_subtitle = etree.fromstring(f"{d['value']['paragraph']}") + # for child in node_subtitle: + # node_title.append(child) + # else: + # node_p.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') + # p_text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') + # append_fragment(node_p, d['value']['paragraph']) + # node_p.text = d['value']['paragraph'].replace('[style name="italic"]', '').replace('[/style]', '') + # p_text = d['value']['paragraph'] + # try: + # node_text = etree.fromstring(f"{p_text}") + # for child in node_text: + # node_p.append(child) + # except Exception as e: + # print(p_text) + # print(e) + + if d["value"]["label"] == "": if subsec: - node_p = etree.SubElement(node_sec, 'p') + node_p = etree.SubElement(node_sec, "p") else: - node_p = etree.SubElement(node, 'p') + node_p = etree.SubElement(node, "p") - p_text = '' - if 'content' in d['value']: - for val in d['value']['content']: - if val['type'] == 'text': - type_val = 'text' + p_text = "" + if "content" in d["value"]: + for val in d["value"]["content"]: + if val["type"] == "text": + type_val = "text" else: - type_val = 'formula' + type_val = "formula" - #if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', val['value']): - if re.search(r'^(.*?)$', val['value']): - node_title.text = '' - #ph = val['value'].replace('[style name="italic"]', '').replace('[/style]', '') - ph = val['value'] + # if re.search(r'^\[style name="italic"\](.*?)\[/style\]$', val['value']): + if re.search(r"^(.*?)$", val["value"]): + node_title.text = "" + # ph = val['value'].replace('[style name="italic"]', '').replace('[/style]', '') + ph = val["value"] node_subtitle = etree.fromstring(f"{ph}") for child in node_subtitle: node_title.append(child) else: - #p_text += val['value'].replace('[style name="italic"]', '').replace('[/style]', '').replace('xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"', '') - p_text += val['value'].replace('xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"', '') + # p_text += val['value'].replace('[style name="italic"]', '').replace('[/style]', '').replace('xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"', '') + p_text += val["value"].replace( + 'xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"', + "", + ) - node_text = etree.fromstring(f"{p_text}") + node_text = etree.fromstring( + f"{p_text}" + ) for child in node_text: node_p.append(child) - + for i, d in enumerate(data_back): - if d['value']['label'] == '': - node_tit = etree.SubElement(node_reflist, 'title') - append_fragment(node_tit, d['value']['paragraph']) - if d['value']['label'] == '

': - values = d['value'] - node_ref = etree.SubElement(node_reflist, 'ref', attrib={"id": values['refid']}) - #node_label = etree.SubElement(node_ref, 'label') - #append_fragment(node_label, values['refid'].replace('B', '')) - node_mix = etree.SubElement(node_ref, 'mixed-citation') - append_fragment(node_mix, values['paragraph']) - - if values['reftype'] == 'journal': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'article-title'), values['title']) - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - append_fragment(etree.SubElement(node_ref, 'year'), str(values['date'])) - append_fragment(etree.SubElement(node_ref, 'volume'), str(values['vol'])) - append_fragment(etree.SubElement(node_ref, 'issue'), str(values['issue'])) - - if values['fpage'] and values['fpage'][0] == 'e': - append_fragment(etree.SubElement(node_ref, 'elocation-id'), values['fpage']) + if d["value"]["label"] == "": + node_tit = etree.SubElement(node_reflist, "title") + append_fragment(node_tit, d["value"]["paragraph"]) + if d["value"]["label"] == "

": + values = d["value"] + node_ref = etree.SubElement( + node_reflist, "ref", attrib={"id": values["refid"]} + ) + # node_label = etree.SubElement(node_ref, 'label') + # append_fragment(node_label, values['refid'].replace('B', '')) + node_mix = etree.SubElement(node_ref, "mixed-citation") + append_fragment(node_mix, values["paragraph"]) + + if values["reftype"] == "journal": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment( + etree.SubElement(node_ref, "article-title"), values["title"] + ) + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + append_fragment(etree.SubElement(node_ref, "year"), str(values["date"])) + append_fragment( + etree.SubElement(node_ref, "volume"), str(values["vol"]) + ) + append_fragment( + etree.SubElement(node_ref, "issue"), str(values["issue"]) + ) + + if values["fpage"] and values["fpage"][0] == "e": + append_fragment( + etree.SubElement(node_ref, "elocation-id"), values["fpage"] + ) else: - append_fragment(etree.SubElement(node_ref, 'fpage'), str(values['fpage'])) - append_fragment(etree.SubElement(node_ref, 'lpage'), str(values['lpage'])) - - append_fragment(etree.SubElement(node_ref, 'pub-id', attrib={"pub-id-type": "doi"}), values['doi']) - - if values['uri']: - append_fragment(etree.SubElement(node_ref, 'ext-link', - attrib={ - "ext-link-type": "uri", - "{http://www.w3.org/1999/xlink}href": values['uri'] - }), - values['uri']) - - if values['reftype'] == 'book': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'part-title'), values['chapter']) - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - append_fragment(etree.SubElement(node_ref, 'edition'), values['edition']) - append_fragment(etree.SubElement(node_ref, 'publisher-loc'), values['location']) - append_fragment(etree.SubElement(node_ref, 'publisher-name'), values['organization']) - append_fragment(etree.SubElement(node_ref, 'year'), str(values['date'])) - append_fragment(etree.SubElement(node_ref, 'fpage'), str(values['fpage'])) - append_fragment(etree.SubElement(node_ref, 'lpage'), str(values['lpage'])) - - - if values['reftype'] == 'data': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'data-title'), values['title']) - append_fragment(etree.SubElement(node_ref, 'version'), values['version']) - append_fragment(etree.SubElement(node_ref, 'year'), str(values['date'])) - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - append_fragment(etree.SubElement(node_ref, 'pub-id', attrib={"pub-id-type": "doi"}), values['doi']) - if values['uri']: - append_fragment(etree.SubElement(node_ref, 'ext-link', - attrib={ - "ext-link-type": "uri", - "{http://www.w3.org/1999/xlink}href": values['uri'] - }), - values['uri']) - - if values['reftype'] == 'webpage': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - if values['uri']: - append_fragment(etree.SubElement(node_ref, 'ext-link', - attrib={ - "ext-link-type": "uri", - "{http://www.w3.org/1999/xlink}href": values['uri'] - }), - values['uri']) - append_fragment(etree.SubElement(node_ref, 'access-date'), values['access_date']) - - if values['reftype'] == 'confproc': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - append_fragment(etree.SubElement(node_ref, 'conf-name'), values['title']) - append_fragment(etree.SubElement(node_ref, 'conf-num'), str(values['issue'])) - append_fragment(etree.SubElement(node_ref, 'conf-date'), str(values['date'])) - append_fragment(etree.SubElement(node_ref, 'conf-loc'), values['location']) - append_fragment(etree.SubElement(node_ref, 'publisher-loc'), values['org_location']) - append_fragment(etree.SubElement(node_ref, 'publisher-name'), values['organization']) - append_fragment(etree.SubElement(node_ref, 'page'), values['pages']) - - if values['reftype'] == 'thesis': - node_elem = etree.SubElement(node_ref, 'element-citation', attrib={"publication-type": values['reftype']}) - node_person = etree.SubElement(node_elem, 'person-group', attrib={"person-group-type": "author"}) - for a in values['authors']: - node_name = etree.SubElement(node_person, 'name') - node_sname = etree.SubElement(node_name, 'surname') - node_gname = etree.SubElement(node_name, 'given-names') - append_fragment(node_sname, a['value']['surname']) - append_fragment(node_gname, a['value']['given_names']) - - append_fragment(etree.SubElement(node_ref, 'source'), values['source']) - append_fragment(etree.SubElement(node_ref, 'publisher-loc'), values['org_location']) - append_fragment(etree.SubElement(node_ref, 'publisher-name'), values['organization']) - append_fragment(etree.SubElement(node_ref, 'year'), str(values['date'])) - append_fragment(etree.SubElement(node_ref, 'page'), values['pages']) + append_fragment( + etree.SubElement(node_ref, "fpage"), str(values["fpage"]) + ) + append_fragment( + etree.SubElement(node_ref, "lpage"), str(values["lpage"]) + ) + + append_fragment( + etree.SubElement(node_ref, "pub-id", attrib={"pub-id-type": "doi"}), + values["doi"], + ) + + if values["uri"]: + append_fragment( + etree.SubElement( + node_ref, + "ext-link", + attrib={ + "ext-link-type": "uri", + "{http://www.w3.org/1999/xlink}href": values["uri"], + }, + ), + values["uri"], + ) + + if values["reftype"] == "book": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment( + etree.SubElement(node_ref, "part-title"), values["chapter"] + ) + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + append_fragment( + etree.SubElement(node_ref, "edition"), values["edition"] + ) + append_fragment( + etree.SubElement(node_ref, "publisher-loc"), values["location"] + ) + append_fragment( + etree.SubElement(node_ref, "publisher-name"), values["organization"] + ) + append_fragment(etree.SubElement(node_ref, "year"), str(values["date"])) + append_fragment( + etree.SubElement(node_ref, "fpage"), str(values["fpage"]) + ) + append_fragment( + etree.SubElement(node_ref, "lpage"), str(values["lpage"]) + ) + + if values["reftype"] == "data": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment( + etree.SubElement(node_ref, "data-title"), values["title"] + ) + append_fragment( + etree.SubElement(node_ref, "version"), values["version"] + ) + append_fragment(etree.SubElement(node_ref, "year"), str(values["date"])) + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + append_fragment( + etree.SubElement(node_ref, "pub-id", attrib={"pub-id-type": "doi"}), + values["doi"], + ) + if values["uri"]: + append_fragment( + etree.SubElement( + node_ref, + "ext-link", + attrib={ + "ext-link-type": "uri", + "{http://www.w3.org/1999/xlink}href": values["uri"], + }, + ), + values["uri"], + ) + + if values["reftype"] == "webpage": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + if values["uri"]: + append_fragment( + etree.SubElement( + node_ref, + "ext-link", + attrib={ + "ext-link-type": "uri", + "{http://www.w3.org/1999/xlink}href": values["uri"], + }, + ), + values["uri"], + ) + append_fragment( + etree.SubElement(node_ref, "access-date"), values["access_date"] + ) + + if values["reftype"] == "confproc": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + append_fragment( + etree.SubElement(node_ref, "conf-name"), values["title"] + ) + append_fragment( + etree.SubElement(node_ref, "conf-num"), str(values["issue"]) + ) + append_fragment( + etree.SubElement(node_ref, "conf-date"), str(values["date"]) + ) + append_fragment( + etree.SubElement(node_ref, "conf-loc"), values["location"] + ) + append_fragment( + etree.SubElement(node_ref, "publisher-loc"), values["org_location"] + ) + append_fragment( + etree.SubElement(node_ref, "publisher-name"), values["organization"] + ) + append_fragment(etree.SubElement(node_ref, "page"), values["pages"]) + + if values["reftype"] == "thesis": + node_elem = etree.SubElement( + node_ref, + "element-citation", + attrib={"publication-type": values["reftype"]}, + ) + node_person = etree.SubElement( + node_elem, "person-group", attrib={"person-group-type": "author"} + ) + for a in values["authors"]: + node_name = etree.SubElement(node_person, "name") + node_sname = etree.SubElement(node_name, "surname") + node_gname = etree.SubElement(node_name, "given-names") + append_fragment(node_sname, a["value"]["surname"]) + append_fragment(node_gname, a["value"]["given_names"]) + + append_fragment(etree.SubElement(node_ref, "source"), values["source"]) + append_fragment( + etree.SubElement(node_ref, "publisher-loc"), values["org_location"] + ) + append_fragment( + etree.SubElement(node_ref, "publisher-name"), values["organization"] + ) + append_fragment(etree.SubElement(node_ref, "year"), str(values["date"])) + append_fragment(etree.SubElement(node_ref, "page"), values["pages"]) # Convertir a una cadena XML xml_como_texto = etree.tostring(root, pretty_print=True, encoding="unicode") - return xml_como_texto, data \ No newline at end of file + return xml_como_texto, data diff --git a/markuplib/function_docx.py b/markuplib/function_docx.py index decf630..f92c31d 100644 --- a/markuplib/function_docx.py +++ b/markuplib/function_docx.py @@ -1,18 +1,20 @@ +import html +import os +import re +import zipfile + import docx +from django.core.files.base import ContentFile +from docx.oxml.ns import qn from docx.oxml.table import CT_Tbl from docx.oxml.text.paragraph import CT_P -from docx.oxml.ns import qn from lxml import etree, objectify -from django.core.files.base import ContentFile -import re, zipfile -import os from wagtail.images import get_image_model ImageModel = get_image_model() class functionsDocx: - def openDocx(filename): doc = docx.Document(filename) return doc @@ -44,28 +46,33 @@ def replace_mfenced_pipe_only(self, mathml_root): parent.replace(mfenced, mrow) return mathml_root - def extract_numbering_info(self, docx_path): # Diccionario para mapear numId a su tipo (numerada o viñeta) numbering_map = {} - namespaces = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' + namespaces = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" # Abrir el archivo DOCX como un archivo ZIP - with zipfile.ZipFile(docx_path, 'r') as docx: + with zipfile.ZipFile(docx_path, "r") as docx: # Verificar si existe el archivo numbering.xml - if 'word/numbering.xml' in docx.namelist(): + if "word/numbering.xml" in docx.namelist(): # Extraer el archivo numbering.xml - numbering_xml = docx.read('word/numbering.xml') + numbering_xml = docx.read("word/numbering.xml") # Parsear el XML numbering_tree = etree.fromstring(numbering_xml) # Buscar todas las definiciones abstractas de numeración - for abstract_num in numbering_tree.findall('.//w:abstractNum', namespaces=numbering_tree.nsmap): - abstract_num_id = abstract_num.get(namespaces+'abstractNumId') + for abstract_num in numbering_tree.findall( + ".//w:abstractNum", namespaces=numbering_tree.nsmap + ): + abstract_num_id = abstract_num.get(namespaces + "abstractNumId") # Revisar los niveles dentro de la definición abstracta - for lvl in abstract_num.findall('.//w:lvl', namespaces=abstract_num.nsmap): - num_fmt = lvl.find('.//w:numFmt', lvl.nsmap).get(namespaces+'val') - ilvl = lvl.get(namespaces+'ilvl') + for lvl in abstract_num.findall( + ".//w:lvl", namespaces=abstract_num.nsmap + ): + num_fmt = lvl.find(".//w:numFmt", lvl.nsmap).get( + namespaces + "val" + ) + ilvl = lvl.get(namespaces + "ilvl") # Asignar el tipo según el valor de numFmt if abstract_num_id not in numbering_map: @@ -74,56 +81,62 @@ def extract_numbering_info(self, docx_path): numbering_map[abstract_num_id][ilvl] = num_fmt # Relacionar numId con su abstractNumId - for num in numbering_tree.findall('.//w:num', namespaces=numbering_tree.nsmap): - num_id = num.get(namespaces+'numId') - abstract_num_id = num.find('.//w:abstractNumId', namespaces=num.nsmap).get(namespaces+'val') + for num in numbering_tree.findall( + ".//w:num", namespaces=numbering_tree.nsmap + ): + num_id = num.get(namespaces + "numId") + abstract_num_id = num.find( + ".//w:abstractNumId", namespaces=num.nsmap + ).get(namespaces + "val") if abstract_num_id in numbering_map: - numbering_map[abstract_num_id]['numId'] = num_id + numbering_map[abstract_num_id]["numId"] = num_id else: numbering_map = None return numbering_map - def extract_hiperlinks_info(self, docx_path): hiperlinks = [] - with zipfile.ZipFile(docx_path, 'r') as docx: + with zipfile.ZipFile(docx_path, "r") as docx: # Leer relaciones del documento - rels_path = 'word/_rels/document.xml.rels' + rels_path = "word/_rels/document.xml.rels" if rels_path in docx.namelist(): rels_data = docx.read(rels_path) rels_root = etree.fromstring(rels_data) # Buscar hipervínculos - for rel in rels_root.findall('{http://schemas.openxmlformats.org/package/2006/relationships}Relationship'): - r_id = rel.attrib['Id'] - target = rel.attrib['Target'] - if rel.attrib['Type'].endswith('/hyperlink'): + for rel in rels_root.findall( + "{http://schemas.openxmlformats.org/package/2006/relationships}Relationship" + ): + r_id = rel.attrib["Id"] + target = rel.attrib["Target"] + if rel.attrib["Type"].endswith("/hyperlink"): hiperlinks.append((r_id, target)) return dict(hiperlinks) - def extract_hiperlink(self, element, rels_map, namespaces): links = [] # 1. Buscar hipervínculos de texto (recursivo con .//) - for hyperlink in element.findall('.//w:hyperlink', namespaces=namespaces): - r_id = hyperlink.attrib.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id') + for hyperlink in element.findall(".//w:hyperlink", namespaces=namespaces): + r_id = hyperlink.attrib.get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id" + ) if r_id and r_id in rels_map: links.append(rels_map[r_id]) # 2. Buscar hipervínculos en imágenes (recursivo con .//) - for hlink in element.findall('.//a:hlinkClick', namespaces=namespaces): - r_id = hlink.attrib.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id') + for hlink in element.findall(".//a:hlinkClick", namespaces=namespaces): + r_id = hlink.attrib.get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}id" + ) if r_id and r_id in rels_map: links.append(rels_map[r_id]) - return ' '.join(links) if links else None - + return " ".join(links) if links else None def extractContent(self, doc, doc_path): - list_types = self.extract_numbering_info(doc_path) hiperlinks_info = self.extract_hiperlinks_info(doc_path) @@ -141,88 +154,91 @@ def extractContent(self, doc, doc_path): transform = etree.XSLT(xslt) def match_paragraph(text): - keywords = r'(?im)^\s*(?:)?\s*(palabra(?:s)?\s*clave|palavras?\s*-?\s*chave|keywords?)\s*(?:)?\s*(?::|\s*:\s*)\s*(.+)$' - #history = r'\d{2}/\d{2}/\d{4}' - #corresp = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' - abstract = r'(?i)^resumen|^resumo|^abstract' - accepted = r'(?i)aceptado|accepted|aceited|aprovado' - received = r'(?i)recibido|received|recebido' + keywords = r"(?im)^\s*(?:)?\s*(palabra(?:s)?\s*clave|palavras?\s*-?\s*chave|keywords?)\s*(?:)?\s*(?::|\s*:\s*)\s*(.+)$" + # history = r'\d{2}/\d{2}/\d{4}' + # corresp = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}' + abstract = r"(?i)^resumen|^resumo|^abstract" + accepted = r"(?i)aceptado|accepted|aceited|aprovado" + received = r"(?i)recibido|received|recebido" if re.search(keywords, text): - return '' - #if re.search(history, text): - #return '' - #if re.search(corresp, text): - #return '' + return "" + # if re.search(history, text): + # return '' + # if re.search(corresp, text): + # return '' if re.search(abstract, text): - return '' + return "" if re.search(accepted, text): - return '' + return "" if re.search(received, text): - return '' + return "" return False def matches_section(a, b): try: return ( - a.get('size') == b.get('size') and - a.get('bold') == b.get('bold') and - a.get('isupper') == b.get('isupper') + a.get("size") == b.get("size") + and a.get("bold") == b.get("bold") + and a.get("isupper") == b.get("isupper") ) except Exception as e: print(f"Error comparando secciones: {e}") return False def section_priority(sections): - return (-sections['size'], not sections['bold'], not sections['isupper']) + return (-sections["size"], not sections["bold"], not sections["isupper"]) def identify_section(sections, size, bold, text): if size == 0: return sections isupper = text.isupper() - s_id = {'size': size, 'bold': bold, 'isupper': isupper, 'count': 0} - + s_id = {"size": size, "bold": bold, "isupper": isupper, "count": 0} + if len(sections) == 0: sections.append(s_id) return sections for section in sections: if matches_section(s_id, section): - section['count'] += 1 + section["count"] += 1 return sections - + sections.append(s_id) return sections def clean_labels(text): # Eliminar etiquetas cuadradas tipo [ ... ] con espacios opcionales - extract_label = re.sub(r'\[\s*/?\s*[\w-]+(?:\s+[^\]]+)?\s*\]', '', text) + extract_label = re.sub(r"\[\s*/?\s*[\w-]+(?:\s+[^\]]+)?\s*\]", "", text) # Reemplazar múltiples espacios por uno solo - clean_text = re.sub(r'\s+', ' ', extract_label) + clean_text = re.sub(r"\s+", " ", extract_label) # Eliminar espacio antes de signos de puntuación - clean_text = re.sub(r'\s+([;:,.])', r'\1', clean_text) + clean_text = re.sub(r"\s+([;:,.])", r"\1", clean_text) return clean_text.strip() def extrae_Tabla(element, rels_map, namespaces): - # Inicializa la estructura HTML de la tabla - html = "

\n" + table_html = "
\n" # Almacena las combinaciones para las celdas rowspan_dict = {} # {(row, col): rowspan_count} colspan_dict = {} # {(row, col): colspan_count} # Itera sobre las filas de la tabla - for i, row in enumerate(element.xpath('.//w:tr')): - hiperlinks = self.extract_hiperlink(row, rels_map, namespaces) if found_hiperlinks else None - - html += " \n" + for i, row in enumerate(element.xpath(".//w:tr")): + hiperlinks = ( + self.extract_hiperlink(row, rels_map, namespaces) + if found_hiperlinks + else None + ) + + table_html += " \n" # Itera sobre las celdas de cada fila j = 0 # índice de columna - for cell in row.xpath('.//w:tc'): + for cell in row.xpath(".//w:tc"): # Revisa si la celda está en una posición afectada por rowspan while (i, j) in rowspan_dict and rowspan_dict[(i, j)] > 0: # Reduce el contador de rowspan @@ -230,29 +246,34 @@ def extrae_Tabla(element, rels_map, namespaces): j += 1 # Mueve a la siguiente columna # Revisa las propiedades de la celda para rowspan y colspan - cell_props = cell.xpath('.//w:tcPr') + cell_props = cell.xpath(".//w:tcPr") rowspan = 1 colspan = 1 # Procesa rowspan (vMerge) v_merge_fin = False - v_merge = cell.xpath('.//w:vMerge') + v_merge = cell.xpath(".//w:vMerge") if v_merge: - v_merge_val = v_merge[0].get(qn('w:val')) + v_merge_val = v_merge[0].get(qn("w:val")) if v_merge_val == "restart": # Es el inicio de una combinación vertical rowspan = 1 # Busca el total de filas combinadas contando hacia abajo k = i + 1 - while k < len(element.xpath('.//w:tr')): + while k < len(element.xpath(".//w:tr")): try: - next_cell = element.xpath('.//w:tr')[k].xpath('.//w:tc')[j] - next_merge = next_cell.xpath('.//w:tcPr//w:vMerge') + next_cell = element.xpath(".//w:tr")[k].xpath( + ".//w:tc" + )[j] + next_merge = next_cell.xpath(".//w:tcPr//w:vMerge") except: next_cell = None next_merge = None - if next_merge and next_merge[0].get(qn('w:val')) is None: + if ( + next_merge + and next_merge[0].get(qn("w:val")) is None + ): rowspan += 1 else: break @@ -264,16 +285,21 @@ def extrae_Tabla(element, rels_map, namespaces): v_merge_fin = True # Procesa colspan (gridSpan) - grid_span = cell.xpath('.//w:gridSpan') + grid_span = cell.xpath(".//w:gridSpan") if grid_span: - colspan = int(grid_span[0].get(qn('w:val'))) + colspan = int(grid_span[0].get(qn("w:val"))) for k in range(colspan): colspan_dict[(i, j + k)] = colspan - k - 1 if not v_merge_fin: # Obtén el contenido del texto de la celda - cell_text = "
".join([t.text for t in cell.xpath('.//w:t')]) - cell_text = clean_labels(cell_text) + (f" {hiperlinks}" if hiperlinks else "") + cell_text = "
".join( + t.text or "" for t in cell.xpath(".//w:t") + ) + cell_text = clean_labels(cell_text) + ( + f" {hiperlinks}" if hiperlinks else "" + ) + cell_text = html.escape(cell_text, quote=False) # Determina el tag a usar (th para el encabezado, td para celdas normales) tag = "th" if i == 0 else "td" @@ -286,379 +312,383 @@ def extrae_Tabla(element, rels_map, namespaces): cell_html += f' colspan="{colspan}"' cell_html += f">{cell_text}\n" - html += cell_html - j += 1 + (colspan - 1) # Avanza las columnas tomando en cuenta el colspan + table_html += cell_html + j += 1 + ( + colspan - 1 + ) # Avanza las columnas tomando en cuenta el colspan - html += " \n" + table_html += " \n" - html += "
" - return html + table_html += "" + return table_html content = [] sections = [] images = [] found_fb = False review_fb = True - #Palabras a buscar como indicador del primer bloque - start_text = ['introducción', 'introduction', 'introdução'] + # Palabras a buscar como indicador del primer bloque + start_text = ["introducción", "introduction", "introdução"] current_list = [] current_num_id = None numId = None - namespaces_p = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' + namespaces_p = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" - for element in doc.element.body: + for element in doc.element.body: + is_numPr = False if isinstance(element, CT_P): obj = {} namespaces = { - 'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main', - 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', - 'r': 'http://schemas.openxmlformats.org/officeDocument/2006/relationships' + "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main", + "a": "http://schemas.openxmlformats.org/drawingml/2006/main", + "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships", } - hiperlinks = self.extract_hiperlink(element, hiperlinks_info, namespaces) if found_hiperlinks else None + hiperlinks = ( + self.extract_hiperlink(element, hiperlinks_info, namespaces) + if found_hiperlinks + else None + ) obj_image = False obj_formula = False - for drawing in element.findall('.//w:drawing', namespaces=namespaces): - if drawing.find('.//a:blip', namespaces=namespaces) is not None: - blip = drawing.find('.//a:blip', namespaces=namespaces) + for drawing in element.findall(".//w:drawing", namespaces=namespaces): + if drawing.find(".//a:blip", namespaces=namespaces) is not None: + blip = drawing.find(".//a:blip", namespaces=namespaces) if blip is not None: obj_image = True - rId = blip.get('{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed') + rId = blip.get( + "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}embed" + ) image_part = doc.part.related_parts[rId] image_data = image_part.blob - image_name = image_part.partname.split('/')[-1] - + image_name = image_part.partname.split("/")[-1] + if image_name not in images: images.append(image_name) # Guardar la imagen en Wagtail wagtail_image = ImageModel.objects.create( title=image_name, - file=ContentFile(image_data, name=image_name) + file=ContentFile(image_data, name=image_name), ) - + # Referenciar la imagen guardada en el objeto - obj['type'] = 'image' - obj['image'] = wagtail_image.id - + obj["type"] = "image" + obj["image"] = wagtail_image.id + ns_math = { - 'm': 'http://schemas.openxmlformats.org/officeDocument/2006/math', - 'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main' + "m": "http://schemas.openxmlformats.org/officeDocument/2006/math", + "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main", } - for formula in element.findall('.//m:oMathPara', namespaces=ns_math): + for formula in element.findall(".//m:oMathPara", namespaces=ns_math): obj_formula = True mathml_result = transform(formula) mathml_root = etree.fromstring(str(mathml_result)) mathml_root = self.replace_mfenced_pipe_only(mathml_root) - obj['type'] = 'formula' - obj['formula'] = etree.tostring(mathml_root, pretty_print=True, encoding='unicode') + obj["type"] = "formula" + obj["formula"] = etree.tostring( + mathml_root, pretty_print=True, encoding="unicode" + ) + if not obj_image: + paragraph = element + text_paragraph = [] - # obtiene id y nivel - if is_numPr: - numPr = paragraph.find('.//w:numPr', namespaces=paragraph.nsmap) - numId = numPr.find('.//w:numId', namespaces=paragraph.nsmap).get(namespaces_p + 'val') - type_matches = [ - (key, objt) - for key, objt in list_types.items() - if objt.get('numId') == numId - ] + # Determina si es parte de una lista + is_numPr = ( + paragraph.find(".//w:numPr", namespaces=paragraph.nsmap) + is not None + ) # obtiene id y nivel if is_numPr: - numPr = paragraph.find('.//w:numPr', namespaces=paragraph.nsmap) - numId = numPr.find('.//w:numId', namespaces=paragraph.nsmap).get(namespaces_p + 'val') - type = [(key, objt) for key, objt in list_types.items() if objt['numId'] == numId] - - #Es una lista diferente + numPr = paragraph.find(".//w:numPr", namespaces=paragraph.nsmap) + numId = numPr.find( + ".//w:numId", namespaces=paragraph.nsmap + ).get(namespaces_p + "val") + type = [ + (key, objt) + for key, objt in list_types.items() + if objt["numId"] == numId + ] + + # Es una lista diferente if numId != current_num_id: current_num_id = numId if len(current_list) > 0: - current_list.append('[/list]') + current_list.append("[/list]") objl = {} - objl['type'] = 'list' - objl['list'] = '\n'.join(current_list) + objl["type"] = "list" + objl["list"] = "\n".join(current_list) current_list = [] content.append(objl) - list_type = 'bullet' - if type[0][1][str(0)] == 'decimal': - list_type = 'order' + list_type = "bullet" + if type[0][1][str(0)] == "decimal": + list_type = "order" current_list.append(f'[list list-type="{list_type}"]') else: - #Se terminaron de agregar elementos a la lista + # Se terminaron de agregar elementos a la lista if len(current_list) > 0: - current_list.append('[/list]') + current_list.append("[/list]") objl = {} - objl['type'] = 'list' - objl['list'] = '\n'.join(current_list) + objl["type"] = "list" + objl["list"] = "\n".join(current_list) current_list = [] content.append(objl) - list_type = 'bullet' - if type_matches and type_matches[0][1].get(str(0)) == 'decimal': - list_type = 'order' - - current_list.append(f'[list list-type="{list_type}"]') - else: - #Se terminaron de agregar elementos a la lista - if len(current_list) > 0: - current_list.append('[/list]') - objl = {} - objl['type'] = 'list' - objl['list'] = '\n'.join(current_list) - current_list = [] - content.append(objl) - - for child in paragraph: - if child.tag == '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hyperlink': - for r in child.findall('w:r', namespaces=child.nsmap): - t_elem = r.find('w:t', namespaces=child.nsmap) - if t_elem is not None and t_elem.text: - text_paragraph.append(t_elem.text) - - elif child.tag == '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}r': - namespaces = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' - sz_element = child.find('.//w:sz', namespaces=child.nsmap) - obj['font_size'] = 0 - - if sz_element is None: - p_pr = paragraph.find('.//w:rPr/w:sz', namespaces=child.nsmap) - if p_pr is not None: - sz_element = p_pr.find('.//w:pPr', namespaces=child.nsmap) - - if sz_element is not None: - xml_string = etree.tostring(sz_element, pretty_print=True, encoding='unicode') - size_element = objectify.fromstring(xml_string) - font_size_value = size_element.get(namespaces+'val') - obj['font_size'] = int(font_size_value)/2 - - color_element = child.find('.//w:color', namespaces=child.nsmap) - - if color_element is None: - p_pr = paragraph.find('.//w:pPr', namespaces=child.nsmap) - if p_pr is not None: - color_element = p_pr.find('.//w:rPr/w:color', namespaces=child.nsmap) - - if color_element is not None: - xml_string_color = etree.tostring(color_element, pretty_print=True, encoding='unicode') - object_element = objectify.fromstring(xml_string_color) - color_value = object_element.get(namespaces + 'val') - obj['color'] = color_value - - b_tag = child.find('.//w:b', namespaces=child.nsmap) - - if b_tag is None: - p_pr = paragraph.find('.//w:rPr/w:b', namespaces=child.nsmap) - if p_pr is not None: - b_tag = p_pr.find('.//w:pPr', namespaces=child.nsmap) - - if b_tag is not None: - val = b_tag.get('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val') - obj['bold'] = (val is None or val in ['1', 'true', 'True']) - else: - obj['bold'] = False - - i_tag = child.find('.//w:i', namespaces=child.nsmap) - - if i_tag is None: - p_pr = paragraph.find('.//w:rPr/w:i', namespaces=child.nsmap) - if p_pr is not None: - i_tag = p_pr.find('.//w:pPr', namespaces=child.nsmap) for child in paragraph: - if child.tag == '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hyperlink': - for r in child.findall('w:r', namespaces=child.nsmap): - t_elem = r.find('w:t', namespaces=child.nsmap) + if ( + child.tag + == "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}hyperlink" + ): + for r in child.findall("w:r", namespaces=child.nsmap): + t_elem = r.find("w:t", namespaces=child.nsmap) if t_elem is not None and t_elem.text: text_paragraph.append(t_elem.text) - elif child.tag == '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}r': - namespaces = '{http://schemas.openxmlformats.org/wordprocessingml/2006/main}' - sz_element = child.find('.//w:sz', namespaces=child.nsmap) - obj['font_size'] = 0 + elif ( + child.tag + == "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}r" + ): + namespaces = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" + sz_element = child.find(".//w:sz", namespaces=child.nsmap) + obj["font_size"] = 0 if sz_element is None: - p_pr = paragraph.find('.//w:rPr/w:sz', namespaces=child.nsmap) + p_pr = paragraph.find( + ".//w:rPr/w:sz", namespaces=child.nsmap + ) if p_pr is not None: - sz_element = p_pr.find('.//w:pPr', namespaces=child.nsmap) + sz_element = p_pr.find( + ".//w:pPr", namespaces=child.nsmap + ) if sz_element is not None: - xml_string = etree.tostring(sz_element, pretty_print=True, encoding='unicode') + xml_string = etree.tostring( + sz_element, pretty_print=True, encoding="unicode" + ) size_element = objectify.fromstring(xml_string) - font_size_value = size_element.get(namespaces+'val') - obj['font_size'] = int(font_size_value)/2 + font_size_value = size_element.get(namespaces + "val") + obj["font_size"] = int(font_size_value) / 2 - color_element = child.find('.//w:color', namespaces=child.nsmap) + color_element = child.find( + ".//w:color", namespaces=child.nsmap + ) if color_element is None: - p_pr = paragraph.find('.//w:pPr', namespaces=child.nsmap) + p_pr = paragraph.find( + ".//w:pPr", namespaces=child.nsmap + ) if p_pr is not None: - color_element = p_pr.find('.//w:rPr/w:color', namespaces=child.nsmap) + color_element = p_pr.find( + ".//w:rPr/w:color", namespaces=child.nsmap + ) if color_element is not None: - xml_string_color = etree.tostring(color_element, pretty_print=True, encoding='unicode') + xml_string_color = etree.tostring( + color_element, pretty_print=True, encoding="unicode" + ) object_element = objectify.fromstring(xml_string_color) - color_value = object_element.get(namespaces + 'val') - obj['color'] = color_value + color_value = object_element.get(namespaces + "val") + obj["color"] = color_value - b_tag = child.find('.//w:b', namespaces=child.nsmap) + b_tag = child.find(".//w:b", namespaces=child.nsmap) if b_tag is None: - p_pr = paragraph.find('.//w:rPr/w:b', namespaces=child.nsmap) + p_pr = paragraph.find( + ".//w:rPr/w:b", namespaces=child.nsmap + ) if p_pr is not None: - b_tag = p_pr.find('.//w:pPr', namespaces=child.nsmap) + b_tag = p_pr.find( + ".//w:pPr", namespaces=child.nsmap + ) if b_tag is not None: - val = b_tag.get('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val') - obj['bold'] = (val is None or val in ['1', 'true', 'True']) + val = b_tag.get( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val" + ) + obj["bold"] = val is None or val in [ + "1", + "true", + "True", + ] else: - obj['bold'] = False + obj["bold"] = False - i_tag = child.find('.//w:i', namespaces=child.nsmap) + i_tag = child.find(".//w:i", namespaces=child.nsmap) if i_tag is None: - p_pr = paragraph.find('.//w:rPr/w:i', namespaces=child.nsmap) + p_pr = paragraph.find( + ".//w:rPr/w:i", namespaces=child.nsmap + ) if p_pr is not None: - i_tag = p_pr.find('.//w:pPr', namespaces=child.nsmap) + i_tag = p_pr.find( + ".//w:pPr", namespaces=child.nsmap + ) if i_tag is not None: - val = i_tag.get('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val') - obj['italic'] = (val is None or val in ['1', 'true', 'True']) + val = i_tag.get( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}val" + ) + obj["italic"] = val is None or val in [ + "1", + "true", + "True", + ] else: - obj['italic'] = False - - s_tag = child.find('.//w:spacing', namespaces=child.nsmap) - + obj["italic"] = False + + s_tag = child.find(".//w:spacing", namespaces=child.nsmap) + if s_tag is None: - p_pr = paragraph.find('.//w:rPr/w:spacing', namespaces=child.nsmap) + p_pr = paragraph.find( + ".//w:rPr/w:spacing", namespaces=child.nsmap + ) if p_pr is not None: - s_tag = p_pr.find('.//w:pPr', namespaces=child.nsmap) + s_tag = p_pr.find( + ".//w:pPr", namespaces=child.nsmap + ) if s_tag is not None: - val = s_tag.get('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}before') - obj['spacing'] = not (val is None) + val = s_tag.get( + "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}before" + ) + obj["spacing"] = not (val is None) else: - obj['spacing'] = False + obj["spacing"] = False clean_text = clean_labels(child.text) - #identifica sección - sections = identify_section(sections, obj['font_size'], obj['bold'] , clean_text) - - if obj['italic']: - text_paragraph.append('' + clean_text + '' + (f" {hiperlinks}" if hiperlinks else "")) + # identifica sección + sections = identify_section( + sections, obj["font_size"], obj["bold"], clean_text + ) + + if obj["italic"]: + text_paragraph.append( + "" + + clean_text + + "" + + (f" {hiperlinks}" if hiperlinks else "") + ) else: - text_paragraph.append(clean_text + (f" {hiperlinks}" if hiperlinks else "")) + text_paragraph.append( + clean_text + + (f" {hiperlinks}" if hiperlinks else "") + ) paraph = match_paragraph(clean_text) if paraph: - obj['paraph'] = paraph - obj['type'] = paraph + obj["paraph"] = paraph + obj["type"] = paraph if review_fb: - found_fb = any(word in clean_text.lower() for word in start_text) - - #Si se encontró alguna palabra, incluye todo lo anterior en un sólo bloque + found_fb = any( + word in clean_text.lower() for word in start_text + ) + + # Si se encontró alguna palabra, incluye todo lo anterior en un sólo bloque if found_fb: found_fb = False review_fb = False found_hiperlinks = False sections = [sections[-1]] - first_block = '' + first_block = "" tmp_content = [] - abstract_mode = False + abstract_mode = False for c in content: if abstract_mode: - if c['text'] == '' or c['spacing'] is True: + if c["text"] == "" or c["spacing"] is True: abstract_mode = False else: tmp_content.append(c) continue - if 'paraph' in c: + if "paraph" in c: tmp_content.append(c) abstract_mode = False - if c['paraph'] == '': + if c["paraph"] == "": abstract_mode = True - continue + continue else: - if 'text' in c: + if "text" in c: first_block = first_block + "\n" + c["text"] - if 'table' in c: - first_block = first_block + "\n" + c["table"] + if "table" in c: + first_block = ( + first_block + "\n" + c["table"] + ) obj_b = {} - obj_b['type'] = 'first_block' - obj_b['text'] = first_block + obj_b["type"] = "first_block" + obj_b["text"] = first_block tmp_content.append(obj_b) content = tmp_content start_text = [] - + if child.tag == f"{{{ns_math['m']}}}oMath": - if 'text' not in obj or not isinstance(obj['text'], list): - obj['type'] = 'compound' - obj['text'] = [] + if "text" not in obj or not isinstance(obj["text"], list): + obj["type"] = "compound" + obj["text"] = [] if len(text_paragraph) > 0: obj2 = {} - obj2['type'] = 'text' - obj2['value'] = ' '.join(text_paragraph) - obj['text'].append(obj2) + obj2["type"] = "text" + obj2["value"] = " ".join(text_paragraph) + obj["text"].append(obj2) text_paragraph = [] mathml_result = transform(child) mathml_root = etree.fromstring(str(mathml_result)) self.replace_mfenced_pipe_only(mathml_root) obj2 = {} - obj2['type'] = 'formula' - obj2['value'] = etree.tostring(mathml_root, pretty_print=True, encoding='unicode') - obj['text'].append(obj2) - - if 'text' not in obj: - obj['text'] = (' '.join(text_paragraph)).strip() - clean_text = clean_labels(obj['text']) - obj['text'] = clean_text - - paraph = match_paragraph(obj['text']) + obj2["type"] = "formula" + obj2["value"] = etree.tostring( + mathml_root, pretty_print=True, encoding="unicode" + ) + obj["text"].append(obj2) + + if "text" not in obj: + obj["text"] = (" ".join(text_paragraph)).strip() + clean_text = clean_labels(obj["text"]) + obj["text"] = clean_text + + paraph = match_paragraph(obj["text"]) if paraph: - obj['paraph'] = paraph - obj['type'] = paraph + obj["paraph"] = paraph + obj["type"] = paraph if is_numPr: - if 'font_size' in obj: - del obj['font_size'] + if "font_size" in obj: + del obj["font_size"] current_list.append(f'[list-item]{obj["text"]}[/list-item]') - if isinstance(obj['text'], list) and len(text_paragraph) > 0: + if isinstance(obj["text"], list) and len(text_paragraph) > 0: obj2 = {} - obj2['type'] = 'text' - obj2['value'] = ' '.join(text_paragraph) - obj['text'].append(obj2) + obj2["type"] = "text" + obj2["value"] = " ".join(text_paragraph) + obj["text"].append(obj2) text_paragraph = [] - + elif isinstance(element, CT_Tbl): namespaces = { - 'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main', - 'a': 'http://schemas.openxmlformats.org/drawingml/2006/main', - 'r': 'http://schemas.openxmlformats.org/officeDocument/2006/relationships' + "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main", + "a": "http://schemas.openxmlformats.org/drawingml/2006/main", + "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships", } - if is_numPr: - if 'font_size' in obj: - del obj['font_size'] - current_list.append(f'[list-item]{obj["text"]}[/list-item]') - if isinstance(obj['text'], list) and len(text_paragraph) > 0: - obj2 = {} - obj2['type'] = 'text' - obj2['value'] = ' '.join(text_paragraph) - obj['text'].append(obj2) - text_paragraph = [] - if not is_numPr: - content.append(obj) + table = element + table_data = extrae_Tabla(element, hiperlinks_info, namespaces) + obj = {} + obj["type"] = "table" + obj["table"] = table_data + + if not is_numPr: + content.append(obj) sections.sort(key=section_priority) return sections, content diff --git a/model_ai/llama.py b/model_ai/llama.py index ae8f443..e9930cb 100644 --- a/model_ai/llama.py +++ b/model_ai/llama.py @@ -3,14 +3,11 @@ import os import time -from config.settings.base import ( - LLAMA_ENABLED, - LLAMA_MODEL_DIR, -) - # Third-party imports import google.generativeai as genai +from config.settings.base import LLAMA_ENABLED, LLAMA_MODEL_DIR + # Local application imports from model_ai import messages from model_ai.exceptions import ( @@ -20,97 +17,116 @@ ) from model_ai.models import LlamaModel +GEMINI_MODEL = "models/gemini-3.1-flash-lite-preview" +logger = logging.getLogger(__name__) + class LlamaService: - # Singleton pattern to cache the LLaMA model instance - _cached_llm = None - - def __init__(self, messages=None, response_format=None, max_tokens=4000, temperature=0.1, top_p=0.1, mode='chat', nthreads=2): - self.messages = messages - self.response_format = response_format - self.max_tokens = max_tokens - self.temperature = temperature - self.top_p = top_p - self.mode = mode - - model_ai = LlamaModel.objects.first() - - # Initialize local LLaMA only when Gemini is not configured - if not model_ai or not model_ai.api_key_gemini: - - if not LLAMA_ENABLED: - raise LlamaDisabledError("LLaMA is disabled in settings.") - - if LlamaService._cached_llm is None: - try: - from llama_cpp import Llama - except ImportError as e: - raise LlamaNotInstalledError("The 'llama-cpp-python' package is not installed. Please use the llama-activated Docker image (Dockerfile.llama).") from e + # Singleton pattern to cache the LLaMA model instance + _cached_llm = None + + def __init__( + self, + messages=None, + response_format=None, + max_tokens=4000, + temperature=0.1, + top_p=0.1, + mode="chat", + nthreads=2, + ): + self.messages = messages + self.response_format = response_format + self.max_tokens = max_tokens + self.temperature = temperature + self.top_p = top_p + self.mode = mode model_ai = LlamaModel.objects.first() - if not model_ai: - raise LlamaModelNotFoundError("No LLaMA model configured in the database. Please add a LLaMA model entry.") - - model_path = os.path.join(LLAMA_MODEL_DIR, model_ai.name_file) - if not os.path.isfile(model_path): - raise LlamaModelNotFoundError(f"LLaMA model file not found at {model_path}. Please ensure the model is downloaded and the path is correct.") - - try: - LlamaService._cached_llm = Llama(model_path=model_path, n_ctx=max_tokens, n_threads=nthreads) - except Exception as e: - raise RuntimeError(f"Failed to initialize LLaMA model: {e}") from e - - self.llm = LlamaService._cached_llm - - def run(self, user_input): - if self.mode == 'chat': - return self._run_as_chat(user_input) - elif self.mode == 'prompt': - return self._run_as_content_generation(user_input) - - def _run_as_chat(self, user_input): - """ Run LLaMA in chat mode.""" - input = self.messages.copy() - input.append({ - 'role': 'user', - 'content': user_input - }) - return self.llm.create_chat_completion( - messages=input, - response_format=self.response_format, - max_tokens=self.max_tokens, - temperature=self.temperature, - top_p=self.top_p - ) - - def _run_as_content_generation(self, user_input): - """ Run LLaMA in completion mode.""" - model_ai = LlamaModel.objects.first() - - # Try to use Gemini if configured - if model_ai and model_ai.api_key_gemini: - - # Setup Gemini API key - genai.configure(api_key=model_ai.api_key_gemini) - - # Fetch the Gemini model - # FIXME: Hardcoded model name - model = genai.GenerativeModel('models/gemini-3.1-flash-lite-preview') - - # Generate content using Gemini - response_gemini = model.generate_content(user_input).text - time.sleep(15) - return response_gemini - - # Gemini not configured, fallback to LLaMA - else: - return self.llm( - user_input, - max_tokens=self.max_tokens, - temperature=self.temperature, - stop=["\n\n"] - ) - + + # Initialize local LLaMA only when Gemini is not configured + if not model_ai or not model_ai.api_key_gemini: + if not LLAMA_ENABLED: + raise LlamaDisabledError("LLaMA is disabled in settings.") + + if LlamaService._cached_llm is None: + try: + from llama_cpp import Llama + except ImportError as e: + raise LlamaNotInstalledError( + "The 'llama-cpp-python' package is not installed. Please use the llama-activated Docker image (Dockerfile.llama)." + ) from e + + model_ai = LlamaModel.objects.first() + if not model_ai: + raise LlamaModelNotFoundError( + "No LLaMA model configured in the database. Please add a LLaMA model entry." + ) + + model_path = os.path.join(LLAMA_MODEL_DIR, model_ai.name_file) + if not os.path.isfile(model_path): + raise LlamaModelNotFoundError( + f"LLaMA model file not found at {model_path}. Please ensure the model is downloaded and the path is correct." + ) + + try: + LlamaService._cached_llm = Llama( + model_path=model_path, n_ctx=max_tokens, n_threads=nthreads + ) + except Exception as e: + raise RuntimeError(f"Failed to initialize LLaMA model: {e}") from e + + self.llm = LlamaService._cached_llm + + def run(self, user_input): + if self.mode == "chat": + return self._run_as_chat(user_input) + elif self.mode == "prompt": + return self._run_as_content_generation(user_input) + + def _run_as_chat(self, user_input): + """Run LLaMA in chat mode.""" + input = self.messages.copy() + input.append({"role": "user", "content": user_input}) + return self.llm.create_chat_completion( + messages=input, + response_format=self.response_format, + max_tokens=self.max_tokens, + temperature=self.temperature, + top_p=self.top_p, + ) + + def _run_as_content_generation(self, user_input): + """Run LLaMA in completion mode.""" + model_ai = LlamaModel.objects.first() + + # Try to use Gemini if configured + if model_ai and model_ai.api_key_gemini: + genai.configure(api_key=model_ai.api_key_gemini) + model = genai.GenerativeModel(GEMINI_MODEL) + logger.info( + "Chamando Gemini (%s), prompt com %d caracteres", + GEMINI_MODEL, + len(user_input), + ) + response_gemini = model.generate_content(user_input).text + logger.info( + "Resposta Gemini recebida (%d caracteres)", + len(response_gemini or ""), + ) + time.sleep(15) + return response_gemini + + else: + logger.info("Gemini não configurado — usando Llama local") + return self.llm( + user_input, + max_tokens=self.max_tokens, + temperature=self.temperature, + stop=["\n\n"], + ) + + class LlamaInputSettings: @staticmethod def get_first_metadata(text): @@ -136,4 +152,3 @@ def get_affiliations(): @staticmethod def get_reference(): return messages.REFERENCE_MESSAGES, messages.REFERENCE_RESPONSE_FORMAT - \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..e203b62 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +DJANGO_SETTINGS_MODULE = config.settings.test +python_files = tests.py test_*.py *_tests.py