diff --git a/nodes/src/nodes/llamaparse/IInstance.py b/nodes/src/nodes/llamaparse/IInstance.py index aabd6b771..868a013fd 100644 --- a/nodes/src/nodes/llamaparse/IInstance.py +++ b/nodes/src/nodes/llamaparse/IInstance.py @@ -27,6 +27,7 @@ from rocketlib import IInstanceBase, Entry, debug from ai.common.schema import Doc from .IGlobal import IGlobal +from .scale import extract_markdown_tables class IInstance(IInstanceBase): @@ -375,31 +376,10 @@ def extract_tables_from_text(self, text: str): try: debug('LlamaParse Instance: Extracting tables from text') - # Simple table detection - look for markdown table patterns - lines = text.split('\n') - tables = [] - current_table = [] - in_table = False - - for line in lines: - line = line.strip() - - # Check if line looks like a table row (contains |) - if '|' in line and len(line.split('|')) > 2: - if not in_table: - in_table = True - current_table = [] - current_table.append(line) - elif in_table: - # End of table - if current_table: - tables.append('\n'.join(current_table)) - current_table = [] - in_table = False - - # Don't forget the last table - if in_table and current_table: - tables.append('\n'.join(current_table)) + # Simple table detection - look for markdown table patterns. + # Shared with the scale-header preservation logic so there is a + # single table heuristic (see scale.extract_markdown_tables). + tables = extract_markdown_tables(text) debug(f'LlamaParse Instance: Found {len(tables)} potential tables') diff --git a/nodes/src/nodes/llamaparse/README.md b/nodes/src/nodes/llamaparse/README.md index 02cec3b01..d4c68ecb0 100644 --- a/nodes/src/nodes/llamaparse/README.md +++ b/nodes/src/nodes/llamaparse/README.md @@ -14,6 +14,29 @@ A single shared parser instance is guarded by a lock, so documents are parsed on --- +## Scale-header preservation + +Financial statements declare their magnitude in a caption above the table ("(In millions)", "$ in thousands", "amounts in millions of USD"). If that caption is separated from its table by downstream chunking or an LLM step, every figure reads as raw units and is silently 1,000x, 1,000,000x, or 1,000,000,000x off. Nothing downstream catches it, because a wrong-by-1e6 number looks exactly like a right one. + +To keep that loss visible, the parser post-processes the extracted Markdown: + +- **Detects scale captions** the parser already emitted ("(in millions)", "in thousands, except per share amounts", "(₹ in crore)", ...) and normalizes each to a unit, a numeric factor, and an optional currency. Prose uses of the same words ("millions of users", "billions served", "hundreds of thousands of dollars") are not treated as captions. +- **Welds a marker to the table.** When a caption is in scope for a *numeric* table (same page/section, no page break between them), a normalized line is injected directly above the table, for example `> Scale: amounts in millions (x1,000,000)`. A later chunker or LLM cannot separate the scale from its figures. A caption is only welded onto a table that actually looks numeric, so a caption sitting over a prose or roster table is never mis-attached. +- **Flags a missing scale, when the context is financial.** When a numeric table has no caption in scope *and* the document shows a financial signal (a caption detected elsewhere, or a currency marking in the table itself), a warning line is injected above it: `> [scale?] Scale not detected for this table. Figures may be in thousands, millions, or billions. Verify against the source.` The warning errs toward caution: a visible warning is far cheaper than a silent 1e6 error. Plain numeric tables with no financial signal (inventory counts, server metrics, sports standings) are left untouched, so ordinary documents pay nothing for this feature. The markers are plain ASCII, so they survive a cp1252 console without an encoding error. + +The welded markers travel in-band with the text and table lanes, so they are the durable downstream signal: whatever chunks or summarizes the output carries the scale (or the warning) with the figures. + +For observability, the same results are also recorded in `parsing_metadata` (debug-logged by the instance; not currently re-emitted onto a lane): + +- `detected_scales`: list of `{phrase, unit, factor, currency}` for each caption found. +- `scale_warnings`: per-table `{table_index, status, ...}` where `status` is `scale_detected`, `scale_missing`, `not_financial`, or `already_annotated` (the table already carried one of our markers and was left as-is). + +Annotation is idempotent (re-running does not double-inject; the injected markers are not themselves read back as captions) and never blocks parsing: any failure falls back to the untouched text. + +**Out of scope.** Recovering a caption that LlamaParse dropped from the source entirely is not handled here. That needs an independent source-text extraction and belongs to the audit-grade **`datalab_parse`** node. This node makes an *emitted-but-separable* scale un-loseable and flags its *absence*; it does not re-parse the source. + +--- + ## Configuration ### Lanes diff --git a/nodes/src/nodes/llamaparse/parser.py b/nodes/src/nodes/llamaparse/parser.py index 1fcfbe957..c89efb588 100644 --- a/nodes/src/nodes/llamaparse/parser.py +++ b/nodes/src/nodes/llamaparse/parser.py @@ -280,6 +280,39 @@ def parse(self, file_data: bytes, file_name: Optional[str] = None) -> dict[str, debug(f'parsing_metadata: {parsing_metadata}, structured_data: {structured_data}') + # Preserve financial scale headers. A caption like "(in millions)" + # sets the magnitude of every figure in the table below it; if a + # downstream chunker or LLM step separates the two, the figures + # read as raw units and are silently 1e3/1e6/1e9 off. Weld a + # normalized marker to each table and flag numeric tables that + # have no scale in scope. Never let this break parsing: on any + # failure, fall back to the untouched text_content. + detected_scales: list = [] + scale_warnings: list = [] + try: + from .scale import annotate_scale, detect_scale_declarations + + declarations = detect_scale_declarations(text_content) + text_content, scale_warnings = annotate_scale(text_content, declarations) + detected_scales = [ + { + 'phrase': d.phrase, + 'unit': d.unit, + 'factor': d.factor, + 'currency': d.currency, + } + for d in declarations + ] + debug( + f'Scale headers: {len(detected_scales)} declaration(s), ' + f'{len(scale_warnings)} table annotation(s)' + ) + except Exception as e: + debug(f'Scale-header annotation skipped: {str(e)}') + + parsing_metadata['detected_scales'] = detected_scales + parsing_metadata['scale_warnings'] = scale_warnings + # Return text, structured data, page count, and metadata return { 'text': text_content, diff --git a/nodes/src/nodes/llamaparse/scale.py b/nodes/src/nodes/llamaparse/scale.py new file mode 100644 index 000000000..66ab2c449 --- /dev/null +++ b/nodes/src/nodes/llamaparse/scale.py @@ -0,0 +1,485 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +"""Financial scale-header detection and preservation for the LlamaParse node. + +A statement's scale caption ("(In millions)", "$ in thousands", ...) declares +the magnitude of every figure in the table below it. When that caption is +separated from its table by downstream chunking or an LLM step, the figures +read as raw units and are silently 1,000x / 1,000,000x / 1,000,000,000x off. +Nothing downstream catches it, because a wrong-by-1e6 number looks exactly like +a right one. + +This module keeps scale loss non-silent. It (1) detects scale declarations that +the parser already emitted, (2) welds a normalized marker to the table so a +later chunker cannot split the header away, and (3) flags any numeric table that +has no scale in scope, turning a silent magnitude error into a visible warning. + +Recovering a caption that the parser dropped from the source entirely is out of +scope; that needs source re-extraction and belongs to the audit-grade +``datalab_parse`` node. See this node's README ("Scale-header preservation"). + +Pure text logic, no external dependencies, so it is unit-testable without the +LlamaParse API or the native engine. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Iterator, List, Optional, Tuple + +# Canonical scale word -> (normalized unit label, multiplier). +_SCALE_UNITS = { + 'thousand': ('thousands', 1_000), + 'thousands': ('thousands', 1_000), + 'lakh': ('lakhs', 100_000), + 'lakhs': ('lakhs', 100_000), + 'million': ('millions', 1_000_000), + 'millions': ('millions', 1_000_000), + 'crore': ('crores', 10_000_000), + 'crores': ('crores', 10_000_000), + 'billion': ('billions', 1_000_000_000), + 'billions': ('billions', 1_000_000_000), + 'trillion': ('trillions', 1_000_000_000_000), + 'trillions': ('trillions', 1_000_000_000_000), +} + +_SCALE_WORD_RE = re.compile( + r'\b(thousands?|lakhs?|millions?|crores?|billions?|trillions?)\b', + re.IGNORECASE, +) + +# Words that, following "... of ", keep it a monetary declaration +# ("in millions of dollars"). Anything else after "of" ("millions of users") +# means the scale word is prose, not a caption. +_CURRENCY_WORDS = { + 'dollar', + 'dollars', + 'usd', + 'euro', + 'euros', + 'eur', + 'pound', + 'pounds', + 'sterling', + 'gbp', + 'rupee', + 'rupees', + 'inr', + 'yen', + 'jpy', + 'franc', + 'francs', + 'chf', + 'cad', + 'aud', + 'won', + 'krw', + 'yuan', + 'renminbi', + 'rmb', + 'cny', + 'real', + 'reais', + 'brl', + 'peso', + 'pesos', +} + +_CURRENCY_WORDS_RE = re.compile( + r'\b(?:' + '|'.join(re.escape(w) for w in sorted(_CURRENCY_WORDS)) + r')\b', + re.IGNORECASE, +) + +_SYMBOL_TO_CODE = {'$': 'USD', '€': 'EUR', '£': 'GBP', '₹': 'INR', '¥': 'JPY'} + +_WORD_TO_CODE = { + 'dollar': 'USD', + 'dollars': 'USD', + 'usd': 'USD', + 'euro': 'EUR', + 'euros': 'EUR', + 'eur': 'EUR', + 'pound': 'GBP', + 'pounds': 'GBP', + 'sterling': 'GBP', + 'gbp': 'GBP', + 'rupee': 'INR', + 'rupees': 'INR', + 'inr': 'INR', + 'yen': 'JPY', + 'jpy': 'JPY', +} + +# Amount of surrounding text inspected to decide whether a scale word is a +# genuine declaration. +_CONTEXT_CHARS = 48 + +# "in" (optionally preceded by a currency symbol) sitting immediately before the +# scale word: "(in millions)", "$ in thousands", "amounts in millions". +_LEADIN_RE = re.compile(r'\bin\s*$', re.IGNORECASE) +# A lone currency symbol immediately before the scale word: "($ millions)". +_CURRENCY_PREFIX_RE = re.compile(r'[$€£₹¥]\s*$') +# "of" immediately before the scale word: prose like "hundreds of thousands". +_TRAILING_OF_RE = re.compile(r'\bof\s*$', re.IGNORECASE) +# "of " immediately after the scale word; the word decides monetary vs +# prose. An optional "U.S." prefix is skipped (outside the capture) so +# "of U.S. dollars" yields "dollars", while "of USD" stays intact. +_OF_CLAUSE_RE = re.compile(r'^\s*of\s+(?:u\.?\s*s\.?\s+)?([a-z]+)', re.IGNORECASE) + +# Verb sitting immediately before the "in" lead-in marks the scale word as prose, +# not a caption: "valued in billions", "grew to ... reaching in ...". A caption +# ("Dollars in millions", "Amounts are in thousands") never leads with these, so +# vetoing them costs no real captions while killing a class of false welds. +_PROSE_LEADIN_WORD_RE = re.compile( + r'\b(?:valued|worth|grew|grown|rose|risen|fell|fallen|reaching|reached|' + r'totaling|totalling|generating|generated|earning|earned|spending|spent|' + r'costing|saving|saved|raising|raised|investing|invested|losing|lost|' + r'measured|priced|estimated)\s+in\s*$', + re.IGNORECASE, +) + +# A page or section boundary between a caption and a table puts the caption out +# of scope: a form feed, a horizontal rule, or an explicit "Page N" marker. +_PAGE_BREAK_RE = re.compile( + r'\f' + r'|^\s*(?:-{3,}|\*{3,}|_{3,})\s*$' + r'|^\s*#{1,6}\s+.*\bpage\s+\d+' + r'|\bpage\s+\d+\s+of\s+\d+\b', + re.IGNORECASE | re.MULTILINE, +) + +# A table cell that is essentially a number: "1,234", "(1,234)", "$1,234.56", +# "45%", "-5.0". Used to decide whether a headerless table looks financial. +_NUMERIC_CELL_RE = re.compile(r'^[(\-+]?\s*[$€£₹¥]?\s*[\d,]+(?:\.\d+)?\s*%?\)?$') + +# Markers this module injects; used for idempotence and for the metadata signal. +# Plain text only: these lines are welded into the parsed text that flows down +# the pipeline (application output), where a non-ASCII glyph can raise a +# UnicodeEncodeError on a cp1252 console. +_SCALE_MARKER_PREFIX = '> Scale:' +_WARNING_MARKER_PREFIX = '> [scale?]' +_WARNING_MARKER = ( + f'{_WARNING_MARKER_PREFIX} Scale not detected for this table. Figures may be in ' + 'thousands, millions, or billions. Verify against the source.' +) + +# Longest gap (in characters) allowed between a caption and the table it scales, +# as a backstop when the parser emits no page break between unrelated sections. +_MAX_SCOPE_DISTANCE = 1500 + + +@dataclass(frozen=True) +class ScaleDeclaration: + """A detected scale caption and its normalized magnitude.""" + + phrase: str + unit: str + factor: int + currency: Optional[str] + start: int + end: int + + +def _classify_of_clause(right: str) -> Tuple[bool, bool]: + """Return (is_currency, is_non_currency) for an "of " tail, if any.""" + m = _OF_CLAUSE_RE.match(right) + if not m: + return (False, False) + if m.group(1).lower() in _CURRENCY_WORDS: + return (True, False) + return (False, True) + + +def _detect_currency(left: str, right: str) -> Optional[str]: + """Best-effort currency label from the text around a scale word.""" + for sym, code in _SYMBOL_TO_CODE.items(): + if sym in left or sym in right: + return code + combined = (left + ' ' + right).lower() + for word, code in _WORD_TO_CODE.items(): + if re.search(r'\b' + re.escape(word) + r'\b', combined): + return code + return None + + +def _build_phrase(text: str, left: str, start: int, end: int, right: str) -> str: + """Reconstruct a readable caption snippet for logging/metadata.""" + p_start = start + lead = re.search(r'((?:[$€£₹¥]\s*)?in\s+)$', left, re.IGNORECASE) + if lead: + p_start = start - len(lead.group(1)) + p_end = end + tail = re.match( + r'\s*(?:of\s+[\w.]+(?:\s+[\w.]+)?)?(?:\s*,\s*except[^)\n.]*)?', + right, + re.IGNORECASE, + ) + if tail and tail.end() > 0: + p_end = end + tail.end() + return ' '.join(text[p_start:p_end].split()) + + +def detect_scale_declarations(text: str) -> List[ScaleDeclaration]: + """Find scale captions in ``text``. + + A scale word ("millions", "thousands", ...) is treated as a declaration only + when it is anchored to a monetary context: preceded by "in" or a currency + symbol, or followed by "of ". Prose uses of the same words are + rejected: "millions of users", "billions served", "hundreds of thousands of + dollars", "$5 million in revenue". + """ + if not text: + return [] + + declarations: List[ScaleDeclaration] = [] + for m in _SCALE_WORD_RE.finditer(text): + if _on_marker_line(text, m.start()): + # A scale word inside a marker we injected ("> Scale: ... in millions", + # "> [scale?] ... in thousands, millions, or billions") is not a source + # caption; skipping it keeps re-annotation idempotent. + continue + word = m.group(1).lower() + unit, factor = _SCALE_UNITS[word] + left = text[max(0, m.start() - _CONTEXT_CHARS) : m.start()] + right = text[m.end() : m.end() + _CONTEXT_CHARS] + + of_currency, of_non_currency = _classify_of_clause(right) + if of_non_currency: + # "millions of ways", "thousands of customers" -> prose. + continue + if _TRAILING_OF_RE.search(left): + # "hundreds of thousands", "tens of millions" -> prose. + continue + if _PROSE_LEADIN_WORD_RE.search(left): + # "valued in billions", "reached ... in millions" -> prose, not a caption. + continue + + has_in = bool(_LEADIN_RE.search(left)) + has_currency_prefix = bool(_CURRENCY_PREFIX_RE.search(left)) + if not (has_in or of_currency or has_currency_prefix): + continue + + declarations.append( + ScaleDeclaration( + phrase=_build_phrase(text, left, m.start(), m.end(), right), + unit=unit, + factor=factor, + currency=_detect_currency(left, right), + start=m.start(), + end=m.end(), + ) + ) + return declarations + + +def _is_table_row(line: str) -> bool: + """A markdown table row: a pipe-delimited line with at least two cells. + + Matches the heuristic the node uses to split tables out of parsed text, so + table detection stays consistent between the two callers. + """ + stripped = line.strip() + return '|' in stripped and len(stripped.split('|')) > 2 + + +def iter_table_blocks(text: str) -> Iterator[Tuple[int, int, List[str]]]: + """Yield (start_line, end_line_exclusive, lines) for each markdown table.""" + lines = text.split('\n') + start: Optional[int] = None + for i, line in enumerate(lines): + if _is_table_row(line): + if start is None: + start = i + elif start is not None: + yield (start, i, lines[start:i]) + start = None + if start is not None: + yield (start, len(lines), lines[start:]) + + +def extract_markdown_tables(text: str) -> List[str]: + """Return each detected markdown table as a string of its stripped rows. + + Shared implementation for the node's ``table`` lane so there is one table + heuristic, not two. + """ + return ['\n'.join(line.strip() for line in block) for _, _, block in iter_table_blocks(text)] + + +def _line_offsets(lines: List[str]) -> List[int]: + offsets = [] + pos = 0 + for line in lines: + offsets.append(pos) + pos += len(line) + 1 # + 1 for the '\n' that split removed + return offsets + + +def find_markdown_table_spans(text: str) -> List[Tuple[int, int]]: + """Return (char_start, char_end) offsets in ``text`` for each table block.""" + lines = text.split('\n') + offsets = _line_offsets(lines) + spans = [] + for start, end, _ in iter_table_blocks(text): + last = end - 1 + spans.append((offsets[start], offsets[last] + len(lines[last]))) + return spans + + +def _table_looks_numeric(table_text: str) -> bool: + """True when a table is mostly numbers, so a missing scale matters.""" + cells = [] + for line in table_text.split('\n'): + for cell in line.strip().strip('|').split('|'): + c = cell.strip() + # Skip separator cells like "---" / ":--:". + if c and set(c) - set('-: '): + cells.append(c) + if not cells: + return False + numeric = sum(1 for c in cells if _NUMERIC_CELL_RE.match(c)) + return numeric >= 3 and numeric / len(cells) >= 0.3 + + +def _nearest_in_scope(declarations: List[ScaleDeclaration], text: str, table_start: int) -> Optional[ScaleDeclaration]: + """Nearest declaration ending before ``table_start`` with nothing (page + break / oversized gap) separating it from the table. + """ + best: Optional[ScaleDeclaration] = None + for d in declarations: + if d.end > table_start: + continue + if table_start - d.end > _MAX_SCOPE_DISTANCE: + continue + if _PAGE_BREAK_RE.search(text[d.end : table_start]): + continue + if best is None or d.end > best.end: + best = d + return best + + +def _on_marker_line(text: str, pos: int) -> bool: + """True when ``pos`` falls on a line that is one of our injected markers.""" + line_start = text.rfind('\n', 0, pos) + 1 + stripped = text[line_start:].lstrip() + return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped.startswith(_WARNING_MARKER_PREFIX) + + +def _already_annotated(prev_line: str) -> bool: + stripped = prev_line.strip() + return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped.startswith(_WARNING_MARKER_PREFIX) + + +def _table_has_currency(table_text: str) -> bool: + """True when a table carries a currency signal of its own (symbol or code). + + Lets a numeric, uncaptioned table still be flagged in an otherwise + non-financial document, without warning on plain numeric tables (inventory + counts, sports standings, server metrics) that carry no financial signal. + """ + if any(sym in table_text for sym in _SYMBOL_TO_CODE): + return True + return _CURRENCY_WORDS_RE.search(table_text) is not None + + +def _scale_marker(decl: ScaleDeclaration) -> str: + marker = f'{_SCALE_MARKER_PREFIX} amounts in {decl.unit} (x{decl.factor:,})' + if decl.currency: + marker += f' [{decl.currency}]' + return marker + + +def annotate_scale( + text: str, + declarations: Optional[List[ScaleDeclaration]] = None, +) -> Tuple[str, List[dict]]: + """Weld scale markers to tables and flag numeric tables missing a scale. + + Returns the annotated text plus a list of per-table warning records + ``{'table_index', 'status', ...}`` where ``status`` is one of: + + - ``scale_detected``: a caption was in scope; its marker was welded above. + - ``scale_missing``: a numeric table with a financial signal but no caption + in scope; a missing-scale warning was welded above. + - ``not_financial``: no marker welded, either because the table is not + numeric or it carries no financial signal (so a warning would be noise). + - ``already_annotated``: the table already carried one of our markers and was + left untouched. + + A marker is welded only above a *numeric* table, so a caption sitting over a + prose/roster table is never mis-attached. The missing-scale warning is gated + on a financial signal (a caption elsewhere in the document, or a currency + marking in the table itself) so ordinary numeric tables are not annotated. + Idempotent: re-running does not double-inject. + """ + if not text or not text.strip(): + return text, [] + if declarations is None: + declarations = detect_scale_declarations(text) + + lines = text.split('\n') + offsets = _line_offsets(lines) + blocks = list(iter_table_blocks(text)) + if not blocks: + return text, [] + + doc_has_declaration = bool(declarations) + warnings: List[dict] = [] + insertions: dict = {} + for idx, (start, _end, block) in enumerate(blocks): + prev_line = lines[start - 1] if start > 0 else '' + if _already_annotated(prev_line): + warnings.append({'table_index': idx, 'status': 'already_annotated'}) + continue + + block_text = '\n'.join(block) + is_numeric = _table_looks_numeric(block_text) + decl = _nearest_in_scope(declarations, text, offsets[start]) + if decl is not None and is_numeric: + insertions[start] = _scale_marker(decl) + warnings.append( + { + 'table_index': idx, + 'status': 'scale_detected', + 'unit': decl.unit, + 'factor': decl.factor, + 'currency': decl.currency, + } + ) + elif is_numeric and (doc_has_declaration or _table_has_currency(block_text)): + insertions[start] = _WARNING_MARKER + warnings.append({'table_index': idx, 'status': 'scale_missing', 'factor': None}) + else: + warnings.append({'table_index': idx, 'status': 'not_financial'}) + + if not insertions: + return text, warnings + + out = [] + for i, line in enumerate(lines): + if i in insertions: + out.append(insertions[i]) + out.append(line) + return '\n'.join(out), warnings diff --git a/nodes/test/test_llamaparse_scale.py b/nodes/test/test_llamaparse_scale.py new file mode 100644 index 000000000..695a5ddd3 --- /dev/null +++ b/nodes/test/test_llamaparse_scale.py @@ -0,0 +1,230 @@ +"""Unit tests for the llamaparse scale-header detector/annotator (scale.py). + +Pure-logic tests, no engine / LlamaParse API / rocketlib required. Cover: +- a labeled detection corpus (positive captions + adversarial negatives), +- table annotation (marker injection) and missing-scale warnings, +- idempotence and empty-input safety, +- parity of the shared table-extraction heuristic. +""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'nodes', 'llamaparse')) +from scale import ( # noqa: E402 + annotate_scale, + detect_scale_declarations, + extract_markdown_tables, + find_markdown_table_spans, +) + + +# --------------------------------------------------------------------------- +# Detection corpus: captions that MUST be detected, with their expected factor. +# --------------------------------------------------------------------------- +POSITIVE_CASES = [ + ('(In millions)', 1_000_000), + ('(in thousands)', 1_000), + ('(In billions)', 1_000_000_000), + ('$ in millions', 1_000_000), + ('$ in thousands', 1_000), + ('($ millions)', 1_000_000), + ('Amounts in thousands of dollars', 1_000), + ('in millions of USD', 1_000_000), + ('In millions of U.S. dollars', 1_000_000), + ('(in billions, except per-share data)', 1_000_000_000), + ('In thousands, except share and per share amounts', 1_000), + ('Dollars in millions', 1_000_000), + ('(Millions of dollars)', 1_000_000), + ('Amounts are in thousands unless otherwise stated', 1_000), + ('(€ in millions)', 1_000_000), + ('(£ in thousands)', 1_000), + ('(₹ in crore)', 10_000_000), + ('in lakhs', 100_000), + ('figures in trillions of dollars', 1_000_000_000_000), +] + +# --------------------------------------------------------------------------- +# Adversarial negatives: prose that MUST NOT be detected as a scale caption. +# --------------------------------------------------------------------------- +NEGATIVE_CASES = [ + 'millions of users signed up last year', + 'in millions of ways this helped customers', + 'thousands of customers rely on the service', + 'billions served worldwide since 1955', + 'a bare million appears in this sentence', + 'the company saved hundreds of thousands of dollars', + 'tens of millions of shares were outstanding', + 'we processed $5 million in revenue', + 'revenue grew to 3.5 billion dollars over the decade', + 'affects millions of people globally', +] + + +def test_positive_captions_all_detected_with_correct_factor(): + misses = [] + for text, factor in POSITIVE_CASES: + decls = detect_scale_declarations(text) + if not decls or decls[0].factor != factor: + misses.append((text, factor, [(d.unit, d.factor) for d in decls])) + assert not misses, f'detection misses: {misses}' + + +def test_no_false_positives_on_prose(): + false_positives = [] + for text in NEGATIVE_CASES: + decls = detect_scale_declarations(text) + if decls: + false_positives.append((text, [(d.phrase, d.factor) for d in decls])) + assert not false_positives, f'false positives: {false_positives}' + + +def test_detection_reports_measurable_accuracy(): + # The resume-honest numbers, asserted so they cannot silently regress. + detected = sum(1 for t, _ in POSITIVE_CASES if detect_scale_declarations(t)) + clean = sum(1 for t in NEGATIVE_CASES if not detect_scale_declarations(t)) + assert detected == len(POSITIVE_CASES) + assert clean == len(NEGATIVE_CASES) + + +def test_currency_is_extracted(): + assert detect_scale_declarations('in millions of USD')[0].currency == 'USD' + assert detect_scale_declarations('(€ in millions)')[0].currency == 'EUR' + assert detect_scale_declarations('(₹ in crore)')[0].currency == 'INR' + assert detect_scale_declarations('(In millions)')[0].currency is None + + +# --------------------------------------------------------------------------- +# Annotation. +# --------------------------------------------------------------------------- +TABLE = '| Item | 2024 | 2023 |\n| --- | --- | --- |\n| Revenue | 1,234 | 1,100 |\n| Net income | 456 | 400 |' +# Same table but the figures carry a currency symbol, i.e. a financial signal in +# the table itself even with no caption in scope. +CURRENCY_TABLE = ( + '| Item | 2024 | 2023 |\n| --- | --- | --- |\n| Revenue | $1,234 | $1,100 |\n| Net income | $456 | $400 |' +) + + +def test_marker_welded_above_table_with_scale_in_scope(): + text = f'Consolidated Statements of Operations\n(In millions)\n\n{TABLE}' + out, warnings = annotate_scale(text) + assert '> Scale: amounts in millions (x1,000,000)' in out + # Marker is ASCII only: it is welded into the pipeline text lane, which may be + # written to a cp1252 console. A non-ASCII glyph there raises UnicodeEncodeError. + out.encode('cp1252') + # Marker sits on the line directly above the first table row. + lines = out.split('\n') + table_start = next(i for i, ln in enumerate(lines) if ln.startswith('| Item')) + assert lines[table_start - 1].startswith('> Scale:') + assert any(w['status'] == 'scale_detected' and w['factor'] == 1_000_000 for w in warnings) + + +def test_warning_injected_for_currency_table_without_scale(): + # A numeric table with a currency signal but no caption in scope: warn. + text = f'Some financial figures follow.\n\n{CURRENCY_TABLE}' + out, warnings = annotate_scale(text) + assert 'Scale not detected for this table' in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_warning_injected_when_document_has_a_caption_elsewhere(): + # A caption out of scope still marks the document financial, so an + # uncaptioned numeric table downstream is warned even without a currency mark. + text = f'(In millions)\n\n---\n\nUnrelated section\n\n{TABLE}' + out, warnings = annotate_scale(text) + assert 'Scale not detected for this table' in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_no_warning_for_numeric_non_financial_table(): + # Server metrics: numeric, but no currency and no caption anywhere. Injecting + # a scale warning here would be misleading hedging in an ordinary document. + table = '| Endpoint | p50 ms | p99 ms | rps |\n| --- | --- | --- | --- |\n| /login | 12 | 88 | 240 |\n| /search | 34 | 210 | 90 |' + text = f'Service latency report\n\n{table}' + out, warnings = annotate_scale(text) + assert 'Scale not detected' not in out + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + +def test_non_numeric_table_gets_no_warning(): + table = '| Name | Role |\n| --- | --- |\n| Ada | Engineer |\n| Grace | Admiral |' + text = f'Team roster\n\n{table}' + out, warnings = annotate_scale(text) + assert 'Scale not detected' not in out + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + +def test_prose_scale_word_does_not_weld_marker(): + # "valued in billions" is prose, not a caption; it must not weld a scale onto + # a following table -- the exact wrong-scale error this feature prevents. + text = f'The market opportunity, valued in billions, keeps growing.\n\n{CURRENCY_TABLE}' + out, warnings = annotate_scale(text) + assert '> Scale: amounts in billions' not in out + assert not any(w['status'] == 'scale_detected' for w in warnings) + + +def test_caption_over_non_numeric_table_is_not_welded(): + # A real caption in scope but the table is a roster, not figures: no weld. + table = '| Name | Role |\n| --- | --- |\n| Ada | Engineer |\n| Grace | Admiral |' + text = f'(In millions)\n\n{table}' + out, warnings = annotate_scale(text) + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + +def test_injected_markers_are_not_detected_as_captions(): + # Both markers contain scale words ("in millions", "thousands, millions, or + # billions"). Neither may be read back as a source caption, or a re-annotation + # pass could weld a scale derived from our own marker. + scale_marker = '> Scale: amounts in millions (x1,000,000)' + warning_marker = ( + '> [scale?] Scale not detected for this table. Figures may be in ' + 'thousands, millions, or billions. Verify against the source.' + ) + assert detect_scale_declarations(scale_marker) == [] + assert detect_scale_declarations(warning_marker) == [] + + +def test_scale_out_of_scope_when_page_break_between(): + text = f'(In millions)\n\n---\n\nUnrelated section\n\n{TABLE}' + out, warnings = annotate_scale(text) + # A page/section break severs the caption from the table -> warned, not welded. + assert '> Scale:' not in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_annotation_is_idempotent(): + text = f'(In millions)\n\n{TABLE}' + once, _ = annotate_scale(text) + twice, warnings = annotate_scale(once) + assert once == twice + assert once.count('> Scale:') == 1 + + +def test_empty_and_tableless_text_are_noops(): + assert annotate_scale('') == ('', []) + assert annotate_scale(' ') == (' ', []) + prose = 'Just a paragraph with no tables in it at all.' + assert annotate_scale(prose) == (prose, []) + + +# --------------------------------------------------------------------------- +# Shared table heuristic. +# --------------------------------------------------------------------------- +def test_extract_markdown_tables_matches_legacy_heuristic(): + text = f'intro\n{TABLE}\noutro\n\n| a | b |\n| 1 | 2 |' + tables = extract_markdown_tables(text) + assert len(tables) == 2 + # Rows are stripped and pipe-delimited, as the table lane expects. + assert tables[0].splitlines()[0] == '| Item | 2024 | 2023 |' + + +def test_table_spans_point_at_table_text(): + text = f'header line\n{TABLE}\nfooter' + spans = find_markdown_table_spans(text) + assert len(spans) == 1 + start, end = spans[0] + assert text[start:end].startswith('| Item') + assert text[start:end].rstrip().endswith('|')