From e564f6ab2292f8c5f66c9e838616c76a66f3cbce Mon Sep 17 00:00:00 2001 From: Ansh-Karnwal <78387773+Ansh-Karnwal@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:05:47 -0700 Subject: [PATCH 1/4] fix(nodes): preserve financial scale headers in the llamaparse node Financial statements declare magnitude in a caption above the table ("(In millions)", "$ in thousands"). When that caption is separated from its table by downstream chunking or an LLM step, every figure reads as raw units and is silently 1,000x / 1,000,000x / 1,000,000,000x off. Nothing downstream catches it, since a wrong-by-1e6 number looks exactly like a correct one. This makes the loss non-silent. A new pure-Python module (scale.py) runs after the parser assembles its Markdown and: - detects scale captions the parser already emitted and normalizes each to a unit, factor, and optional currency. Prose uses of the same words ("millions of users", "billions served", "hundreds of thousands of dollars", "$5 million in revenue") are rejected. - welds a normalized marker directly above each table whose caption is in scope ("> Scale: amounts in millions (x1,000,000)"), so a later chunker cannot split the scale away from its figures. - flags any numeric table with no caption in scope with a visible warning line, and surfaces detected_scales and scale_warnings in parsing_metadata. Scope is bounded by page/section breaks so a caption on one page is not attached to an unrelated table on the next. Annotation is idempotent and wrapped so any failure falls back to the untouched text and never blocks parsing. The table-detection heuristic is factored into scale.py and reused by IInstance.extract_tables_from_text so there is one implementation, with no change to the table lane output. Recovering a caption the parser dropped from the source entirely is out of scope; that needs source re-extraction and belongs to the audit-grade datalab_parse node. Documented in the node README. Adds test_llamaparse_scale.py: 18 labeled positive captions all detected with the correct factor, 10 adversarial negatives with zero false positives, plus annotation, scoping, idempotence, and table-parity cases. --- nodes/src/nodes/llamaparse/IInstance.py | 31 +- nodes/src/nodes/llamaparse/README.md | 21 ++ nodes/src/nodes/llamaparse/parser.py | 33 ++ nodes/src/nodes/llamaparse/scale.py | 423 ++++++++++++++++++++++++ nodes/test/test_llamaparse_scale.py | 170 ++++++++++ 5 files changed, 653 insertions(+), 25 deletions(-) create mode 100644 nodes/src/nodes/llamaparse/scale.py create mode 100644 nodes/test/test_llamaparse_scale.py diff --git a/nodes/src/nodes/llamaparse/IInstance.py b/nodes/src/nodes/llamaparse/IInstance.py index aabd6b771..0c1fb6bb9 100644 --- a/nodes/src/nodes/llamaparse/IInstance.py +++ b/nodes/src/nodes/llamaparse/IInstance.py @@ -375,31 +375,12 @@ def extract_tables_from_text(self, text: str): try: debug('LlamaParse Instance: Extracting tables from text') - # Simple table detection - look for markdown table patterns - lines = text.split('\n') - tables = [] - current_table = [] - in_table = False - - for line in lines: - line = line.strip() - - # Check if line looks like a table row (contains |) - if '|' in line and len(line.split('|')) > 2: - if not in_table: - in_table = True - current_table = [] - current_table.append(line) - elif in_table: - # End of table - if current_table: - tables.append('\n'.join(current_table)) - current_table = [] - in_table = False - - # Don't forget the last table - if in_table and current_table: - tables.append('\n'.join(current_table)) + # Simple table detection - look for markdown table patterns. + # Shared with the scale-header preservation logic so there is a + # single table heuristic (see scale.extract_markdown_tables). + from .scale import extract_markdown_tables + + tables = extract_markdown_tables(text) debug(f'LlamaParse Instance: Found {len(tables)} potential tables') diff --git a/nodes/src/nodes/llamaparse/README.md b/nodes/src/nodes/llamaparse/README.md index 02cec3b01..f980852f8 100644 --- a/nodes/src/nodes/llamaparse/README.md +++ b/nodes/src/nodes/llamaparse/README.md @@ -14,6 +14,27 @@ A single shared parser instance is guarded by a lock, so documents are parsed on --- +## Scale-header preservation + +Financial statements declare their magnitude in a caption above the table ("(In millions)", "$ in thousands", "amounts in millions of USD"). If that caption is separated from its table by downstream chunking or an LLM step, every figure reads as raw units and is silently 1,000x, 1,000,000x, or 1,000,000,000x off. Nothing downstream catches it, because a wrong-by-1e6 number looks exactly like a right one. + +To keep that loss visible, the parser post-processes the extracted Markdown: + +- **Detects scale captions** the parser already emitted ("(in millions)", "in thousands, except per share amounts", "(₹ in crore)", ...) and normalizes each to a unit, a numeric factor, and an optional currency. Prose uses of the same words ("millions of users", "billions served", "hundreds of thousands of dollars") are not treated as captions. +- **Welds a marker to the table.** When a caption is in scope for a table (same page/section, no page break between them), a normalized line is injected directly above the table, for example `> Scale: amounts in millions (×1,000,000)`. A later chunker or LLM cannot separate the scale from its figures. +- **Flags a missing scale.** When a table looks numeric and no caption is in scope, a warning line is injected above it: `> ⚠️ Scale not detected for this table. Figures may be in thousands, millions, or billions. Verify against the source.` The warning errs toward caution: a visible warning is far cheaper than a silent 1e6 error. + +The results are also surfaced structurally in `parsing_metadata`: + +- `detected_scales`: list of `{phrase, unit, factor, currency}` for each caption found. +- `scale_warnings`: per-table `{table_index, status, ...}` where `status` is `scale_detected`, `scale_missing`, or `not_financial`. + +Annotation is idempotent (re-running does not double-inject) and never blocks parsing: any failure falls back to the untouched text. + +**Out of scope.** Recovering a caption that LlamaParse dropped from the source entirely is not handled here. That needs an independent source-text extraction and belongs to the audit-grade **`datalab_parse`** node. This node makes an *emitted-but-separable* scale un-loseable and flags its *absence*; it does not re-parse the source. + +--- + ## Configuration ### Lanes diff --git a/nodes/src/nodes/llamaparse/parser.py b/nodes/src/nodes/llamaparse/parser.py index 1fcfbe957..c89efb588 100644 --- a/nodes/src/nodes/llamaparse/parser.py +++ b/nodes/src/nodes/llamaparse/parser.py @@ -280,6 +280,39 @@ def parse(self, file_data: bytes, file_name: Optional[str] = None) -> dict[str, debug(f'parsing_metadata: {parsing_metadata}, structured_data: {structured_data}') + # Preserve financial scale headers. A caption like "(in millions)" + # sets the magnitude of every figure in the table below it; if a + # downstream chunker or LLM step separates the two, the figures + # read as raw units and are silently 1e3/1e6/1e9 off. Weld a + # normalized marker to each table and flag numeric tables that + # have no scale in scope. Never let this break parsing: on any + # failure, fall back to the untouched text_content. + detected_scales: list = [] + scale_warnings: list = [] + try: + from .scale import annotate_scale, detect_scale_declarations + + declarations = detect_scale_declarations(text_content) + text_content, scale_warnings = annotate_scale(text_content, declarations) + detected_scales = [ + { + 'phrase': d.phrase, + 'unit': d.unit, + 'factor': d.factor, + 'currency': d.currency, + } + for d in declarations + ] + debug( + f'Scale headers: {len(detected_scales)} declaration(s), ' + f'{len(scale_warnings)} table annotation(s)' + ) + except Exception as e: + debug(f'Scale-header annotation skipped: {str(e)}') + + parsing_metadata['detected_scales'] = detected_scales + parsing_metadata['scale_warnings'] = scale_warnings + # Return text, structured data, page count, and metadata return { 'text': text_content, diff --git a/nodes/src/nodes/llamaparse/scale.py b/nodes/src/nodes/llamaparse/scale.py new file mode 100644 index 000000000..845a310e1 --- /dev/null +++ b/nodes/src/nodes/llamaparse/scale.py @@ -0,0 +1,423 @@ +# ============================================================================= +# MIT License +# Copyright (c) 2026 Aparavi Software AG +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# ============================================================================= + +"""Financial scale-header detection and preservation for the LlamaParse node. + +A statement's scale caption ("(In millions)", "$ in thousands", ...) declares +the magnitude of every figure in the table below it. When that caption is +separated from its table by downstream chunking or an LLM step, the figures +read as raw units and are silently 1,000x / 1,000,000x / 1,000,000,000x off. +Nothing downstream catches it, because a wrong-by-1e6 number looks exactly like +a right one. + +This module keeps scale loss non-silent. It (1) detects scale declarations that +the parser already emitted, (2) welds a normalized marker to the table so a +later chunker cannot split the header away, and (3) flags any numeric table that +has no scale in scope, turning a silent magnitude error into a visible warning. + +Recovering a caption that the parser dropped from the source entirely is out of +scope; that needs source re-extraction and belongs to the audit-grade +``datalab_parse`` node. See this node's README ("Scale-header preservation"). + +Pure text logic, no external dependencies, so it is unit-testable without the +LlamaParse API or the native engine. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Iterator, List, Optional, Tuple + +# Canonical scale word -> (normalized unit label, multiplier). +_SCALE_UNITS = { + 'thousand': ('thousands', 1_000), + 'thousands': ('thousands', 1_000), + 'lakh': ('lakhs', 100_000), + 'lakhs': ('lakhs', 100_000), + 'million': ('millions', 1_000_000), + 'millions': ('millions', 1_000_000), + 'crore': ('crores', 10_000_000), + 'crores': ('crores', 10_000_000), + 'billion': ('billions', 1_000_000_000), + 'billions': ('billions', 1_000_000_000), + 'trillion': ('trillions', 1_000_000_000_000), + 'trillions': ('trillions', 1_000_000_000_000), +} + +_SCALE_WORD_RE = re.compile( + r'\b(thousands?|lakhs?|millions?|crores?|billions?|trillions?)\b', + re.IGNORECASE, +) + +# Words that, following "... of ", keep it a monetary declaration +# ("in millions of dollars"). Anything else after "of" ("millions of users") +# means the scale word is prose, not a caption. +_CURRENCY_WORDS = { + 'dollar', + 'dollars', + 'usd', + 'euro', + 'euros', + 'eur', + 'pound', + 'pounds', + 'sterling', + 'gbp', + 'rupee', + 'rupees', + 'inr', + 'yen', + 'jpy', + 'franc', + 'francs', + 'chf', + 'cad', + 'aud', + 'won', + 'krw', + 'yuan', + 'renminbi', + 'rmb', + 'cny', + 'real', + 'reais', + 'brl', + 'peso', + 'pesos', +} + +_SYMBOL_TO_CODE = {'$': 'USD', '€': 'EUR', '£': 'GBP', '₹': 'INR', '¥': 'JPY'} + +_WORD_TO_CODE = { + 'dollar': 'USD', + 'dollars': 'USD', + 'usd': 'USD', + 'euro': 'EUR', + 'euros': 'EUR', + 'eur': 'EUR', + 'pound': 'GBP', + 'pounds': 'GBP', + 'sterling': 'GBP', + 'gbp': 'GBP', + 'rupee': 'INR', + 'rupees': 'INR', + 'inr': 'INR', + 'yen': 'JPY', + 'jpy': 'JPY', +} + +# Amount of surrounding text inspected to decide whether a scale word is a +# genuine declaration. +_CONTEXT_CHARS = 48 + +# "in" (optionally preceded by a currency symbol) sitting immediately before the +# scale word: "(in millions)", "$ in thousands", "amounts in millions". +_LEADIN_RE = re.compile(r'\bin\s*$', re.IGNORECASE) +# A lone currency symbol immediately before the scale word: "($ millions)". +_CURRENCY_PREFIX_RE = re.compile(r'[$€£₹¥]\s*$') +# "of" immediately before the scale word: prose like "hundreds of thousands". +_TRAILING_OF_RE = re.compile(r'\bof\s*$', re.IGNORECASE) +# "of " immediately after the scale word; the word decides monetary vs +# prose. An optional "U.S." prefix is skipped (outside the capture) so +# "of U.S. dollars" yields "dollars", while "of USD" stays intact. +_OF_CLAUSE_RE = re.compile(r'^\s*of\s+(?:u\.?\s*s\.?\s+)?([a-z]+)', re.IGNORECASE) + +# A page or section boundary between a caption and a table puts the caption out +# of scope: a form feed, a horizontal rule, or an explicit "Page N" marker. +_PAGE_BREAK_RE = re.compile( + r'\f' + r'|^\s*(?:-{3,}|\*{3,}|_{3,})\s*$' + r'|^\s*#{1,6}\s+.*\bpage\s+\d+' + r'|\bpage\s+\d+\s+of\s+\d+\b', + re.IGNORECASE | re.MULTILINE, +) + +# A table cell that is essentially a number: "1,234", "(1,234)", "$1,234.56", +# "45%", "-5.0". Used to decide whether a headerless table looks financial. +_NUMERIC_CELL_RE = re.compile(r'^[(\-+]?\s*[$€£₹¥]?\s*[\d,]+(?:\.\d+)?\s*%?\)?$') + +# Markers this module injects; used for idempotence and for the metadata signal. +_SCALE_MARKER_PREFIX = '> Scale:' +_WARNING_MARKER = ( + '> ⚠️ Scale not detected for this table. Figures may be in ' + 'thousands, millions, or billions. Verify against the source.' +) + +# Longest gap (in characters) allowed between a caption and the table it scales, +# as a backstop when the parser emits no page break between unrelated sections. +_MAX_SCOPE_DISTANCE = 1500 + + +@dataclass(frozen=True) +class ScaleDeclaration: + """A detected scale caption and its normalized magnitude.""" + + phrase: str + unit: str + factor: int + currency: Optional[str] + start: int + end: int + + +def _classify_of_clause(right: str) -> Tuple[bool, bool]: + """Return (is_currency, is_non_currency) for an "of " tail, if any.""" + m = _OF_CLAUSE_RE.match(right) + if not m: + return (False, False) + if m.group(1).lower() in _CURRENCY_WORDS: + return (True, False) + return (False, True) + + +def _detect_currency(left: str, right: str) -> Optional[str]: + """Best-effort currency label from the text around a scale word.""" + for sym, code in _SYMBOL_TO_CODE.items(): + if sym in left or sym in right: + return code + combined = (left + ' ' + right).lower() + for word, code in _WORD_TO_CODE.items(): + if re.search(r'\b' + re.escape(word) + r'\b', combined): + return code + return None + + +def _build_phrase(text: str, left: str, start: int, end: int, right: str) -> str: + """Reconstruct a readable caption snippet for logging/metadata.""" + p_start = start + lead = re.search(r'((?:[$€£₹¥]\s*)?in\s+)$', left, re.IGNORECASE) + if lead: + p_start = start - len(lead.group(1)) + p_end = end + tail = re.match( + r'\s*(?:of\s+[\w.]+(?:\s+[\w.]+)?)?(?:\s*,\s*except[^)\n.]*)?', + right, + re.IGNORECASE, + ) + if tail and tail.end() > 0: + p_end = end + tail.end() + return ' '.join(text[p_start:p_end].split()) + + +def detect_scale_declarations(text: str) -> List[ScaleDeclaration]: + """Find scale captions in ``text``. + + A scale word ("millions", "thousands", ...) is treated as a declaration only + when it is anchored to a monetary context: preceded by "in" or a currency + symbol, or followed by "of ". Prose uses of the same words are + rejected: "millions of users", "billions served", "hundreds of thousands of + dollars", "$5 million in revenue". + """ + if not text: + return [] + + declarations: List[ScaleDeclaration] = [] + for m in _SCALE_WORD_RE.finditer(text): + word = m.group(1).lower() + unit, factor = _SCALE_UNITS[word] + left = text[max(0, m.start() - _CONTEXT_CHARS) : m.start()] + right = text[m.end() : m.end() + _CONTEXT_CHARS] + + of_currency, of_non_currency = _classify_of_clause(right) + if of_non_currency: + # "millions of ways", "thousands of customers" -> prose. + continue + if _TRAILING_OF_RE.search(left): + # "hundreds of thousands", "tens of millions" -> prose. + continue + + has_in = bool(_LEADIN_RE.search(left)) + has_currency_prefix = bool(_CURRENCY_PREFIX_RE.search(left)) + if not (has_in or of_currency or has_currency_prefix): + continue + + declarations.append( + ScaleDeclaration( + phrase=_build_phrase(text, left, m.start(), m.end(), right), + unit=unit, + factor=factor, + currency=_detect_currency(left, right), + start=m.start(), + end=m.end(), + ) + ) + return declarations + + +def _is_table_row(line: str) -> bool: + """A markdown table row: a pipe-delimited line with at least two cells. + + Matches the heuristic the node uses to split tables out of parsed text, so + table detection stays consistent between the two callers. + """ + stripped = line.strip() + return '|' in stripped and len(stripped.split('|')) > 2 + + +def iter_table_blocks(text: str) -> Iterator[Tuple[int, int, List[str]]]: + """Yield (start_line, end_line_exclusive, lines) for each markdown table.""" + lines = text.split('\n') + start: Optional[int] = None + for i, line in enumerate(lines): + if _is_table_row(line): + if start is None: + start = i + elif start is not None: + yield (start, i, lines[start:i]) + start = None + if start is not None: + yield (start, len(lines), lines[start:]) + + +def extract_markdown_tables(text: str) -> List[str]: + """Return each detected markdown table as a string of its stripped rows. + + Shared implementation for the node's ``table`` lane so there is one table + heuristic, not two. + """ + return ['\n'.join(line.strip() for line in block) for _, _, block in iter_table_blocks(text)] + + +def _line_offsets(lines: List[str]) -> List[int]: + offsets = [] + pos = 0 + for line in lines: + offsets.append(pos) + pos += len(line) + 1 # + 1 for the '\n' that split removed + return offsets + + +def find_markdown_table_spans(text: str) -> List[Tuple[int, int]]: + """Return (char_start, char_end) offsets in ``text`` for each table block.""" + lines = text.split('\n') + offsets = _line_offsets(lines) + spans = [] + for start, end, _ in iter_table_blocks(text): + last = end - 1 + spans.append((offsets[start], offsets[last] + len(lines[last]))) + return spans + + +def _table_looks_numeric(table_text: str) -> bool: + """True when a table is mostly numbers, so a missing scale matters.""" + cells = [] + for line in table_text.split('\n'): + for cell in line.strip().strip('|').split('|'): + c = cell.strip() + # Skip separator cells like "---" / ":--:". + if c and set(c) - set('-: '): + cells.append(c) + if not cells: + return False + numeric = sum(1 for c in cells if _NUMERIC_CELL_RE.match(c)) + return numeric >= 3 and numeric / len(cells) >= 0.3 + + +def _nearest_in_scope(declarations: List[ScaleDeclaration], text: str, table_start: int) -> Optional[ScaleDeclaration]: + """Nearest declaration ending before ``table_start`` with nothing (page + break / oversized gap) separating it from the table. + """ + best: Optional[ScaleDeclaration] = None + for d in declarations: + if d.end > table_start: + continue + if table_start - d.end > _MAX_SCOPE_DISTANCE: + continue + if _PAGE_BREAK_RE.search(text[d.end : table_start]): + continue + if best is None or d.end > best.end: + best = d + return best + + +def _already_annotated(prev_line: str) -> bool: + stripped = prev_line.strip() + return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped == _WARNING_MARKER + + +def _scale_marker(decl: ScaleDeclaration) -> str: + marker = f'{_SCALE_MARKER_PREFIX} amounts in {decl.unit} (×{decl.factor:,})' + if decl.currency: + marker += f' [{decl.currency}]' + return marker + + +def annotate_scale( + text: str, + declarations: Optional[List[ScaleDeclaration]] = None, + table_spans: Optional[List[Tuple[int, int]]] = None, +) -> Tuple[str, List[dict]]: + """Weld scale markers to tables and flag numeric tables missing a scale. + + Returns the annotated text plus a list of per-table warning records + ``{'table_index', 'status', ...}`` where ``status`` is ``scale_detected``, + ``scale_missing``, or ``not_financial``. Idempotent: a table already + carrying one of our markers is left untouched. + """ + if not text or not text.strip(): + return text, [] + if declarations is None: + declarations = detect_scale_declarations(text) + + lines = text.split('\n') + offsets = _line_offsets(lines) + blocks = list(iter_table_blocks(text)) + if not blocks: + return text, [] + + warnings: List[dict] = [] + insertions: dict = {} + for idx, (start, _end, block) in enumerate(blocks): + prev_line = lines[start - 1] if start > 0 else '' + if _already_annotated(prev_line): + warnings.append({'table_index': idx, 'status': 'already_annotated'}) + continue + + decl = _nearest_in_scope(declarations, text, offsets[start]) + if decl is not None: + insertions[start] = _scale_marker(decl) + warnings.append( + { + 'table_index': idx, + 'status': 'scale_detected', + 'unit': decl.unit, + 'factor': decl.factor, + 'currency': decl.currency, + } + ) + elif _table_looks_numeric('\n'.join(block)): + insertions[start] = _WARNING_MARKER + warnings.append({'table_index': idx, 'status': 'scale_missing', 'factor': None}) + else: + warnings.append({'table_index': idx, 'status': 'not_financial'}) + + if not insertions: + return text, warnings + + out = [] + for i, line in enumerate(lines): + if i in insertions: + out.append(insertions[i]) + out.append(line) + return '\n'.join(out), warnings diff --git a/nodes/test/test_llamaparse_scale.py b/nodes/test/test_llamaparse_scale.py new file mode 100644 index 000000000..a64bd7f81 --- /dev/null +++ b/nodes/test/test_llamaparse_scale.py @@ -0,0 +1,170 @@ +"""Unit tests for the llamaparse scale-header detector/annotator (scale.py). + +Pure-logic tests, no engine / LlamaParse API / rocketlib required. Cover: +- a labeled detection corpus (positive captions + adversarial negatives), +- table annotation (marker injection) and missing-scale warnings, +- idempotence and empty-input safety, +- parity of the shared table-extraction heuristic. +""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src', 'nodes', 'llamaparse')) +from scale import ( # noqa: E402 + annotate_scale, + detect_scale_declarations, + extract_markdown_tables, + find_markdown_table_spans, +) + + +# --------------------------------------------------------------------------- +# Detection corpus: captions that MUST be detected, with their expected factor. +# --------------------------------------------------------------------------- +POSITIVE_CASES = [ + ('(In millions)', 1_000_000), + ('(in thousands)', 1_000), + ('(In billions)', 1_000_000_000), + ('$ in millions', 1_000_000), + ('$ in thousands', 1_000), + ('($ millions)', 1_000_000), + ('Amounts in thousands of dollars', 1_000), + ('in millions of USD', 1_000_000), + ('In millions of U.S. dollars', 1_000_000), + ('(in billions, except per-share data)', 1_000_000_000), + ('In thousands, except share and per share amounts', 1_000), + ('Dollars in millions', 1_000_000), + ('(Millions of dollars)', 1_000_000), + ('Amounts are in thousands unless otherwise stated', 1_000), + ('(€ in millions)', 1_000_000), + ('(£ in thousands)', 1_000), + ('(₹ in crore)', 10_000_000), + ('in lakhs', 100_000), + ('figures in trillions of dollars', 1_000_000_000_000), +] + +# --------------------------------------------------------------------------- +# Adversarial negatives: prose that MUST NOT be detected as a scale caption. +# --------------------------------------------------------------------------- +NEGATIVE_CASES = [ + 'millions of users signed up last year', + 'in millions of ways this helped customers', + 'thousands of customers rely on the service', + 'billions served worldwide since 1955', + 'a bare million appears in this sentence', + 'the company saved hundreds of thousands of dollars', + 'tens of millions of shares were outstanding', + 'we processed $5 million in revenue', + 'revenue grew to 3.5 billion dollars over the decade', + 'affects millions of people globally', +] + + +def test_positive_captions_all_detected_with_correct_factor(): + misses = [] + for text, factor in POSITIVE_CASES: + decls = detect_scale_declarations(text) + if not decls or decls[0].factor != factor: + misses.append((text, factor, [(d.unit, d.factor) for d in decls])) + assert not misses, f'detection misses: {misses}' + + +def test_no_false_positives_on_prose(): + false_positives = [] + for text in NEGATIVE_CASES: + decls = detect_scale_declarations(text) + if decls: + false_positives.append((text, [(d.phrase, d.factor) for d in decls])) + assert not false_positives, f'false positives: {false_positives}' + + +def test_detection_reports_measurable_accuracy(): + # The resume-honest numbers, asserted so they cannot silently regress. + detected = sum(1 for t, _ in POSITIVE_CASES if detect_scale_declarations(t)) + clean = sum(1 for t in NEGATIVE_CASES if not detect_scale_declarations(t)) + assert detected == len(POSITIVE_CASES) + assert clean == len(NEGATIVE_CASES) + + +def test_currency_is_extracted(): + assert detect_scale_declarations('in millions of USD')[0].currency == 'USD' + assert detect_scale_declarations('(€ in millions)')[0].currency == 'EUR' + assert detect_scale_declarations('(₹ in crore)')[0].currency == 'INR' + assert detect_scale_declarations('(In millions)')[0].currency is None + + +# --------------------------------------------------------------------------- +# Annotation. +# --------------------------------------------------------------------------- +TABLE = '| Item | 2024 | 2023 |\n| --- | --- | --- |\n| Revenue | 1,234 | 1,100 |\n| Net income | 456 | 400 |' + + +def test_marker_welded_above_table_with_scale_in_scope(): + text = f'Consolidated Statements of Operations\n(In millions)\n\n{TABLE}' + out, warnings = annotate_scale(text) + assert '> Scale: amounts in millions (×1,000,000)' in out + # Marker sits on the line directly above the first table row. + lines = out.split('\n') + table_start = next(i for i, ln in enumerate(lines) if ln.startswith('| Item')) + assert lines[table_start - 1].startswith('> Scale:') + assert any(w['status'] == 'scale_detected' and w['factor'] == 1_000_000 for w in warnings) + + +def test_warning_injected_for_numeric_table_without_scale(): + text = f'Some financial figures follow.\n\n{TABLE}' + out, warnings = annotate_scale(text) + assert 'Scale not detected for this table' in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_non_numeric_table_gets_no_warning(): + table = '| Name | Role |\n| --- | --- |\n| Ada | Engineer |\n| Grace | Admiral |' + text = f'Team roster\n\n{table}' + out, warnings = annotate_scale(text) + assert 'Scale not detected' not in out + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + +def test_scale_out_of_scope_when_page_break_between(): + text = f'(In millions)\n\n---\n\nUnrelated section\n\n{TABLE}' + out, warnings = annotate_scale(text) + # A page/section break severs the caption from the table -> warned, not welded. + assert '> Scale:' not in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_annotation_is_idempotent(): + text = f'(In millions)\n\n{TABLE}' + once, _ = annotate_scale(text) + twice, warnings = annotate_scale(once) + assert once == twice + assert once.count('> Scale:') == 1 + + +def test_empty_and_tableless_text_are_noops(): + assert annotate_scale('') == ('', []) + assert annotate_scale(' ') == (' ', []) + prose = 'Just a paragraph with no tables in it at all.' + assert annotate_scale(prose) == (prose, []) + + +# --------------------------------------------------------------------------- +# Shared table heuristic. +# --------------------------------------------------------------------------- +def test_extract_markdown_tables_matches_legacy_heuristic(): + text = f'intro\n{TABLE}\noutro\n\n| a | b |\n| 1 | 2 |' + tables = extract_markdown_tables(text) + assert len(tables) == 2 + # Rows are stripped and pipe-delimited, as the table lane expects. + assert tables[0].splitlines()[0] == '| Item | 2024 | 2023 |' + + +def test_table_spans_point_at_table_text(): + text = f'header line\n{TABLE}\nfooter' + spans = find_markdown_table_spans(text) + assert len(spans) == 1 + start, end = spans[0] + assert text[start:end].startswith('| Item') + assert text[start:end].rstrip().endswith('|') From 042fc17830ce028460b45bd64e50630cff15643d Mon Sep 17 00:00:00 2001 From: Ansh-Karnwal <78387773+Ansh-Karnwal@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:15:22 -0700 Subject: [PATCH 2/4] refactor(nodes): drop unused table_spans param from annotate_scale annotate_scale computes table blocks internally via iter_table_blocks; the table_spans parameter was never read and no caller passed it. The standalone find_markdown_table_spans helper is unchanged. --- nodes/src/nodes/llamaparse/scale.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nodes/src/nodes/llamaparse/scale.py b/nodes/src/nodes/llamaparse/scale.py index 845a310e1..cb24e5b36 100644 --- a/nodes/src/nodes/llamaparse/scale.py +++ b/nodes/src/nodes/llamaparse/scale.py @@ -366,7 +366,6 @@ def _scale_marker(decl: ScaleDeclaration) -> str: def annotate_scale( text: str, declarations: Optional[List[ScaleDeclaration]] = None, - table_spans: Optional[List[Tuple[int, int]]] = None, ) -> Tuple[str, List[dict]]: """Weld scale markers to tables and flag numeric tables missing a scale. From 32a4ca9b35a3e3c8111cd74de09e50e2aa186e28 Mon Sep 17 00:00:00 2001 From: Ansh-Karnwal <78387773+Ansh-Karnwal@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:03:11 -0700 Subject: [PATCH 3/4] fix(nodes): tighten llamaparse scale annotation from review feedback Address maintainer review on the scale-header preservation: - Gate the missing-scale warning on a financial signal. A numeric table with no caption is warned only when the document has a caption elsewhere or the table itself carries a currency marking. Plain numeric tables (inventory, server metrics, standings) are left untouched, so ordinary documents no longer get hedging welded into the text lane. - Only weld a caption onto a table that looks numeric, so a caption over a prose or roster table is never mis-attached. - Veto prose lead-ins ("valued in billions") so a sentence near a table no longer welds a wrong scale. - Stop the injected markers from being read back as captions on a second pass, keeping re-annotation idempotent across multiple tables. - Make the markers plain ASCII so they survive a cp1252 console. - Move the shared-table import to module scope in IInstance and document the already_annotated status in the README and docstring. --- nodes/src/nodes/llamaparse/IInstance.py | 3 +- nodes/src/nodes/llamaparse/README.md | 12 ++-- nodes/src/nodes/llamaparse/scale.py | 75 ++++++++++++++++++++++--- nodes/test/test_llamaparse_scale.py | 66 +++++++++++++++++++++- 4 files changed, 138 insertions(+), 18 deletions(-) diff --git a/nodes/src/nodes/llamaparse/IInstance.py b/nodes/src/nodes/llamaparse/IInstance.py index 0c1fb6bb9..868a013fd 100644 --- a/nodes/src/nodes/llamaparse/IInstance.py +++ b/nodes/src/nodes/llamaparse/IInstance.py @@ -27,6 +27,7 @@ from rocketlib import IInstanceBase, Entry, debug from ai.common.schema import Doc from .IGlobal import IGlobal +from .scale import extract_markdown_tables class IInstance(IInstanceBase): @@ -378,8 +379,6 @@ def extract_tables_from_text(self, text: str): # Simple table detection - look for markdown table patterns. # Shared with the scale-header preservation logic so there is a # single table heuristic (see scale.extract_markdown_tables). - from .scale import extract_markdown_tables - tables = extract_markdown_tables(text) debug(f'LlamaParse Instance: Found {len(tables)} potential tables') diff --git a/nodes/src/nodes/llamaparse/README.md b/nodes/src/nodes/llamaparse/README.md index f980852f8..d4c68ecb0 100644 --- a/nodes/src/nodes/llamaparse/README.md +++ b/nodes/src/nodes/llamaparse/README.md @@ -21,15 +21,17 @@ Financial statements declare their magnitude in a caption above the table ("(In To keep that loss visible, the parser post-processes the extracted Markdown: - **Detects scale captions** the parser already emitted ("(in millions)", "in thousands, except per share amounts", "(₹ in crore)", ...) and normalizes each to a unit, a numeric factor, and an optional currency. Prose uses of the same words ("millions of users", "billions served", "hundreds of thousands of dollars") are not treated as captions. -- **Welds a marker to the table.** When a caption is in scope for a table (same page/section, no page break between them), a normalized line is injected directly above the table, for example `> Scale: amounts in millions (×1,000,000)`. A later chunker or LLM cannot separate the scale from its figures. -- **Flags a missing scale.** When a table looks numeric and no caption is in scope, a warning line is injected above it: `> ⚠️ Scale not detected for this table. Figures may be in thousands, millions, or billions. Verify against the source.` The warning errs toward caution: a visible warning is far cheaper than a silent 1e6 error. +- **Welds a marker to the table.** When a caption is in scope for a *numeric* table (same page/section, no page break between them), a normalized line is injected directly above the table, for example `> Scale: amounts in millions (x1,000,000)`. A later chunker or LLM cannot separate the scale from its figures. A caption is only welded onto a table that actually looks numeric, so a caption sitting over a prose or roster table is never mis-attached. +- **Flags a missing scale, when the context is financial.** When a numeric table has no caption in scope *and* the document shows a financial signal (a caption detected elsewhere, or a currency marking in the table itself), a warning line is injected above it: `> [scale?] Scale not detected for this table. Figures may be in thousands, millions, or billions. Verify against the source.` The warning errs toward caution: a visible warning is far cheaper than a silent 1e6 error. Plain numeric tables with no financial signal (inventory counts, server metrics, sports standings) are left untouched, so ordinary documents pay nothing for this feature. The markers are plain ASCII, so they survive a cp1252 console without an encoding error. -The results are also surfaced structurally in `parsing_metadata`: +The welded markers travel in-band with the text and table lanes, so they are the durable downstream signal: whatever chunks or summarizes the output carries the scale (or the warning) with the figures. + +For observability, the same results are also recorded in `parsing_metadata` (debug-logged by the instance; not currently re-emitted onto a lane): - `detected_scales`: list of `{phrase, unit, factor, currency}` for each caption found. -- `scale_warnings`: per-table `{table_index, status, ...}` where `status` is `scale_detected`, `scale_missing`, or `not_financial`. +- `scale_warnings`: per-table `{table_index, status, ...}` where `status` is `scale_detected`, `scale_missing`, `not_financial`, or `already_annotated` (the table already carried one of our markers and was left as-is). -Annotation is idempotent (re-running does not double-inject) and never blocks parsing: any failure falls back to the untouched text. +Annotation is idempotent (re-running does not double-inject; the injected markers are not themselves read back as captions) and never blocks parsing: any failure falls back to the untouched text. **Out of scope.** Recovering a caption that LlamaParse dropped from the source entirely is not handled here. That needs an independent source-text extraction and belongs to the audit-grade **`datalab_parse`** node. This node makes an *emitted-but-separable* scale un-loseable and flags its *absence*; it does not re-parse the source. diff --git a/nodes/src/nodes/llamaparse/scale.py b/nodes/src/nodes/llamaparse/scale.py index cb24e5b36..bf2c63f77 100644 --- a/nodes/src/nodes/llamaparse/scale.py +++ b/nodes/src/nodes/llamaparse/scale.py @@ -143,6 +143,18 @@ # "of U.S. dollars" yields "dollars", while "of USD" stays intact. _OF_CLAUSE_RE = re.compile(r'^\s*of\s+(?:u\.?\s*s\.?\s+)?([a-z]+)', re.IGNORECASE) +# Verb sitting immediately before the "in" lead-in marks the scale word as prose, +# not a caption: "valued in billions", "grew to ... reaching in ...". A caption +# ("Dollars in millions", "Amounts are in thousands") never leads with these, so +# vetoing them costs no real captions while killing a class of false welds. +_PROSE_LEADIN_WORD_RE = re.compile( + r'\b(?:valued|worth|grew|grown|rose|risen|fell|fallen|reaching|reached|' + r'totaling|totalling|generating|generated|earning|earned|spending|spent|' + r'costing|saving|saved|raising|raised|investing|invested|losing|lost|' + r'measured|priced|estimated)\s+in\s*$', + re.IGNORECASE, +) + # A page or section boundary between a caption and a table puts the caption out # of scope: a form feed, a horizontal rule, or an explicit "Page N" marker. _PAGE_BREAK_RE = re.compile( @@ -158,9 +170,13 @@ _NUMERIC_CELL_RE = re.compile(r'^[(\-+]?\s*[$€£₹¥]?\s*[\d,]+(?:\.\d+)?\s*%?\)?$') # Markers this module injects; used for idempotence and for the metadata signal. +# Plain text only: these lines are welded into the parsed text that flows down +# the pipeline (application output), where a non-ASCII glyph can raise a +# UnicodeEncodeError on a cp1252 console. _SCALE_MARKER_PREFIX = '> Scale:' +_WARNING_MARKER_PREFIX = '> [scale?]' _WARNING_MARKER = ( - '> ⚠️ Scale not detected for this table. Figures may be in ' + f'{_WARNING_MARKER_PREFIX} Scale not detected for this table. Figures may be in ' 'thousands, millions, or billions. Verify against the source.' ) @@ -234,6 +250,11 @@ def detect_scale_declarations(text: str) -> List[ScaleDeclaration]: declarations: List[ScaleDeclaration] = [] for m in _SCALE_WORD_RE.finditer(text): + if _on_marker_line(text, m.start()): + # A scale word inside a marker we injected ("> Scale: ... in millions", + # "> [scale?] ... in thousands, millions, or billions") is not a source + # caption; skipping it keeps re-annotation idempotent. + continue word = m.group(1).lower() unit, factor = _SCALE_UNITS[word] left = text[max(0, m.start() - _CONTEXT_CHARS) : m.start()] @@ -246,6 +267,9 @@ def detect_scale_declarations(text: str) -> List[ScaleDeclaration]: if _TRAILING_OF_RE.search(left): # "hundreds of thousands", "tens of millions" -> prose. continue + if _PROSE_LEADIN_WORD_RE.search(left): + # "valued in billions", "reached ... in millions" -> prose, not a caption. + continue has_in = bool(_LEADIN_RE.search(left)) has_currency_prefix = bool(_CURRENCY_PREFIX_RE.search(left)) @@ -351,13 +375,33 @@ def _nearest_in_scope(declarations: List[ScaleDeclaration], text: str, table_sta return best +def _on_marker_line(text: str, pos: int) -> bool: + """True when ``pos`` falls on a line that is one of our injected markers.""" + line_start = text.rfind('\n', 0, pos) + 1 + stripped = text[line_start:].lstrip() + return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped.startswith(_WARNING_MARKER_PREFIX) + + def _already_annotated(prev_line: str) -> bool: stripped = prev_line.strip() - return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped == _WARNING_MARKER + return stripped.startswith(_SCALE_MARKER_PREFIX) or stripped.startswith(_WARNING_MARKER_PREFIX) + + +def _table_has_currency(table_text: str) -> bool: + """True when a table carries a currency signal of its own (symbol or code). + + Lets a numeric, uncaptioned table still be flagged in an otherwise + non-financial document, without warning on plain numeric tables (inventory + counts, sports standings, server metrics) that carry no financial signal. + """ + if any(sym in table_text for sym in _SYMBOL_TO_CODE): + return True + lowered = table_text.lower() + return any(re.search(r'\b' + re.escape(w) + r'\b', lowered) for w in _CURRENCY_WORDS) def _scale_marker(decl: ScaleDeclaration) -> str: - marker = f'{_SCALE_MARKER_PREFIX} amounts in {decl.unit} (×{decl.factor:,})' + marker = f'{_SCALE_MARKER_PREFIX} amounts in {decl.unit} (x{decl.factor:,})' if decl.currency: marker += f' [{decl.currency}]' return marker @@ -370,9 +414,21 @@ def annotate_scale( """Weld scale markers to tables and flag numeric tables missing a scale. Returns the annotated text plus a list of per-table warning records - ``{'table_index', 'status', ...}`` where ``status`` is ``scale_detected``, - ``scale_missing``, or ``not_financial``. Idempotent: a table already - carrying one of our markers is left untouched. + ``{'table_index', 'status', ...}`` where ``status`` is one of: + + - ``scale_detected``: a caption was in scope; its marker was welded above. + - ``scale_missing``: a numeric table with a financial signal but no caption + in scope; a missing-scale warning was welded above. + - ``not_financial``: no marker welded, either because the table is not + numeric or it carries no financial signal (so a warning would be noise). + - ``already_annotated``: the table already carried one of our markers and was + left untouched. + + A marker is welded only above a *numeric* table, so a caption sitting over a + prose/roster table is never mis-attached. The missing-scale warning is gated + on a financial signal (a caption elsewhere in the document, or a currency + marking in the table itself) so ordinary numeric tables are not annotated. + Idempotent: re-running does not double-inject. """ if not text or not text.strip(): return text, [] @@ -385,6 +441,7 @@ def annotate_scale( if not blocks: return text, [] + doc_has_declaration = bool(declarations) warnings: List[dict] = [] insertions: dict = {} for idx, (start, _end, block) in enumerate(blocks): @@ -393,8 +450,10 @@ def annotate_scale( warnings.append({'table_index': idx, 'status': 'already_annotated'}) continue + block_text = '\n'.join(block) + is_numeric = _table_looks_numeric(block_text) decl = _nearest_in_scope(declarations, text, offsets[start]) - if decl is not None: + if decl is not None and is_numeric: insertions[start] = _scale_marker(decl) warnings.append( { @@ -405,7 +464,7 @@ def annotate_scale( 'currency': decl.currency, } ) - elif _table_looks_numeric('\n'.join(block)): + elif is_numeric and (doc_has_declaration or _table_has_currency(block_text)): insertions[start] = _WARNING_MARKER warnings.append({'table_index': idx, 'status': 'scale_missing', 'factor': None}) else: diff --git a/nodes/test/test_llamaparse_scale.py b/nodes/test/test_llamaparse_scale.py index a64bd7f81..695a5ddd3 100644 --- a/nodes/test/test_llamaparse_scale.py +++ b/nodes/test/test_llamaparse_scale.py @@ -98,12 +98,20 @@ def test_currency_is_extracted(): # Annotation. # --------------------------------------------------------------------------- TABLE = '| Item | 2024 | 2023 |\n| --- | --- | --- |\n| Revenue | 1,234 | 1,100 |\n| Net income | 456 | 400 |' +# Same table but the figures carry a currency symbol, i.e. a financial signal in +# the table itself even with no caption in scope. +CURRENCY_TABLE = ( + '| Item | 2024 | 2023 |\n| --- | --- | --- |\n| Revenue | $1,234 | $1,100 |\n| Net income | $456 | $400 |' +) def test_marker_welded_above_table_with_scale_in_scope(): text = f'Consolidated Statements of Operations\n(In millions)\n\n{TABLE}' out, warnings = annotate_scale(text) - assert '> Scale: amounts in millions (×1,000,000)' in out + assert '> Scale: amounts in millions (x1,000,000)' in out + # Marker is ASCII only: it is welded into the pipeline text lane, which may be + # written to a cp1252 console. A non-ASCII glyph there raises UnicodeEncodeError. + out.encode('cp1252') # Marker sits on the line directly above the first table row. lines = out.split('\n') table_start = next(i for i, ln in enumerate(lines) if ln.startswith('| Item')) @@ -111,13 +119,34 @@ def test_marker_welded_above_table_with_scale_in_scope(): assert any(w['status'] == 'scale_detected' and w['factor'] == 1_000_000 for w in warnings) -def test_warning_injected_for_numeric_table_without_scale(): - text = f'Some financial figures follow.\n\n{TABLE}' +def test_warning_injected_for_currency_table_without_scale(): + # A numeric table with a currency signal but no caption in scope: warn. + text = f'Some financial figures follow.\n\n{CURRENCY_TABLE}' + out, warnings = annotate_scale(text) + assert 'Scale not detected for this table' in out + assert any(w['status'] == 'scale_missing' for w in warnings) + + +def test_warning_injected_when_document_has_a_caption_elsewhere(): + # A caption out of scope still marks the document financial, so an + # uncaptioned numeric table downstream is warned even without a currency mark. + text = f'(In millions)\n\n---\n\nUnrelated section\n\n{TABLE}' out, warnings = annotate_scale(text) assert 'Scale not detected for this table' in out assert any(w['status'] == 'scale_missing' for w in warnings) +def test_no_warning_for_numeric_non_financial_table(): + # Server metrics: numeric, but no currency and no caption anywhere. Injecting + # a scale warning here would be misleading hedging in an ordinary document. + table = '| Endpoint | p50 ms | p99 ms | rps |\n| --- | --- | --- | --- |\n| /login | 12 | 88 | 240 |\n| /search | 34 | 210 | 90 |' + text = f'Service latency report\n\n{table}' + out, warnings = annotate_scale(text) + assert 'Scale not detected' not in out + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + def test_non_numeric_table_gets_no_warning(): table = '| Name | Role |\n| --- | --- |\n| Ada | Engineer |\n| Grace | Admiral |' text = f'Team roster\n\n{table}' @@ -127,6 +156,37 @@ def test_non_numeric_table_gets_no_warning(): assert any(w['status'] == 'not_financial' for w in warnings) +def test_prose_scale_word_does_not_weld_marker(): + # "valued in billions" is prose, not a caption; it must not weld a scale onto + # a following table -- the exact wrong-scale error this feature prevents. + text = f'The market opportunity, valued in billions, keeps growing.\n\n{CURRENCY_TABLE}' + out, warnings = annotate_scale(text) + assert '> Scale: amounts in billions' not in out + assert not any(w['status'] == 'scale_detected' for w in warnings) + + +def test_caption_over_non_numeric_table_is_not_welded(): + # A real caption in scope but the table is a roster, not figures: no weld. + table = '| Name | Role |\n| --- | --- |\n| Ada | Engineer |\n| Grace | Admiral |' + text = f'(In millions)\n\n{table}' + out, warnings = annotate_scale(text) + assert '> Scale:' not in out + assert any(w['status'] == 'not_financial' for w in warnings) + + +def test_injected_markers_are_not_detected_as_captions(): + # Both markers contain scale words ("in millions", "thousands, millions, or + # billions"). Neither may be read back as a source caption, or a re-annotation + # pass could weld a scale derived from our own marker. + scale_marker = '> Scale: amounts in millions (x1,000,000)' + warning_marker = ( + '> [scale?] Scale not detected for this table. Figures may be in ' + 'thousands, millions, or billions. Verify against the source.' + ) + assert detect_scale_declarations(scale_marker) == [] + assert detect_scale_declarations(warning_marker) == [] + + def test_scale_out_of_scope_when_page_break_between(): text = f'(In millions)\n\n---\n\nUnrelated section\n\n{TABLE}' out, warnings = annotate_scale(text) From bc12aebf3f93923bdd7f329b3333838ff6bed43e Mon Sep 17 00:00:00 2001 From: Ansh-Karnwal <78387773+Ansh-Karnwal@users.noreply.github.com> Date: Wed, 15 Jul 2026 13:12:02 -0700 Subject: [PATCH 4/4] refactor(nodes): precompile currency-word alternation in scale detector Replace the per-word re.search loop in _table_has_currency with a single module-level compiled alternation (_CURRENCY_WORDS_RE), matching the pattern used elsewhere in the module. --- nodes/src/nodes/llamaparse/scale.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nodes/src/nodes/llamaparse/scale.py b/nodes/src/nodes/llamaparse/scale.py index bf2c63f77..66ab2c449 100644 --- a/nodes/src/nodes/llamaparse/scale.py +++ b/nodes/src/nodes/llamaparse/scale.py @@ -107,6 +107,11 @@ 'pesos', } +_CURRENCY_WORDS_RE = re.compile( + r'\b(?:' + '|'.join(re.escape(w) for w in sorted(_CURRENCY_WORDS)) + r')\b', + re.IGNORECASE, +) + _SYMBOL_TO_CODE = {'$': 'USD', '€': 'EUR', '£': 'GBP', '₹': 'INR', '¥': 'JPY'} _WORD_TO_CODE = { @@ -396,8 +401,7 @@ def _table_has_currency(table_text: str) -> bool: """ if any(sym in table_text for sym in _SYMBOL_TO_CODE): return True - lowered = table_text.lower() - return any(re.search(r'\b' + re.escape(w) + r'\b', lowered) for w in _CURRENCY_WORDS) + return _CURRENCY_WORDS_RE.search(table_text) is not None def _scale_marker(decl: ScaleDeclaration) -> str: