diff --git a/.env.example b/.env.example index d04f753..1c8d267 100644 --- a/.env.example +++ b/.env.example @@ -8,7 +8,7 @@ MESA_RUNTIME_PROFILE=combined MESA_STORAGE_ROOT=/var/lib/mesa/data MESA_LOAD_DOTENV=false MESA_MODEL_ENABLED=false -# Each defaults to MESA_MODEL_ENABLED when unset. +# Embedding defaults to MESA_MODEL_ENABLED when unset; extraction defaults to false. # MESA_EMBEDDING_ENABLED=false # MESA_EXTRACTION_ENABLED=false # Graph is independent of model flags; false never opens Kùzu. diff --git a/CHANGELOG.md b/CHANGELOG.md index ab55dff..eb1e8f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ are historical; consult the [current guides](docs/README.md) for present behavio The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Queries matching a decision's own docket/decision reference promote its + first chunk through the existing exact-match slot, shared with articles + (at most three chunks total). Only stored references beginning in the first + 300 characters of an authorized first chunk count; later quotations do not. + +- Newly admitted chunks keep versioned Turkish legal rule items with chunk + offsets, returned in search provenance separately from SPO facts. Item storage + preserves fusion scores, and document purge erases their source quotes. The existing + `MESA_LEGAL_RULES_ENABLED` switch controls admission. + +- Constitutional Court names (`Anayasa Mahkemesi`, its suffix forms and `AYM`) + no longer activate Constitution citation ranking. Separate statute citations + in the same query are preserved. + +- Exact law/article matches now occupy at most three leading search positions + in document order, preserving the remaining order. Authorized document titles + and law headers provide scope-cached law names, including the `K.` abbreviation. +- Document article plans assign the existing typed legal identity to ordinary + articles and their continuation chunks. Generic 3–5 digit law-number + citations use the existing retrieval lane; `MESA_LEGAL_RULES_ENABLED=false` + disables automatic identity assignment for new chunks. Document fallback + assertions preserve their existing document/chunk graph and lexical shape. +- Open-ended SPO fact extraction (`MESA_EXTRACTION_ENABLED`) defaults to false + regardless of `MESA_MODEL_ENABLED`; an explicit true still enables it (D46). + ## [0.9.1] - 2026-11-01 This release combines Round 13 (speed) and Round 14 (retrieval quality and diff --git a/README.md b/README.md index 78f4671..64eaba9 100644 --- a/README.md +++ b/README.md @@ -56,14 +56,16 @@ mesa mcp-config ## Internal models (optional) -Models stay off by default. Documents are stored and found through BM25 lexical search without any model running. `FactExtractionService` owns canonical V4 extraction when models are enabled. +Models stay off by default. Documents are stored and found through BM25 lexical search without any model running. `FactExtractionService` owns canonical V4 extraction when explicitly enabled (`MESA_EXTRACTION_ENABLED=true`). -To enable semantic vector search and fact extraction with local Ollama, add to `~/.mesa/.env`: +To enable semantic vector search with local Ollama, add to `~/.mesa/.env`: ```bash MESA_MODEL_ENABLED=true -MESA_LLM_MODEL=ollama:qwen2.5:7b MESA_EMBEDDING_MODEL=ollama:bge-m3 +# To also enable open-ended fact extraction (defaults to false): +# MESA_EXTRACTION_ENABLED=true +# MESA_LLM_MODEL=ollama:qwen2.5:7b ``` Local `openai_compatible` servers (such as vLLM started with `--port 8001`, at `MESA_LLM_BASE_URL=http://127.0.0.1:8001/v1`; MESA itself uses port 8000) are also allowed with `MESA_EXTERNAL_PROVIDER_ENABLED=false` on a local endpoint (`localhost`, `127.0.0.1`, `::1`, `host.docker.internal`) and require no API key (D31). For cloud model providers (OpenAI, Anthropic), set `MESA_EXTERNAL_PROVIDER_ENABLED=true` and provide the required API keys. diff --git a/docker-compose.v4.yml b/docker-compose.v4.yml index dc99fb9..c6b4560 100644 --- a/docker-compose.v4.yml +++ b/docker-compose.v4.yml @@ -30,7 +30,7 @@ x-mesa-v4-runtime: &mesa-v4-runtime MESA_LOAD_DOTENV: "false" MESA_MODEL_ENABLED: ${MESA_MODEL_ENABLED:-false} MESA_EMBEDDING_ENABLED: ${MESA_EMBEDDING_ENABLED:-${MESA_MODEL_ENABLED:-false}} - MESA_EXTRACTION_ENABLED: ${MESA_EXTRACTION_ENABLED:-${MESA_MODEL_ENABLED:-false}} + MESA_EXTRACTION_ENABLED: ${MESA_EXTRACTION_ENABLED:-false} MESA_GRAPH_ENABLED: ${MESA_GRAPH_ENABLED:-true} MESA_EXTERNAL_PROVIDER_ENABLED: ${MESA_EXTERNAL_PROVIDER_ENABLED:-false} MESA_LLM_MODEL: ${MESA_LLM_MODEL:-} diff --git a/docs/configuration.md b/docs/configuration.md index 48702b2..4d37a34 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -40,6 +40,45 @@ even after restarting or changing this setting. Active revisions are unchanged. If an admitted prefix fits two different plans, the retry returns 409 without appending chunks, because the stored text cannot identify the original plan. +`MESA_LEGAL_RULES_ENABLED=true` enables Turkish legal rule processing. In auto +chunking mode, ordinary article chunks inherit the document's law number from +`metadata.law_number` or a `Kanun Numarası`, `Kanunun Numarası` or `Kanun No.` +header in the first 3,000 characters. Continuation pieces keep that article's +identity; preambles, GEÇİCİ/EK articles and plain plans receive none. Setting the +switch to false disables automatic identity assignment for new chunks. Stored +identities are immutable on resume. Any 3–5 digit law number is accepted by the +typed identity API; known numbers retain their existing statute code (6098 → +TBK). Unknown law names and inconsistent number/code pairs are rejected. +Numbered queries such as `7201 sayılı Kanunun 21. maddesi` use the existing +typed legal candidate lane and ranking factors. Document fallback assertions +keep their document subject, `madde` relation and chunk object; automatic +article identity changes only the typed columns. + +Newly admitted chunks also keep versioned citation, decision-reference, date, +amount, duration and outcome items. Search returns them in +`retrieval_provenance.legal_items` as `{version, items}`. Item `start`/`end` +offsets refer to the original chunk, including when a result quotes a shorter +extracted fact. Items remain separate from SPO facts and do not enter fusion +scores. Decision references can use the bounded exact-match slot below. +Disabling the switch prevents new item snapshots; existing +snapshots remain. Purging a document erases these source quotes too. + +An exact law-and-article query puts at most three matching chunks first, in +document order. Other results retain their fused order (or reranked order when +reranking is enabled). The match must exist in the authorized search scope. +A query's docket and decision numbers can also match a decision's own künye: +a stored `decision_ref` starting within the first 300 characters of chunk zero. +Its first chunk shares the same three-chunk limit with article matches, in +document order. Later quotations do not teach a decision's identity. The map +uses the same tenant, dataset, agent and time scope cache as law names; an +unmatched reference leaves result order unchanged. +The scope cache learns law names from titles and first-chunk law headers of +documents with trusted article identities; it never learns them from another +tenant, dataset, agent or time scope. Names accept case and Turkish ı/i variants, +possessive suffixes, and `K.` for `Kanunu`. The longest exact name from that scope +takes precedence over an overlapping ontology alias. Ambiguous store names +shared by different law numbers are ignored. No static law-name table is added. + `MESA_RERANK_ENABLED=false` keeps the existing fused ranking and does not import ML packages. Enabling it requires `pip install "mesa-memory[ml]"`; startup checks that the package is installed, and the model loads lazily on the first search. @@ -53,8 +92,8 @@ The first search includes model-load latency; subsequent searches reuse it. Model and external-provider flags default false. Validation defaults to mode 0; mode 1 or 2 must be set explicitly via `MESA_TIER3_MODE`. Mode 2 needs distinct A/B validator identities. Extraction and validation are separate. -`MESA_EMBEDDING_ENABLED` and `MESA_EXTRACTION_ENABLED` each default to -`MESA_MODEL_ENABLED`. A combined runtime accepts writes with both off and finds +`MESA_EMBEDDING_ENABLED` defaults to `MESA_MODEL_ENABLED`; `MESA_EXTRACTION_ENABLED` +defaults to `false` (D46). A combined runtime accepts writes with both off and finds them through BM25. Disabled VECTOR lanes end `SKIPPED`; enabling embedding later backfills them at startup using the same embedding identity. Backfill failures leave committed records searchable. diff --git a/mesa_api/v4_router.py b/mesa_api/v4_router.py index 9fba898..87971db 100644 --- a/mesa_api/v4_router.py +++ b/mesa_api/v4_router.py @@ -11,11 +11,21 @@ import asyncio import hashlib import logging +import re from datetime import datetime -from typing import Any, Callable, Literal +from typing import Annotated, Any, Callable, Literal from fastapi import APIRouter, Depends, Header, HTTPException, Request -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from fastapi.exceptions import RequestValidationError +from pydantic import ( + BaseModel, + ConfigDict, + Field, + ValidationError, + WrapValidator, + field_validator, + model_validator, +) from mesa_api.admission import require_mutation_admission as _require_mutation_admission from mesa_api.identifier_validation import PublicIdentifier, SourceIdentifier @@ -23,7 +33,12 @@ from mesa_memory.config import config, configured_embedding_identity from mesa_memory.consolidation.policy import ValidationPolicy from mesa_memory.context_builder import ContextBuilder -from mesa_memory.document_chunking import chunk_document, chunk_legal_document +from mesa_memory.document_chunking import ( + _ARTICLE_HEADING, + _CASE_LAW_PREAMBLE, + chunk_document, + chunk_legal_document, +) from mesa_memory.embedding.service import ( EmbeddingGenerationError, EmbeddingIdentityMismatchError, @@ -1148,12 +1163,22 @@ async def rebuild_index( request, idempotency_key, dao, access_control ) - @router.post("/memory/insert", status_code=202) - async def insert_memory( + @router.post("/memory/insert", status_code=202, name="insert_memory") + async def insert_memory_request( request: Request, payload: V4MemoryInsertRequest, dao: MemoryDAO = Depends(get_dao), access_control: AccessControl = Depends(get_access_control), + ) -> dict: + return await insert_memory(request, payload, dao, access_control) + + async def insert_memory( + request: Request, + payload: V4MemoryInsertRequest, + dao: MemoryDAO, + access_control: AccessControl, + *, + document_chunk: bool = False, ) -> dict: session = await _authorized_v4_session( request, @@ -1190,6 +1215,7 @@ async def insert_memory( chunk_ordinal=payload.chunk_ordinal, supersedes_revision_id=payload.supersedes_revision_id, metadata=payload.metadata, + document_chunk=document_chunk, legal_identity=( payload.legal_identity.model_dump(exclude_none=True) if payload.legal_identity is not None @@ -1232,10 +1258,23 @@ async def insert_memory( @router.post("/documents", response_model=V4DocumentIngestResponse, status_code=202) async def ingest_document( request: Request, - payload: V4DocumentIngestRequest, + payload: Annotated[ + V4DocumentIngestRequest, WrapValidator(lambda value, _handler: value) + ], dao: MemoryDAO = Depends(get_dao), access_control: AccessControl = Depends(get_access_control), ) -> dict: + # Defer, rather than omit, the same model validation. The annotation + # retains its public schema; no payload is used before validation. + try: + payload = await asyncio.to_thread( + V4DocumentIngestRequest.model_validate, payload + ) + except ValidationError as exc: + errors = exc.errors() + for error in errors: + error["loc"] = ("body", *error["loc"]) + raise RequestValidationError(errors) from exc session = await _authorized_v4_session( request, dao, access_control, payload.session_id, level="WRITE" ) @@ -1289,7 +1328,8 @@ async def ingest_document( raise ValueError( f"content equals revision {revision['revision_number']} " f"({revision['status']}); restoring an earlier revision " - "is not supported in v0.9" + "is not supported in v0.9. Add it under a new document_id " + "to store it again." ) admitted = await dao.list_v4_revision_mutations( **scope, revision_id=revision_id @@ -1305,10 +1345,11 @@ async def ingest_document( "mutation_ids": [admitted[i] for i in sorted(admitted)], "duplicate": True, } + legal_plan = config.chunking == "auto" chunks = ( - chunk_legal_document(payload.content) + await asyncio.to_thread(chunk_legal_document, payload.content) if config.chunking == "auto" - else chunk_document(payload.content) + else await asyncio.to_thread(chunk_document, payload.content) ) if admitted: # Admissions are already scoped above. Their durable payloads, @@ -1327,22 +1368,68 @@ def matches(plan: list[str]) -> bool: ) alternative = ( - chunk_document(payload.content) + await asyncio.to_thread(chunk_document, payload.content) if config.chunking == "auto" - else chunk_legal_document(payload.content) + else await asyncio.to_thread(chunk_legal_document, payload.content) ) preferred_matches = matches(chunks) alternative_matches = matches(alternative) if preferred_matches and alternative_matches and chunks != alternative: raise HTTPException( - 409, "admitted chunking plan is ambiguous; no chunks appended" + 409, + "admitted chunking plan is ambiguous; no chunks appended. " + "Retry with the MESA_CHUNKING value used when this revision was " + "first added, or add the content under a new document_id.", ) if not preferred_matches: if not alternative_matches: raise HTTPException( - 409, "admitted chunks do not match a supported chunking plan" + 409, + "admitted chunks do not match a supported chunking plan. " + "Retry with the MESA_CHUNKING value used when this revision was " + "first added, or add the content under a new document_id.", ) chunks = alternative + legal_plan = not legal_plan + identities: list[dict[str, str] | None] = [None] * len(chunks) + if config.legal_rules_enabled and legal_plan: + headings = list(_ARTICLE_HEADING.finditer(payload.content)) + header = re.search( + r"\bKanun(?:un)?\s+(?:Numarası\s*:|No\.?\s*:?)\s*([0-9]{3,5})\b", + payload.content[:3000], + re.IGNORECASE, + ) + law_number = str( + payload.metadata.get("law_number") or (header[1] if header else "") + ) + if ( + re.fullmatch(r"[0-9]{3,5}", law_number) + and len(headings) >= 3 + and not _CASE_LAW_PREAMBLE.search( + payload.content[: headings[0].start()] + ) + ): + heading_index = -1 + offset = 0 + for ordinal, chunk in enumerate(chunks): + while ( + heading_index + 1 < len(headings) + and headings[heading_index + 1].start() <= offset + ): + heading_index += 1 + if heading_index >= 0: + # Special/range/suffixed articles have no ordinary article identity. + ordinary = re.fullmatch( + r"\s*MADDE\s+([0-9]+)\s*[-–—−:]", + headings[heading_index][0], + re.IGNORECASE, + ) + if ordinary: + identities[ordinal] = { + "law_id": law_number, + "article_id": ordinary[1], + } + offset += len(chunk) duplicate = len(admitted) == len(chunks) for ordinal, chunk in enumerate(chunks): if ordinal in admitted: @@ -1353,6 +1440,7 @@ def matches(plan: list[str]) -> bool: V4MemoryInsertRequest( **payload.model_dump(exclude={"content"}), content=chunk, + legal_identity=identities[ordinal], revision_id=revision_id, chunk_id=chunk_id, revision_number=revision["revision_number"], @@ -1363,6 +1451,7 @@ def matches(plan: list[str]) -> bool: ), dao, access_control, + document_chunk=True, ) admitted[ordinal] = response["mutation_id"] return { diff --git a/mesa_memory/api/server.py b/mesa_memory/api/server.py index d69cfad..2edc2fd 100644 --- a/mesa_memory/api/server.py +++ b/mesa_memory/api/server.py @@ -378,6 +378,7 @@ async def _runtime_lifespan(app: FastAPI, runtime: RuntimeProfileConfig): "graph": config.rrf_graph_weight, }, search_scope_cache_enabled=config.search_scope_cache, + legal_rules_enabled=config.legal_rules_enabled, reranker=reranker, rerank_depth=config.rerank_depth, ) diff --git a/mesa_memory/config.py b/mesa_memory/config.py index 7f2994c..d047956 100644 --- a/mesa_memory/config.py +++ b/mesa_memory/config.py @@ -189,7 +189,7 @@ def load_runtime_profile( extraction_enabled = _parse_runtime_bool( values.get("MESA_EXTRACTION_ENABLED"), name="MESA_EXTRACTION_ENABLED", - default=model_enabled, + default=False, ) graph_enabled = _parse_runtime_bool( values.get("MESA_GRAPH_ENABLED"), name="MESA_GRAPH_ENABLED", default=True @@ -616,6 +616,7 @@ def vector_worker_limit(self) -> int: chunking: Literal["auto", "plain"] = Field( "auto", validation_alias="MESA_CHUNKING" ) + legal_rules_enabled: bool = Field(True, validation_alias="MESA_LEGAL_RULES_ENABLED") metrics_admission_threshold: float = 0.8 metrics_divergence_threshold: float = 0.5 diff --git a/mesa_storage/dao.py b/mesa_storage/dao.py index c7364f3..7d20ebe 100644 --- a/mesa_storage/dao.py +++ b/mesa_storage/dao.py @@ -34,10 +34,12 @@ import aiosqlite +from mesa_storage import legal_rules from mesa_storage.kuzu_provider import GraphSearchError, KuzuGraphProvider from mesa_storage.legal_identity import ( LegalEntityResolver, canonicalize_trusted_legal_identity, + normalize_law_name, ) from mesa_storage.rebuild_health import RebuildHealthReader from mesa_storage.repositories.catalog import CatalogRepository, CatalogRepositoryPort @@ -541,6 +543,7 @@ class MemoryDAO: "_rrf_k", "_rrf_weights", "_search_scope_cache_enabled", + "_legal_rules_enabled", "_search_scope_cache", "_search_cache_db", "_search_data_version", @@ -561,6 +564,7 @@ def __init__( rrf_k: int = V4_RRF_DEFAULT_K, rrf_weights: Mapping[str, float] | None = None, search_scope_cache_enabled: bool = True, + legal_rules_enabled: bool = True, reranker: Any = None, rerank_depth: int = 50, ) -> None: @@ -578,6 +582,7 @@ def __init__( self._rrf_k = rrf_k self._rrf_weights = dict(rrf_weights or V4_RRF_LANE_WEIGHTS) self._search_scope_cache_enabled = search_scope_cache_enabled + self._legal_rules_enabled = legal_rules_enabled self._search_scope_cache: OrderedDict[tuple[Any, ...], dict[str, Any]] = ( OrderedDict() ) @@ -1432,7 +1437,7 @@ async def purge_v4_document( key: value for key, value in metadata.items() if key.startswith("_mesa_") - and key != "_mesa_v4_projection_triplets" + and key not in {"_mesa_v4_projection_triplets", "_mesa_legal_items"} }, sort_keys=True, ) @@ -1914,6 +1919,7 @@ async def admit_v4_memory( policy: QueueAdmissionPolicy, validation_mode: int, legal_identity: Mapping[str, Any] | None = None, + document_chunk: bool = False, embedding_space_id: str | None = None, embedding_model_revision: str | None = None, embedding_normalized: bool = True, @@ -1965,6 +1971,8 @@ async def admit_v4_memory( for key, value in (metadata.items() if isinstance(metadata, dict) else ()) if not str(key).startswith("_mesa_") } + if document_chunk: + effective_metadata["_mesa_document_chunk"] = True normalized_legal_identity = canonicalize_trusted_legal_identity( {**legal_identity, "identity_source": "verified_ingestion"} if legal_identity is not None @@ -1992,6 +2000,11 @@ async def admit_v4_memory( "dimension": embedding_dimension, "normalized": embedding_normalized, } + if self._legal_rules_enabled: + effective_metadata["_mesa_legal_items"] = { + "version": legal_rules.RULES_VERSION, + "items": await asyncio.to_thread(legal_rules.extract, content_payload), + } raw_payload = { "tenant_id": tenant_id, @@ -3569,6 +3582,7 @@ async def get_projection_mutation(self, mutation_id: str) -> dict[str, Any] | No record["legal_identity"] = canonicalize_trusted_legal_identity( metadata.get("_mesa_legal_identity") ) + record["document_chunk"] = metadata.get("_mesa_document_chunk") is True record["projection_triplets"] = metadata.get("_mesa_v4_projection_triplets", []) return record @@ -5910,7 +5924,12 @@ async def _rerank_results( for row, score in zip(results, scores): row["rerank_score"] = float(score) # Python's stable sort retains fused order for tied scores. - return sorted(results, key=lambda row: -row["rerank_score"])[:limit] + ordered = sorted(results, key=lambda row: -row["rerank_score"]) + exact = sorted( + (row for row in ordered if "exact_match" in row["retrieval_provenance"]), + key=lambda row: row["retrieval_provenance"]["exact_match"]["order"], + )[:3] + return (exact + [row for row in ordered if row not in exact])[:limit] async def _clear_search_scope_cache(self, db: aiosqlite.Connection) -> None: for scope in self._search_scope_cache.values(): @@ -5968,10 +5987,6 @@ async def _search_v4_memory_uncached( scope = _scope if _scope is not None else {} scoped_fts = scope.get("fts_table", "v4_scoped_assertions_fts") text_features = scope.setdefault("text_features", {}) - legal_resolver = LegalEntityResolver() - legal_citations = legal_resolver.extract_citations(query) - target_statutes = {c.statute_code for c in legal_citations} - historical_query = bool(valid_at or valid_from or valid_to) assertion_status_sql = ( "(a.status = 'ACTIVE' OR (a.status = 'SUPERSEDED' AND a.valid_to != ''))" @@ -6254,6 +6269,180 @@ async def _search_v4_memory_uncached( } in_scope_assertions = scope["assertions"] in_scope_assertion_ids = set(in_scope_assertions) + if "legal_context" not in scope: + # Only authorized assertions select revisions. Header/title names and + # chunk ordinals share the existing data_version scope-cache lifetime. + revisions = sorted( + {str(a["revision_id"]) for a in in_scope_assertions.values()} + ) + revision_laws: dict[str, set[str]] = {} + identities: dict[str, dict[str, str]] = {} + for assertion in in_scope_assertions.values(): + identity = _assertion_legal_identity(assertion) + if identity: + identities[str(assertion["assertion_id"])] = identity + revision_laws.setdefault(str(assertion["revision_id"]), set()).add( + identity["law_id"] + ) + names: dict[str, set[str]] = {} + decisions: dict[tuple[str, str], set[str]] = {} + authorized_mutations: dict[str, set[str]] = {} + for aid, assertion in in_scope_assertions.items(): + authorized_mutations.setdefault( + str(assertion["mutation_id"]), set() + ).add(aid) + chunk_order: dict[str, tuple[str, str, int]] = {} + authorized_chunks = {str(a["chunk_id"]) for a in in_scope_assertions.values()} + named_revisions: set[str] = set() + async with self._search_read_connection(_search_db) as db: + for offset in range(0, len(revisions), 500): + batch = revisions[offset : offset + 500] + marks = ",".join("?" for _ in batch) + async with db.execute( + "SELECT c.chunk_id,c.ordinal,c.revision_id,r.document_id,d.title," + "CASE WHEN c.ordinal=0 THEN c.content_payload ELSE '' END AS header," + "m.mutation_id,CASE WHEN c.ordinal=0 THEN m.metadata_json ELSE '{}' END AS rule_metadata " + "FROM source_chunks c JOIN document_revisions r ON r.revision_id=c.revision_id " + "JOIN documents d ON d.document_id=r.document_id " + "LEFT JOIN memory_mutations m ON m.chunk_id=c.chunk_id AND m.tenant_id=c.tenant_id " + f"WHERE c.tenant_id=? AND c.revision_id IN ({marks})", + (tenant_id, *batch), + ) as cursor: + for row in await cursor.fetchall(): + if str(row["chunk_id"]) not in authorized_chunks: + continue + revision = str(row["revision_id"]) + chunk_order[str(row["chunk_id"])] = ( + str(row["document_id"]), + revision, + int(row["ordinal"]), + ) + # Only the authorized first chunk can teach its own + # künye; later quotations and out-of-scope mutations + # of the same revision cannot enter the map. + mid = str(row["mutation_id"]) + if row["ordinal"] == 0 and mid in authorized_mutations: + item_set = _decode_registry_metadata( + row["rule_metadata"] + ).get("_mesa_legal_items") + items = ( + item_set.get("items", []) + if isinstance(item_set, dict) + else [] + ) + for item in items if isinstance(items, list) else (): + if not isinstance(item, dict): + continue + start = item.get("start") + docket = item.get("docket_no") + decision = item.get("decision_no") + if ( + item.get("type") == "decision_ref" + and isinstance(start, int) + and 0 <= start < 300 + and isinstance(docket, str) + and docket + and isinstance(decision, str) + and decision + ): + decisions.setdefault( + (docket, decision), set() + ).update(authorized_mutations[mid]) + laws = revision_laws.get(revision, set()) + if len(laws) != 1: + continue + header = re.split( + r"\bKanun(?:un)?\s+(?:Numarası|No\.)", + str(row["header"] or "")[:3000], + maxsplit=1, + flags=re.IGNORECASE, + ) + header_name = "" + if len(header) == 2: + header_tail = header[0].rstrip() + end = len(header_tail) + while end and header_tail[end - 1].isdecimal(): + end -= 1 + header_name = header_tail[:end].strip() + revision_names = [header_name] + if revision not in named_revisions: + revision_names.append(str(row["title"] or "")) + named_revisions.add(revision) + for name in revision_names: + if name and len(name) <= 200: + names.setdefault( + normalize_law_name(name), set() + ).update(laws) + scope["legal_context"] = { + "names": { + name: next(iter(laws)) + for name, laws in names.items() + if len(laws) == 1 + }, + "chunk_order": chunk_order, + "identities": identities, + "decisions": decisions, + "ordered_assertions": sorted( + in_scope_assertions, + key=lambda aid: ( + *chunk_order.get( + str(in_scope_assertions[aid]["chunk_id"]), + ( + str(in_scope_assertions[aid]["document_id"]), + str(in_scope_assertions[aid]["revision_id"]), + 0, + ), + ), + aid, + ), + ), + } + legal_context = scope["legal_context"] + legal_resolver = LegalEntityResolver(law_names=legal_context["names"]) + # Resolver positions stay internal; outer whitespace carries no identity. + # Avoid retrying its boundary patterns along an empty query prefix. + legal_citations = await asyncio.to_thread( + legal_resolver.extract_citations, query.strip() + ) + target_statutes = {c.statute_code for c in legal_citations} + exact_keys = {(c.statute_code, c.article) for c in legal_citations if c.article} + decision_ids: set[str] = set() + query_items = ( + await asyncio.to_thread(legal_rules.extract, query) + if legal_context["decisions"] + else () + ) + for item in query_items: + if item["type"] == "decision_ref": + decision_ids.update( + legal_context["decisions"].get( + (item.get("docket_no"), item.get("decision_no")), () + ) + ) + exact_ids: list[str] = [] + exact_kinds: dict[str, str] = {} + seen_chunks: set[str] = set() + for aid in ( + legal_context["ordered_assertions"] if exact_keys or decision_ids else () + ): + assertion = in_scope_assertions[aid] + identity = legal_context["identities"].get(aid) + chunk = str(assertion["chunk_id"]) + if ( + identity + and (identity["statute_code"], identity["article_id"]) in exact_keys + or aid in decision_ids + ) and chunk not in seen_chunks: + exact_ids.append(aid) + exact_kinds[aid] = ( + "article" + if identity + and (identity["statute_code"], identity["article_id"]) in exact_keys + else "decision" + ) + seen_chunks.add(chunk) + if len(exact_ids) == 3: + break typed_legal_candidate_ids: set[str] = set() if legal_citations: identity_clauses: list[str] = [] @@ -6298,6 +6487,9 @@ async def _search_v4_memory_uncached( typed_legal_candidate_ids = { str(row["assertion_id"]) for row in await cursor.fetchall() } + typed_legal_candidate_ids.update( + aid for aid in exact_ids if exact_kinds[aid] == "article" + ) in_scope_entity_ids_from_assertions = { str(row["subject_id"]) for row in in_scope_assertion_rows @@ -6527,6 +6719,10 @@ async def _search_v4_memory_uncached( continue content_tokens.append(t) + # Keep the existing filters/order and substring matching, but bound + # repeated scans of API-sized candidate text. + scoring_content_tokens = content_tokens[:64] + lexical_lane: list[str] = [] lexical_assertions: dict[str, dict[str, Any]] = {} lexical_scores: dict[str, float] = {} @@ -6649,14 +6845,16 @@ async def _search_v4_memory_uncached( p_score = -raw_bm25 matched_tokens = 0 - for token in content_tokens: + for token in scoring_content_tokens: if token in p_text: matched_tokens += 1 p_score += 10.0 + min(len(token), 10) - if content_tokens and matched_tokens == len(content_tokens): + if scoring_content_tokens and matched_tokens == len( + scoring_content_tokens + ): p_score += 50.0 - clean_query = " ".join(content_tokens) + clean_query = " ".join(scoring_content_tokens) if clean_query and clean_query in p_text: p_score += 100.0 @@ -6678,7 +6876,9 @@ async def _search_v4_memory_uncached( else: if "passage_citations" not in features: features["passage_citations"] = ( - legal_resolver.extract_citations(p_raw_text) + await asyncio.to_thread( + legal_resolver.extract_citations, p_raw_text + ) ) row_citations = features["passage_citations"] row_identities = { @@ -6740,7 +6940,7 @@ async def _search_v4_memory_uncached( lexical_entity_lane.append(eid) # Assertion Lane Redesign (Phase 5) - tokens_to_match = content_tokens if content_tokens else raw_tokens + tokens_to_match = (content_tokens if content_tokens else raw_tokens)[:64] # Reuse the scoped FTS candidate pool. This avoids admitting rows # through SQLite's ASCII-only LIKE casing before Turkish-aware scoring. assertion_rows = passage_candidate_rows @@ -6835,7 +7035,8 @@ async def _search_v4_memory_uncached( else: if "assertion_citations" not in features: features["assertion_citations"] = ( - legal_resolver.extract_citations( + await asyncio.to_thread( + legal_resolver.extract_citations, " ".join( str(a_row.get(field) or "") for field in ( @@ -7151,6 +7352,9 @@ async def _search_v4_memory_uncached( "assertion": assertion_lane, "graph": graph_lane, } + lane_rankings["assertion"] = assertion_lane + [ + aid for aid in exact_ids if aid not in assertion_lane + ] for candidate_id, fused_score, lane_ranks in rrf_fuse_lanes( lane_rankings, k=effective_k, @@ -7300,7 +7504,8 @@ async def _search_v4_memory_uncached( tuple(cand_texts), ) if citation_key not in text_features: - text_features[citation_key] = legal_resolver.extract_citations( + text_features[citation_key] = await asyncio.to_thread( + legal_resolver.extract_citations, " ".join(cand_texts) ) candidate_citations = text_features[citation_key] @@ -7332,6 +7537,11 @@ async def _search_v4_memory_uncached( ordered_candidates = sorted( candidates.values(), key=lambda c: (-(c["rrf_score"] * c["legal_factor"]), c["candidate_id"]), + ) + exact_candidates = [candidates[aid] for aid in exact_ids if aid in candidates] + ordered_candidates = ( + exact_candidates + + [c for c in ordered_candidates if c["candidate_id"] not in exact_ids] )[:limit] needed_entity_ids = sorted( @@ -7415,6 +7625,7 @@ async def _search_v4_memory_uncached( } ) metadata_by_mutation: dict[str, dict[str, Any]] = {} + legal_items_by_mutation: dict[str, dict[str, Any]] = {} rerank_texts: dict[str, str] = {} async with self._search_read_connection(_search_db) as db: if mutation_ids: @@ -7434,6 +7645,9 @@ async def _search_v4_memory_uncached( except (TypeError, json.JSONDecodeError): metadata = {} if isinstance(metadata, dict): + item_set = metadata.get("_mesa_legal_items") + if isinstance(item_set, dict): + legal_items_by_mutation[str(row["mutation_id"])] = item_set metadata_by_mutation[str(row["mutation_id"])] = { key: value for key, value in metadata.items() @@ -7494,6 +7708,11 @@ async def _search_v4_memory_uncached( "lane_ranks": cand["lane_ranks"], "raw_scores": cand["raw_scores"], } + if cand["candidate_id"] in exact_ids: + retrieval_provenance["exact_match"] = { + "kind": exact_kinds[cand["candidate_id"]], + "order": exact_ids.index(cand["candidate_id"]), + } if cand["assertion_id"]: retrieval_provenance["assertion_id"] = cand["assertion_id"] if cand["source_chunk_id"]: @@ -7526,6 +7745,9 @@ async def _search_v4_memory_uncached( cand_prov = cand["materialized_provenance"] if not cand_prov: continue + item_set = legal_items_by_mutation.get(str(cand_prov[0]["mutation_id"])) + if item_set is not None: + retrieval_provenance["legal_items"] = item_set result = { "entity": entities.get(eid, {}), "candidate_id": cand["candidate_id"], diff --git a/mesa_storage/kuzu_provider.py b/mesa_storage/kuzu_provider.py index ee9ad3f..12597ce 100644 --- a/mesa_storage/kuzu_provider.py +++ b/mesa_storage/kuzu_provider.py @@ -1654,6 +1654,7 @@ def _compute_path_score( # 3. Query, predicate, and evidence relevance pred_bonus = 0.0 + query_tokens = list(dict.fromkeys(query_tokens))[:64] if query_tokens: for pred, ev, jur in zip(predicates, evidence_spans, jurisdictions): p_norm = _normalize_text(pred) diff --git a/mesa_storage/legal_identity.py b/mesa_storage/legal_identity.py index 6a7cead..243a1f4 100644 --- a/mesa_storage/legal_identity.py +++ b/mesa_storage/legal_identity.py @@ -23,6 +23,12 @@ def normalize_turkish(text: str) -> str: return unicodedata.normalize("NFKC", cleaned).lower() +def normalize_law_name(text: str) -> str: + """Normalize exact store names, including the conventional Kanunu abbreviation.""" + text = normalize_turkish(text).replace("ı", "i") + return " ".join(re.sub(r"\bk\.(?=\W|$)", "kanunu", text).split()) + + @dataclass(frozen=True) class LegalCitation: statute_code: str @@ -222,11 +228,17 @@ def canonicalize_trusted_legal_identity( raise ValueError("legal identity requires law_id and article_id") def resolve_code(reference: str) -> str | None: - return resolver.alias_to_code.get(normalize_turkish(reference)) + if re.fullmatch(r"[0-9]{3,5}", reference): + return next( + (code for code, number in LAW_IDS.items() if number == reference), + reference, + ) + code = resolver.alias_to_code.get(normalize_turkish(reference)) + return code if code in resolver.laws else None law_code = resolve_code(law_ref) statute_code = resolve_code(statute_ref) if statute_ref else law_code - if law_code not in resolver.laws or statute_code != law_code: + if law_code is None or statute_code != law_code: raise ValueError("legal identity law aliases are unknown or inconsistent") article_match = _ARTICLE_PATTERN.fullmatch(article_ref) @@ -238,7 +250,7 @@ def resolve_code(reference: str) -> str | None: raise ValueError("legal identity source is not trusted") normalized: dict[str, str] = { - "law_id": LAW_IDS[law_code], + "law_id": LAW_IDS.get(law_code, law_code), "statute_code": law_code, "article_id": article_match.group("article"), "version_id": str(value.get("version_id") or "").strip(), @@ -254,7 +266,7 @@ def resolve_code(reference: str) -> str | None: class LegalEntityResolver: - def __init__(self) -> None: + def __init__(self, *, law_names: Mapping[str, str] | None = None) -> None: self.ontology = ONTOLOGY self.laws = { "TBK", @@ -269,36 +281,165 @@ def __init__(self) -> None: "Anayasa", } self.alias_to_code: dict[str, str] = {} + self._law_names = { + normalize_law_name(name): number + for name, number in (law_names or {}).items() + if name and re.fullmatch(r"[0-9]{3,5}", number) + } + name_patterns = [] + for name in sorted(self._law_names, key=len, reverse=True): + escaped = re.escape(name) + if name.endswith("kanunu"): + escaped = re.escape(name[:-1]) + r"u?" + name_patterns.append(escaped) + self._scope_name_pattern = ( + re.compile( + r"(?" + "|".join(name_patterns) + r")" + r"(?:['’]?(?:nun|un|nin|in))?(?!\w)" + r"(?:\s{0,128}+(?:(?:m\.|md\.?|madde)\s{0,128}+(?P[0-9]{1,12}+(?![0-9]))" + r"|(?P[0-9]{1,12}+(?![0-9]))(?:\.|\s{1,128}+(?:inci|uncu|üncü|nci|ncu|ncü))?\s{1,128}+madde(?:si)?))?" + ) + if name_patterns + else None + ) for code, info in self.ontology.items(): for alias in info["aliases"]: self.alias_to_code[normalize_turkish(alias)] = code aliases = sorted(self.alias_to_code, key=len, reverse=True) alias_pattern = "|".join(re.escape(alias) for alias in aliases) + # A legal token has at most 12 digits and each layout gap at most 128 + # characters. Possessive runs cannot backtrack or parse a digit suffix + # as a different article. Long input is still scanned in full. self._forward_pattern = re.compile( rf"(?:(?<=\W)|^)(?P{alias_pattern})(?:(?=\W)|$)" rf"(?P['’]n?[ıiuü]n)?" - rf"(?:\s*(?Pm\.|md\.?|madde))?\s*" - rf"(?P\d+)?(?:\.|\'[a-zçğıöşü]+)?(?:\s*madde(?:si)?)?", + rf"\s{{0,128}}+(?:(?Pm\.|md\.?|madde)\s{{0,128}}+)?" + rf"(?P\d{{1,12}}+(?!\d))?(?:\.|\'[a-zçğıöşü]{{1,12}}+)?(?:\s{{0,128}}+madde(?:si)?)?", re.IGNORECASE, ) self._reverse_pattern = re.compile( - rf"(?Pm\.|md\.?|madde)?\s*(?P\d+)(?:\.|\'[a-zçğıöşü]+)?(?:\s*madde(?:si)?)?(?:\s+(?:uyarınca|gereğince|kapsamında|hükmü|hükmünce|düzenlemesi))?\s+(?P{alias_pattern})(?:(?=\W)|$)", + rf"(?Pm\.|md\.?|madde)?\s{{0,128}}+(?\d{{1,12}}+(?!\d))(?:\.|\'[a-zçğıöşü]{{1,12}}+)?(?:\s{{0,128}}+madde(?:si)?)?(?:\s{{1,128}}+(?:uyarınca|gereğince|kapsamında|hükmü|hükmünce|düzenlemesi))?\s{{1,128}}+(?P{alias_pattern})(?:(?=\W)|$)", re.IGNORECASE, ) self._continued_article_pattern = re.compile( - r"\s*(?:(?:-|–|—|ila)\s*|(?:,|;|ve|ile)\s*(?:m\.|md\.?|madde)\s*)" - r"(?P\d+)(?:\.|\'[a-zçğıöşü]+)?(?:\s*madde(?:si)?)?", + r"\s{0,128}+(?:(?:-|–|—|ila)\s{0,128}+|(?:,|;|ve|ile)\s{0,128}+(?:m\.|md\.?|madde)\s{0,128}+)" + r"(?P\d{1,12}+(?!\d))(?:\.|\'[a-zçğıöşü]{1,12}+)?(?:\s{0,128}+madde(?:si)?)?", re.IGNORECASE, ) + self._numbered_law_pattern = re.compile( + r"\b(?P[0-9]{3,5})\s{1,128}+sayılı\s{1,128}+" + r"(?:[^\n.;!?]{0,160}?(?<=\S)\s{1,128}+)?kanun(?:u)?(?:['’]?(?:nun|un|nın|ın))?\s{1,128}+" + r"(?:(?:m\.|md\.?|madde)\s{0,128}+(?P[0-9]{1,12}+(?![0-9]))" + r"|(?P[0-9]{1,12}+(?![0-9]))(?:\.|\s{1,128}+(?:ıncı|inci|uncu|üncü|nci|ncı|ncu|ncü))?\s{1,128}+madde(?:si)?)", + ) + self._duration_pattern = re.compile( + r"\s{0,128}+(?:aylık|yıllık|günlük|haftalık|saatlik|dakikalık|" + r"ay|yıl|gün|hafta|saat|dakika)(?:\W|$)" + ) def extract_citations(self, text: str) -> list[LegalCitation]: if not text: return [] + # Mask court mentions before both store-name and ontology matching. + # Keep offsets and independent Constitution/article citations intact. + text = re.sub( + r"\banayasa\s+mahkemesi\w*|\baym\b", + lambda match: " " * len(match[0]), + text, + flags=re.IGNORECASE, + ) + if self._scope_name_pattern is not None: + normalized_name_text = normalize_law_name(text) + matches = list(self._scope_name_pattern.finditer(normalized_name_text)) + if matches: + scoped_citations = [] + # Name normalization folds Turkish letters, expands K. and + # collapses whitespace. Map its spans back before masking so + # independent citations retain their spelling and grammar. + source_text = normalize_turkish(text) + source_spans: list[tuple[int, int]] = [] + for piece in re.finditer( + r"\s+|\bk\.(?=\W|$)|.", source_text, re.DOTALL + ): + value = piece[0] + if value.isspace(): + if not source_spans: + continue + width = 1 + else: + width = len("kanunu") if value == "k." else 1 + source_spans.extend([piece.span()] * width) + if source_spans and source_text[-1].isspace(): + source_spans.pop() + remainder = list(source_text) + for match in matches: + name = match["name"] + number = self._law_names.get(name) + if number is None: + number = self._law_names.get(name + "u") + if number is None: + continue + scoped_code = next( + (c for c, n in LAW_IDS.items() if n == number), number + ) + scoped_article = match["marked"] or match["ordinal"] + scoped_citations.append( + LegalCitation( + statute_code=scoped_code, + statute_canonical=name, + article=scoped_article, + canonical_name=f"{scoped_code} m.{scoped_article}" + if scoped_article + else scoped_code, + aliases=(name,), + ) + ) + start = source_spans[match.start()][0] + end = source_spans[match.end() - 1][1] + remainder[start:end] = " " * (end - start) + # A store name consumes its entire span before ontology matching, + # so a shorter ontology alias cannot reinterpret that same name. + scoped_citations.extend( + LegalEntityResolver().extract_citations("".join(remainder)) + ) + return list(dict.fromkeys(scoped_citations)) normalized = normalize_turkish(text) raw_matches: list[dict[str, Any]] = [] + # Numbered statutes use the same typed candidate lane as ontology aliases. + numbered_matches = list(self._numbered_law_pattern.finditer(normalized)) + for match in numbered_matches: + number = match.group("number") + code = next((code for code, n in LAW_IDS.items() if n == number), number) + raw_matches.append( + { + "code": code, + "article": match.group("marked") or match.group("ordinal"), + "start": match.start(), + "end": match.end(), + "statute_match": number, + "article_span": match.span("marked") + if match.group("marked") + else match.span("ordinal"), + } + ) + + # Both streams advance left to right; never re-scan numbered matches. + numbered_index = 0 + article_codes: dict[tuple[int, int], set[str]] = {} + # Find forward matches (statute [article]) for match in self._forward_pattern.finditer(normalized): + while ( + numbered_index < len(numbered_matches) + and numbered_matches[numbered_index].end() <= match.start() + ): + numbered_index += 1 + if ( + numbered_index < len(numbered_matches) + and numbered_matches[numbered_index].start() <= match.start() + ): + continue statute_match = match.group("statute") code = self.alias_to_code.get(statute_match) if not code: @@ -308,11 +449,7 @@ def extract_citations(self, text: str) -> list[LegalCitation]: article and match.group("possessive") and not match.group("forward_art_marker") - and re.match( - r"\s*(?:aylık|yıllık|günlük|haftalık|saatlik|dakikalık|" - r"ay|yıl|gün|hafta|saat|dakika)(?:\W|$)", - normalized[match.end() :], - ) + and self._duration_pattern.match(normalized, match.end()) ): article = None @@ -360,6 +497,10 @@ def extract_citations(self, text: str) -> list[LegalCitation]: ) continuation_start = continued.end() + for item in raw_matches: + if item.get("article_span") is not None: + article_codes.setdefault(item["article_span"], set()).add(item["code"]) + # Find reverse matches ([article] statute) for match in self._reverse_pattern.finditer(normalized): statute_match = match.group("statute") @@ -373,10 +514,7 @@ def extract_citations(self, text: str) -> list[LegalCitation]: # A number already attached to a preceding statute cannot also # become the next statute's reverse citation (TBK 117 CMK 86). - if any( - prior.get("article_span") == match.span("art") and prior["code"] != code - for prior in raw_matches - ): + if article_codes.get(match.span("art"), set()) - {code}: continue # Guard: bare numbers before "ay" (e.g. "3 ay") are durations, NOT Anayasa citations! @@ -402,33 +540,47 @@ def extract_citations(self, text: str) -> list[LegalCitation]: if not raw_matches: return [] - # Semantic Deduplication: - # If a match with an article overlaps with or subsumes a match without an article for the same statute, - # keep the specific article citation (e.g. "117 TBK" -> TBK m.117, NOT both TBK and TBK m.117). - filtered_matches: list[dict[str, Any]] = [] - for i, m1 in enumerate(raw_matches): - is_subsumed = False - for j, m2 in enumerate(raw_matches): - if i == j: - continue - if m1["code"] != m2["code"]: - continue - # If m1 has no article but m2 has an article and their spans overlap or touch - if m1["article"] is None and m2["article"] is not None: - # Check overlap or containment - if max(m1["start"], m2["start"]) <= min(m1["end"], m2["end"]): - is_subsumed = True - break - # If both have the same article and overlap, keep the larger span - if m1["article"] == m2["article"] and m1["article"] is not None: - if max(m1["start"], m2["start"]) < min(m1["end"], m2["end"]): - span1 = m1["end"] - m1["start"] - span2 = m2["end"] - m2["start"] - if span1 < span2 or (span1 == span2 and i > j): - is_subsumed = True - break - if not is_subsumed: - filtered_matches.append(m1) + # Bucket by source offset to preserve a linear sweep (no match sorting). + # All raw patterns have bounded spans. At any offset only matches from + # that fixed-size window can overlap, so the active set is bounded too. + # Keep the original raw order and the old strict/touching overlap rules. + # Empty per-character lists trigger thousands of GC-tracked allocations + # even for a single citation. Allocate buckets only where matches start. + starts: list[list[int] | None] = [None] * (len(normalized) + 1) + for index, item in enumerate(raw_matches): + bucket = starts[item["start"]] + if bucket is None: + starts[item["start"]] = [index] + else: + bucket.append(index) + subsumed: set[int] = set() + active: list[int] = [] + for offset, indexes in enumerate(starts): + if not indexes: + continue + active = [i for i in active if raw_matches[i]["end"] >= offset] + for index in indexes: + item = raw_matches[index] + for prior_index in active: + prior = raw_matches[prior_index] + if item["code"] != prior["code"]: + continue + if item["article"] is None and prior["article"] is not None: + subsumed.add(index) + elif prior["article"] is None and item["article"] is not None: + subsumed.add(prior_index) + elif ( + item["article"] is not None + and item["article"] == prior["article"] + and offset < prior["end"] + ): + priority = (item["end"] - offset, -index) + prior_priority = (prior["end"] - prior["start"], -prior_index) + subsumed.add( + prior_index if priority > prior_priority else index + ) + active.append(index) + filtered_matches = [m for i, m in enumerate(raw_matches) if i not in subsumed] citations: list[LegalCitation] = [] seen: set[tuple[str, str | None]] = set() @@ -437,7 +589,10 @@ def extract_citations(self, text: str) -> list[LegalCitation]: if key in seen: continue seen.add(key) - info = self.ontology[m["code"]] + info = self.ontology.get( + m["code"], + {"canonical": f"{m['code']} sayılı Kanun", "aliases": [m["code"]]}, + ) article = m["article"] citations.append( LegalCitation( @@ -446,7 +601,7 @@ def extract_citations(self, text: str) -> list[LegalCitation]: article=article, canonical_name=( f"{m['code']} m.{article}" - if article and m["code"] in self.laws + if article and (m["code"] in self.laws or m["code"].isdigit()) else m["code"] ), aliases=tuple(info["aliases"]), diff --git a/mesa_storage/legal_rules.py b/mesa_storage/legal_rules.py new file mode 100644 index 0000000..42c74a9 --- /dev/null +++ b/mesa_storage/legal_rules.py @@ -0,0 +1,1128 @@ +"""Turkish legal rule-based item extractor for MESA. + +Standard library only. Layer 0 (mesa_storage). +Schema: mesa_bench/extract/schema.json v1.3. +Extracts RULE types: citation, decision_ref, date, amount, duration, outcome. +""" + +import re +from collections.abc import Iterator +from typing import Any, Dict, List, Optional, Tuple, cast + +RULES_VERSION = "tr-legal-rules-3" + +def _up_to(piece: str, count: int) -> str: + """Spell out a small bounded sequence without nested regex repetitions. + + Each piece has bounded terminal runs separated by delimiters. Sharing the + prefix avoids rescanning it for every possible list length. Empty branches + let the following clause select the end, without a repeated regex group. + """ + tail = "" + for _ in range(count): + tail = "(?:" + piece + tail + "|)" + return tail + + +def _span_ends(length: int, spans: list[tuple[int, int]]) -> list[int]: + """Maximum end of any span starting at or before each character, O(n).""" + ends = [0] * (length + 1) + if not spans: + return ends + for start, end in spans: + ends[start] = max(ends[start], end) + furthest = 0 + for i, end in enumerate(ends): + if end > furthest: + furthest = end + ends[i] = furthest + return ends + + +_ARTICLE_NUMBER = r"\d{1,12}+" + _up_to(r"\.\d{1,12}+", 5) + +# ---------------------------------------------------------------------- +# 1. Turkish number parsing +# ---------------------------------------------------------------------- +_NUM_TOKENS: List[Tuple[str, int]] = [ + ("milyar", 1000000000), + ("milyon", 1000000), + ("bin", 1000), + ("yüz", 100), + ("yuz", 100), + ("doksan", 90), + ("seksen", 80), + ("yetmiş", 70), + ("yetmis", 70), + ("altmış", 60), + ("altmis", 60), + ("elli", 50), + ("kırk", 40), + ("kirk", 40), + ("otuz", 30), + ("yirmi", 20), + ("on", 10), + ("dokuz", 9), + ("sekiz", 8), + ("yedi", 7), + ("altı", 6), + ("alti", 6), + ("beş", 5), + ("bes", 5), + ("dört", 4), + ("dort", 4), + ("üç", 3), + ("uc", 3), + ("iki", 2), + ("bir", 1), +] + + +def parse_turkish_number(s: str) -> Optional[int]: + """Parse integer or Turkish number words into int.""" + s = s.strip() + if s.isdigit(): + return int(s) + words = s.lower().replace("’", "'").split() + flat_tokens: List[Tuple[str, int]] = [] + for w in words: + w_clean = re.sub(r"[^a-zçğıöşü]", "", w) + if not w_clean: + continue + i = 0 + while i < len(w_clean): + matched = False + for tok, val in _NUM_TOKENS: + if w_clean.startswith(tok, i): + flat_tokens.append((tok, val)) + i += len(tok) + matched = True + break + if not matched: + return None + if not flat_tokens: + return None + + total = 0 + current_scale_acc = 0 + current_hundreds_acc = 0 + + for tok, val in flat_tokens: + if val in (1000000000, 1000000, 1000): + scale_multiplier = val + num = current_scale_acc + current_hundreds_acc + if num == 0: + num = 1 + total += num * scale_multiplier + current_scale_acc = 0 + current_hundreds_acc = 0 + elif val == 100: + h_mult = current_hundreds_acc if current_hundreds_acc > 0 else 1 + current_scale_acc += h_mult * 100 + current_hundreds_acc = 0 + else: + current_hundreds_acc += val + + total += current_scale_acc + current_hundreds_acc + return total + + +# ---------------------------------------------------------------------- +# 2. Durations +# ---------------------------------------------------------------------- +_NUM_ROOTS_STR = ( + r"(?:bir|iki|üç|uc|dört|dort|beş|bes|altı|alti|yedi|sekiz|dokuz|" + r"on|yirmi|otuz|kırk|kirk|elli|altmış|altmis|yetmiş|yetmis|seksen|doksan|" + r"yüz|yuz|bin|milyon|milyar)" +) +_NUM_WORD_TOKEN_STR = r"(?:" + _NUM_ROOTS_STR + r"[a-zçğıöşü]{0,64}+)" +_NUMBER_TOKEN = re.compile(r"\b(?:\d{1,12}+|" + _NUM_WORD_TOKEN_STR + r"\b)", re.IGNORECASE) + + +def _number_spans(text: str, *, words_only: bool = False) -> Iterator[tuple[int, int]]: + """Collect at most twelve adjacent number words in a single token pass.""" + chain: List[int] = [] + previous_end = -1 + for token in _NUMBER_TOKEN.finditer(text): + start, end = token.span() + word = not token.group().isdigit() + gap = text[previous_end:start] if previous_end >= 0 else "" + if not word or not chain or not (0 < len(gap) <= 4 and gap.isspace()): + chain = [] + if word: + chain.append(start) + chain = chain[-12:] + if word or not words_only: + yield (chain[0] if word else start), end + previous_end = end + + +_DURATION_PAT = re.compile( + r"(?:(?<=\d)\s{0,4}+|(?<=[a-zçğıöşü])\s{1,4}+)" + r"(?P" + r"(?Pi\s{0,4}+ş\s{1,4}+g\s{0,4}+ü\s{0,4}+n\s{0,4}+ü|g\s{0,4}+ü\s{0,4}+n|h\s{0,4}+a\s{0,4}+f\s{0,4}+t\s{0,4}+a|a\s{0,4}+y|y\s{0,4}+ı\s{0,4}+l|s\s{0,4}+a\s{0,4}+a\s{0,4}+t|(?:eğitim-öğretim\s{1,4}+)?y\s{0,4}+a\s{0,4}+r\s{0,4}+ı\s{0,4}+y\s{0,4}+ı\s{0,4}+l)" + r"[a-zçğıöşü]{0,64}+" + r")\b", + re.IGNORECASE, +) + +_MONTH_NAMES = { + "ocak", "şubat", "mart", "nisan", "mayıs", "haziran", "temmuz", + "ağustos", "eylül", "ekim", "kasım", "aralık" +} + + +def extract_durations(text: str) -> List[Dict[str, Any]]: + items = [] + for start_pos, num_end in _number_spans(text): + m = _DURATION_PAT.match(text, num_end) + if m is None: + continue + # Suppress match if preceded by date like 14.6.1982 gün + prefix = text[max(0, start_pos - 10):start_pos] + if re.search(r"\d{1,12}+[./]\s{0,4}+$", prefix): + continue + + num_str = text[start_pos:num_end].strip() + words = num_str.split() + parsed_val = None + best_words: List[str] = [] + for i in range(len(words), 0, -1): + cand = " ".join(words[:i]) + val_num = parse_turkish_number(cand) + if val_num is not None and val_num > 0: + parsed_val = val_num + best_words = words[:i] + break + if parsed_val is None: + continue + + unit_raw = re.sub(r"\s{1,4}+", "", m.group("unit_lemma").lower()) + if "yarıyıl" in unit_raw or "yariyil" in unit_raw: + unit = "yarıyıl" + elif "işgünü" in unit_raw or "isgunu" in unit_raw: + unit = "iş günü" + elif "gün" in unit_raw or "gun" in unit_raw: + unit = "gün" + elif "hafta" in unit_raw: + unit = "hafta" + elif "ay" in unit_raw: + p = text[max(0, start_pos - 15):start_pos].lower().strip() + if any(p.endswith(mon) for mon in _MONTH_NAMES): + continue + unit = "ay" + elif "yıl" in unit_raw or "yil" in unit_raw: + unit = "yıl" + elif "saat" in unit_raw: + unit = "saat" + else: + continue + + unit_lemma_end = m.start("unit_lemma") + len(m.group("unit_lemma")) + cand_str = " ".join(best_words) + sub_num = text[start_pos:m.start("unit_full")] + cand_offset = sub_num.find(cand_str) + span_start = start_pos + (cand_offset if cand_offset != -1 else 0) + span_end = unit_lemma_end + span_text = text[span_start:span_end] + + items.append({ + "start": span_start, + "end": span_end, + "text": span_text, + "type": "duration", + "value": parsed_val, + "unit": unit, + }) + return items + + +# ---------------------------------------------------------------------- +# 3. Amounts +# ---------------------------------------------------------------------- +_DIGIT_NUM_STR = r"(?:\d{1,3}\.\d{3}" + _up_to(r"\.\d{3}", 5) + r"(?:,\d{1,2})?|\d{1,12}+(?:,\d{1,2})?)" + +_CURRENCY_PAT_STR = ( + r"(?:" + r"TL|T\.L\.|₺|" + r"(?:Türk\s{1,4}+)?[Ll]\s{0,4}+i\s{0,4}+r\s{0,4}+a\s{0,4}+s\s{0,4}+ı[a-zçğıöşü]{0,64}+|" + r"(?:Türk\s{1,4}+)?[Ll]\s{0,4}+i\s{0,4}+r\s{0,4}+a[a-zçğıöşü]{0,64}+|" + r"avro|euro|eur|€|" + r"dolar[a-zçğıöşü]{0,64}+|ABD\s{0,4}+Doları[a-zçğıöşü]{0,64}+|usd|\$" + r")\b" +) + +_DIGIT_AMOUNT_PAT = re.compile( + r"\b(?P" + _DIGIT_NUM_STR + r")(?:\s{0,4}+-\s{0,4}+|\s{0,4}+)(?P" + _CURRENCY_PAT_STR + r")", + re.IGNORECASE, +) + +_WORD_AMOUNT_PAT = re.compile(r"\s{1,4}+(?P" + _CURRENCY_PAT_STR + r")", re.IGNORECASE) + + +def extract_amounts(text: str) -> List[Dict[str, Any]]: + items = [] + seen = set() + + # 1. Digits + for m in _DIGIT_AMOUNT_PAT.finditer(text): + num_str = m.group("num").strip() + curr_str = m.group("curr").strip().lower() + + if "," in num_str: + parts = num_str.split(",") + val = f"{parts[0].replace('.', '')}.{parts[1]}" + else: + val = num_str.replace(".", "") + + if any(c in curr_str for c in ["tl", "türk", "t.l.", "₺", "lira"]): + currency = "TRY" + elif any(c in curr_str for c in ["dolar", "usd", "$"]): + currency = "USD" + elif any(c in curr_str for c in ["avro", "euro", "eur", "€"]): + currency = "EUR" + else: + currency = "TRY" + + items.append({ + "start": m.start(), + "end": m.end(), + "text": text[m.start():m.end()], + "type": "amount", + "value": val, + "currency": currency, + }) + seen.add((m.start(), m.end())) + + # 2. Written-out words + digit_spans = _span_ends(len(text), list(seen)) + for num_start, num_end in _number_spans(text, words_only=True): + word_match = _WORD_AMOUNT_PAT.match(text, num_end) + if word_match is None: + continue + m = word_match + num_str = text[num_start:num_end].strip() + curr_str = m.group("curr").strip().lower() + + words = num_str.split() + parsed_val = None + best_words: List[str] = [] + for i in range(len(words), 0, -1): + cand = " ".join(words[:i]) + val_num = parse_turkish_number(cand) + if val_num is not None: + parsed_val = val_num + best_words = words[:i] + break + if parsed_val is None: + continue + + sub_text = text[num_start:m.end()] + curr_match = re.search(_CURRENCY_PAT_STR, sub_text, re.IGNORECASE) + if not curr_match: + continue + end = num_start + curr_match.end() + cand_idx = sub_text.find(best_words[0]) + if cand_idx == -1: + continue + start = num_start + cand_idx + + # Strip any leading quote characters + span_str = text[start:end] + lstrip_len = len(span_str) - len(span_str.lstrip("“\"'‘` ")) + start += lstrip_len + span_str = text[start:end] + + if digit_spans[end - 1] > start: + continue + + if any(c in curr_str for c in ["tl", "türk", "t.l.", "₺", "lira"]): + currency = "TRY" + elif any(c in curr_str for c in ["dolar", "usd", "$"]): + currency = "USD" + elif any(c in curr_str for c in ["avro", "euro", "eur", "€"]): + currency = "EUR" + else: + currency = "TRY" + + items.append({ + "start": start, + "end": end, + "text": span_str, + "type": "amount", + "value": str(parsed_val), + "currency": currency, + }) + seen.add((start, end)) + + items.sort(key=lambda x: cast(int, x["start"])) + return items + + +# ---------------------------------------------------------------------- +# 4. Dates +# ---------------------------------------------------------------------- +_MONTHS = { + "ocak": "01", "şubat": "02", "subat": "02", "mart": "03", "nisan": "04", + "mayıs": "05", "mayis": "05", "haziran": "06", "temmuz": "07", + "ağustos": "08", "agustos": "08", "eylül": "09", "eylul": "09", + "ekim": "10", "kasım": "11", "kasim": "11", "aralık": "12", "aralik": "12", +} + +_NUMERIC_DATE_PAT = re.compile(r"\b(\d{1,2})[./](\d{1,2})[./](\d{4})\b") +_TEXT_DATE_PAT = re.compile( + r"\b(\d{1,2})\s{1,4}+(Ocak|Şubat|Subat|Mart|Nisan|Mayıs|Mayis|Haziran|Temmuz|Ağustos|Agustos|Eylül|Eylul|Ekim|Kasım|Kasim|Aralık|Aralik)\s{1,4}+(\d{4})\b", + re.IGNORECASE, +) +_PARTIAL_DATE_PAT = re.compile( + r"\b(Ocak|Şubat|Subat|Mart|Nisan|Mayıs|Mayis|Haziran|Temmuz|Ağustos|Agustos|Eylül|Eylul|Ekim|Kasım|Kasim|Aralık|Aralik)\s{1,4}+(\d{4})\b", + re.IGNORECASE, +) + + +def extract_dates(text: str) -> List[Dict[str, Any]]: + items = [] + # 1. Numeric dates + for m in _NUMERIC_DATE_PAT.finditer(text): + d_str, m_str, y_str = m.groups() + day, month, year = int(d_str), int(m_str), int(y_str) + if 1 <= day <= 31 and 1 <= month <= 12 and 1900 <= year <= 2099: + items.append({ + "start": m.start(), + "end": m.end(), + "text": text[m.start():m.end()], + "type": "date", + "value": f"{year:04d}-{month:02d}-{day:02d}", + }) + + # 2. Text dates + for m in _TEXT_DATE_PAT.finditer(text): + d_str, m_name, y_str = m.groups() + day = int(d_str) + month_code = _MONTHS.get(m_name.lower()) + year = int(y_str) + if month_code and 1 <= day <= 31 and 1900 <= year <= 2099: + items.append({ + "start": m.start(), + "end": m.end(), + "text": text[m.start():m.end()], + "type": "date", + "value": f"{year:04d}-{month_code}-{day:02d}", + }) + + # 3. Partial dates + text_date_spans: List[Tuple[int, int]] = [ + (cast(int, it["start"]), cast(int, it["end"])) for it in items + ] + date_ends = _span_ends(len(text), text_date_spans) + for m in _PARTIAL_DATE_PAT.finditer(text): + m_start, m_end = m.start(), m.end() + if m_end <= date_ends[m_start]: + continue + m_name, y_str = m.groups() + month_code = _MONTHS.get(m_name.lower()) + year = int(y_str) + if month_code and 1900 <= year <= 2099: + items.append({ + "start": m_start, + "end": m_end, + "text": text[m_start:m_end], + "type": "date", + "value": f"{year:04d}-{month_code}", + }) + + return items + + +# ---------------------------------------------------------------------- +# 5. Decision Refs +# ---------------------------------------------------------------------- +_DOCKET_PAT_STR = r"(?:\d{4}/(?:\(\d{1,12}+\))?[\d-]{1,64}+|\d{4}/\d{1,12}+)" +_DECISION_PAT_STR = r"\d{4}/\d{1,12}+" +_DATE_PAT_STR = r"\d{1,2}[./]\d{1,2}[./]\d{4}" +_DATE_CLAUSE_STR = r"(?P" + _DATE_PAT_STR + r")\s{0,4}+(?:t\s{0,4}+a\s{0,4}+r\s{0,4}+i?\s{0,4}+h(?:l\s{0,4}+i?)?|g\s{0,4}+ü\s{0,4}+n)(?:[,\s]{1,4}+(?:ve\s{0,4}+)?)?" + +_LOC_PREFIX_STR = r"(?-i:[A-ZÇĞİÖŞÜ][a-zçğıöşü]{1,64}+(?:\s{1,4}+[A-ZÇĞİÖŞÜ][a-zçğıöşü]{1,64}+)?\s{1,4}+)" + +_COURT_CHAM_PAT_STR = ( + r"(?P" + r"(?:(?:Yargıtay|Danıştay|Y\.C\.G\.K)\s{1,4}+)?" + r"(?:\((?:kapanan|kapatılan)\)\s{1,4}+)?(?:\d{1,12}+\.\s{1,4}+)?(?:Ceza|Hukuk)?\s{0,4}+(?:Genel\s{1,4}+Kurulu?[\w]{0,64}+|Dairesi[\w]{0,64}+)|" + r"(?:(?:" + _LOC_PREFIX_STR + r")?\d{0,12}+\.?\s{0,4}+)?(?:Asliye\s{1,4}+(?:Hukuk|Ceza)|İş|Ticaret|Sulh\s{1,4}+(?:Hukuk|Ceza)|Bölge\s{1,4}+Adliye|Ağır\s{1,4}+Ceza)\s{1,4}+Mahkemes[a-zçğıöşü]{0,64}+|" + r"Özel\s{1,4}+Dairece|Dairece|Mahkemece|Anayasa\s{1,4}+M\s{0,4}+a\s{0,4}+h\s{0,4}+k?\s{0,4}+e\s{0,4}+m\s{0,4}+e\s{0,4}+s\s{0,4}+i[’'\w]{0,64}+|Y\.C\.G\.K" + r")\s{0,4}+(?:verilen\s{1,4}+)?" +) + +# Pattern 1: Docket E., Decision K. +_DEC_REF_PAT1 = re.compile( + r"(?:" + _COURT_CHAM_PAT_STR + r")?" + r"(?:" + _DATE_CLAUSE_STR + r")?" + r"(?P" + _DOCKET_PAT_STR + r")\s{0,4}+(?:E\.|Esas)" + r"[,\s]{1,4}+" + r"(?P" + _DECISION_PAT_STR + r")\s{0,4}+(?:K\.|Karar)" + r"(?:[,\s]{1,4}+(?P" + _DATE_PAT_STR + r")\s{0,4}+(?:t\s{0,4}+a\s{0,4}+r\s{0,4}+i?\s{0,4}+h(?:l\s{0,4}+i?)?|g\s{0,4}+ü\s{0,4}+n))?", + re.IGNORECASE, +) + +# Pattern 2: Anayasa Mahkemesi style: E. , K. +_DEC_REF_PAT2 = re.compile( + r"(?:" + _COURT_CHAM_PAT_STR + r")?" + r"(?:" + _DATE_CLAUSE_STR + r")?" + r"(?:E\.|E\.:|Esas\s{1,4}+No:?)\s{0,4}+(?P" + _DOCKET_PAT_STR + r")" + r"[,\s]{1,4}+" + r"(?:K\.|K\.:|Karar\s{1,4}+No:?)\s{0,4}+(?P" + _DECISION_PAT_STR + r")", + re.IGNORECASE, +) + +# Pattern 3: Single K pattern (e.g. 'Y.C.G.K 2020/527 K') +_DEC_REF_PAT3 = re.compile( + r"(?PY\.C\.G\.K)\s{1,4}+(?P\d{4}/\d{1,12}+)\s{0,4}+K\b", + re.IGNORECASE, +) + +# Pattern 4: Short form ? tarih(li) ve - (sayılı|sayı)? +_DEC_REF_PAT4 = re.compile( + r"(?:(?P" + r"(?:(?:(?:" + _LOC_PREFIX_STR + r")?Bölge\s{1,4}+Adliye\s{1,4}+Mahkemesi|Yargıtay|Danıştay|Y\.C\.G\.K)\s{1,4}+)?\s{0,4}+" + r"(?:\((?:kapanan|kapatılan)\)\s{1,4}+)?(?:\d{1,12}+\.\s{1,4}+)?(?:Ceza|Hukuk)?\s{0,4}+(?:Genel\s{1,4}+Kurulu?[\w]{0,64}+|Dairesi[\w]{0,64}+)|" + r"(?:(?:" + _LOC_PREFIX_STR + r")?\d{0,12}+\.?\s{0,4}+)?(?:Asliye\s{1,4}+(?:Hukuk|Ceza)|İş|Ticaret|Sulh\s{1,4}+(?:Hukuk|Ceza)|Bölge\s{1,4}+Adliye|Ağır\s{1,4}+Ceza)\s{1,4}+Mahkemes[a-zçğıöşü]{0,64}+|" + r"Özel\s{1,4}+Dairece|Dairece|Mahkemece|Anayasa\s{1,4}+M\s{0,4}+a\s{0,4}+h\s{0,4}+k?\s{0,4}+e\s{0,4}+m\s{0,4}+e\s{0,4}+s\s{0,4}+i[’'\w]{0,64}+|Y\.C\.G\.K" + r")\s{0,4}+(?:verilen\s{1,4}+)?)?" + r"(?P" + _DATE_PAT_STR + r")\s{0,4}+(?:t\s{0,4}+a\s{0,4}+r\s{0,4}+i?\s{0,4}+h(?:l\s{0,4}+i?)?|g\s{0,4}+ü\s{0,4}+n)[,\s]{1,4}+(?:ve\s{0,4}+)?" + r"(?P\d{1,12}+)\s{0,4}+-\s{0,4}+(?P\d{1,12}+)(?:\s{0,4}+(?:sayılı|sayı))?", + re.IGNORECASE, +) + +# Pattern 5: Single Esas with court (e.g. 'Bandırma Sulh Hukuk Mahkemesinde görülen 2009/183 Esas') +_DEC_REF_PAT5 = re.compile( + r"(?P" + r"(?:(?:" + _LOC_PREFIX_STR + r")?\d{0,12}+\.?\s{0,4}+)?(?:Asliye\s{1,4}+(?:Hukuk|Ceza)|İş|Ticaret|Sulh\s{1,4}+(?:Hukuk|Ceza)|Bölge\s{1,4}+Adliye|Ağır\s{1,4}+Ceza)\s{1,4}+Mahkemes[a-zçğıöşü]{0,64}+|" + r"(?:Yargıtay|Danıştay)\s{1,4}+(?:\d{1,12}+\.\s{1,4}+)?(?:Ceza|Hukuk)\s{1,4}+Dairesi[\w]{0,64}+" + r")\s{0,4}+(?:verilen\s{1,4}+|kayıtlı\s{1,4}+|görülen\s{1,4}+)(?P\d{4}/\d{1,12}+)\s{0,4}+(?:E\.|Esas)\b(?!\s{0,4}+,\s{0,4}+\d{4}/\d{1,12}+\s{0,4}+(?:K\.|Karar))", + re.IGNORECASE, +) + + +def _clean_court_chamber(raw: Optional[str]) -> Tuple[Optional[str], Optional[str]]: + """Split and clean court and chamber names from matched court_cham text.""" + if not raw: + return None, None + raw_str = re.sub(r"\s{1,4}+", " ", raw.strip()) + raw_str = re.sub(r"\((?:kapatılan|kapanan)\)\s{0,4}+", "", raw_str, flags=re.IGNORECASE).strip() + raw_str = re.sub(r"[’'\x27\u2032][a-zçğıöşü]{1,64}+$", "", raw_str, flags=re.IGNORECASE).strip() + raw_str = re.sub(r"Mahkemes(?:i[’'\x27\u2032]?(?:nce|nin|ne|nde)|ince|inin)$", "Mahkemesi", raw_str, flags=re.IGNORECASE) + raw_str = re.sub(r"Daires(?:i[’'\x27\u2032]?(?:nce|nin|ne|nde)|ince|inin)$", "Dairesi", raw_str, flags=re.IGNORECASE) + raw_str = re.sub(r"Kurul(?:u[’'\x27\u2032]?(?:nca|nun|na|nda)|unca|unun)$", "Kurulu", raw_str, flags=re.IGNORECASE) + raw_str = re.sub(r"Dairece$", "Dairesi", raw_str, flags=re.IGNORECASE) + raw_str = re.sub(r"Mahkemece$", "Mahkemesi", raw_str, flags=re.IGNORECASE) + raw_str = raw_str.strip() + + if "anayasa mahkemesi" in raw_str.lower(): + return "Anayasa Mahkemesi", None + elif "danıştay" in raw_str.lower() or "danistay" in raw_str.lower(): + court = "Danıştay" + cham = re.sub(r"danıştay|danistay", "", raw_str, flags=re.IGNORECASE).strip() + return court, (cham if cham else None) + elif "bölge adliye mahkemesi" in raw_str.lower(): + m = re.match(r"^(.{0,256}?Bölge\s{1,4}+Adliye\s{1,4}+Mahkemesi)\s{0,4}+(.{0,256})$", raw_str, re.IGNORECASE) + if m: + court = m.group(1).strip() + cham = m.group(2).strip() + return court, (cham if cham else None) + return raw_str, None + elif "yargıtay" in raw_str.lower() or "y.c.g.k" in raw_str.lower() or raw_str.lower().startswith("ceza genel") or raw_str.lower().startswith("hukuk genel"): + court = "Yargıtay" + chamber: Optional[str] = None + if raw_str.upper() == "Y.C.G.K": + chamber = "Ceza Genel Kurulu" + else: + cham = re.sub(r"yargıtay", "", raw_str, flags=re.IGNORECASE).strip() + chamber = cham if cham else None + return court, chamber + elif "mahkeme" in raw_str.lower(): + return None, raw_str + else: + return raw_str, None + + +_DECISION_HINT = re.compile(r"E\.|K\b|Esas|Karar", re.IGNORECASE) + + +def extract_decision_refs(text: str) -> List[Dict[str, Any]]: + if "/" not in text and "-" not in text: + return [] + if not _DECISION_HINT.search(text) and not _NUMERIC_DATE_PAT.search(text): + return [] + items = [] + seen_spans: set[Tuple[int, int]] = set() + + for pat in [_DEC_REF_PAT2, _DEC_REF_PAT1, _DEC_REF_PAT4, _DEC_REF_PAT5, _DEC_REF_PAT3]: + previous_ends = _span_ends(len(text), list(seen_spans)) + for m in pat.finditer(text): + start, end = m.start(), m.end() + span_str = text[start:end] + + # Strip leading whitespace + lstrip_len = len(span_str) - len(span_str.lstrip()) + start += lstrip_len + span_str = text[start:end] + + # Strip leading unrelated words ('verilen', 'ilişkin', 'sayılı') + for bad_prefix in ["verilen ", "ilişkin ", "sayılı "]: + if span_str.lower().startswith(bad_prefix): + start += len(bad_prefix) + span_str = text[start:end] + lstrip_len = len(span_str) - len(span_str.lstrip()) + start += lstrip_len + span_str = text[start:end] + + rstrip_len = len(span_str) - len(span_str.rstrip(" ,.")) + end -= rstrip_len + span_str = text[start:end] + + if not span_str: + continue + + span = (start, end) + if end <= previous_ends[start]: + continue + + groups = m.groupdict() + court_cham_raw = groups.get("court_cham") + docket = groups.get("docket") + decision = groups.get("decision") + date_raw = groups.get("date_pre") or groups.get("date_post") or groups.get("date") + + court, chamber = _clean_court_chamber(court_cham_raw) + + date_val = None + if date_raw: + d_m = _NUMERIC_DATE_PAT.search(date_raw) + if d_m: + day, month, year = int(d_m.group(1)), int(d_m.group(2)), int(d_m.group(3)) + date_val = f"{year:04d}-{month:02d}-{day:02d}" + + items.append({ + "start": start, + "end": end, + "text": span_str, + "type": "decision_ref", + "court": court, + "chamber": chamber, + "docket_no": docket, + "decision_no": decision, + "date": date_val, + }) + seen_spans.add(span) + + return items + + +# ---------------------------------------------------------------------- +# 6. Citations +# ---------------------------------------------------------------------- +_ORDINAL_MAP = { + "birinci": "1", "ikinci": "2", "üçüncü": "3", "ucuncu": "3", + "dördüncü": "4", "dorduncu": "4", "beşinci": "5", "besinci": "5", + "altıncı": "6", "altinci": "6", "yedinci": "7", "sekizinci": "8", + "dokuzuncu": "9", "onuncu": "10", "onbeşinci": "15", +} + +_KNOWN_ACRONYMS = { + "TCK", "TMK", "HMK", "İİK", "IIK", "CMK", "CMUK", "İYUK", "IYUK", + "TTK", "VUK", "DMK", "PVSK", "AİHS", "TBK" +} + +_APOSTROPHE_STR = r"[’'\x27\u2032\u2018\u201b]" + + +def _clean_law_name(header: str) -> Optional[str]: + if not header: + return None + header = re.sub(r"\s{1,4}+", " ", header.strip()) + header = re.sub(r"^Mülga\s{1,4}+", "", header) + header = re.sub(r"\s{0,4}+\([^\)\n]{1,60}\)\s{0,4}+", " ", header).strip() + header = re.sub(r"[’'\x27\u2032\u2018\u201b]\s{0,4}+[a-zçğıöşü]{1,64}+$", "", header).strip() + + lower = header.lower() + if lower.startswith("aynı ") or lower.startswith("bu ") or lower == "aynı" or lower == "bu" or lower.startswith("anılan "): + return None + + for sfx in ["nun", "nün", "nin", "nın", "unun", "ünün", "inin", "ının"]: + if lower.endswith(sfx) and len(header) > len(sfx) + 3: + header = header[:-len(sfx)] + break + + if header.endswith("Kanu"): + header += "n" + elif header.endswith("Yönetmeliğ"): + header += "i" + elif header.endswith("Tebliğ"): + header += "i" + elif header.endswith("Kararname"): + header += "si" + return header.strip() + + +def _parse_articles_and_paragraphs(body_text: str) -> Tuple[Optional[str], Optional[str]]: + lower = body_text.lower() + if "geçici" in lower: + m_range = re.search(r"geçici\s{1,4}+(\d{1,12}+)\s{1,4}+il[aâ]\s{1,4}+geçici\s{1,4}+(\d{1,12}+)", body_text, re.IGNORECASE) + if m_range: + start_n, end_n = int(m_range.group(1)), int(m_range.group(2)) + # Never allocate an input-controlled range of millions of articles. + if end_n - start_n > 128: + return f"geçici {start_n}, geçici {end_n}", None + arts = [f"geçici {i}" for i in range(start_n, end_n + 1)] + return ", ".join(arts), None + + m_gm = re.search(r"Geçici\s{1,4}+madde\s{1,4}+(\d{1,12}+)", body_text, re.IGNORECASE) + if m_gm: + return f"Geçici {m_gm.group(1)}", None + + if "il" in lower: + m_ila = re.search(r"\b(\d{1,12}+)\s{1,4}+il[aâ]\s{1,4}+(\d{1,12}+)\s{0,4}+(?:[ıiuü]?nc[ıiuü]|\.)?\s{0,4}+madde", body_text, re.IGNORECASE) + if m_ila: + return f"{m_ila.group(1)}, {m_ila.group(2)}", None + + if "tbk" in lower: + m_tbk = re.search(r"\bTBK/(\d{1,12}+)\b", body_text) + if m_tbk: + return m_tbk.group(1), None + + articles = [] + paras = [] + + m_list = re.findall(r"\bm\.\s{0,4}+(" + _ARTICLE_NUMBER + r")(?:/(\d{1,12}+))?", body_text) + if len(m_list) >= 2: + for a, p in m_list: + articles.append(a) + paras.append(p if p else None) + else: + clauses = re.split(r"(?:,\s{0,4}+|\s{1,4}+(?:ve|veya|ile|ilâ|ila)\s{1,4}+)", body_text) + for cl in clauses: + cl = cl.strip() + if not cl: + continue + art_m = re.search( + r"\b(?:m\.\s{0,4}+)?((?:[Gg]eçici\s{1,4}+|[Ee]k\s{1,4}+|[Mm]ükerrer\s{1,4}+|[Mm]ülga\s{1,4}+)?" + _ARTICLE_NUMBER + r"(?:/[A-Za-zÇĞİÖŞÜçğıöşü])?)(?:/(\d{1,12}+))?", + cl, + re.IGNORECASE, + ) + if art_m: + a_val = re.sub(r"\s{1,4}+", " ", art_m.group(1).strip()) + if a_val.lower().startswith("mükerrer "): + a_val = "mükerrer " + a_val.split()[1] + p_val = art_m.group(2) + if not p_val: + pw_m = re.search( + r"\b(birinci|ikinci|üçüncü|dördüncü|beşinci|altıncı|yedinci|sekizinci|dokuzuncu|onuncu|onbeşinci|\d{1,12}+)\s{0,4}+(?:[ıiuü]?nc[ıiuü]|\.)?\s{0,4}+f\s{0,4}+ı\s{0,4}+k\s{0,4}+r\s{0,4}+a", + cl, + re.IGNORECASE, + ) + if pw_m: + w = pw_m.group(1).lower() + p_val = _ORDINAL_MAP.get(w, w) + if not p_val: + sl_m = re.search(r"\b\d{1,12}+/(\d{1,12}+)\b", cl) + if sl_m: + p_val = sl_m.group(1) + + if a_val not in articles: + articles.append(a_val) + paras.append(p_val) + + if not articles: + single_m = re.search( + r"\b((?:[Gg]eçici\s{1,4}+|[Ee]k\s{1,4}+|[Mm]ükerrer\s{1,4}+)?" + _ARTICLE_NUMBER + r"(?:/[A-Za-zÇĞİÖŞÜçğıöşü])?)\b", + body_text, + re.IGNORECASE, + ) + if single_m: + articles = [single_m.group(1).strip()] + paras = [None] + + if not articles: + return None, None + + art_str = ", ".join(articles) + non_null_paras = [p for p in paras if p is not None] + para_str = non_null_paras[0] if len(non_null_paras) == 1 else None + return art_str, para_str + + +_LEAD_DATE_STR = ( + r"(?P" + r"(?:\d{1,2}[./]\d{1,2}[./]\d{4}|\d{1,2}\s{1,4}+(?:Ocak|Şubat|Subat|Mart|Nisan|Mayıs|Mayis|Haziran|Temmuz|Ağustos|Agustos|Eylül|Eylul|Ekim|Kasım|Kasim|Aralık|Aralik)\s{1,4}+\d{4})" + r"\s{0,4}+(?:t\s{0,4}+a\s{0,4}+r\s{0,4}+i?\s{0,4}+h(?:l\s{0,4}+i?)?|g\s{0,4}+ü\s{0,4}+n(?:ü)?)\s{1,4}+ve\s{1,4}+" + r")" +) + +_ACRONYMS = r"TCK|TMK|HMK|İİK|IIK|CMK|CMUK|İYUK|IYUK|TTK|VUK|DMK|PVSK|AİHS|TBK" +_TITLE_WORD = r"(?:[A-ZÇĞİÖŞÜ][A-Za-zÇĞİÖŞÜçğıöşü0-9/-]{0,64}+|ve|veya|ile|dair|hakkında|için|ilişkin)" +_TITLE_END = r"(?:Kanun[a-zçğıöşü]{0,64}+|KHK[a-zçğıöşü]{0,64}+|Yasa[a-zçğıöşü]{0,64}+|Yönetmeli[a-zçğıöşü]{0,64}+|Tebliğ[a-zçğıöşü]{0,64}+|Kararname[a-zçğıöşü]{0,64}+|Anayasa[a-zçğıöşü]{0,64}+)" + +# Law header regex +_LAW_HEADER_STR = ( + r"(?:(?:\b" + _LEAD_DATE_STR + r")|\b)" + r"(?:" + r"(?:(?:Mülga\s{1,4}+)?(?P\d{1,12}+)\s{1,4}+sayılı\s{1,4}+(?P" + _up_to(_TITLE_WORD + r"\s{1,4}+", 15) + _TITLE_END + r"|" + _ACRONYMS + r"))|" + r"(?:(?:Mülga\s{1,4}+)?(?P\d{1,12}+)\s{1,4}+sayılı\s{1,4}+(?PKanun[a-zçğıöşü]{0,64}+|Yasa[a-zçğıöşü]{0,64}+))|" + r"(?P" + _ACRONYMS + r")\b|" + r"(?P(?:[Aa]ynı|[Bb]u|[Aa]nılan)\s{1,4}+" + _TITLE_END + r")\b|" + r"(?P" + _up_to(_TITLE_WORD + r"\s{1,4}+", 15) + _TITLE_END + r")\b|" + r"(?PYönetmeliğin|Kanunun|Kararnamenin|Tebliğin)\b" + r")" + r"(?:\s{0,4}+" + _APOSTROPHE_STR + r"\s{0,4}+[a-zçğıöşü]{1,64}+)?" + + _up_to(r"\s{0,4}+\([^\)\n]{1,60}\)", 2) + + r"(?:\s{0,4}+" + _APOSTROPHE_STR + r"\s{0,4}+[a-zçğıöşü]{1,64}+)?" + r"(?:\s{0,4}+(?:eklenen|değiştirilen)\s{1,4}+)?" + r"(?:\s{0,4}+[\"“'][^\"”'\n]{1,80}[\"”']\s{1,4}+başlıklı)?" + r"(?:\s{0,4}+(?:Ek|EK-?)\s{0,4}+[IVX\d]{1,12}+(?:[’'][a-zçğıöşü]{1,64}+)?(?:\s{1,4}+yer\s{1,4}+alan)?\s{0,4}+)?" +) + +_PAT_LAW_HEADER = re.compile(_LAW_HEADER_STR) + +_ART_ATOM = r"(?:(?:[Gg]eçici|[Ee]k|[Mm]ükerrer|[Mm]ülga)\s{1,4}+)?" + _ARTICLE_NUMBER + r"(?:/[A-Za-zÇĞİÖŞÜçğıöşü\d-]{1,64}+)?" +_ORD_WORD = ( + r"(?:birinci|ikinci|üçüncü|dördüncü|beşinci|altıncı|yedinci|sekizinci|dokuzuncu|onuncu|onbeşinci|ilk|\d{1,12}+)" + r"(?:\s{0,4}+(?:[ıiuü]?nc[ıiuü]|\.))?" +) +_PARA_STR = ( + r"(?:\s{0,4}+(?:'?[a-zçğıöşü]{1,64}+\s{1,4}+)?" + _ORD_WORD + + _up_to(r"\s{0,4}+(?:ve|veya|,)\s{0,4}+" + _ORD_WORD, 3) + r"\s{0,4}+f\s{0,4}+ı\s{0,4}+k\s{0,4}+r\s{0,4}+a[a-zçğıöşü]{0,64}+)" +) +_BENT_STR = ( + r"(?:\s{0,4}+\([a-zçğıöşü\d]\)" + _up_to(r"\s{0,4}+(?:ve|veya|,)\s{0,4}+\([a-zçğıöşü\d]\)", 5) + r"\s{0,4}+bend[a-zçğıöşü]{0,64}+|" + r"\s{0,4}+\([a-zçğıöşü\d]\)(?:\s{1,4}+(?:ve|veya)\s{1,4}+\([a-zçğıöşü\d]\))?)" +) + +_ART_UNIT_A = ( + _ART_ATOM + r"(?:\s{0,4}+(?:[ıiuü]?nc[ıiuü]|\.))?" + + _up_to(r"(?:\s{0,4}+,\s{0,4}+|\s{1,4}+(?:ve|veya|ile|ila|ilâ)\s{1,4}+)" + _ART_ATOM + r"(?:\s{0,4}+(?:[ıiuü]?nc[ıiuü]|\.))?", 10) + + r"\s{0,4}+madde[a-zçğıöşü]{0,64}+" + r"(?:" + _PARA_STR + _up_to(r"\s{0,4}+(?:ve|veya|,)\s{0,4}+" + _PARA_STR, 3) + r")?" + r"(?:" + _BENT_STR + r")?" +) + +_ART_UNIT_B = ( + r"m\.\s{0,4}+" + _ART_ATOM + + _up_to(r"(?:\s{0,4}+,\s{0,4}+|\s{1,4}+(?:ve|veya|ile|ila|ilâ)\s{1,4}+)" + _ART_ATOM, 10) + + r"(?:" + _PARA_STR + r")?" + r"(?:" + _BENT_STR + r")?" +) + +_ART_UNIT_C = ( + r"[\"“']?(?:[Gg]eçici|[Ee]k|[Mm]ülga)\s{1,4}+madde\s{1,4}+\d{1,12}+[\"”']?" + + _up_to(r"(?:\s{0,4}+,\s{0,4}+|\s{1,4}+(?:ve|veya|ile|ila|ilâ)\s{1,4}+)\d{1,12}+", 10) + + r"(?:" + _PARA_STR + r")?" + r"(?:" + _BENT_STR + r")?" +) + +_ARTICLE_CLAUSE_STR = ( + r"(?:" + _ART_UNIT_A + r"|" + _ART_UNIT_B + r"|" + _ART_UNIT_C + r")" + r"(?:\s{1,4}+(?:ve|veya|ile)\s{1,4}+(?:" + _ART_UNIT_A + r"|" + _ART_UNIT_B + r"|" + _ART_UNIT_C + r"))?" +) + +_PAT_ARTICLE_CLAUSE = re.compile(r"\b(?:" + _ARTICLE_CLAUSE_STR + r")") + +# Short follow pattern: e.g. 'un 65/1 or 442/3 or 125. +_PAT_SHORT_ARTICLE = re.compile( + r"^(?:[,\s]{0,4}+" + _APOSTROPHE_STR + r"\s{0,4}+[a-zçğıöşü]{1,64}+)?" + r"(?:[,\s]{1,4}+)" + r"(?P\d{1,4})(?:/(?P\d{1,2}))?" + + _up_to(r"(?:\s{0,4}+,\s{0,4}+|\s{1,4}+(?:ve|veya|ile)\s{1,4}+)\d{1,4}(?:/\d{1,2})?", 5) + + r"(?:\.|\b)" +) + +_PAT_SLASH_SHORT = re.compile( + r"\b(?P" + _ACRONYMS + r")/(?P\d{1,12}+)\b", + re.IGNORECASE, +) + +def extract_citations(text: str) -> List[Dict[str, Any]]: + items = [] + seen_spans: set[Tuple[int, int]] = set() + covered = bytearray(len(text)) + + # 1. Slash short: TCK/141 + for m in _PAT_SLASH_SHORT.finditer(text): + start, end = m.start(), m.end() + span = (start, end) + seen_spans.add(span) + covered[start:end] = b"\1" * (end - start) + groups = m.groupdict() + items.append({ + "start": start, + "end": end, + "text": text[start:end], + "type": "citation", + "law_no": None, + "law_name": groups["acronym"].upper(), + "article": groups["art"], + "paragraph": None, + }) + + # Header matches cannot overlap one another, but can overlap slash forms. + slash_ends = _span_ends(len(text), list(seen_spans)) + # 2. Law headers (and attached articles) + for m in _PAT_LAW_HEADER.finditer(text): + h_start, h_end = m.start(), m.end() + + # Check if preceded or followed by Anayasa Mahkemesi (court, not constitution citation) + full_match_text = text[h_start:h_end] + if "Anayasa" in full_match_text: + lookahead = text[h_end:h_end+25].lower() + if lookahead.startswith(" mahkemesi") or lookahead.startswith("mahkemesi"): + continue + + groups = m.groupdict() + law_no = groups.get("law_no") or groups.get("law_no_bare") + raw_name = ( + groups.get("law_title_1") or + groups.get("law_title_bare") or + groups.get("acronym") or + groups.get("law_title_2") + ) + anaphoric = groups.get("anaphoric") or groups.get("anaph_noun") + + law_name = _clean_law_name(raw_name) if raw_name else None + if anaphoric: + law_name = None + law_no = None + + # Check for Tebliğ No inside parenthetical: (Tebliğ No: 2024/53) + if not law_no: + teb_m = re.search(r"\((?:Tebliğ\s{1,4}+No:\s{0,4}+|Karar\s{1,4}+No:\s{0,4}+)([\w/-]{1,64}+)\)", full_match_text) + if teb_m: + law_no = teb_m.group(1) + + # Check what follows immediately after the law header + # The grammar has a fixed maximum clause length; do not copy the entire + # remaining document for every header in a reference-heavy chunk. + after_text = text[h_end:h_end + 8192] + m_art = _PAT_ARTICLE_CLAUSE.match(after_text.lstrip(" ,\t\r\n")) + + art_span_text = None + full_span_end = h_end + + if m_art: + offset = len(after_text) - len(after_text.lstrip(" ,\t\r\n")) + m_art.end() + full_span_end = h_end + offset + art_span_text = m_art.group(0) + else: + m_short = _PAT_SHORT_ARTICLE.match(after_text) + if m_short: + # Ensure it's not a date like 12/9/2010 + look_ahead_short = after_text[m_short.end():m_short.end()+20] + if not re.match(r"^\s{0,4}+/\d{4}", look_ahead_short) and not re.match(r"^\s{0,4}+tarih", look_ahead_short): + full_span_end = h_end + m_short.end() + art_span_text = m_short.group(0) + + # Strip trailing punctuation from citation span + span_str = text[h_start:full_span_end] + rstrip_len = len(span_str) - len(span_str.rstrip(" ,.")) + full_span_end -= rstrip_len + span_str = text[h_start:full_span_end] + + if not art_span_text: + continue + + art_str, para_str = _parse_articles_and_paragraphs(art_span_text) + + span = (h_start, full_span_end) + if full_span_end <= slash_ends[h_start]: + continue + + items.append({ + "start": h_start, + "end": full_span_end, + "text": span_str, + "type": "citation", + "law_no": law_no, + "law_name": law_name, + "article": art_str, + "paragraph": para_str, + }) + seen_spans.add(span) + covered[h_start:full_span_end] = b"\1" * (full_span_end - h_start) + + # 3. Standalone article clauses + for m in _PAT_ARTICLE_CLAUSE.finditer(text): + start, end = m.start(), m.end() + span_text = text[start:end] + + rstrip_len = len(span_text) - len(span_text.rstrip(" ,.")) + end -= rstrip_len + span_text = text[start:end] + + span = (start, end) + if any(covered[start:end]): + continue + + art_str, para_str = _parse_articles_and_paragraphs(span_text) + if not art_str: + continue + + items.append({ + "start": start, + "end": end, + "text": span_text, + "type": "citation", + "law_no": None, + "law_name": None, + "article": art_str, + "paragraph": para_str, + }) + seen_spans.add(span) + covered[start:end] = b"\1" * (end - start) + + items.sort(key=lambda x: (x["start"], x["end"])) + return items + + + +# ---------------------------------------------------------------------- +# 7. Outcome +# ---------------------------------------------------------------------- +_OUTCOME_RULES = [ + ("gönderme", re.compile(r"\b((?:(?:YARGITAY\s{1,4}+)?(?:\d{1,12}+\.\s{1,4}+)?(?:HUKUK|CEZA)\s{1,4}+DA[İI]RES[İI]NE\s{1,4}+GÖNDER[İI]LMES[İI]NE)|(?:(?:Özel\s{1,4}+)?Dairesine\s{1,4}+GER[İI]\s{1,4}+ÇEVR[İI]LMES[İI]NE))\b")), + ("düzeltilerek onama", re.compile(r"\b(DÜZELT[İI]LEREK\s{1,4}+ONANMASINA)\b")), + ("kısmen kabul", re.compile(r"\b(KISMEN\s{1,4}+KABUL[ÜU]NE)\b")), + ("onama", re.compile(r"\b(ONANMASINA)\b")), + ("bozma", re.compile(r"\b(BOZULMASINA)\b")), + ("kabul", re.compile(r"\b(KABUL[ÜU]NE)\b")), + ("red", re.compile(r"\b(REDD[İI]NE)\b")), + ("direnme", re.compile(r"\b(D[İI]REN[İI]LMES[İI]NE)\b")), +] + +_OUTCOME_END = re.compile(r"\bTEVD[İI][İI]NE\b") +_OUTCOME_WORD = re.compile(r"[A-Za-zÇĞİÖŞÜçğıöşü0-9.]{1,64}+") + + +def _forwarding_spans(text: str) -> Iterator[tuple[int, int]]: + # Start at a terminator; examine only a bounded preceding word sequence. + # Retrying every possible prefix of a failed gönderilmek clause was cubic. + for end in _OUTCOME_END.finditer(text): + words = [] + pos = end.start() + for _ in range(26): + gap_end = pos + while pos > 0 and gap_end - pos < 4 and text[pos - 1].isspace(): + pos -= 1 + if pos == gap_end: + break + word_end = pos + while pos > 0 and word_end - pos < 64 and ( + text[pos - 1].isalnum() or text[pos - 1] == "." + ): + pos -= 1 + word = text[pos:word_end] + if not _OUTCOME_WORD.fullmatch(word): + break + words.append((pos, word)) + words.reverse() + for i, (_, word) in enumerate(words): + if word == "gönderilmek" and i + 1 < len(words) and words[i + 1][1] == "üzere": + if i <= 12 and len(words) - i - 2 <= 12: + yield words[0][0], end.end() + break + + +def extract_outcomes(text: str) -> List[Dict[str, Any]]: + items = [] + seen_spans: set[Tuple[int, int]] = set() + for start, end in _forwarding_spans(text): + items.append({"start": start, "end": end, "text": text[start:end], + "type": "outcome", "value": "gönderme"}) + seen_spans.add((start, end)) + for val, pat in _OUTCOME_RULES: + previous_ends = _span_ends(len(text), list(seen_spans)) + for m in pat.finditer(text): + span = (m.start(), m.end()) + if m.end() <= previous_ends[m.start()]: + continue + items.append({ + "start": m.start(), + "end": m.end(), + "text": text[m.start():m.end()], + "type": "outcome", + "value": val, + }) + seen_spans.add(span) + return items + + +# ---------------------------------------------------------------------- +# Master extractor +# ---------------------------------------------------------------------- +def extract_all_rules(text: str) -> List[Dict[str, Any]]: + """Extract all RULE items from chunk text according to schema.json.""" + decision_refs = extract_decision_refs(text) + citations = extract_citations(text) + amounts = extract_amounts(text) + durations = extract_durations(text) + outcomes = extract_outcomes(text) + dates = extract_dates(text) + + # Filter out dates inside citation or decision_ref spans, or inside amendment markers + if dates: + ref_spans = [(it["start"], it["end"]) for it in (decision_refs + citations)] + amendment_marker_spans = [] + markers = { + m.start() for m in re.finditer(r"\((?:(?:Başlığı\s{1,4}+ile\s{1,4}+birlikte\s{1,4}+)?(?:Değişik|Ek|Mülga|İptal))\b", text, re.IGNORECASE) + } + opens: list[int] = [] + for i, char in enumerate(text): + if char == "(": + opens.append(i) + elif char == ")" and opens: + start = opens.pop() + if start in markers: + amendment_marker_spans.append((start, i + 1)) + + suppressed_spans = ref_spans + amendment_marker_spans + suppressed_ends = _span_ends(len(text), suppressed_spans) + + valid_dates = [] + for d in dates: + d_start, d_end = d["start"], d["end"] + if d_end > suppressed_ends[d_start]: + valid_dates.append(d) + else: + valid_dates = [] + + candidates = decision_refs + citations + amounts + durations + outcomes + valid_dates + candidates.sort(key=lambda x: (int(x["start"]), -int(x["end"]))) + + deduped = [] + seen_spans: set[Tuple[int, int, str]] = set() + for it in candidates: + key = (int(it["start"]), int(it["end"]), str(it["type"])) + if key in seen_spans: + continue + seen_spans.add(key) + if text[it["start"]:it["end"]] != it["text"]: + continue + deduped.append(it) + + deduped.sort(key=lambda x: (int(x["start"]), int(x["end"]))) + return deduped + + +def extract(text: str) -> List[Dict[str, Any]]: + """Extract schema-shaped Turkish legal rule items from text. + + Guarantees text[start:end] == item['text'] for every returned item. + """ + if not text: + return [] + items = extract_all_rules(text) + valid_items = [] + for item in items: + start = item.get("start") + end = item.get("end") + if start is not None and end is not None and 0 <= start < end <= len(text): + if text[start:end] == item.get("text"): + valid_items.append(item) + return valid_items diff --git a/mesa_workers/projection_worker.py b/mesa_workers/projection_worker.py index e96e780..ac6d4f7 100644 --- a/mesa_workers/projection_worker.py +++ b/mesa_workers/projection_worker.py @@ -61,7 +61,13 @@ def _triplets(record: dict[str, Any]) -> list[dict[str, Any]]: # and irreversibly hide later clauses even though the source chunk is # still durable in SQLite. evidence = str(record.get("evidence_span") or content) - if isinstance(legal_identity, dict): + # Automatic document identities affect typed retrieval only. Preserve + # the original document/chunk graph and lexical subject for marked + # document chunks. Explicit identities retain main's article fallback. + if ( + isinstance(legal_identity, dict) + and record.get("document_chunk") is not True + ): statute_code = str(legal_identity["statute_code"]) article_id = str(legal_identity["article_id"]) head = f"{statute_code} m.{article_id}" diff --git a/tests/test_d008_model_enabled_runtime_e2e.py b/tests/test_d008_model_enabled_runtime_e2e.py index a1b704d..caf0fb0 100644 --- a/tests/test_d008_model_enabled_runtime_e2e.py +++ b/tests/test_d008_model_enabled_runtime_e2e.py @@ -112,6 +112,7 @@ async def test_d008_model_enabled_combined_runtime_survives_restart( monkeypatch.setenv("MESA_STORAGE_ROOT", str(storage)) monkeypatch.setenv("MESA_LOAD_DOTENV", "false") monkeypatch.setenv("MESA_MODEL_ENABLED", "true") + monkeypatch.setenv("MESA_EXTRACTION_ENABLED", "true") monkeypatch.setenv("MESA_EXTERNAL_PROVIDER_ENABLED", "true") monkeypatch.setenv("MESA_TIER3_MODE", "2") monkeypatch.setenv("MESA_TIER3_LLM_PROVIDER_A", "mock") @@ -227,6 +228,7 @@ async def test_r4_mode_zero_combined_runtime_has_no_validator_dependency( monkeypatch.setenv("MESA_STORAGE_ROOT", str(storage)) monkeypatch.setenv("MESA_LOAD_DOTENV", "false") monkeypatch.setenv("MESA_MODEL_ENABLED", "true") + monkeypatch.setenv("MESA_EXTRACTION_ENABLED", "true") monkeypatch.setenv("MESA_EXTERNAL_PROVIDER_ENABLED", "true") monkeypatch.setenv("MESA_TIER3_MODE", "0") monkeypatch.setenv("MESA_REBEL_ENABLED", "false") @@ -325,6 +327,7 @@ async def test_r4_mode_one_combined_runtime_uses_only_validator_a( monkeypatch.setenv("MESA_STORAGE_ROOT", str(storage)) monkeypatch.setenv("MESA_LOAD_DOTENV", "false") monkeypatch.setenv("MESA_MODEL_ENABLED", "true") + monkeypatch.setenv("MESA_EXTRACTION_ENABLED", "true") monkeypatch.setenv("MESA_EXTERNAL_PROVIDER_ENABLED", "true") monkeypatch.setenv("MESA_TIER3_MODE", "1") monkeypatch.setenv("MESA_TIER3_LLM_PROVIDER_A", "mock") @@ -425,6 +428,7 @@ async def test_r4_invalid_validation_composition_fails_server_startup( monkeypatch.setenv("MESA_STORAGE_ROOT", str(storage)) monkeypatch.setenv("MESA_LOAD_DOTENV", "false") monkeypatch.setenv("MESA_MODEL_ENABLED", "true") + monkeypatch.setenv("MESA_EXTRACTION_ENABLED", "true") monkeypatch.setenv("MESA_EXTERNAL_PROVIDER_ENABLED", "true") monkeypatch.setenv("MESA_TIER3_MODE", str(mode)) monkeypatch.setenv("MESA_API_KEY", "r4-invalid-composition-key") diff --git a/tests/test_document_ingestion.py b/tests/test_document_ingestion.py index c5353ef..e1fb5de 100644 --- a/tests/test_document_ingestion.py +++ b/tests/test_document_ingestion.py @@ -140,7 +140,8 @@ async def test_changed_document_supersedes_active_revision(model_runtime): assert response.status_code == 409, response.text assert response.json()["detail"] == ( "content equals revision 1 (SUPERSEDED); " - "restoring an earlier revision is not supported in v0.9" + "restoring an earlier revision is not supported in v0.9. " + "Add it under a new document_id to store it again." ) assert before == {t: await _rows(env, t) for t in before} @@ -391,7 +392,8 @@ def sync_add(): await _consume(env, 3) detail = ( "content equals revision 1 (SUPERSEDED); " - "restoring an earlier revision is not supported in v0.9" + "restoring an earlier revision is not supported in v0.9. " + "Add it under a new document_id to store it again." ) with pytest.raises(MesaAPIError) as error: await asyncio.to_thread( diff --git a/tests/test_f36_performance.py b/tests/test_f36_performance.py new file mode 100644 index 0000000..e4e7313 --- /dev/null +++ b/tests/test_f36_performance.py @@ -0,0 +1,93 @@ +"""Performance and linear-time regression tests for legal_rules extractor (F-36).""" + +import signal +import time +from pathlib import Path + +import pytest + +from mesa_memory.document_chunking import chunk_legal_document +from mesa_storage.legal_rules import extract + +REPRO_CHUNK = ( + "5/6/2008 tarihli ve\n" + "E.: 2008/16,\n" + "K.: 2008/116 sayılı Kararı\n" + "5982\n" + "10, 20, 23, 41, 51, 53, 54, 74, 84, 94,\n" + "125, 128, 129, 144, 145, 146, 147,148,\n" + "149, 156, 157, 159, 166, Geçici Madde\n" + "15, Geçici Madde 18, Geçici Madde\n" + "19, İşlenemeyen Hüküm\n" + "Bu Kanun 12/9/2010 tarihinde yapılan\n" + "halkoylaması sonucu kabul edilmiş ve\n" + "buna ilişkin Yüksek Seçim Kurulu Kararı\n" + "23/9/2010 tarihli ve 27708 sayılı Resmî\n" + "Gazete’de yayımlanmıştır." +) + + +def test_repro_chunk_under_50ms(): + """1. The repro chunk extracts in under 50 ms.""" + def _timeout_handler(signum, frame): + raise TimeoutError("Extraction timed out (catastrophic backtracking)") + + old_handler = signal.signal(signal.SIGALRM, _timeout_handler) + signal.alarm(1) + try: + t0 = time.perf_counter() + extract(REPRO_CHUNK) + elapsed = time.perf_counter() - t0 + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + assert elapsed < 0.05, f"Extraction took {elapsed * 1000:.1f}ms, expected < 50ms" + + +def test_tier_m_corpus_chunks_under_100ms(): + """2. Every chunk of Tier M corpus extracts in under 100 ms.""" + corpus_dir = Path("/home/yasin/Desktop/mesa_bench/v1/corpus") + if not corpus_dir.is_dir(): + pytest.skip("bench folder absent") + + def _timeout_handler(signum, frame): + raise TimeoutError("Chunk extraction timed out (>100ms)") + + old_handler = signal.signal(signal.SIGALRM, _timeout_handler) + try: + for doc_path in sorted(corpus_dir.glob("*.txt")): + text = doc_path.read_text(encoding="utf-8") + chunks = chunk_legal_document(text) + for i, chunk in enumerate(chunks): + signal.alarm(1) + t0 = time.perf_counter() + try: + extract(chunk) + elapsed = time.perf_counter() - t0 + finally: + signal.alarm(0) + assert elapsed < 0.1, f"Chunk {i} of {doc_path.name} took {elapsed * 1000:.1f}ms" + finally: + signal.signal(signal.SIGALRM, old_handler) + + +def test_adversarial_string_under_100ms(): + """3. A 20,000-character adversarial string extracts in under 100 ms.""" + pattern = "1, 2 ve 3 üncü maddelerinin ve 4, 5 ve 6 ncı maddeleri uyarınca " + adversarial = (pattern * (20000 // len(pattern) + 1))[:20000] + + def _timeout_handler(signum, frame): + raise TimeoutError("Adversarial extraction timed out") + + old_handler = signal.signal(signal.SIGALRM, _timeout_handler) + signal.alarm(2) + try: + t0 = time.perf_counter() + extract(adversarial) + elapsed = time.perf_counter() - t0 + finally: + signal.alarm(0) + signal.signal(signal.SIGALRM, old_handler) + + assert elapsed < 0.1, f"Adversarial extraction took {elapsed * 1000:.1f}ms, expected < 100ms" diff --git a/tests/test_legal_rules.py b/tests/test_legal_rules.py new file mode 100644 index 0000000..a8c3597 --- /dev/null +++ b/tests/test_legal_rules.py @@ -0,0 +1,148 @@ +"""Tests for mesa_storage.legal_rules module.""" + +import unittest + +from mesa_storage.legal_rules import RULES_VERSION, extract + + +class TestLegalRulesExtractor(unittest.TestCase): + def test_rules_version(self): + self.assertEqual(RULES_VERSION, "tr-legal-rules-3") + + def test_span_invariant(self): + sample = ( + "Yargıtay 3. Hukuk Dairesi 2022/1235 E., 2022/1899 K. sayılı ilamı ile " + "5237 sayılı TCK m. 141 uyarınca verilen 5.000 TL adli para cezasına ve " + "iki yıl hapis cezasına ilişkin karar ONANMASINA 15/04/2023 tarihinde karar verildi." + ) + items = extract(sample) + self.assertGreater(len(items), 0) + for item in items: + start = item["start"] + end = item["end"] + self.assertEqual(sample[start:end], item["text"]) + self.assertIn("type", item) + + def test_empty_and_whitespace_text(self): + self.assertEqual(extract(""), []) + self.assertEqual(extract(" \n\t "), []) + + def test_examples_per_type(self): + # 1. decision_ref + t_dec = "Yargıtay Hukuk Genel Kurulu 2022/1235 E., 2022/1899 K." + items = extract(t_dec) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "decision_ref") + self.assertEqual(items[0]["docket_no"], "2022/1235") + self.assertEqual(items[0]["decision_no"], "2022/1899") + + # 2. citation + t_cit = "6098 sayılı Türk Borçlar Kanununun 49 uncu maddesinin birinci fıkrası" + items = extract(t_cit) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "citation") + self.assertEqual(items[0]["law_no"], "6098") + self.assertEqual(items[0]["article"], "49") + self.assertEqual(items[0]["paragraph"], "1") + + # 3. date + t_date = "Dava 15.04.2023 tarihinde açılmıştır." + items = extract(t_date) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "date") + self.assertEqual(items[0]["value"], "2023-04-15") + + # 4. amount + t_amt = "Davalıdan 5.000 TL tahsiline karar verildi." + items = extract(t_amt) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "amount") + self.assertEqual(items[0]["value"], "5000") + self.assertEqual(items[0]["currency"], "TRY") + + # 5. duration + t_dur = "Sözleşme süresi iki yıl olarak kararlaştırıldı." + items = extract(t_dur) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "duration") + self.assertEqual(items[0]["value"], 2) + self.assertEqual(items[0]["unit"], "yıl") + + # 6. outcome + t_out = "Hükmün DÜZELTİLEREK ONANMASINA karar verildi." + items = extract(t_out) + self.assertEqual(len(items), 1) + self.assertEqual(items[0]["type"], "outcome") + self.assertEqual(items[0]["value"], "düzeltilerek onama") + + def test_error_cause_1_leading_date_in_citation(self): + # Dates emitted inside a citation starting with tarihli ve must not be separate dates + sample = "21/1/2017 tarihli ve 6771 sayılı Kanunun 16 ncı maddesi uyarınca işlem yapılmıştır." + items = extract(sample) + # Citation should start at the date + citations = [it for it in items if it["type"] == "citation"] + dates = [it for it in items if it["type"] == "date"] + self.assertEqual(len(citations), 1) + self.assertTrue(citations[0]["text"].startswith("21/1/2017 tarihli ve 6771 sayılı")) + self.assertEqual(citations[0]["law_no"], "6771") + self.assertEqual(citations[0]["article"], "16") + # Leading date must be suppressed from separate date extraction + self.assertEqual(len(dates), 0) + + def test_error_cause_2_onay_is_not_duration(self): + # 'onay' must not be recognized as 'on ay' (10 months) + sample = "Bakanlık tarafından onay verilen her türlü masrafları kurumca ödenir." + items = extract(sample) + durations = [it for it in items if it["type"] == "duration"] + self.assertEqual(durations, []) + + def test_error_cause_3_compound_number_words(self): + # Compound number words written together (e.g. yirmibeş) + sample1 = "Müebbet hapis cezasını gerektiren suçlarda yirmibeş yıl hapis cezası uygulanır." + items1 = extract(sample1) + durations = [it for it in items1 if it["type"] == "duration"] + self.assertEqual(len(durations), 1) + self.assertEqual(durations[0]["value"], 25) + self.assertEqual(durations[0]["unit"], "yıl") + self.assertEqual(durations[0]["text"], "yirmibeş yıl") + + def test_error_cause_4_comparative_durations(self): + # Comparative durations per schema v1.3: item is number and unit only ('beş yıl', value 5) + sample = "Beş yıldan fazla ve yirmi yıldan aşağı olmamak üzere hapis cezası verilir." + items = extract(sample) + durations = [it for it in items if it["type"] == "duration"] + self.assertEqual(len(durations), 2) + self.assertEqual(durations[0]["text"], "Beş yıl") + self.assertEqual(durations[0]["value"], 5) + self.assertEqual(durations[0]["unit"], "yıl") + self.assertEqual(durations[1]["text"], "yirmi yıl") + self.assertEqual(durations[1]["value"], 20) + self.assertEqual(durations[1]["unit"], "yıl") + + def test_error_cause_5_written_out_and_glued_amounts(self): + # Written-out amounts (yüz milyon lira, ikiyüzyirmi Türk Lirası) and glued 20.000,00-TL + sample1 = "Bu fıkrada yer alan yüz milyon lira para cezası ibaresi değiştirilmiştir." + items1 = extract(sample1) + amts1 = [it for it in items1 if it["type"] == "amount"] + self.assertEqual(len(amts1), 1) + self.assertEqual(amts1[0]["value"], "100000000") + self.assertEqual(amts1[0]["currency"], "TRY") + + sample2 = "İlgili fıkraya ikiyüzyirmi Türk Lirası idari para cezası eklenmiştir." + items2 = extract(sample2) + amts2 = [it for it in items2 if it["type"] == "amount"] + self.assertEqual(len(amts2), 1) + self.assertEqual(amts2[0]["value"], "220") + self.assertEqual(amts2[0]["currency"], "TRY") + + sample3 = "Manevi tazminata yönelik davanın kısmen kabulü ile 20.000,00-TL tahsiline karar verildi." + items3 = extract(sample3) + amts3 = [it for it in items3 if it["type"] == "amount"] + self.assertEqual(len(amts3), 1) + self.assertEqual(amts3[0]["text"], "20.000,00-TL") + self.assertEqual(amts3[0]["value"], "20000.00") + self.assertEqual(amts3[0]["currency"], "TRY") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_model_optional_runtime.py b/tests/test_model_optional_runtime.py index 52c2caf..e4dfa16 100644 --- a/tests/test_model_optional_runtime.py +++ b/tests/test_model_optional_runtime.py @@ -262,7 +262,9 @@ async def test_failing_backfill_keeps_committed_record_searchable(model_runtime) @pytest.mark.asyncio async def test_legacy_model_flag_enables_both_models(model_runtime): - async with model_runtime(MESA_MODEL_ENABLED=True) as env: + async with model_runtime( + MESA_MODEL_ENABLED=True, MESA_EXTRACTION_ENABLED=True + ) as env: runtime = server.state.runtime_profile assert getattr(runtime, "embedding_enabled", None) is True assert getattr(runtime, "extraction_enabled", None) is True diff --git a/tests/test_p403_local_cli.py b/tests/test_p403_local_cli.py index a4a260e..336eeac 100644 --- a/tests/test_p403_local_cli.py +++ b/tests/test_p403_local_cli.py @@ -373,7 +373,8 @@ def test_cli_add_prints_server_conflict_detail_without_traceback(tmp_path, capsy detail = ( "content equals revision 1 (SUPERSEDED); " - "restoring an earlier revision is not supported in v0.9" + "restoring an earlier revision is not supported in v0.9. " + "Add it under a new document_id to store it again." ) document = tmp_path / "a.txt" document.write_text("A") diff --git a/tests/test_p801_article_chunking.py b/tests/test_p801_article_chunking.py index 48e718d..08995bc 100644 --- a/tests/test_p801_article_chunking.py +++ b/tests/test_p801_article_chunking.py @@ -142,6 +142,11 @@ async def interrupt(self, **kwargs): monkeypatch.setattr(type(env.dao), "admit_v4_memory", original) response = await env.client.post("/v4/documents", json=_payload(env, text)) assert response.status_code == 409, response.text + assert response.json()["detail"] == ( + "admitted chunks do not match a supported chunking plan. " + "Retry with the MESA_CHUNKING value used when this revision was " + "first added, or add the content under a new document_id." + ) assert await _rows(env, "memory_mutations") == before @@ -172,7 +177,11 @@ async def interrupt(self, **kwargs): await env.restart(MESA_CHUNKING="auto") response = await env.client.post("/v4/documents", json=_payload(env, text)) assert response.status_code == 409 - assert "ambiguous" in response.json()["detail"] + assert response.json()["detail"] == ( + "admitted chunking plan is ambiguous; no chunks appended. " + "Retry with the MESA_CHUNKING value used when this revision was " + "first added, or add the content under a new document_id." + ) assert await _rows(env, "memory_mutations") == before diff --git a/tests/test_p901_court_names.py b/tests/test_p901_court_names.py new file mode 100644 index 0000000..f11c699 --- /dev/null +++ b/tests/test_p901_court_names.py @@ -0,0 +1,28 @@ +"""Court names must not activate Constitution ranking factors.""" + +import pytest + +from mesa_storage.legal_identity import LegalEntityResolver + + +@pytest.mark.parametrize("law_names", [None, {"Anayasa": "2709"}]) +@pytest.mark.parametrize( + "court", + ["Anayasa Mahkemesi", "Anayasa Mahkemesinin", "Anayasa Mahkemesi'nin", + "ANAYASA MAHKEMESİNİN", "AYM"], +) +def test_constitutional_court_is_not_a_statute(court, law_names): + resolver = LegalEntityResolver(law_names=law_names) + assert resolver.extract_citations(f"{court} 10. madde hakkında ne karar verdi?") == [] + assert resolver.extract_citations(f"10. madde uyarınca {court} karar verir.") == [] + + +@pytest.mark.parametrize("law_names", [None, {"Anayasa": "2709"}]) +def test_court_does_not_hide_a_separate_real_citation(law_names): + resolver = LegalEntityResolver(law_names=law_names) + assert [(c.statute_code, c.article) for c in resolver.extract_citations( + "Anayasa Mahkemesinin Anayasa m. 10 ve TBK m. 49 hakkındaki kararı" + )] == [("Anayasa", "10"), ("TBK", "49")] + assert [(c.statute_code, c.article) for c in resolver.extract_citations( + "AYM, AY m. 10 hükmünü uyguladı." + )] == [("Anayasa", "10")] diff --git a/tests/test_p901_document_identity.py b/tests/test_p901_document_identity.py new file mode 100644 index 0000000..88aeab8 --- /dev/null +++ b/tests/test_p901_document_identity.py @@ -0,0 +1,268 @@ +"""Generic law identities must come from the selected lossless article plan.""" + +import json + +import pytest + +from mesa_memory.config import MesaConfig +from mesa_storage.dao import QueueUnavailableError +from mesa_storage.legal_identity import ( + LegalEntityResolver, + canonicalize_trusted_legal_identity, +) +from mesa_workers.projection_worker import _triplets +from tests.test_document_ingestion import _add, _payload, _rows +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +@pytest.mark.parametrize( + "number,code", [("7201", "7201"), ("6098", "TBK"), ("7596", "7596")] +) +def test_numeric_law_identity(number, code): + identity = canonicalize_trusted_legal_identity( + {"law_id": number, "article_id": "21"} + ) + assert identity["law_id"] == number + assert identity["statute_code"] == code + for bad in ("UNKNOWN", "12", "123456", "7201x"): + with pytest.raises(ValueError): + canonicalize_trusted_legal_identity({"law_id": bad, "article_id": "21"}) + with pytest.raises(ValueError): + canonicalize_trusted_legal_identity( + {"law_id": number, "statute_code": "TCK", "article_id": "21"} + ) + + +@pytest.mark.parametrize( + "query,expected", + [ + ("7201 sayılı Kanunun 21. maddesi", [("7201", "21")]), + ("6098 sayılı Kanunun 49 uncu maddesi", [("TBK", "49")]), + ("TBK m. 49", [("TBK", "49")]), + ("5237 sayılı Kanun m. 81", [("TCK", "81")]), + ("7201 sayılı Tebligat Kanunu'nun 21. maddesi", [("7201", "21")]), + ("MADDE 19 uyarınca işlem yapılır", []), + ], +) +def test_generic_numbered_query(query, expected): + assert [ + (c.statute_code, c.article) + for c in LegalEntityResolver().extract_citations(query) + ] == expected + + +def _text(header="Kanun Numarası: 7201"): + return ( + header + + "\nÖnsöz.\n" + + ( + "MADDE 21- " + "Tebligat usulü uygulanır. " * 160 + "\n" + "MADDE 22- Başka hüküm.\n" + "GEÇİCİ MADDE 1- Geçiş hükmü.\n" + "EK MADDE 1- Ek hüküm.\n" + ) + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "flags,header,metadata,number", + [ + ({}, "Kanun Numarası: 7201", {}, "7201"), + ({}, "Kanunun Numarası: 4734", {}, "4734"), + ({}, "Kanun\nNo. 7596", {}, "7596"), + ({}, "Başlık", {"law_number": "6098"}, "6098"), + ({"MESA_CHUNKING": "plain"}, "Kanun Numarası: 7201", {}, None), + ({"MESA_LEGAL_RULES_ENABLED": False}, "Kanun Numarası: 7201", {}, None), + ], +) +async def test_document_plan_identity(model_runtime, flags, header, metadata, number): + async with model_runtime(**flags) as env: + result = await _add(env, _text(header), metadata=metadata) + by_id = {m["mutation_id"]: m for m in await _rows(env, "memory_mutations")} + mutations = [by_id[mid] for mid in result["mutation_ids"]] + current = None + counts = {} + for mutation in mutations: + text = mutation["content_payload"] + if text.startswith("MADDE "): + current = text.split()[1].rstrip("-") + elif text.startswith((header, "GEÇİCİ", "EK")): + current = None + identity = json.loads(mutation["metadata_json"]).get("_mesa_legal_identity") + if number and current: + assert identity["law_id"] == number + assert identity["article_id"] == current + counts[current] = counts.get(current, 0) + 1 + else: + assert identity is None + if number: + assert counts["21"] > 1 + await _consume(env, result["chunk_count"] + 2) + for assertion in await _rows(env, "v4_assertions"): + mutation = next( + m for m in mutations if m["mutation_id"] == assertion["mutation_id"] + ) + identity = json.loads(mutation["metadata_json"]).get("_mesa_legal_identity") + projected = await env.dao.get_projection_mutation(mutation["mutation_id"]) + fallback = _triplets(projected)[0] + assert (fallback["head"], fallback["relation"], fallback["tail"]) == ( + mutation["document_id"], "madde", mutation["chunk_id"] + ) + assert fallback["literal_value"] is None + assert assertion["predicate"] == "madde" + assert assertion["object_entity_id"] + assert (assertion["law_id"] or "") == ( + identity["law_id"] if identity else "" + ) + + +def test_rules_config(monkeypatch): + monkeypatch.delenv("MESA_LEGAL_RULES_ENABLED", raising=False) + assert MesaConfig(_env_file=None).legal_rules_enabled is True + monkeypatch.setenv("MESA_LEGAL_RULES_ENABLED", "false") + assert MesaConfig(_env_file=None).legal_rules_enabled is False + + +@pytest.mark.asyncio +async def test_resume_does_not_rewrite_admitted_identity(model_runtime, monkeypatch): + async with model_runtime() as env: + original = type(env.dao).admit_v4_memory + calls = 0 + + async def interrupt(self, **kwargs): + nonlocal calls + calls += 1 + if calls == 3: + raise QueueUnavailableError("injected interruption") + return await original(self, **kwargs) + + monkeypatch.setattr(type(env.dao), "admit_v4_memory", interrupt) + response = await env.client.post("/v4/documents", json=_payload(env, _text())) + assert response.status_code == 503 + before = await _rows(env, "memory_mutations") + assert all(json.loads(m["metadata_json"])["_mesa_document_chunk"] is True for m in before) + assert any( + json.loads(m["metadata_json"]).get("_mesa_legal_identity") for m in before + ) + monkeypatch.setattr(type(env.dao), "admit_v4_memory", original) + await env.restart(MESA_CHUNKING="plain", MESA_LEGAL_RULES_ENABLED=False) + await _add(env, _text()) + after = {m["mutation_id"]: m for m in await _rows(env, "memory_mutations")} + assert all(after[m["mutation_id"]]["metadata_json"] == m["metadata_json"] for m in before) + + +def test_catalog_memory_fallback_keeps_article_subject(): + record = { + "document_id": "catalog-law", + "chunk_id": "catalog-article", + "content_payload": "MADDE 49- Tazminat hükmü.", + "legal_identity": canonicalize_trusted_legal_identity({ + "law_id": "6098", "article_id": "49", "identity_source": "catalog" + }), + } + triplet = _triplets(record)[0] + assert (triplet["head"], triplet["relation"], triplet["tail"]) == ( + "TBK m.49", "hükmü", None + ) + assert triplet["literal_value"] == record["content_payload"] + + +@pytest.mark.parametrize("source", ["catalog", "verified_ingestion"]) +@pytest.mark.parametrize("document_chunk", [False, True]) +def test_fallback_origin_is_independent_of_identity_source(source, document_chunk): + record = { + "document_id": "document", + "chunk_id": "chunk", + "content_payload": "MADDE 49- Tazminat hükmü.", + "document_chunk": document_chunk, + "legal_identity": canonicalize_trusted_legal_identity({ + "law_id": "6098", "article_id": "49", "identity_source": source + }), + } + triplet = _triplets(record)[0] + if document_chunk: + assert (triplet["head"], triplet["relation"], triplet["tail"]) == ( + "document", "madde", "chunk" + ) + assert triplet["literal_value"] is None + else: + assert (triplet["head"], triplet["relation"], triplet["tail"]) == ( + "TBK m.49", "hükmü", None + ) + assert triplet["literal_value"] == record["content_payload"] + + +@pytest.mark.asyncio +async def test_document_origin_marker_is_internal_and_durable(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + added = await _add( + env, + "Kanun Numarası: 6098\nMADDE 49- Tazminat hükmü.\n" + "MADDE 50- Başka hüküm.\nMADDE 51- Son hüküm.\n", + ) + content = "MADDE 49- Tazminat hükmü." + response = await env.client.post( + "/v4/memory/insert?document_chunk=true&_document_chunk=true", + json={ + "session_id": env.session_id, + "dataset_id": "main", + "document_id": "explicit-memory", + "revision_id": "explicit-revision", + "chunk_id": "explicit-chunk", + "title": "Explicit article", + "source_ref": "file://explicit", + "content": content, + "legal_identity": {"law_id": "6098", "article_id": "49"}, + "metadata": {"_mesa_document_chunk": True}, + "document_chunk": True, + }, + ) + assert response.status_code == 422 + payload = json.loads(response.request.content) + payload["metadata"] = {} + response = await env.client.post( + "/v4/memory/insert?document_chunk=true&_document_chunk=true", json=payload + ) + assert response.status_code == 202, response.text + manual_mid = response.json()["mutation_id"] + mutations = {m["mutation_id"]: m for m in await _rows(env, "memory_mutations")} + for mid in added["mutation_ids"]: + assert json.loads(mutations[mid]["metadata_json"])["_mesa_document_chunk"] is True + assert "_mesa_document_chunk" not in json.loads(mutations[manual_mid]["metadata_json"]) + raw_mutations = {m["raw_log_id"]: m["mutation_id"] for m in mutations.values()} + for raw in await _rows(env, "raw_logs"): + payload = json.loads(raw["payload"]) + assert payload["metadata"].get("_mesa_document_chunk", False) is ( + raw_mutations[raw["id"]] in added["mutation_ids"] + ) + await _consume(env, added["chunk_count"] + 3) + article_mid = next( + mid for mid in added["mutation_ids"] + if (json.loads(mutations[mid]["metadata_json"]).get("_mesa_legal_identity") or {}).get("article_id") == "49" + ) + doc = await env.dao.get_projection_mutation(article_mid) + manual = await env.dao.get_projection_mutation(manual_mid) + assert doc["document_chunk"] is True + assert manual["document_chunk"] is False + assert doc["legal_identity"] == manual["legal_identity"] + assert "_mesa_document_chunk" not in doc["metadata"] + doc_fallback = _triplets(doc)[0] + assert (doc_fallback["head"], doc_fallback["relation"], doc_fallback["tail"]) == ( + doc["document_id"], "madde", doc["chunk_id"] + ) + manual_fallback = _triplets(manual)[0] + assert (manual_fallback["head"], manual_fallback["relation"], manual_fallback["tail"]) == ( + "TBK m.49", "hükmü", None + ) + assertions = {a["mutation_id"]: a for a in await _rows(env, "v4_assertions")} + assert assertions[article_mid]["predicate"] == "madde" + assert assertions[article_mid]["literal_value"] is None + assert assertions[manual_mid]["predicate"] == "hükmü" + assert assertions[manual_mid]["literal_value"] == content + from mesa_memory.api import server + + operation = server.app.openapi()["paths"]["/v4/memory/insert"]["post"] + assert operation["operationId"] == "insert_memory_v4_memory_insert_post" + assert not any(p["name"] in {"document_chunk", "_document_chunk"} for p in operation.get("parameters", [])) diff --git a/tests/test_p901_exact_scope.py b/tests/test_p901_exact_scope.py new file mode 100644 index 0000000..fd923eb --- /dev/null +++ b/tests/test_p901_exact_scope.py @@ -0,0 +1,149 @@ +"""Store-derived law names and the bounded exact-match-first slot.""" + +import pytest + +from mesa_storage.legal_identity import LegalEntityResolver +from tests.test_document_ingestion import _add +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +@pytest.mark.parametrize( + "query,expected", + [ + ("Kamulaştırma Kanunu 35. madde", [("2942", "35")]), + ("TEBLİGAT K. 21. madde", [("7201", "21")]), + ("Tebligat Kanunu'nun 21. maddesi", [("7201", "21")]), + ("Tebligat Kanununun 21 inci maddesi", [("7201", "21")]), + ("Borçlar Kanunu m. 49", [("818", "49")]), + ("Türk Borçlar Kanunu m. 49", [("TBK", "49")]), + ("2017 yılında 5 maddelik bir düzenleme", []), + ], +) +def test_scope_names_override_only_the_longest_exact_name(query, expected): + resolver = LegalEntityResolver( + law_names={ + "Kamulaştırma Kanunu": "2942", + "Tebligat Kanunu": "7201", + "Borçlar Kanunu": "818", + "Türk Borçlar Kanunu": "6098", + } + ) + assert [ + (c.statute_code, c.article) for c in resolver.extract_citations(query) + ] == expected + + +def test_no_store_name_no_new_name_resolution(): + assert LegalEntityResolver().extract_citations("Tebligat K. 21. madde") == [] + assert [ + (c.statute_code, c.article) + for c in LegalEntityResolver().extract_citations("Borçlar Kanunu 49") + ] == [("TBK", "49")] + + +@pytest.mark.asyncio +async def test_exact_slot_is_bounded_in_document_order_and_preserves_other_order( + model_runtime, monkeypatch +): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + text = "TEBLİGAT KANUNU1\nKanun Numarası: 7201\nÖnsöz.\n" + ( + "MADDE 21- " + "Tebligat hükmü. " * 800 + "\n" + "MADDE 22- Tebligat hükmü.\nMADDE 23- Tebligat hükmü.\n" + ) + added = await _add(env, text, title="opaque.txt") + await _consume(env, added["chunk_count"] + 2) + + async def search(query): + return await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query=query, + limit=50, + graph_enabled=False, + ) + + result = await search("Tebligat K. 21. madde") + expected = [ + m + for m in added["mutation_ids"] + if ( + (await env.dao.get_projection_mutation(m)).get("legal_identity") or {} + ).get("article_id") + == "21" + ] + assert len(expected) > 3 + assert [r["provenance"][0]["mutation_id"] for r in result[:3]] == expected[:3] + assert [ + r["retrieval_provenance"]["exact_match"]["order"] for r in result[:3] + ] == [0, 1, 2] + assert sum("exact_match" in r["retrieval_provenance"] for r in result) == 3 + remainder = [ + r for r in result if "exact_match" not in r["retrieval_provenance"] + ] + assert remainder == sorted( + remainder, key=lambda r: (-r["final_score"], r["candidate_id"]) + ) + for query in ("Tebligat K. 9999. madde", "99999 sayılı Kanunun 21. maddesi"): + assert all( + "exact_match" not in r["retrieval_provenance"] + for r in await search(query) + ) + # The revision name/order cache is reused, and purging invalidates it. + scope = next(iter(env.dao._search_scope_cache.values())) + assert "legal_context" in scope + before = await search("Tebligat K. 21. madde") + assert [r["candidate_id"] for r in before] == [ + r["candidate_id"] for r in result + ] + # An out-of-time first chunk must not teach its header to this scope. + async with env.dao._sql.transaction() as db: + await db.execute( + "UPDATE v4_assertions SET valid_from='2030-01-01' WHERE mutation_id=?", + (added["mutation_ids"][0],), + ) + await db.commit() + historical = await env.dao.search_v4_memory( + tenant_id="model-tenant", agent_id="model-agent", dataset_ids=["main"], + query="Tebligat K. 21. madde", graph_enabled=False, valid_at="2020-01-01", + ) + assert all("exact_match" not in r["retrieval_provenance"] for r in historical) + await env.dao.purge_v4_document( + tenant_id="model-tenant", dataset_id="main", document_id="whole-document" + ) + assert await search("Tebligat K. 21. madde") == [] + + +@pytest.mark.asyncio +async def test_store_names_cannot_cross_dataset_scope(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + content = "Kanun Numarası: 2942\nMADDE 1- Kural.\nMADDE 2- Kural.\nMADDE 35- Kamulaştırma.\n" + added = await _add(env, content, title="Kamulaştırma Kanunu") + await _consume(env, added["chunk_count"] + 2) + matching = await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="Kamulaştırma Kanunu 35. madde", + graph_enabled=False, + ) + assert matching[0]["retrieval_provenance"]["legal_identity"]["law_id"] == "2942" + assert ( + matching[0]["retrieval_provenance"]["legal_identity"]["article_id"] == "35" + ) + await env.dao.ensure_v4_catalog_scope( + tenant_id="model-tenant", workspace_id="default", dataset_id="empty" + ) + result = await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["empty"], + query="Kamulaştırma Kanunu 35. madde", + graph_enabled=False, + ) + assert result == [] + empty_scope = next( + s for k, s in env.dao._search_scope_cache.items() if k[1] == ("empty",) + ) + assert not empty_scope["legal_context"]["names"] diff --git a/tests/test_p902_rule_items.py b/tests/test_p902_rule_items.py new file mode 100644 index 0000000..fcfa76b --- /dev/null +++ b/tests/test_p902_rule_items.py @@ -0,0 +1,194 @@ +"""Runtime-owned rule items are source evidence, never ranking inputs.""" + +import copy +import json + +import pytest + +from mesa_memory.config import config +from mesa_storage.legal_rules import RULES_VERSION, extract +from mesa_workers.projection_worker import _triplets +from tests.test_document_ingestion import _add, _rows +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + +TEXT = "15.03.2024 tarihinde TBK m. 49 uygulanır." + + +async def _admit(env, metadata): + return await env.dao.admit_v4_memory( + tenant_id="model-tenant", + workspace_id="default", + dataset_id="main", + agent_id="model-agent", + session_id=env.session_id, + document_id="rules-document", + revision_id="rules-revision", + chunk_id="rules-chunk", + title="Rules", + content_payload=TEXT, + source_ref="file://rules.txt", + evidence_span=TEXT, + revision_number=1, + chunk_ordinal=0, + supersedes_revision_id=None, + metadata=metadata, + legal_identity={"law_id": "6098", "article_id": "1"}, + embedding_provider="test", + embedding_model="rules", + embedding_version="v1", + embedding_dimension=2, + validation_mode=0, + policy=config.queue_admission_policy, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("enabled", [True, False]) +async def test_items_are_stored_and_returned_with_chunk_offsets(model_runtime, enabled): + async with model_runtime( + MESA_LEGAL_RULES_ENABLED=enabled, MESA_GRAPH_ENABLED=False + ) as env: + text = ( + "Kanun Numarası: 6098\nMADDE 1- " + + TEXT + + "\nMADDE 2- Kural.\nMADDE 3- Kural.\n" + ) + added = await _add(env, text) + await _consume(env, added["chunk_count"] + 2) + mutations = await _rows(env, "memory_mutations") + stored = {} + for mutation in mutations: + item_set = json.loads(mutation["metadata_json"]).get("_mesa_legal_items") + if enabled: + assert item_set == { + "version": RULES_VERSION, + "items": extract(mutation["content_payload"]), + } + for item in item_set["items"]: + assert ( + mutation["content_payload"][item["start"] : item["end"]] + == item["text"] + ) + else: + assert item_set is None + stored[mutation["mutation_id"]] = item_set + if enabled: + assert { + item["type"] for value in stored.values() for item in value["items"] + } >= {"date", "citation"} + for raw in await _rows(env, "raw_logs"): + assert ( + json.loads(raw["payload"])["metadata"].get("_mesa_legal_items") + in stored.values() + ) + results = await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="TBK m. 49 15.03.2024", + limit=50, + graph_enabled=False, + ) + assert results + for result in results: + mid = result["provenance"][0]["mutation_id"] + assert result["retrieval_provenance"].get("legal_items") == stored[mid] + if not enabled: + return + # Remove only stored items: preserve identities, candidates, scores and order. + before = copy.deepcopy(results) + for result in before: + result["retrieval_provenance"].pop("legal_items", None) + async with env.dao._sql.transaction() as db: + await db.execute( + "UPDATE memory_mutations SET metadata_json=json_remove(metadata_json,'$._mesa_legal_items')" + ) + await db.commit() + after = await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="TBK m. 49 15.03.2024", + limit=50, + graph_enabled=False, + ) + assert after == before + + +@pytest.mark.asyncio +async def test_admission_replaces_forged_items_and_extraction_keeps_them(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + admitted = await _admit( + env, {"_mesa_legal_items": {"version": "forged", "items": []}} + ) + mid = admitted["response"]["mutation_id"] + expected = {"version": RULES_VERSION, "items": extract(TEXT)} + metadata = json.loads( + (await _rows(env, "memory_mutations"))[0]["metadata_json"] + ) + assert metadata["_mesa_legal_items"] == expected + assert await env.dao.record_mutation_extraction( + "model-agent", + mid, + [ + { + "head": "Kişi", + "relation": "uygular", + "tail": None, + "literal_value": "ilk hüküm", + "source_span": TEXT, + }, + { + "head": "Kişi", + "relation": "uygular", + "tail": None, + "literal_value": "ikinci hüküm", + "source_span": TEXT, + }, + ], + ) + metadata = json.loads( + (await _rows(env, "memory_mutations"))[0]["metadata_json"] + ) + assert metadata["_mesa_legal_items"] == expected + mutation = await env.dao.get_projection_mutation(mid) + triplets = _triplets(mutation) + assert len(triplets) == 2 + for triplet in triplets: + await env.dao.project_v4_sql_entity( + mutation=mutation, entity_name=triplet["head"] + ) + await env.dao.project_v4_sql_assertion(mutation=mutation, triplet=triplet) + assertions = await env.dao.list_v4_assertions_for_mutation(mid) + assert len(assertions) == 2 + assert { + (a["law_id"], a["statute_code"], a["article_id"]) for a in assertions + } == {("6098", "TBK", "1")} + + +@pytest.mark.asyncio +async def test_document_purge_erases_rule_quotes_in_both_ledgers(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + await _add(env, TEXT) + await _consume(env, 3) + # Seed a runtime-owned item snapshot explicitly so this privacy test also + # covers upgraded stores independently of the admission implementation. + item_set = json.dumps({"version": RULES_VERSION, "items": extract(TEXT)}) + async with env.dao._sql.transaction() as db: + await db.execute( + "UPDATE memory_mutations SET metadata_json=json_set(metadata_json,'$._mesa_legal_items',json(?))", + (item_set,), + ) + await db.execute( + "UPDATE raw_logs SET payload=json_set(payload,'$.metadata._mesa_legal_items',json(?))", + (item_set,), + ) + await db.commit() + await env.dao.purge_v4_document( + tenant_id="model-tenant", dataset_id="main", document_id="whole-document" + ) + for row in await _rows(env, "memory_mutations"): + assert "_mesa_legal_items" not in json.loads(row["metadata_json"]) + for row in await _rows(env, "raw_logs"): + assert "_mesa_legal_items" not in json.loads(row["payload"])["metadata"] diff --git a/tests/test_p903_decision_slot.py b/tests/test_p903_decision_slot.py new file mode 100644 index 0000000..6a65bb8 --- /dev/null +++ b/tests/test_p903_decision_slot.py @@ -0,0 +1,142 @@ +"""Only an authorized decision's own first-chunk header can use the exact slot.""" + +import copy + +import pytest + +from tests.test_document_ingestion import _add +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + +REF = "Ceza Genel Kurulu 2017/885 E., 2021/693 K." +OTHER = "Hukuk Genel Kurulu 2000/11 E., 2001/22 K." + + +async def _search(env, query, **scope): + params = dict( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query=query, + limit=50, + graph_enabled=False, + ) + params.update(scope) + return await env.dao.search_v4_memory(**params) + + +@pytest.mark.asyncio +async def test_own_header_first_chunk_wins_and_quoted_reference_does_not(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + own = await _add( + env, REF + "\n" + "Gerekçe açıklanmıştır. " * 600, document_id="own" + ) + quoted = await _add( + env, + OTHER + "\n" + "Açıklama devam eder. " * 100 + REF + " kararı " * 30, + document_id="quoted", + ) + await _consume(env, own["chunk_count"] + quoted["chunk_count"] + 3) + results = await _search(env, REF + " kararı") + assert results[0]["provenance"][0]["mutation_id"] == own["mutation_ids"][0] + marked = [r for r in results if "exact_match" in r["retrieval_provenance"]] + assert len(marked) == 1 + assert marked[0]["retrieval_provenance"]["exact_match"] == { + "kind": "decision", + "order": 0, + } + before = copy.deepcopy(results) + context = next(iter(env.dao._search_scope_cache.values()))["legal_context"] + assert await _search(env, REF + " kararı") == before + assert ( + next(iter(env.dao._search_scope_cache.values()))["legal_context"] is context + ) + assert await _search(env, REF, agent_id="other-agent") == [] + await env.dao.ensure_v4_catalog_scope( + tenant_id="model-tenant", workspace_id="default", dataset_id="other-dataset" + ) + assert await _search(env, REF, dataset_ids=["other-dataset"]) == [] + await env.dao.ensure_v4_catalog_scope( + tenant_id="other-tenant", workspace_id="default", dataset_id="main" + ) + assert await _search(env, REF, tenant_id="other-tenant") == [] + # Out-of-time first chunks must not teach their own reference via a + # still-authorized continuation or another decision's quotation. + async with env.dao._sql.transaction() as db: + await db.execute( + "UPDATE v4_assertions SET valid_from='2030-01-01' WHERE mutation_id=?", + (own["mutation_ids"][0],), + ) + await db.commit() + historical = await _search(env, REF + " kararı", valid_at="2020-01-01") + assert all("exact_match" not in r["retrieval_provenance"] for r in historical) + await env.dao.purge_v4_document( + tenant_id="model-tenant", dataset_id="main", document_id="own" + ) + assert all( + "exact_match" not in r["retrieval_provenance"] + for r in await _search(env, REF + " kararı") + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("offset,expected", [(299, True), (300, False)]) +async def test_own_reference_starts_within_first_300_characters( + model_runtime, offset, expected +): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + await _add(env, " " * offset + REF + "\nGerekçe.") + await _consume(env, 3) + results = await _search(env, REF + " kararı") + assert ( + any("exact_match" in r["retrieval_provenance"] for r in results) is expected + ) + + +@pytest.mark.asyncio +async def test_unmatched_reference_preserves_results_without_items(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + await _add(env, OTHER + "\n" + "Yargılama kararı gerekçesi. " * 80) + await _consume(env, 3) + query = REF + " kararı gerekçesi" + before = copy.deepcopy(await _search(env, query)) + assert before and all( + "exact_match" not in r["retrieval_provenance"] for r in before + ) + for row in before: + row["retrieval_provenance"].pop("legal_items", None) + async with env.dao._sql.transaction() as db: + await db.execute( + "UPDATE memory_mutations SET metadata_json=json_remove(metadata_json,'$._mesa_legal_items')" + ) + await db.commit() + assert await _search(env, query) == before + + +@pytest.mark.asyncio +async def test_articles_and_decisions_share_three_unique_chunk_slots(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + law = await _add( + env, + "Kanun Numarası: 6098\nMADDE 49- " + + "Tazminat hükmü uygulanır. " * 800 + + "\nMADDE 50- Kural.\nMADDE 51- Kural.\n", + document_id="law", + ) + decision = await _add(env, REF + "\nGerekçe.", document_id="decision") + await _consume(env, law["chunk_count"] + decision["chunk_count"] + 3) + results = await _search(env, "TBK m. 49 ve " + REF + " kararı") + marked = [r for r in results if "exact_match" in r["retrieval_provenance"]] + assert len(marked) == 3 + assert len({r["source_chunk_id"] for r in marked}) == 3 + assert [r["retrieval_provenance"]["exact_match"]["order"] for r in marked] == [ + 0, + 1, + 2, + ] + remainder = [ + r for r in results if "exact_match" not in r["retrieval_provenance"] + ] + assert remainder == sorted( + remainder, key=lambda r: (-r["final_score"], r["candidate_id"]) + ) diff --git a/tests/test_p905_rule_limits.py b/tests/test_p905_rule_limits.py new file mode 100644 index 0000000..8e760c4 --- /dev/null +++ b/tests/test_p905_rule_limits.py @@ -0,0 +1,306 @@ +"""The public input limits must also bound legal extraction latency.""" + +import asyncio +import random +import signal +import threading +import time + +import pytest + +from mesa_storage import legal_rules +from tests.test_document_ingestion import _add +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +# Wall-clock tests use no_cover so tracing does not consume the 200 ms budget. +# They still run in the coverage job; functional/structural tests stay traced. +def _timed_extract(text): + def expired(*_): + raise AssertionError("legal extraction exceeded 200 ms") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, 0.2) + start = time.perf_counter() + try: + items = legal_rules.extract(text) + assert time.perf_counter() - start < 0.2 + assert all(text[it["start"] : it["end"]] == it["text"] for it in items) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +@pytest.mark.parametrize( + "stem", + [ + " " * 1000 + "2020/1 x", + "bir ", + "1.000.", + "m. 1 m. 2 ", + " \n\t", + "dosyanın gönderilmek üzere yerine ", + "(Değişik ", + ], + ids=[ + "spaces-docket", + "number-words", + "dotted-digits", + "article-list", + "whitespace", + "forwarding-prefix", + "open-amendments", + ], +) +@pytest.mark.no_cover +def test_opus_repros_at_maximum_insert_size(stem): + text = (stem * (32768 // len(stem) + 1))[:32767] + "/" + _timed_extract(text) + + +@pytest.mark.no_cover +def test_seeded_grammar_fuzz_at_api_limits(): + rng = random.Random(905) + tokens = [ + " ", + "\t", + "\n", + "1", + "2020", + "/", + ".", + "-", + "E.", + "K.", + "Esas", + "Karar", + "madde", + "sayılı", + "bir", + "yüz", + "milyon", + "TL", + "lira", + "dosyanın", + "gönderilmek", + "üzere", + "TEVDİİNE", + ] + for size in (4096, 32768): + for _ in range(24): + parts = [] + length = 0 + while length < size: + token = rng.choice(tokens) + parts.append(token) + length += len(token) + _timed_extract("".join(parts)[:size]) + + +def test_bounded_lists_preserve_normal_values_and_source_spans(): + text = "5gün ve yirmibeş yıl. 1.234.567,89-TL. TCK m. 17.3.2." + items = legal_rules.extract(text) + assert [(x["value"], x["unit"]) for x in items if x["type"] == "duration"] == [ + (5, "gün"), (25, "yıl") + ] + assert [x["value"] for x in items if x["type"] == "amount"] == ["1234567.89"] + assert [x["article"] for x in items if x["type"] == "citation"] == ["17.3.2"] + assert all(text[x["start"]:x["end"]] == x["text"] for x in items) + + +@pytest.mark.parametrize( + "stem", + [ + "ONANMASINA ", + "01.01.2020 ", + "TCK m. 1 ", + "2020/1 E., 2020/2 K. ", + "bir TL ", + "(Değişik 01.01.2020 ", + "geçici 1 ila geçici 999999999999 madde ", + " " * 120 + "2020/1 E., 2020/2 K.\n", + ], +) +@pytest.mark.no_cover +def test_many_matches_and_unclosed_markers_remain_bounded(stem): + _timed_extract((stem * (32768 // len(stem) + 1))[:32768]) + + +def test_compiled_patterns_have_only_bounded_terminal_repetitions(): + # Guard the regex structure as well as observed runtimes. Single optional + # clauses are allowed; repeating a group containing another repeat is not. + import re + from re import _constants, _parser + + def audit(tree): + for op, args in tree: + if op in ( + _constants.MAX_REPEAT, + _constants.MIN_REPEAT, + _constants.POSSESSIVE_REPEAT, + ): + _, maximum, body = args + assert maximum != _constants.MAXREPEAT + if maximum > 1: + assert all( + child + in ( + _constants.LITERAL, + _constants.NOT_LITERAL, + _constants.IN, + _constants.ANY, + ) + for child, _ in body + ), (op, args) + audit(body) + elif op is _constants.SUBPATTERN: + audit(args[-1]) + elif op is _constants.BRANCH: + for branch in args[1]: + audit(branch) + elif op in (_constants.ASSERT, _constants.ASSERT_NOT): + audit(args[1]) + + patterns = [p for p in vars(legal_rules).values() if isinstance(p, re.Pattern)] + patterns += [p for _, p in legal_rules._OUTCOME_RULES] + assert len(patterns) >= 20 + for pattern in patterns: + audit(_parser.parse(pattern.pattern, pattern.flags)) + + +@pytest.mark.asyncio +async def test_admission_and_decision_query_extract_off_the_event_loop( + model_runtime, monkeypatch +): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + from mesa_storage.legal_identity import LegalEntityResolver + + main_thread = threading.get_ident() + original = legal_rules.extract + calls = [] + + def checked(text): + calls.append(threading.get_ident()) + assert threading.get_ident() != main_thread + return original(text) + + monkeypatch.setattr(legal_rules, "extract", checked) + await _add(env, "Ceza Genel Kurulu 2017/885 E., 2021/693 K.\nTBK m. 49 uygulanır.") + await _consume(env, 3) + admitted = len(calls) + assert admitted + resolver_extract = LegalEntityResolver.extract_citations + resolved = [] + + def checked_resolver(self, text): + assert threading.get_ident() != main_thread + resolved.append(True) + return resolver_extract(self, text) + + monkeypatch.setattr(LegalEntityResolver, "extract_citations", checked_resolver) + await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="2017/885 E., 2021/693 K.", + limit=5, + graph_enabled=False, + ) + assert len(calls) > admitted + assert resolved + # A legal query also parses untyped passage/assertion/candidate text. + await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="TBK m. 49", + limit=5, + graph_enabled=False, + ) + await asyncio.sleep(0) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("plan", ["auto", "plain"]) +async def test_document_chunking_and_resume_alternative_run_off_loop( + model_runtime, monkeypatch, plan +): + from mesa_api import v4_router + from mesa_storage.dao import QueueUnavailableError + from tests.test_document_ingestion import _payload + + calls = [] + loop_thread = threading.get_ident() + for name in ("chunk_legal_document", "chunk_document"): + original = getattr(v4_router, name) + + def checked(text, name=name, original=original): + assert threading.get_ident() != loop_thread + calls.append(name) + return original(text) + + monkeypatch.setattr(v4_router, name, checked) + async with model_runtime(MESA_CHUNKING=plan, MESA_GRAPH_ENABLED=False) as env: + original_admit = type(env.dao).admit_v4_memory + count = 0 + + async def interrupt(self, **kwargs): + nonlocal count + count += 1 + if count == 3: + raise QueueUnavailableError("injected partial admission") + return await original_admit(self, **kwargs) + + monkeypatch.setattr(type(env.dao), "admit_v4_memory", interrupt) + content = "Durable memory preserves evidence. " * 650 + response = await env.client.post("/v4/documents", json=_payload(env, content)) + assert response.status_code == 503 + monkeypatch.setattr(type(env.dao), "admit_v4_memory", original_admit) + await _consume(env, 4) + calls.clear() + await env.restart(MESA_CHUNKING="plain" if plan == "auto" else "auto") + await _add(env, content) + assert sorted(calls) == ["chunk_document", "chunk_legal_document"] + + +@pytest.mark.asyncio +async def test_whole_document_validation_runs_off_loop(model_runtime, monkeypatch): + from mesa_api import v4_router + + original = v4_router.validate_write_payload + loop_thread = threading.get_ident() + large_checks = [] + + def checked(content, metadata): + if len(content) > 32768: + assert threading.get_ident() != loop_thread + large_checks.append(len(content)) + return original(content, metadata) + + monkeypatch.setattr(v4_router, "validate_write_payload", checked) + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + await _add(env, " " * 32768 + "2020/1 x") + assert large_checks == [32776] + + +@pytest.mark.asyncio +async def test_deferred_document_validation_keeps_rejection_contract(model_runtime): + from tests.test_document_ingestion import _payload, _rows + + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + cases = [ + ({"content": " "}, 422, "content must not be empty"), + ({"content": "secret: example-placeholder"}, 422, "appears to contain a secret"), + ({"metadata": {"_mesa_document_chunk": True}}, 422, "reserved"), + ({"title": 123}, 422, "body.title"), + ({"content": "x" * 5_000_001}, 413, "exceeds 5,000,000"), + ] + for changes, status, detail in cases: + response = await env.client.post( + "/v4/documents", json={**_payload(env, "Valid text."), **changes} + ) + assert response.status_code == status, response.text + assert detail in response.json()["detail"] + assert await _rows(env, "memory_mutations") == [] + assert await _rows(env, "documents") == [] diff --git a/tests/test_p906_mixed_citations.py b/tests/test_p906_mixed_citations.py new file mode 100644 index 0000000..06e5bdc --- /dev/null +++ b/tests/test_p906_mixed_citations.py @@ -0,0 +1,95 @@ +"""Mask store names without changing the spelling of independent citations.""" + +import pytest + +from mesa_storage.legal_identity import LegalEntityResolver +from tests.test_document_ingestion import _add +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +def _pairs(resolver, query): + return {(c.statute_code, c.article) for c in resolver.extract_citations(query)} + + +@pytest.mark.parametrize( + "query", + [ + "Tebligat Kanunu 21. madde ve 6325 sayılı Kanunun 2. maddesi", + " \tTEBLİGAT K.\n21. madde ve 6325 sayılı Kanunun 2. maddesi ", + "6325 sayılı Kanunun 2. maddesi ile Tebligat K. 21. madde", + ], +) +def test_numbered_citation_survives_store_name_and_offset_changes(query): + resolver = LegalEntityResolver(law_names={"Tebligat Kanunu": "7201"}) + assert _pairs(resolver, query) == {("7201", "21"), ("6325", "2")} + + +def test_ontology_alias_number_and_store_name_are_all_kept(): + resolver = LegalEntityResolver(law_names={"Tebligat Kanunu": "7201"}) + query = "TCK m. 81 ve Tebligat Kanunu 21. madde ile 6325 sayılı Kanunun 2. maddesi" + assert _pairs(resolver, query) == {("TCK", "81"), ("7201", "21"), ("6325", "2")} + + +def test_numbered_law_and_decomposed_turkish_store_name(): + resolver = LegalEntityResolver(law_names={"Zümrüt Taşları Kanunu": "8888"}) + query = "5237 sayılı Kanunun 81. maddesi ile Zu\u0308mru\u0308t Tas\u0327ları K. 3. madde" + assert _pairs(resolver, query) == {("TCK", "81"), ("8888", "3")} + + +@pytest.mark.parametrize( + "numbered", + [ + "6325 sayılı Kanunun 2. maddesi", + "6325 sayılı Kanun m. 2", + "6325 sayılı Kanun’un 2. maddesi", + "6325 sayılı Hukuk Uyuşmazlıklarında Arabuluculuk Kanununun 2 nci maddesi", + # A newline inside the title must keep the same rejection as without a store name. + "6325 sayılı Hukuk\nUyuşmazlıklarında Arabuluculuk Kanununun 2 nci maddesi", + ], +) +def test_numbered_grammar_is_identical_with_and_without_a_store_name(numbered): + expected = _pairs(LegalEntityResolver(), numbered) + resolver = LegalEntityResolver(law_names={"Tebligat Kanunu": "7201"}) + combined = _pairs(resolver, numbered + " ve Tebligat K. 21. madde") + assert combined == expected | {("7201", "21")} + + +@pytest.mark.asyncio +async def test_both_typed_articles_use_the_existing_exact_slot(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + first = await _add( + env, + "Kanun Numarası: 7201\nMADDE 21- Tebligat hükmü.\n" + "MADDE 22- Kural.\nMADDE 23- Kural.\n", + document_id="tebligat", + title="Tebligat Kanunu", + ) + second = await _add( + env, + "Kanun Numarası: 6325\nMADDE 2- Arabuluculuk hükmü.\n" + "MADDE 3- Kural.\nMADDE 4- Kural.\n", + document_id="arabuluculuk", + title="Arabuluculuk Kanunu", + ) + await _consume(env, first["chunk_count"] + second["chunk_count"] + 3) + results = await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query="Tebligat Kanunu 21. madde ve 6325 sayılı Kanunun 2. maddesi", + graph_enabled=False, + limit=10, + ) + promoted = [r for r in results if "exact_match" in r["retrieval_provenance"]] + assert len(promoted) == 2 + assert { + ( + r["retrieval_provenance"]["legal_identity"]["law_id"], + r["retrieval_provenance"]["legal_identity"]["article_id"], + ) + for r in promoted + } == {("7201", "21"), ("6325", "2")} + assert [ + r["retrieval_provenance"]["exact_match"]["order"] for r in promoted + ] == [0, 1] diff --git a/tests/test_p907_citation_limits.py b/tests/test_p907_citation_limits.py new file mode 100644 index 0000000..59f6f36 --- /dev/null +++ b/tests/test_p907_citation_limits.py @@ -0,0 +1,232 @@ +"""Citation parsing must stay responsive for queries and stored API-sized text.""" + +import json +import random +import signal +import subprocess +import sys +import time + +import pytest + +from mesa_storage.legal_identity import LegalEntityResolver + + +# Wall-clock tests use no_cover so tracing does not consume the 200 ms budget. +# They still run in the coverage job; functional/structural tests stay traced. +def _timed_extract(resolver, text): + def expired(*_): + raise AssertionError("citation resolution exceeded 200 ms") + + previous = signal.signal(signal.SIGALRM, expired) + signal.setitimer(signal.ITIMER_REAL, 0.2) + start = time.perf_counter() + try: + citations = resolver.extract_citations(text) + assert time.perf_counter() - start < 0.2 + return citations + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +@pytest.mark.parametrize("size", [4096, 32768]) +@pytest.mark.parametrize( + "stem", + [ + "1", + " ", + "TCK/1 ", + "TCK m. 81 ", + "TBK 117 CMK 86 ", + "7201 sayılı Kanunun 21. maddesi ", + "Tebligat K. 21. madde ", + ], +) +@pytest.mark.parametrize("scoped", [False, True]) +@pytest.mark.no_cover +def test_opus_repros_and_dense_matches(size, stem, scoped): + resolver = LegalEntityResolver( + law_names={"Tebligat Kanunu": "7201"} if scoped else None + ) + text = (stem * (size // len(stem) + 1))[:size] + _timed_extract(resolver, text) + + +@pytest.mark.parametrize("size", [4096, 32768]) +@pytest.mark.parametrize("tail", ["1 x", "1 TCK", "2020/1 E."]) +@pytest.mark.no_cover +def test_long_internal_whitespace(size, tail): + _timed_extract(LegalEntityResolver(), "TBK" + " " * (size - 3 - len(tail)) + tail) + + +@pytest.mark.no_cover +def test_stored_content_repro_preserves_real_citation(): + citations = _timed_extract(LegalEntityResolver(), "TCK m. 81 " + "1" * (32768 - 10)) + assert [(c.statute_code, c.article) for c in citations] == [("TCK", "81")] + + +@pytest.mark.no_cover +def test_seeded_citation_grammar_fuzz_at_api_limits(): + rng = random.Random(907) + tokens = [ + "1", + "2020", + "12345678901234567890", + " ", + "\t", + "\n", + " " * 500, + "m.", + "md.", + "madde", + "TCK", + "TBK", + "AY", + "5237", + "7201", + "Tebligat Kanunu", + "sayılı", + "Kanunun", + "inci", + "ıncı", + "uncu", + "üncü", + "nci", + "maddesi", + "uyarınca", + ".", + ",", + ";", + "/", + "-", + "ve", + ] + resolvers = [ + LegalEntityResolver(), + LegalEntityResolver( + law_names={"Tebligat Kanunu": "7201", "Zümrüt Taşları Kanunu": "8888"} + ), + ] + for size in (4096, 32768): + for _ in range(24): + parts = [] + length = 0 + while length < size: + token = rng.choice(tokens) + parts.append(token) + length += len(token) + for resolver in resolvers: + _timed_extract(resolver, "".join(parts)[:size]) + + +@pytest.mark.no_cover +def test_many_distinct_articles_and_possessive_duration_guards(): + # Exercise the match loops, including duration lookahead at every alias. + for stem in ( + "7201 sayılı Kanunun {i}. maddesi ", + "TBK'nın {i} ay ", + "TBK m. {i} - {i} CMK m. {i}; ", + ): + text = "".join(stem.format(i=i) for i in range(1600))[:32768] + _timed_extract(LegalEntityResolver(), text) + + +@pytest.mark.parametrize( + "text,expected", + [ + ("117 TBK, CMK m. 86", [("CMK", "86"), ("TBK", "117")]), + ("TBK 117 CMK 86", [("TBK", "117"), ("CMK", "86")]), + ("TCK m. 81 ve m. 82 - 83", [("TCK", "81"), ("TCK", "82"), ("TCK", "83")]), + ("TBK'nın 5 ay süre şartı ve 3 ay", [("TBK", None)]), + ( + "7201 sayılı Kanunun 21. maddesi ve TCK m. 81", + [("7201", "21"), ("TCK", "81")], + ), + ("TBK ve 49. madde uyarınca TBK", [("TBK", None), ("TBK", "49")]), + ("21 1 TBK 1 CMK", [("CMK", None), ("TBK", "1")]), + ("6098 81 madde 6098 TCK", [("TCK", None), ("TBK", "81")]), + ("TBK TBK 81.", [("TBK", "81")]), + ("117 TBK 117 CMK", [("TBK", "117"), ("CMK", None)]), + ], +) +def test_normal_citations_keep_order_and_overlap_semantics(text, expected): + assert [ + (c.statute_code, c.article) + for c in LegalEntityResolver().extract_citations(text) + ] == expected + + +@pytest.mark.parametrize("text", ["1" * 32760 + " TCK", "TCK m. " + "1" * 32760]) +@pytest.mark.no_cover +def test_overlong_digit_runs_do_not_become_partial_articles(text): + assert all(c.article is None for c in _timed_extract(LegalEntityResolver(), text)) + + +def test_citation_patterns_have_no_unbounded_repetitions(): + # Timing repros cover observed costs; this also guards the cause of the + # single regex call that held the GIL. Optional clauses can contain bounded + # tokens, but no run can grow with the entire input. + from re import _constants, _parser + + def audit(tree): + for op, args in tree: + if op in ( + _constants.MAX_REPEAT, + _constants.MIN_REPEAT, + _constants.POSSESSIVE_REPEAT, + ): + assert args[1] != _constants.MAXREPEAT + audit(args[2]) + elif op is _constants.SUBPATTERN: + audit(args[-1]) + elif op is _constants.BRANCH: + for branch in args[1]: + audit(branch) + elif op in (_constants.ASSERT, _constants.ASSERT_NOT): + audit(args[1]) + + resolver = LegalEntityResolver(law_names={"Tebligat Kanunu": "7201"}) + for name in ( + "_forward_pattern", + "_reverse_pattern", + "_continued_article_pattern", + "_numbered_law_pattern", + "_scope_name_pattern", + "_duration_pattern", + ): + pattern = getattr(resolver, name) + audit(_parser.parse(pattern.pattern, pattern.flags)) + + +@pytest.mark.no_cover +def test_sparse_api_sized_citation_has_bounded_allocations(): + # Run in a fresh process so earlier integration tests cannot contaminate + # the peak. Per-character GC-tracked objects caused CI latency spikes even + # with coverage paused; sparse 32 KB input should need less than 1 MB. + result = subprocess.run( + [ + sys.executable, + "-c", + """ +import json +import tracemalloc +from mesa_storage.legal_identity import LegalEntityResolver +resolver = LegalEntityResolver() +text = 'TCK m. 81 ' + ' ' * (32768 - 10) +tracemalloc.start() +citations = resolver.extract_citations(text) +_, peak = tracemalloc.get_traced_memory() +tracemalloc.stop() +print(json.dumps({'citations': [(c.statute_code, c.article) for c in citations], + 'peak': peak})) +""", + ], + capture_output=True, + text=True, + check=True, + timeout=10, + ) + measured = json.loads(result.stdout) + assert measured["citations"] == [["TCK", "81"]] + assert measured["peak"] < 1_000_000, measured diff --git a/tests/test_p908_search_limits.py b/tests/test_p908_search_limits.py new file mode 100644 index 0000000..a9547f5 --- /dev/null +++ b/tests/test_p908_search_limits.py @@ -0,0 +1,178 @@ +"""Opus's writer-controlled scope and API-sized token scoring repros.""" + +import asyncio +import itertools +import re +import time + +import pytest + +import mesa_storage.dao as dao_module +import mesa_storage.kuzu_provider as graph_module +from mesa_storage.kuzu_provider import KuzuGraphProvider +from tests.test_model_optional_runtime import _consume +from tests.test_model_optional_runtime import model_runtime as model_runtime + + +def _query_tokens(): + letters = "abcçdefghıijklmnoöprsştuüvy0123456789" + return [a + b for a, b in itertools.product(letters, repeat=2)][:1359] + + +async def _insert(env, index, content, title="Law", legal=False): + payload = dict( + session_id=env.session_id, + dataset_id="main", + document_id=f"doc-{index}", + revision_id=f"rev-{index}", + chunk_id=f"chunk-{index}", + title=title, + source_ref=f"file://{index}", + content=content, + ) + if legal: + payload["legal_identity"] = {"law_id": "6098", "article_id": "1"} + response = await env.client.post("/v4/memory/insert", json=payload) + assert response.status_code == 202, response.text + await _consume(env, 3) + + +async def _search(env, query): + return await env.dao.search_v4_memory( + tenant_id="model-tenant", + agent_id="model-agent", + dataset_ids=["main"], + query=query, + limit=50, + graph_enabled=False, + ) + + +@pytest.mark.asyncio +async def test_ten_crafted_law_headers_do_not_block_after_each_write(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + for index in range(10): + await _insert( + env, + index, + "1" * 2975 + "x Kanun Numarası: 6098\nMADDE 1- Kural.", + legal=True, + ) + gaps = [] + + async def heartbeat(): + last = time.perf_counter() + while True: + await asyncio.sleep(0.002) + now = time.perf_counter() + gaps.append(now - last - 0.002) + last = now + + for index in range(3): + await _insert(env, 10 + index, "Benign note to invalidate scope.") + pulse = asyncio.create_task(heartbeat()) + await asyncio.sleep(0.005) + try: + assert await _search(env, "TBK m. 49") + await asyncio.sleep(0.005) + finally: + pulse.cancel() + await asyncio.gather(pulse, return_exceptions=True) + assert max(gaps) < 0.2, f"scope build blocked the loop for {max(gaps):.3f}s" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("suffix", ["123 ", "١٢٣\t", "１２３\n", "1 2 ", " "]) +async def test_header_cleanup_keeps_previous_output(model_runtime, suffix): + prefix = "Özel Zümrüt Kanunu " + suffix + expected = dao_module.normalize_law_name(re.sub(r"\d+\s*$", "", prefix).strip()) + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + await _insert( + env, 0, prefix + "Kanun Numarası: 6098\nMADDE 1- Kural.", legal=True + ) + await _search(env, "TBK m. 1") + scope = next(iter(env.dao._search_scope_cache.values())) + assert scope["legal_context"]["names"][expected] == "6098" + + +@pytest.mark.asyncio +async def test_skip_overlong_stored_names_but_keep_200_character_names(model_runtime): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + for index, size in enumerate((200, 201, 2970)): + name = "a" * (size - len(" kanunu")) + " kanunu" + await _insert( + env, + index, + name + " Kanun Numarası: 6098\nKural.", + title=name if size <= 512 else "a" * 193 + " kanunu", + legal=True, + ) + await _search(env, "TBK m. 1") + scope = next(iter(env.dao._search_scope_cache.values())) + names = scope["legal_context"]["names"] + assert len(names) == 1 + assert len(next(iter(names))) == 200 + + +@pytest.mark.asyncio +async def test_40_large_memories_bound_both_dao_scoring_loops( + model_runtime, monkeypatch +): + async with model_runtime(MESA_GRAPH_ENABLED=False) as env: + for index in range(40): + await _insert(env, index, ("zz " * 11000)[:32768]) + original = dao_module._normalize_identity_text + scans = [] + tokens = set(_query_tokens()) | {"zz"} + + class TrackedText(str): + def __contains__(self, token): + if token in tokens: + scans.append(token) + return super().__contains__(token) + + def track(text): + normalized = original(text) + return TrackedText(normalized) if len(normalized) > 10000 else normalized + + monkeypatch.setattr(dao_module, "_normalize_identity_text", track) + query = ("zz " + " ".join(_query_tokens())).ljust(4096) + assert len(query) == 4096 + assert await _search(env, query) + # Both passage and assertion evidence are visited. No candidate scan may + # admit tokens past the first 64 after the existing stopword filters. + assert scans + assert len(set(scans)) == 64 + assert "zz" in scans + assert _query_tokens()[-1] not in scans + + +def test_graph_scoring_deduplicates_and_caps_tokens_in_order(monkeypatch): + original = graph_module._normalize_text + scans = [] + + class TrackedText(str): + def __contains__(self, token): + scans.append(token) + return super().__contains__(token) + + monkeypatch.setattr( + graph_module, "_normalize_text", lambda text: TrackedText(original(text)) + ) + tokens = ["zz", "zz", *_query_tokens()] + KuzuGraphProvider._compute_path_score( + hop=1, + seed_id="seed", + seed_scores=None, + query_tokens=tokens, + predicates=["!"], + evidence_spans=["!" * 32768], + jurisdictions=[""], + confidences=[1.0], + edge_directions=["forward"], + target_statutes=set(), + competing_statutes=[], + resolver=None, + ) + assert list(dict.fromkeys(scans)) == list(dict.fromkeys(tokens))[:64] + assert len(scans) == 128 # Each token checks predicate, then evidence. diff --git a/tests/test_runtime_profiles_contract.py b/tests/test_runtime_profiles_contract.py index b399f78..cfade08 100644 --- a/tests/test_runtime_profiles_contract.py +++ b/tests/test_runtime_profiles_contract.py @@ -153,18 +153,22 @@ def test_separate_flags_default_to_legacy_and_can_override_it(tmp_path, legacy): legacy_bool = legacy == "true" runtime = load_runtime_profile(env) assert runtime.embedding_enabled is legacy_bool - assert runtime.extraction_enabled is legacy_bool + assert runtime.extraction_enabled is False override_str = "false" if legacy_bool else "true" runtime_embed_only = load_runtime_profile( {**env, "MESA_EMBEDDING_ENABLED": override_str} ) assert runtime_embed_only.embedding_enabled is (not legacy_bool) - assert runtime_embed_only.extraction_enabled is legacy_bool + assert runtime_embed_only.extraction_enabled is False runtime_extract_only = load_runtime_profile( - {**env, "MESA_EXTRACTION_ENABLED": override_str} + {**env, "MESA_EXTRACTION_ENABLED": "true"} ) assert runtime_extract_only.embedding_enabled is legacy_bool - assert runtime_extract_only.extraction_enabled is (not legacy_bool) + assert runtime_extract_only.extraction_enabled is True + runtime_extract_explicit_false = load_runtime_profile( + {**env, "MESA_EXTRACTION_ENABLED": "false"} + ) + assert runtime_extract_explicit_false.extraction_enabled is False runtime = load_runtime_profile( {**env, "MESA_EMBEDDING_ENABLED": "false", "MESA_EXTRACTION_ENABLED": "true"} ) diff --git a/tests/test_soak_profile_a.py b/tests/test_soak_profile_a.py index c54f42f..6379d09 100644 --- a/tests/test_soak_profile_a.py +++ b/tests/test_soak_profile_a.py @@ -657,8 +657,11 @@ async def test_purge_case_c_all_searches_fail_unconfirmed() -> None: @pytest.mark.asyncio -async def test_embedded_soak_run_integration(tmp_path: Path) -> None: +async def test_embedded_soak_run_integration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: """Run a fast 5-second in-process soak test exercising real V4 lifecycle.""" + monkeypatch.setenv("MESA_EXTRACTION_ENABLED", "true") log_file = tmp_path / "soak_test_fast.jsonl" storage_root = tmp_path / "storage" diff --git a/tests/test_v4_retry_lifecycle.py b/tests/test_v4_retry_lifecycle.py index debd677..d0d9cdf 100644 --- a/tests/test_v4_retry_lifecycle.py +++ b/tests/test_v4_retry_lifecycle.py @@ -54,6 +54,7 @@ async def retry_runtime(tmp_path, monkeypatch): "MESA_STORAGE_ROOT": str(tmp_path / "storage"), "MESA_LOAD_DOTENV": "false", "MESA_MODEL_ENABLED": "true", + "MESA_EXTRACTION_ENABLED": "true", "MESA_EXTERNAL_PROVIDER_ENABLED": "true", "MESA_TIER3_MODE": "0", "MESA_EMBEDDING_DIMENSION": "384", diff --git a/tests/test_vector_failure_semantics.py b/tests/test_vector_failure_semantics.py index 26a954e..fac28eb 100644 --- a/tests/test_vector_failure_semantics.py +++ b/tests/test_vector_failure_semantics.py @@ -75,6 +75,10 @@ def execute(self, query: str, _params: object = ()) -> _Cursor: "assertion_id": "assertion-1", "subject_id": "subject-1", "object_entity_id": None, + "revision_id": "revision-1", + "document_id": "document-1", + "chunk_id": "chunk-1", + "mutation_id": "mutation-1", } ] )