From 0a5cd5cc9edfd7c46d3f7a31cec6d92792125711 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:31:54 +0300 Subject: [PATCH 1/8] fix: preserve Turkish legal identity in parsing and chunks --- src/mesa_legal_data/ids.py | 2 + src/mesa_legal_data/parsers/citations.py | 33 +++-- src/mesa_legal_data/parsers/legislation.py | 9 +- src/mesa_legal_data/publisher/chunker.py | 78 ++++++++-- .../unit/test_comprehensive_legal_identity.py | 136 ++++++++++++++++++ 5 files changed, 238 insertions(+), 20 deletions(-) create mode 100644 tests/unit/test_comprehensive_legal_identity.py diff --git a/src/mesa_legal_data/ids.py b/src/mesa_legal_data/ids.py index 1e4c7dc..7ed74e2 100644 --- a/src/mesa_legal_data/ids.py +++ b/src/mesa_legal_data/ids.py @@ -35,6 +35,8 @@ def build_article_id(legislation_id: str, article_number: str, article_kind: str prefix = "ek-" elif article_kind == "temporary": prefix = "gecici-" + elif article_kind == "repeated": + prefix = "mukerrer-" else: prefix = "" return f"{legislation_id}:article:{prefix}{norm_num}" diff --git a/src/mesa_legal_data/parsers/citations.py b/src/mesa_legal_data/parsers/citations.py index 4a85103..cf9f999 100644 --- a/src/mesa_legal_data/parsers/citations.py +++ b/src/mesa_legal_data/parsers/citations.py @@ -52,9 +52,6 @@ if alias: ALIAS_MAP[alias.upper()] = (num, kind) -KNOWN_LEGISLATION_NUMBERS = {num for _, _, num, _ in LEGAL_CODES} - - class Citation(BaseModel): model_config = ConfigDict(frozen=True) @@ -70,7 +67,7 @@ class Citation(BaseModel): # Pattern 1: Numbered Law Citations with explicit context # e.g. "4857 sayılı Kanun'un 1. maddesi", "6698 sayılı Kişisel Verilerin Korunması Kanunu m. 9", "7500 sayılı Kanun 3. maddesi" NUMBERED_LAW_PATTERN = re.compile( - r"\b(?P[1-9]\d{0,4})\s+sayılı\s+(?:[A-Za-zÇĞİÖŞÜçğıöşü\s]{0,50}?)(?:Kanun(?:u|un|una|unda|undan|a|da|dan|lar|ları)?|KHK(?:'nin|'ye|'de)?|Cumhurbaşkanlığı\s+Kararnamesi(?:'nin)?|Yasa(?:sı|nın)?)\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?:ek|geçici)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+|[A-ZÇĞİÖŞÜ]+)|(?P\d+)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+)\b))?", + r"\b(?P[1-9]\d{0,4})\s+sayılı\s+(?:[A-Za-zÇĞİÖŞÜçğıöşü\s]{0,50}?)(?:Kanun(?:u|un|una|unda|undan|a|da|dan|lar|ları)?|KHK(?:'nin|'ye|'de)?|Cumhurbaşkanlığı\s+Kararnamesi(?:'nin)?|Yasa(?:sı|nın)?)\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?Pek|geçici|mükerrer)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?|[A-ZÇĞİÖŞÜ]+)|(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?)\b))?", re.IGNORECASE, ) @@ -80,7 +77,7 @@ class Citation(BaseModel): ALIAS_PATTERN_STR = "|".join(re.escape(k) for k in ALIAS_NAMES) ALIAS_LAW_PATTERN = re.compile( - rf"\b(?P{ALIAS_PATTERN_STR})\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?:ek|geçici)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+|[A-ZÇĞİÖŞÜ]+)|(?P\d+)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+)\b))?", + rf"\b(?P{ALIAS_PATTERN_STR})\b(?:\s*['’][A-Za-zçğıöşüÇĞİÖŞÜ]+)?(?:\s+(?:(?:(?Pek|geçici|mükerrer)\s+)?(?:madde|maddesi|m\.)\s*\.?\s*(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?|[A-ZÇĞİÖŞÜ]+)|(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?)(?:\.|\s*['’]?(?:nci|üncü|inci|ıncı|uncu))?\s*(?:maddesi|madde)|(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?)\b))?", re.IGNORECASE, ) @@ -115,6 +112,20 @@ def _detect_relation_hint(context_window: str) -> RelationHint | None: return None +def _article_target_id(legislation_id: str, article_number: str | None, article_kind: str | None) -> str | None: + if not article_number: + return None + base_number, separator, suffix = article_number.partition("/") + # Turkish citations commonly use ``27/2`` and ``25/II`` for a paragraph + # or clause inside article 27/25. Alphabetic suffixes such as ``1/A`` are + # part of the article's legal identity and must remain in the target ID. + is_subdivision = bool(separator) and (suffix.isdigit() or set(suffix.upper()) <= set("IVXLCDM")) + normalized_number = base_number.lower() if is_subdivision else article_number.lower().replace("/", "-") + normalized_kind = (article_kind or "").lower() + prefix = {"ek": "ek-", "geçici": "gecici-", "mükerrer": "mukerrer-"}.get(normalized_kind, "") + return f"{legislation_id}:article:{prefix}{normalized_number}" + + def extract_citations(text: str) -> list[Citation]: """ Extracts deterministic legal citations and references with exact canonical coordinate spans. @@ -135,10 +146,11 @@ def extract_citations(text: str) -> list[Citation]: raw = text[start:end] law_num = match.group("law_num") art_num = match.group("art_num_m") or match.group("art_num_num") or match.group("art_num_bare") + art_kind = match.group("art_kind_m") leg_type = "constitution" if law_num == "2709" else "law" leg_id = f"tr:legislation:{leg_type}:{law_num}" - art_id = f"{leg_id}:article:{art_num}" if art_num else None + art_id = _article_target_id(leg_id, art_num, art_kind) window_start = max(0, start - 50) window_end = min(len(text), end + 100) @@ -153,7 +165,9 @@ def extract_citations(text: str) -> list[Citation]: char_end=end, # A syntactically valid number is only a resolution candidate # unless it belongs to the deterministic legislation registry. - citation_status=("RESOLVED" if law_num in KNOWN_LEGISLATION_NUMBERS else "EXTRACTED"), + # A registry match establishes a typed candidate, not that the + # cited article/version exists in the current legal corpus. + citation_status="EXTRACTED", relation_hint=relation, ) ) @@ -177,8 +191,9 @@ def extract_citations(text: str) -> list[Citation]: law_num, leg_type = law_tuple art_num = match.group("art_num_m") or match.group("art_num_num") or match.group("art_num_bare") + art_kind = match.group("art_kind_m") leg_id = f"tr:legislation:{leg_type}:{law_num}" - art_id = f"{leg_id}:article:{art_num}" if art_num else None + art_id = _article_target_id(leg_id, art_num, art_kind) window_start = max(0, start - 50) window_end = min(len(text), end + 100) @@ -191,7 +206,7 @@ def extract_citations(text: str) -> list[Citation]: target_article_id=art_id, char_start=start, char_end=end, - citation_status="RESOLVED", + citation_status="EXTRACTED", relation_hint=relation, ) ) diff --git a/src/mesa_legal_data/parsers/legislation.py b/src/mesa_legal_data/parsers/legislation.py index 89df9d7..2a8f3fc 100644 --- a/src/mesa_legal_data/parsers/legislation.py +++ b/src/mesa_legal_data/parsers/legislation.py @@ -5,14 +5,14 @@ from mesa_legal_data.parsers.text_normalizer import normalize_text PARSER_NAME = "legislation_parser" -PARSER_VERSION = "1.0.0" +PARSER_VERSION = "1.1.0" class ParsedArticle(BaseModel): model_config = ConfigDict(frozen=True) article_number: str - article_kind: str # "standard", "additional", "temporary" + article_kind: str # "standard", "additional", "temporary", "repeated" heading: str | None = None text: str char_start: int | None = None @@ -29,7 +29,8 @@ class ParsedLegislation(BaseModel): ARTICLE_PATTERN = re.compile( - r"^(?PEK MADDE|GEÇİCİ MADDE|MADDE)\s+(?P\d+|[A-ZÇĞİÖŞÜ]+)\s*[-–—:]?\s*(?P.*)$", + r"^(?PMÜKERRER MADDE|EK MADDE|GEÇİCİ MADDE|MADDE)\s+" + r"(?P\d+(?:/[A-ZÇĞİÖŞÜ0-9]+)?|[A-ZÇĞİÖŞÜ]+)\s*[-–—:]?\s*(?P.*)$", re.IGNORECASE | re.MULTILINE, ) @@ -66,6 +67,8 @@ def parse_legislation_text(text: str, auto_normalize: bool = True) -> ParsedLegi kind_str = "additional" elif raw_kind == "GEÇİCİ MADDE": kind_str = "temporary" + elif raw_kind == "MÜKERRER MADDE": + kind_str = "repeated" match_start = match.start() next_match_start = matches[idx].start() if idx < len(matches) else len(text) diff --git a/src/mesa_legal_data/publisher/chunker.py b/src/mesa_legal_data/publisher/chunker.py index 53c9bc6..ed82aa9 100644 --- a/src/mesa_legal_data/publisher/chunker.py +++ b/src/mesa_legal_data/publisher/chunker.py @@ -67,6 +67,7 @@ def plan_source_chunks( canonical_text: str, records: list[dict[str, Any]], content_limit_chars: int = 32768, + document_metadata: dict[str, Any] | None = None, ) -> list[SourceChunk]: """ Deterministically transforms canonical text and version records into ordered SourceChunks. @@ -87,10 +88,15 @@ def plan_source_chunks( # payload for its deterministic fallback assertion, so the semantic limit # is stricter than the 32,768-character transport-content limit. chunk_limit_chars = min(content_limit_chars, MESA_V4_EVIDENCE_SPAN_MAX_CHARS) + document_identity = dict(document_metadata or {}) - # Auto-resolve char_start and char_end if content is provided + # Work on shallow copies so coordinate recovery never mutates canonical + # records owned by the caller. + normalized_records = [dict(record) for record in records] + + # Auto-resolve char_start and char_end if content is provided. search_cursor = 0 - for r in records: + for r in normalized_records: if r.get("char_start") is None and r.get("content"): pos = canonical_text.find(r["content"], search_cursor) if pos != -1: @@ -98,14 +104,24 @@ def plan_source_chunks( r["char_end"] = pos + len(r["content"]) search_cursor = r["char_end"] - # Filter and sort article records by char_start + # Filter and sort article records by char_start. article_records = [ r - for r in records + for r in normalized_records if r.get("record_type") == "article" and r.get("char_start") is not None and r.get("char_end") is not None ] article_records.sort(key=lambda r: (r.get("char_start", 0), r.get("ordinal", 0))) + previous_end = 0 + for article in article_records: + start = article.get("char_start") + end = article.get("char_end") + if not isinstance(start, int) or not isinstance(end, int) or start < 0 or end > len(canonical_text) or start >= end: + raise ValueError(f"Invalid article source span: [{start}, {end}]") + if start < previous_end: + raise ValueError(f"Article source spans overlap at offset {start}") + previous_end = end + chunks: list[SourceChunk] = [] ordinal_counter = 1 @@ -131,7 +147,7 @@ def plan_source_chunks( ordinal=ordinal_counter, content=piece_text, content_hash=c_hash, - metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + metadata={**document_identity, "piece_index": piece_idx, "total_pieces": len(splits)}, ) ) ordinal_counter += 1 @@ -162,15 +178,48 @@ def plan_source_chunks( ordinal=ordinal_counter, content=piece_text, content_hash=c_hash, - metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + metadata={**document_identity, "piece_index": piece_idx, "total_pieces": len(splits)}, ) ) ordinal_counter += 1 - # 2. Article Chunks + # 2. Article Chunks and meaningful structural gaps. Parser boundaries may + # deliberately exclude headings such as BÖLÜM/KISIM between articles; those + # bytes remain canonical evidence and must not disappear from publication. + previous_end = first_art_start for art in article_records: a_start = int(art["char_start"]) a_end = int(art["char_end"]) + if a_start > previous_end: + gap_text = canonical_text[previous_end:a_start] + if gap_text.strip(): + gap_splits = _split_oversized_text(gap_text, previous_end, chunk_limit_chars) + for piece_idx, (c_start, c_end, piece_text) in enumerate(gap_splits, start=1): + chunk_id = ( + f"{version_id}:chunk:{ordinal_counter:04d}" + if len(gap_splits) == 1 + else f"{version_id}:chunk:{ordinal_counter:04d}:p{piece_idx}" + ) + chunks.append( + SourceChunk( + chunk_id=chunk_id, + document_id=document_id, + version_id=version_id, + chunk_type="general", + title="Yapısal Başlık", + char_start=c_start, + char_end=c_end, + ordinal=ordinal_counter, + content=piece_text, + content_hash=calculate_content_hash(piece_text), + metadata={ + **document_identity, + "piece_index": piece_idx, + "total_pieces": len(gap_splits), + }, + ) + ) + ordinal_counter += 1 art_text = canonical_text[a_start:a_end] art_id = art.get("record_id", "") art_num = art.get("article_number", "") @@ -197,14 +246,27 @@ def plan_source_chunks( content=piece_text, content_hash=c_hash, metadata={ + **document_identity, "article_id": art_id, "article_number": art_num, + **{ + key: art[key] + for key in ( + "article_kind", + "law_number", + "law_title", + "effective_from", + "effective_to", + ) + if key in art + }, "piece_index": piece_idx, "total_pieces": len(splits), }, ) ) ordinal_counter += 1 + previous_end = a_end # 3. Annex / Trailing Chunk (if meaningful content exists after last article) last_art_end = article_records[-1].get("char_end", len(canonical_text)) @@ -231,7 +293,7 @@ def plan_source_chunks( ordinal=ordinal_counter, content=piece_text, content_hash=c_hash, - metadata={"piece_index": piece_idx, "total_pieces": len(splits)}, + metadata={**document_identity, "piece_index": piece_idx, "total_pieces": len(splits)}, ) ) ordinal_counter += 1 diff --git a/tests/unit/test_comprehensive_legal_identity.py b/tests/unit/test_comprehensive_legal_identity.py new file mode 100644 index 0000000..24e82be --- /dev/null +++ b/tests/unit/test_comprehensive_legal_identity.py @@ -0,0 +1,136 @@ +import pytest + +from mesa_legal_data.parsers.citations import extract_citations +from mesa_legal_data.parsers.legislation import parse_legislation_text +from mesa_legal_data.publisher.chunker import plan_source_chunks + + +def test_article_parser_preserves_extended_turkish_article_identity() -> None: + parsed = parse_legislation_text( + "\n".join( + [ + "MADDE 1- Olağan hüküm.", + "EK MADDE 1- Ek hüküm.", + "GEÇİCİ MADDE 1- Geçici hüküm.", + "MÜKERRER MADDE 1- Mükerrer hüküm.", + "MADDE 1/A- Harf ekli hüküm.", + ] + ) + ) + + assert [(article.article_kind, article.article_number) for article in parsed.articles] == [ + ("standard", "1"), + ("additional", "1"), + ("temporary", "1"), + ("repeated", "1"), + ("standard", "1/A"), + ] + + +def test_chunk_plan_is_lossless_and_rejects_overlapping_article_identity() -> None: + canonical = "KANUN BAŞLIĞI\nMADDE 1- Bir.\nBİRİNCİ BÖLÜM\nMADDE 2- İki." + first_start = canonical.index("MADDE 1") + gap_start = canonical.index("BİRİNCİ BÖLÜM") + second_start = canonical.index("MADDE 2") + records = [ + { + "record_id": "law:article:1", + "record_type": "article", + "article_number": "1", + "article_kind": "standard", + "char_start": first_start, + "char_end": gap_start, + "ordinal": 1, + }, + { + "record_id": "law:article:2", + "record_type": "article", + "article_number": "2", + "article_kind": "standard", + "char_start": second_start, + "char_end": len(canonical), + "ordinal": 2, + }, + ] + + chunks = plan_source_chunks( + document_id="tr:legislation:law:6098", + version_id="tr:legislation:law:6098:version:2026-01-01:a", + canonical_text=canonical, + records=records, + ) + assert "".join(chunk.content for chunk in chunks) == canonical + assert any(chunk.chunk_type == "general" and "BİRİNCİ BÖLÜM" in chunk.content for chunk in chunks) + + overlapping = [dict(records[0]), dict(records[1], char_start=gap_start - 2)] + with pytest.raises(ValueError, match="overlap"): + plan_source_chunks( + document_id="tr:legislation:law:6098", + version_id="tr:legislation:law:6098:version:2026-01-01:a", + canonical_text=canonical, + records=overlapping, + ) + + +def test_chunk_plan_carries_structured_law_article_and_temporal_identity() -> None: + canonical = "MADDE 1/A- Hüküm." + chunks = plan_source_chunks( + document_id="tr:legislation:law:6098", + version_id="tr:legislation:law:6098:version:2026-01-01:a", + canonical_text=canonical, + records=[ + { + "record_id": "tr:legislation:law:6098:article:1-a", + "record_type": "article", + "article_number": "1/A", + "article_kind": "standard", + "char_start": 0, + "char_end": len(canonical), + "ordinal": 1, + "law_number": "6098", + "law_title": "TÜRK BORÇLAR KANUNU", + "effective_from": "2026-01-01", + "effective_to": None, + } + ], + ) + + assert chunks[0].metadata == { + "article_id": "tr:legislation:law:6098:article:1-a", + "article_number": "1/A", + "article_kind": "standard", + "law_number": "6098", + "law_title": "TÜRK BORÇLAR KANUNU", + "effective_from": "2026-01-01", + "effective_to": None, + "piece_index": 1, + "total_pieces": 1, + } + + +def test_chunk_coordinate_recovery_does_not_mutate_canonical_records() -> None: + records = [{"record_id": "a1", "record_type": "article", "article_number": "1", "content": "MADDE 1- A."}] + plan_source_chunks(document_id="law", version_id="law:v1", canonical_text="MADDE 1- A.", records=records) + assert "char_start" not in records[0] + assert "char_end" not in records[0] + + +@pytest.mark.parametrize( + ("source", "expected"), + [ + ("6098 sayılı Kanunun 1/A maddesi değiştirilmiştir.", "tr:legislation:law:6098:article:1-a"), + ("TBK geçici madde 2 uyarınca", "tr:legislation:law:6098:article:gecici-2"), + ("TCK mükerrer madde 5 uyarınca", "tr:legislation:law:5237:article:mukerrer-5"), + ], +) +def test_citation_article_identity_preserves_suffix_and_kind(source: str, expected: str) -> None: + assert extract_citations(source)[0].target_article_id == expected + + +def test_syntactic_citation_candidate_is_not_falsely_resolved() -> None: + citation = extract_citations("6098 sayılı Kanunun 1 inci maddesi değiştirilmiştir.")[0] + + assert citation.target_legislation_id == "tr:legislation:law:6098" + assert citation.target_article_id == "tr:legislation:law:6098:article:1" + assert citation.citation_status == "EXTRACTED" + assert citation.relation_hint == "amends" From 9500542cc4085e7c76983b3d287e6a4f28ef084d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:32:19 +0300 Subject: [PATCH 2/8] fix: enforce authoritative legal chronology and effective intervals --- src/mesa_legal_data/catalog.py | 84 +++++++- src/mesa_legal_data/pipeline.py | 91 +++++++-- src/mesa_legal_data/publisher/engine.py | 42 ++++ src/mesa_legal_data/quality.py | 29 +++ .../test_post_master_independent_audit.py | 13 +- .../test_comprehensive_temporal_integrity.py | 180 ++++++++++++++++++ 6 files changed, 422 insertions(+), 17 deletions(-) create mode 100644 tests/unit/test_comprehensive_temporal_integrity.py diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index 58397b5..4a612e5 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -268,6 +268,7 @@ def recompute_document_current_version( return None parsed_versions = [] + as_of = datetime.now(UTC).date().isoformat() for r in rows: meta_dict: dict[str, Any] = {} if r[13]: @@ -285,6 +286,17 @@ def recompute_document_current_version( ) legal_date = _normalize_legal_date(legal_date_str) has_known_date = legal_date is not None + effective_from = _normalize_legal_date(r[3]) + effective_to = _normalize_legal_date(r[4]) + selection_start = effective_from or legal_date + if selection_start is None: + temporal_rank = 0 # unknown legal chronology + elif selection_start > as_of: + temporal_rank = 1 # enacted/published but not yet in force + elif effective_to is not None and effective_to < as_of: + temporal_rank = 2 # historical, no longer in force + else: + temporal_rank = 3 # in force as of today parsed_versions.append( { @@ -292,6 +304,10 @@ def recompute_document_current_version( "version_kind": r[1], "legal_date": legal_date or "0000-00-00", "has_known_date": has_known_date, + "effective_from": effective_from, + "effective_to": effective_to, + "selection_start": selection_start or "0000-00-00", + "temporal_rank": temporal_rank, "canonical_sha256": r[5], "revision_number": r[6] or 1, "supersedes_version_id": r[7], @@ -305,7 +321,8 @@ def recompute_document_current_version( def sort_key(item): return ( - 1 if item["has_known_date"] else 0, + item["temporal_rank"], + item["selection_start"], item["legal_date"], item["revision_number"], item["created_at"], @@ -314,6 +331,64 @@ def sort_key(item): sorted_versions = sorted(parsed_versions, key=sort_key, reverse=True) chosen = sorted_versions[0] + # Two bounded, explicitly declared effective intervals with different + # content must not silently compete for the same legal instant. Open-ended + # consolidated snapshots are intentionally excluded: their chronology is + # already ordered by legal date unless a bounded contradiction is declared. + conflicting_ids: set[str] = set() + for index, left in enumerate(parsed_versions): + if not left["effective_from"] or not left["effective_to"]: + continue + for right in parsed_versions[index + 1 :]: + if not right["effective_from"] or not right["effective_to"]: + continue + overlaps = max(left["effective_from"], right["effective_from"]) <= min( + left["effective_to"], right["effective_to"] + ) + explicitly_ordered = left.get("supersedes_version_id") == right["version_id"] or right.get( + "supersedes_version_id" + ) == left["version_id"] + if overlaps and left["canonical_sha256"] != right["canonical_sha256"] and not explicitly_ordered: + conflicting_ids.update((left["version_id"], right["version_id"])) + + if conflicting_ids: + now = datetime.now(UTC).isoformat() + for conflicting in parsed_versions: + if conflicting["version_id"] not in conflicting_ids: + continue + conflicting_id = conflicting["version_id"] + issue_id = "iss-version-interval-" + hashlib.sha256(conflicting_id.encode("utf-8")).hexdigest()[:16] + conn.execute( + """INSERT OR IGNORE INTO validation_issues ( + issue_id, subject_type, subject_id, version_id, record_instance_id, + severity, code, message, details_json, status, opened_at + ) VALUES (?, 'version', ?, ?, NULL, 'error', 'VERSION_EFFECTIVE_INTERVAL_AMBIGUITY', + 'Different content has a contradictory bounded effective interval without explicit supersession', + ?, 'open', ?)""", + ( + issue_id, + conflicting_id, + conflicting_id, + json.dumps( + { + "effective_from": conflicting["effective_from"], + "effective_to": conflicting["effective_to"], + "document_id": document_id, + } + ), + now, + ), + ) + conn.execute( + """UPDATE versions + SET approval_status = 'pending', auto_approved = 0, quality_status = 'REVIEW' + WHERE version_id = ?""", + (conflicting_id,), + ) + conn.execute("UPDATE records SET approval_status = 'pending' WHERE version_id = ?", (conflicting_id,)) + conflicting["approval_status"] = "pending" + conflicting["quality_status"] = "REVIEW" + same_date_candidates = [ v for v in sorted_versions @@ -322,7 +397,10 @@ def sort_key(item): and v["canonical_sha256"] != chosen["canonical_sha256"] ] ambiguity_group = [chosen, *same_date_candidates] - has_explicit_supersession = any(item.get("supersedes_version_id") for item in ambiguity_group) + ambiguity_ids = {item["version_id"] for item in ambiguity_group} + has_explicit_supersession = any( + item.get("supersedes_version_id") in ambiguity_ids for item in ambiguity_group + ) if same_date_candidates and not has_explicit_supersession: doc_cur = conn.cursor() doc_cur.execute("SELECT current_version_id FROM documents WHERE document_id = ?", (document_id,)) @@ -987,6 +1065,8 @@ def iter_records_for_release( AND v.validation_status = 'valid' AND v.quality_status = 'PASS' AND v.privacy_status IN ('clean', 'approved') + AND (v.effective_from IS NULL OR date(v.effective_from) <= date('now')) + AND (v.effective_to IS NULL OR date(v.effective_to) >= date('now')) ) SELECT r.record_id, r.record_type, r.record_sha256, r.canonical_path, r.canonical_line, r.version_id, v.document_id diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index 52a3145..6b8b3d6 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -221,11 +221,13 @@ def process_artifact_pipeline( ret_at = art_row.get("retrieved_at") if art_row else None pub_date_missing = False + legal_date_missing = False legal_date_invalid = False if v_kind == "original_publication": raw_ver_date = meta_dict.get("publication_date") or meta_dict.get("source_date") if not raw_ver_date: pub_date_missing = True + legal_date_missing = True ver_date = "unknown-date" else: normalized_date = _valid_legal_date(raw_ver_date) @@ -240,10 +242,7 @@ def process_artifact_pipeline( or meta_dict.get("publication_date") or meta_dict.get("snapshot_date") ) - if not raw_ver_date and last_mod: - raw_ver_date = str(last_mod)[:10] - elif not raw_ver_date and ret_at: - raw_ver_date = str(ret_at)[:10] + legal_date_missing = not bool(raw_ver_date) normalized_date = _valid_legal_date(raw_ver_date) legal_date_invalid = bool(raw_ver_date) and normalized_date is None ver_date = normalized_date or "unknown-date" @@ -252,13 +251,22 @@ def process_artifact_pipeline( meta_dict.get("snapshot_date") or meta_dict.get("publication_date") or meta_dict.get("source_date") - or (str(last_mod)[:10] if last_mod else None) - or (str(ret_at)[:10] if ret_at else None) ) + legal_date_missing = not bool(raw_ver_date) normalized_date = _valid_legal_date(raw_ver_date) legal_date_invalid = bool(raw_ver_date) and normalized_date is None ver_date = normalized_date or "unknown-date" + raw_effective_from = meta_dict.get("effective_from") + raw_effective_to = meta_dict.get("effective_to") or meta_dict.get("effective_until") + effective_from = _valid_legal_date(raw_effective_from) + effective_to = _valid_legal_date(raw_effective_to) + temporal_interval_invalid = ( + (bool(raw_effective_from) and effective_from is None) + or (bool(raw_effective_to) and effective_to is None) + or (effective_from is not None and effective_to is not None and effective_from > effective_to) + ) + # Deterministic Version ID based on document_id, version_date, and artifact_sha256 if fam == "legislation": version_id = build_legislation_version_id(doc_id or "tr:legislation:unknown", ver_date, expected_sha) @@ -358,8 +366,8 @@ def process_artifact_pipeline( "version_id": version_id, "version_kind": v_kind, "snapshot_date": ver_date, - "effective_from": None, - "effective_to": None, + "effective_from": effective_from, + "effective_to": effective_to, }, "full_text": canonical_text, "schema_version": "1.0.0", @@ -395,8 +403,8 @@ def process_artifact_pipeline( "text": a.text, "structure": None, "status": "active", - "effective_from": None, - "effective_to": None, + "effective_from": effective_from, + "effective_to": effective_to, "source_span": span_obj, "schema_version": "1.0.0", "created_at": art_row["retrieved_at"], @@ -602,6 +610,56 @@ def process_artifact_pipeline( details_json=json.dumps({"raw_date": str(raw_ver_date)}), ) + if legal_date_missing: + if quality_status == "PASS": + quality_status = "REVIEW" + quality_data["decision"] = quality_status + quality_data["checks"].append( + { + "group": "METADATA", + "name": "legal_date_present", + "status": "REVIEW", + "message": "Authoritative legal date is missing; acquisition time was not substituted", + "details": {}, + } + ) + open_issue( + conn, + issue_id=f"iss-{uuid.uuid4().hex[:8]}", + subject_type="version", + subject_id=version_id, + version_id=version_id, + severity="error", + code="LEGAL_DATE_MISSING", + message="Authoritative legal date is missing; acquisition time cannot establish legal chronology", + details_json=json.dumps({"retrieved_at": ret_at, "last_modified": last_mod}), + ) + + if temporal_interval_invalid: + if quality_status == "PASS": + quality_status = "REVIEW" + quality_data["decision"] = quality_status + quality_data["checks"].append( + { + "group": "METADATA", + "name": "effective_interval_valid", + "status": "REVIEW", + "message": "Legal effective interval is invalid or contradictory", + "details": {"effective_from": raw_effective_from, "effective_to": raw_effective_to}, + } + ) + open_issue( + conn, + issue_id=f"iss-{uuid.uuid4().hex[:8]}", + subject_type="version", + subject_id=version_id, + version_id=version_id, + severity="error", + code="EFFECTIVE_INTERVAL_INVALID", + message="Legal effective interval is invalid or contradictory", + details_json=json.dumps({"effective_from": raw_effective_from, "effective_to": raw_effective_to}), + ) + quality_json = json.dumps(quality_data) # Release Guard & Lifecycle Determination @@ -667,8 +725,8 @@ def process_artifact_pipeline( artifact_id=artifact_id, version_kind=v_kind, snapshot_date=ver_date, - effective_from=None, - effective_to=None, + effective_from=effective_from, + effective_to=effective_to, canonical_path=c_path, canonical_line=1, canonical_sha256=c_sha, @@ -702,7 +760,14 @@ def process_artifact_pipeline( # Step 9b: Safe Auto-Approval Evaluation auto_approved = False auto_reason = "" - if quality_status != "BLOCK" and val_status == "valid" and not pub_date_missing and not legal_date_invalid: + if ( + quality_status != "BLOCK" + and val_status == "valid" + and not pub_date_missing + and not legal_date_missing + and not legal_date_invalid + and not temporal_interval_invalid + ): try: auto_approved, auto_reason = evaluate_auto_approval( conn, diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py index 4cf5e45..faee7cb 100644 --- a/src/mesa_legal_data/publisher/engine.py +++ b/src/mesa_legal_data/publisher/engine.py @@ -149,6 +149,8 @@ def get_ready_versions_and_content(conn) -> tuple[list[dict[str, Any]], int]: AND v.validation_status = 'valid' AND v.privacy_status IN ('clean', 'approved') AND v.quality_status = 'PASS' + AND (v.effective_from IS NULL OR date(v.effective_from) <= date('now')) + AND (v.effective_to IS NULL OR date(v.effective_to) >= date('now')) ORDER BY v.created_at ASC""") rows = cursor.fetchall() version_items = [ @@ -200,8 +202,25 @@ def build_delivery_plan( v_id = v_info["version_id"] canonical_text = "" records: list[dict[str, Any]] = [] + document_identity: dict[str, Any] = {} if release_id: packaged_records = v_info["packaged_records"] + legislation_item = next( + (item for item in packaged_records if item.get("record_type") == "legislation"), + None, + ) + if legislation_item: + version_meta = legislation_item.get("version") or {} + document_identity = { + "law_id": legislation_item.get("id"), + "law_number": legislation_item.get("number"), + "law_title": legislation_item.get("title"), + "law_short_title": legislation_item.get("short_title"), + "version_kind": version_meta.get("version_kind"), + "snapshot_date": version_meta.get("snapshot_date"), + "effective_from": version_meta.get("effective_from"), + "effective_to": version_meta.get("effective_to"), + } for item in packaged_records: record_type = item.get("record_type") record_id = item.get("id") @@ -220,6 +239,9 @@ def build_delivery_plan( "ordinal": item.get("ordinal", 0), "title": item.get("heading"), "article_number": item.get("article_number"), + "article_kind": item.get("article_kind"), + "effective_from": item.get("effective_from"), + "effective_to": item.get("effective_to"), "content": item.get("text") or "", } ) @@ -256,6 +278,17 @@ def build_delivery_plan( if record_type == "legislation": canonical_text = item.get("full_text") or "" + version_meta = item.get("version") or {} + document_identity = { + "law_id": item.get("id"), + "law_number": item.get("number"), + "law_title": item.get("title"), + "law_short_title": item.get("short_title"), + "version_kind": version_meta.get("version_kind"), + "snapshot_date": version_meta.get("snapshot_date"), + "effective_from": version_meta.get("effective_from"), + "effective_to": version_meta.get("effective_to"), + } elif record_type == "decision": canonical_text = item.get("text") or "" elif record_type == "article": @@ -269,6 +302,9 @@ def build_delivery_plan( "ordinal": item.get("ordinal", line_number), "title": item.get("heading"), "article_number": item.get("article_number"), + "article_kind": item.get("article_kind"), + "effective_from": item.get("effective_from"), + "effective_to": item.get("effective_to"), "content": item.get("text") or "", } ) @@ -280,12 +316,18 @@ def build_delivery_plan( unique_docs.add(doc_id) + for article in records: + for key in ("law_number", "law_title"): + if key in document_identity: + article[key] = document_identity[key] + chunks = plan_source_chunks( document_id=doc_id, version_id=v_id, canonical_text=canonical_text, records=records, content_limit_chars=target_settings.content_limit_chars, + document_metadata=document_identity, ) provenance = conn.execute( diff --git a/src/mesa_legal_data/quality.py b/src/mesa_legal_data/quality.py index 2a84b6e..d459889 100644 --- a/src/mesa_legal_data/quality.py +++ b/src/mesa_legal_data/quality.py @@ -235,6 +235,35 @@ def evaluate_quality( elif is_legislation: checks.append(CheckResult("STRUCTURE", "article_plausibility", "PASS", f"Parsed {len(art_recs)} articles")) + fallback_only_articles = [ + article + for article in art_recs + if str(article.get("text") or "").strip().casefold() + == f"madde {str(article.get('article_number') or '').strip()}".casefold() + ] + if len(fallback_only_articles) == len(art_recs): + checks.append( + CheckResult( + "STRUCTURE", + "article_body_presence", + "BLOCK", + "Every parsed article is header-only; substantive article text is missing", + {"article_ids": [article.get("id") for article in fallback_only_articles]}, + ) + ) + elif fallback_only_articles: + checks.append( + CheckResult( + "STRUCTURE", + "article_body_presence", + "REVIEW", + "One or more parsed articles contain only a header fallback", + {"article_ids": [article.get("id") for article in fallback_only_articles]}, + ) + ) + else: + checks.append(CheckResult("STRUCTURE", "article_body_presence", "PASS", "Article bodies are present")) + # Coverage evaluation cov_dict = None if coverage: diff --git a/tests/integration/test_post_master_independent_audit.py b/tests/integration/test_post_master_independent_audit.py index 0ca9f2f..d1d60e7 100644 --- a/tests/integration/test_post_master_independent_audit.py +++ b/tests/integration/test_post_master_independent_audit.py @@ -34,6 +34,7 @@ from mesa_legal_data.hashing import hash_stream from mesa_legal_data.parsers.citations import extract_citations from mesa_legal_data.parsers.coverage import compute_parsing_coverage +from mesa_legal_data.parsers.legislation import PARSER_VERSION as LEGISLATION_PARSER_VERSION from mesa_legal_data.pipeline import process_artifact_pipeline from mesa_legal_data.publisher.client import MesaClient, MesaClientError from mesa_legal_data.publisher.engine import build_delivery_plan, execute_publish_delivery, retry_delivery_failures @@ -367,8 +368,16 @@ def test_control_11_and_12_certified_and_uncertified_parser_auto_approval(tmp_pa upsert_source_operational_settings( conn, "cert_src", enabled=True, auto_approval_enabled=True, weekly_sample_count=0 ) - # Certify legislation_parser 1.0.0 for cert_src - set_parser_certification(conn, "cert_src", "legislation_parser", "1.0.0", certified=True, certified_by="auditor") + # Certification is deliberately exact-version scoped; parser changes require + # an explicit recertification instead of inheriting trust from older output. + set_parser_certification( + conn, + "cert_src", + "legislation_parser", + LEGISLATION_PARSER_VERSION, + certified=True, + certified_by="auditor", + ) conn.close() # Ingest doc with certified parser -> auto_approved == True diff --git a/tests/unit/test_comprehensive_temporal_integrity.py b/tests/unit/test_comprehensive_temporal_integrity.py new file mode 100644 index 0000000..3c2ba79 --- /dev/null +++ b/tests/unit/test_comprehensive_temporal_integrity.py @@ -0,0 +1,180 @@ +import hashlib +import json + +from mesa_legal_data.catalog import ( + get_connection, + get_db_path, + get_document, + get_version_for_artifact, + insert_artifact, + migrate, + upsert_document, +) +from mesa_legal_data.pipeline import process_artifact_pipeline + + +def _ingest_law(tmp_path, *, document_id: str, artifact_id: str, body: str, metadata: dict) -> dict: + migrate(None, get_db_path()) + raw = f"

TEST KANUNU

MADDE 1- {body}

".encode() + raw_path = tmp_path / "raw" / f"{artifact_id}.html" + raw_path.parent.mkdir(parents=True, exist_ok=True) + raw_path.write_bytes(raw) + conn = get_connection() + upsert_document(conn, document_id, "legislation", "law", "TR", "Test Kanunu", document_id, "fetched") + insert_artifact( + conn, + artifact_id, + document_id, + "mevzuat", + f"https://example.test/{artifact_id}.html", + "2026-10-11T12:00:00Z", + "manual", + 200, + "text/html", + "text/html", + len(raw), + hashlib.sha256(raw).hexdigest(), + str(raw_path.relative_to(tmp_path)), + None, + None, + "fetched", + None, + json.dumps(metadata), + ) + conn.close() + process_artifact_pipeline(artifact_id) + conn = get_connection() + version = get_version_for_artifact(conn, artifact_id) + conn.close() + assert version is not None + return version + + +def test_acquisition_time_is_not_used_as_legal_snapshot_date(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + version = _ingest_law( + tmp_path, + document_id="tr:legislation:law:9998", + artifact_id="art-no-legal-date", + body="Hukuk.", + metadata={"source_role": "consolidated_snapshot"}, + ) + assert version["snapshot_date"] == "unknown-date" + assert version["quality_status"] == "REVIEW" + conn = get_connection() + assert conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'LEGAL_DATE_MISSING'", + (version["version_id"],), + ).fetchone()[0] == 1 + conn.close() + + +def test_declared_effective_interval_is_preserved_and_validated(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + version = _ingest_law( + tmp_path, + document_id="tr:legislation:law:9997", + artifact_id="art-effective-interval", + body="Hukuk.", + metadata={ + "source_role": "original_publication", + "publication_date": "2025-01-01", + "effective_from": "2025-02-01", + "effective_to": "2026-01-31", + }, + ) + assert version["effective_from"] == "2025-02-01" + assert version["effective_to"] == "2026-01-31" + + +def test_contradictory_declared_effective_interval_requires_review(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + version = _ingest_law( + tmp_path, + document_id="tr:legislation:law:9996", + artifact_id="art-invalid-effective-interval", + body="Hukuk.", + metadata={ + "publication_date": "2025-01-01", + "effective_from": "2026-02-01", + "effective_to": "2026-01-31", + }, + ) + assert version["quality_status"] == "REVIEW" + conn = get_connection() + assert conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'EFFECTIVE_INTERVAL_INVALID'", + (version["version_id"],), + ).fetchone()[0] == 1 + conn.close() + + +def test_future_effective_version_does_not_replace_active_version(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + document_id = "tr:legislation:law:future-guard" + active = _ingest_law( + tmp_path, + document_id=document_id, + artifact_id="active-version", + body="Bugün yürürlükte.", + metadata={"publication_date": "2025-01-01", "effective_from": "2025-01-01"}, + ) + _ingest_law( + tmp_path, + document_id=document_id, + artifact_id="future-version", + body="Gelecekte yürürlüğe girecek.", + metadata={"publication_date": "2026-09-01", "effective_from": "2027-01-01"}, + ) + conn = get_connection() + assert get_document(conn, document_id)["current_version_id"] == active["version_id"] + conn.close() + + +def test_overlapping_bounded_effective_intervals_force_review(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + document_id = "tr:legislation:law:interval-conflict" + first = _ingest_law( + tmp_path, + document_id=document_id, + artifact_id="interval-a", + body="Birinci içerik.", + metadata={ + "publication_date": "2025-01-01", + "effective_from": "2025-01-01", + "effective_to": "2027-01-31", + }, + ) + second = _ingest_law( + tmp_path, + document_id=document_id, + artifact_id="interval-b", + body="Çelişkili içerik.", + metadata={ + "publication_date": "2026-01-01", + "effective_from": "2026-01-01", + "effective_to": "2028-01-31", + }, + ) + conn = get_connection() + rows = conn.execute( + "SELECT version_id, approval_status, quality_status FROM versions WHERE version_id IN (?, ?) ORDER BY version_id", + (first["version_id"], second["version_id"]), + ).fetchall() + assert all(row[1:] == ("pending", "REVIEW") for row in rows) + assert conn.execute( + "SELECT count(*) FROM validation_issues WHERE code = 'VERSION_EFFECTIVE_INTERVAL_AMBIGUITY' AND status = 'open'" + ).fetchone()[0] == 2 + conn.close() + + +def test_header_only_legislation_cannot_pass_quality_gate(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MESA_DATA_DATA_ROOT", str(tmp_path)) + version = _ingest_law( + tmp_path, + document_id="tr:legislation:law:header-only", + artifact_id="header-only", + body="", + metadata={"publication_date": "2026-01-01"}, + ) + assert version["quality_status"] == "BLOCK" From 17ce8f8fa45e1d814df4729fc354f839ad030af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:32:50 +0300 Subject: [PATCH 3/8] fix: bind native publish authorization to actor and frozen scope --- src/mesa_legal_data/web/api.py | 25 ++++++++++++++++++-- tests/integration/test_master_mvp_closure.py | 21 +++++++++++++++- 2 files changed, 43 insertions(+), 3 deletions(-) diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index 7df7261..9af8c38 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -2242,7 +2242,7 @@ async def prepare_publisher_release_endpoint(target_key: str = "default"): @router.post("/publisher/publish") -def start_publisher_delivery_endpoint(req: MesaPublishRequest): +def start_publisher_delivery_endpoint(req: MesaPublishRequest, request: Request): from mesa_legal_data.operations import submit_operation conn = get_connection() @@ -2279,17 +2279,38 @@ def start_publisher_delivery_endpoint(req: MesaPublishRequest): error_response("PREFLIGHT_FAILED", f"Preflight checks failed: {'; '.join(failed_msgs)}", status_code=400) delivery_id = f"del-{uuid.uuid4().hex[:12]}" + actor = extract_actor(request) op_id = submit_operation( conn, operation_type="mesa_v4_delivery", - requested_by="web-user", + requested_by=actor, input_dict={ "delivery_id": delivery_id, "release_id": req.release_id, + "manifest_sha256": req.manifest_sha256, "target_key": req.target_key, "target_config_sha256": req.target_config_sha256, + "scope": summary.model_dump(), }, ) + log_audit_event( + conn, + actor=actor, + action="mesa_publish_authorize", + subject_type="release", + subject_id=req.release_id, + new_sha256=req.manifest_sha256, + details_json=json.dumps( + { + "operation_id": op_id, + "delivery_id": delivery_id, + "target_key": req.target_key, + "target_config_sha256": req.target_config_sha256, + "scope": summary.model_dump(), + }, + sort_keys=True, + ), + ) conn.close() return ok_response({"operation_id": op_id, "delivery_id": delivery_id, "summary": summary.model_dump()}) diff --git a/tests/integration/test_master_mvp_closure.py b/tests/integration/test_master_mvp_closure.py index 9365d52..c20e671 100644 --- a/tests/integration/test_master_mvp_closure.py +++ b/tests/integration/test_master_mvp_closure.py @@ -158,6 +158,7 @@ def test_final_panel_confirmation_binds_exact_release_manifest_and_target(tmp_pa def fake_submit(conn, *, operation_type, requested_by, input_dict): captured.update(input_dict) + captured["requested_by"] = requested_by return "op-frozen-final" monkeypatch.setattr("mesa_legal_data.operations.submit_operation", fake_submit) @@ -167,7 +168,7 @@ def fake_submit(conn, *, operation_type, requested_by, input_dict): ) response = client.post( "/api/publisher/publish", - headers=WEB_HEADERS, + headers={**WEB_HEADERS, "X-MESA-Actor": "release-authorizer"}, json={ "target_key": prepared["target_key"], "release_id": prepared["release_id"], @@ -177,6 +178,24 @@ def fake_submit(conn, *, operation_type, requested_by, input_dict): ) assert response.status_code == 200 assert captured["release_id"] == prepared["release_id"] + assert captured["manifest_sha256"] == prepared["manifest_sha256"] + assert captured["requested_by"] == "release-authorizer" + assert captured["scope"]["estimated_chunks"] > 0 + conn = get_connection() + authorization = conn.execute( + """SELECT actor, new_sha256, details_json + FROM audit_events + WHERE action = 'mesa_publish_authorize' AND subject_id = ? + ORDER BY created_at DESC LIMIT 1""", + (prepared["release_id"],), + ).fetchone() + conn.close() + assert authorization is not None + assert authorization[0] == "release-authorizer" + assert authorization[1] == prepared["manifest_sha256"] + authorization_details = json.loads(authorization[2]) + assert authorization_details["target_config_sha256"] == prepared["target_config_sha256"] + assert authorization_details["scope"]["estimated_chunks"] > 0 frozen_chunks, _ = build_delivery_plan(release_id=captured["release_id"]) assert {chunk.version_id for chunk, _ in frozen_chunks} == {v1["version_id"]} From 78174b392d97761b3e6018a1b3a798264918722f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:35:23 +0300 Subject: [PATCH 4/8] test: declare legal dates in releasable synthetic fixtures --- tests/acceptance/test_e2e_mvp.py | 2 +- tests/acceptance/test_feature_freeze_gate.py | 2 +- tests/acceptance/test_final_hardening_gate.py | 1 + tests/acceptance/test_security_and_scale_gate.py | 1 + tests/acceptance/test_web_e2e.py | 1 + tests/integration/test_audit_coverage.py | 2 +- tests/integration/test_bulk_review_streaming.py | 2 +- tests/integration/test_bulk_version_review.py | 2 +- tests/integration/test_downloads.py | 2 +- tests/integration/test_exports.py | 2 +- tests/integration/test_large_release_streaming.py | 2 +- tests/integration/test_mesa_import.py | 2 +- tests/integration/test_prompt4_integration_audit.py | 4 +++- tests/integration/test_release_atomicity.py | 2 +- tests/integration/test_release_builder.py | 2 +- tests/integration/test_release_lifecycle.py | 2 +- tests/integration/test_release_source_snapshot.py | 4 ++-- tests/integration/test_release_tamper.py | 2 +- tests/integration/test_review_flow.py | 2 +- tests/integration/test_rollback_and_provenance.py | 2 +- tests/integration/test_web_release.py | 2 +- tests/integration/test_web_review.py | 2 +- tests/security/test_security_negative.py | 2 +- tests/unit/test_catalog_release_iterator.py | 2 +- tests/unit/test_release_canonical_scan.py | 2 +- 25 files changed, 28 insertions(+), 23 deletions(-) diff --git a/tests/acceptance/test_e2e_mvp.py b/tests/acceptance/test_e2e_mvp.py index 52a9f8a..1148556 100644 --- a/tests/acceptance/test_e2e_mvp.py +++ b/tests/acceptance/test_e2e_mvp.py @@ -85,7 +85,7 @@ def test_e2e_mvp_acceptance_flow(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/acceptance/test_feature_freeze_gate.py b/tests/acceptance/test_feature_freeze_gate.py index e7cac05..884c1ea 100644 --- a/tests/acceptance/test_feature_freeze_gate.py +++ b/tests/acceptance/test_feature_freeze_gate.py @@ -74,7 +74,7 @@ def test_feature_freeze_gate_complete_workflow(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) log_audit_event( conn, diff --git a/tests/acceptance/test_final_hardening_gate.py b/tests/acceptance/test_final_hardening_gate.py index 78f64e6..9d1c737 100644 --- a/tests/acceptance/test_final_hardening_gate.py +++ b/tests/acceptance/test_final_hardening_gate.py @@ -130,6 +130,7 @@ def test_final_hardening_acceptance_gate(tmp_path, monkeypatch): document_id="tr:legislation:law:4721", family="legislation", title="Türk Medeni Kanunu", + publication_date="2026-01-01", ) assert art.artifact_id.startswith("sha256:") diff --git a/tests/acceptance/test_security_and_scale_gate.py b/tests/acceptance/test_security_and_scale_gate.py index 49faa4e..50785da 100644 --- a/tests/acceptance/test_security_and_scale_gate.py +++ b/tests/acceptance/test_security_and_scale_gate.py @@ -103,6 +103,7 @@ def test_master_security_and_scale_acceptance_gate(tmp_path, monkeypatch): document_id="tr:legislation:constitution:2709", family="legislation", title="Anayasa", + publication_date="2026-01-01", ) assert art.artifact_id.startswith("sha256:") diff --git a/tests/acceptance/test_web_e2e.py b/tests/acceptance/test_web_e2e.py index 6c13326..2b50a37 100644 --- a/tests/acceptance/test_web_e2e.py +++ b/tests/acceptance/test_web_e2e.py @@ -44,6 +44,7 @@ def test_web_e2e_acceptance_flow(tmp_path, monkeypatch): "family": "legislation", "document_type": "law", "title": "Türk Medeni Kanunu", + "publication_date": "2026-01-01", }, files={"file": ("payload.html", law_text.encode("utf-8"), "text/html")}, headers={"X-MESA-Requested-With": "web-admin"}, diff --git a/tests/integration/test_audit_coverage.py b/tests/integration/test_audit_coverage.py index 048f920..e0fc337 100644 --- a/tests/integration/test_audit_coverage.py +++ b/tests/integration/test_audit_coverage.py @@ -47,7 +47,7 @@ def test_audit_event_recording_and_coverage(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) # 1. Direct log_audit_event check diff --git a/tests/integration/test_bulk_review_streaming.py b/tests/integration/test_bulk_review_streaming.py index e4fe219..27e5aaf 100644 --- a/tests/integration/test_bulk_review_streaming.py +++ b/tests/integration/test_bulk_review_streaming.py @@ -57,7 +57,7 @@ def test_bulk_review_streaming_scale_and_single_pass(tmp_path, monkeypatch): None, "fetched", None, - "{}", + '{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_bulk_version_review.py b/tests/integration/test_bulk_version_review.py index 9010fe0..2fec4e2 100644 --- a/tests/integration/test_bulk_version_review.py +++ b/tests/integration/test_bulk_version_review.py @@ -66,7 +66,7 @@ def _setup_multi_record_version(tmp_path, doc_num="100"): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_downloads.py b/tests/integration/test_downloads.py index 5c41679..23d7be3 100644 --- a/tests/integration/test_downloads.py +++ b/tests/integration/test_downloads.py @@ -46,7 +46,7 @@ def test_all_download_endpoints(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_exports.py b/tests/integration/test_exports.py index b052e25..04fc86b 100644 --- a/tests/integration/test_exports.py +++ b/tests/integration/test_exports.py @@ -48,7 +48,7 @@ def test_export_types_and_filters(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_large_release_streaming.py b/tests/integration/test_large_release_streaming.py index 2575138..9d24329 100644 --- a/tests/integration/test_large_release_streaming.py +++ b/tests/integration/test_large_release_streaming.py @@ -66,7 +66,7 @@ def test_large_release_streaming_and_corrupt_rollback(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_mesa_import.py b/tests/integration/test_mesa_import.py index 1315549..89dfe7a 100644 --- a/tests/integration/test_mesa_import.py +++ b/tests/integration/test_mesa_import.py @@ -50,7 +50,7 @@ def setup_synthetic_release(tmp_path, release_id: str): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_prompt4_integration_audit.py b/tests/integration/test_prompt4_integration_audit.py index d096ffc..74e90ee 100644 --- a/tests/integration/test_prompt4_integration_audit.py +++ b/tests/integration/test_prompt4_integration_audit.py @@ -420,7 +420,9 @@ def test_kontrol_4_citations_lifecycle_and_semantics(): cits_num = extract_citations("5237 sayılı Türk Ceza Kanunu madde 81 uyarınca") assert len(cits_num) >= 1 assert cits_num[0].target_legislation_id == "tr:legislation:law:5237" - assert cits_num[0].citation_status == "RESOLVED" + # A syntactically mapped target remains a candidate until the referenced + # law/article/version is verified against the corpus. + assert cits_num[0].citation_status == "EXTRACTED" assert cits_num[0].char_start is not None assert "5237 sayılı" in cits_num[0].raw_text diff --git a/tests/integration/test_release_atomicity.py b/tests/integration/test_release_atomicity.py index 5b587f2..5093d8a 100644 --- a/tests/integration/test_release_atomicity.py +++ b/tests/integration/test_release_atomicity.py @@ -45,7 +45,7 @@ def test_release_state_machine_and_doctor(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_release_builder.py b/tests/integration/test_release_builder.py index adb311d..83634e5 100644 --- a/tests/integration/test_release_builder.py +++ b/tests/integration/test_release_builder.py @@ -57,7 +57,7 @@ def test_release_builder_real_counts(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_release_lifecycle.py b/tests/integration/test_release_lifecycle.py index b383062..49eeb2c 100644 --- a/tests/integration/test_release_lifecycle.py +++ b/tests/integration/test_release_lifecycle.py @@ -51,7 +51,7 @@ def test_release_lifecycle_transitions(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_release_source_snapshot.py b/tests/integration/test_release_source_snapshot.py index 99faade..02e8b7f 100644 --- a/tests/integration/test_release_source_snapshot.py +++ b/tests/integration/test_release_source_snapshot.py @@ -53,7 +53,7 @@ def test_release_source_snapshot_exact(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) upsert_document(conn, "tr:decision:aym:1", "decision", "aym_decision", "TR", "AYM 1", "2026/1", "fetched") @@ -75,7 +75,7 @@ def test_release_source_snapshot_exact(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_release_tamper.py b/tests/integration/test_release_tamper.py index af1ace5..832a736 100644 --- a/tests/integration/test_release_tamper.py +++ b/tests/integration/test_release_tamper.py @@ -60,7 +60,7 @@ def test_release_tamper_detection(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_review_flow.py b/tests/integration/test_review_flow.py index 38ecd99..5b079bb 100644 --- a/tests/integration/test_review_flow.py +++ b/tests/integration/test_review_flow.py @@ -58,7 +58,7 @@ def test_review_flow_integration(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_rollback_and_provenance.py b/tests/integration/test_rollback_and_provenance.py index cb875f6..0716c63 100644 --- a/tests/integration/test_rollback_and_provenance.py +++ b/tests/integration/test_rollback_and_provenance.py @@ -65,7 +65,7 @@ def test_rollback_and_provenance_integration(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_web_release.py b/tests/integration/test_web_release.py index df14f3f..ecefbdc 100644 --- a/tests/integration/test_web_release.py +++ b/tests/integration/test_web_release.py @@ -51,7 +51,7 @@ def test_web_release_and_import_api(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/integration/test_web_review.py b/tests/integration/test_web_review.py index c84b155..854bb0d 100644 --- a/tests/integration/test_web_review.py +++ b/tests/integration/test_web_review.py @@ -49,7 +49,7 @@ def test_web_review_api(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/security/test_security_negative.py b/tests/security/test_security_negative.py index ec5099e..7e0eae4 100644 --- a/tests/security/test_security_negative.py +++ b/tests/security/test_security_negative.py @@ -83,7 +83,7 @@ def test_release_tamper_detection(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/unit/test_catalog_release_iterator.py b/tests/unit/test_catalog_release_iterator.py index 984f40f..b935153 100644 --- a/tests/unit/test_catalog_release_iterator.py +++ b/tests/unit/test_catalog_release_iterator.py @@ -72,7 +72,7 @@ def test_catalog_release_iterator_no_fetchall(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() diff --git a/tests/unit/test_release_canonical_scan.py b/tests/unit/test_release_canonical_scan.py index 855e987..9e384d9 100644 --- a/tests/unit/test_release_canonical_scan.py +++ b/tests/unit/test_release_canonical_scan.py @@ -54,7 +54,7 @@ def test_canonical_file_opened_only_once_per_part(tmp_path, monkeypatch): last_modified=None, transport_status="fetched", error_code=None, - metadata_json="{}", + metadata_json='{"publication_date":"2026-01-01"}', ) conn.close() From ce27a7869b8cc9ff878cf81620a6e4c9a7b9ee6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:39:17 +0300 Subject: [PATCH 5/8] style: apply repository formatter to audit repairs --- src/mesa_legal_data/catalog.py | 11 +++---- src/mesa_legal_data/parsers/citations.py | 1 + src/mesa_legal_data/pipeline.py | 4 +-- src/mesa_legal_data/publisher/chunker.py | 8 ++++- .../test_comprehensive_temporal_integrity.py | 31 ++++++++++++------- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/src/mesa_legal_data/catalog.py b/src/mesa_legal_data/catalog.py index 4a612e5..267a319 100644 --- a/src/mesa_legal_data/catalog.py +++ b/src/mesa_legal_data/catalog.py @@ -345,9 +345,10 @@ def sort_key(item): overlaps = max(left["effective_from"], right["effective_from"]) <= min( left["effective_to"], right["effective_to"] ) - explicitly_ordered = left.get("supersedes_version_id") == right["version_id"] or right.get( - "supersedes_version_id" - ) == left["version_id"] + explicitly_ordered = ( + left.get("supersedes_version_id") == right["version_id"] + or right.get("supersedes_version_id") == left["version_id"] + ) if overlaps and left["canonical_sha256"] != right["canonical_sha256"] and not explicitly_ordered: conflicting_ids.update((left["version_id"], right["version_id"])) @@ -398,9 +399,7 @@ def sort_key(item): ] ambiguity_group = [chosen, *same_date_candidates] ambiguity_ids = {item["version_id"] for item in ambiguity_group} - has_explicit_supersession = any( - item.get("supersedes_version_id") in ambiguity_ids for item in ambiguity_group - ) + has_explicit_supersession = any(item.get("supersedes_version_id") in ambiguity_ids for item in ambiguity_group) if same_date_candidates and not has_explicit_supersession: doc_cur = conn.cursor() doc_cur.execute("SELECT current_version_id FROM documents WHERE document_id = ?", (document_id,)) diff --git a/src/mesa_legal_data/parsers/citations.py b/src/mesa_legal_data/parsers/citations.py index cf9f999..756baf9 100644 --- a/src/mesa_legal_data/parsers/citations.py +++ b/src/mesa_legal_data/parsers/citations.py @@ -52,6 +52,7 @@ if alias: ALIAS_MAP[alias.upper()] = (num, kind) + class Citation(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index 6b8b3d6..bd644d6 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -248,9 +248,7 @@ def process_artifact_pipeline( ver_date = normalized_date or "unknown-date" else: raw_ver_date = ( - meta_dict.get("snapshot_date") - or meta_dict.get("publication_date") - or meta_dict.get("source_date") + meta_dict.get("snapshot_date") or meta_dict.get("publication_date") or meta_dict.get("source_date") ) legal_date_missing = not bool(raw_ver_date) normalized_date = _valid_legal_date(raw_ver_date) diff --git a/src/mesa_legal_data/publisher/chunker.py b/src/mesa_legal_data/publisher/chunker.py index ed82aa9..dd5a0e1 100644 --- a/src/mesa_legal_data/publisher/chunker.py +++ b/src/mesa_legal_data/publisher/chunker.py @@ -116,7 +116,13 @@ def plan_source_chunks( for article in article_records: start = article.get("char_start") end = article.get("char_end") - if not isinstance(start, int) or not isinstance(end, int) or start < 0 or end > len(canonical_text) or start >= end: + if ( + not isinstance(start, int) + or not isinstance(end, int) + or start < 0 + or end > len(canonical_text) + or start >= end + ): raise ValueError(f"Invalid article source span: [{start}, {end}]") if start < previous_end: raise ValueError(f"Article source spans overlap at offset {start}") diff --git a/tests/unit/test_comprehensive_temporal_integrity.py b/tests/unit/test_comprehensive_temporal_integrity.py index 3c2ba79..0aaa3bb 100644 --- a/tests/unit/test_comprehensive_temporal_integrity.py +++ b/tests/unit/test_comprehensive_temporal_integrity.py @@ -62,10 +62,13 @@ def test_acquisition_time_is_not_used_as_legal_snapshot_date(tmp_path, monkeypat assert version["snapshot_date"] == "unknown-date" assert version["quality_status"] == "REVIEW" conn = get_connection() - assert conn.execute( - "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'LEGAL_DATE_MISSING'", - (version["version_id"],), - ).fetchone()[0] == 1 + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'LEGAL_DATE_MISSING'", + (version["version_id"],), + ).fetchone()[0] + == 1 + ) conn.close() @@ -102,10 +105,13 @@ def test_contradictory_declared_effective_interval_requires_review(tmp_path, mon ) assert version["quality_status"] == "REVIEW" conn = get_connection() - assert conn.execute( - "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'EFFECTIVE_INTERVAL_INVALID'", - (version["version_id"],), - ).fetchone()[0] == 1 + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE version_id = ? AND code = 'EFFECTIVE_INTERVAL_INVALID'", + (version["version_id"],), + ).fetchone()[0] + == 1 + ) conn.close() @@ -162,9 +168,12 @@ def test_overlapping_bounded_effective_intervals_force_review(tmp_path, monkeypa (first["version_id"], second["version_id"]), ).fetchall() assert all(row[1:] == ("pending", "REVIEW") for row in rows) - assert conn.execute( - "SELECT count(*) FROM validation_issues WHERE code = 'VERSION_EFFECTIVE_INTERVAL_AMBIGUITY' AND status = 'open'" - ).fetchone()[0] == 2 + assert ( + conn.execute( + "SELECT count(*) FROM validation_issues WHERE code = 'VERSION_EFFECTIVE_INTERVAL_AMBIGUITY' AND status = 'open'" + ).fetchone()[0] + == 2 + ) conn.close() From dfc752a386b9f1287ee42256aa4013b24309ac6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 15:41:40 +0300 Subject: [PATCH 6/8] fix: carry versioned citation evidence into native chunks --- schemas/citation.schema.json | 119 +++++++++---------- src/mesa_legal_data/pipeline.py | 4 + src/mesa_legal_data/publisher/engine.py | 37 ++++++ tests/integration/test_master_mvp_closure.py | 18 ++- 4 files changed, 117 insertions(+), 61 deletions(-) diff --git a/schemas/citation.schema.json b/schemas/citation.schema.json index b2eeab6..3e48c69 100644 --- a/schemas/citation.schema.json +++ b/schemas/citation.schema.json @@ -1,63 +1,62 @@ { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": [ - "id", - "record_type", - "source_record_id", - "target_legislation_id", - "raw_text", - "extraction_method", - "validation_status", - "schema_version", - "created_at", - "source", - "provenance" - ], - "properties": { - "id": { "type": "string" }, - "record_type": { "type": "string", "const": "citation" }, - "source_record_id": { "type": "string" }, - "target_legislation_id": { "type": ["string", "null"] }, - "target_article_id": { "type": ["string", "null"] }, - "raw_text": { "type": "string" }, - "citation_status": { "type": "string", "enum": ["EXTRACTED", "RESOLVED", "UNRESOLVED", "HUMAN_VERIFIED"] }, - "relation_hint": { "type": ["string", "null"], "enum": ["amends", "repeals", "adds", null] }, - "source_span": { - "type": ["object", "null"], - "properties": { - "char_start": { "type": ["integer", "null"] }, - "char_end": { "type": ["integer", "null"] } - }, - "additionalProperties": false + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": [ + "id", + "record_type", + "source_record_id", + "target_legislation_id", + "raw_text", + "extraction_method", + "validation_status", + "schema_version", + "created_at", + "source", + "provenance" + ], + "properties": { + "id": {"type": "string"}, + "record_type": {"type": "string", "const": "citation"}, + "source_record_id": {"type": "string"}, + "source_version_id": {"type": "string"}, + "target_legislation_id": {"type": ["string", "null"]}, + "target_article_id": {"type": ["string", "null"]}, + "target_version_id": {"type": ["string", "null"]}, + "raw_text": {"type": "string"}, + "citation_status": {"type": "string", "enum": ["EXTRACTED", "RESOLVED", "UNRESOLVED", "HUMAN_VERIFIED"]}, + "relation_hint": {"type": ["string", "null"], "enum": ["amends", "repeals", "adds", null]}, + "source_span": { + "type": ["object", "null"], + "properties": {"char_start": {"type": ["integer", "null"]}, "char_end": {"type": ["integer", "null"]}}, + "additionalProperties": false + }, + "extraction_method": {"type": "string"}, + "validation_status": {"type": "string"}, + "schema_version": {"type": "string"}, + "created_at": {"type": "string"}, + "source": { + "type": "object", + "required": ["source_id", "source_url", "retrieved_at", "artifact_sha256"], + "properties": { + "source_id": {"type": "string"}, + "source_url": {"type": "string"}, + "retrieved_at": {"type": "string"}, + "publication_date": {"type": ["string", "null"]}, + "artifact_sha256": {"type": "string", "pattern": "^[a-fA-F0-9]{64}$"}, + "artifact_path": {"type": ["string", "null"]} + }, + "additionalProperties": false + }, + "provenance": { + "type": "object", + "required": ["parser_name", "parser_version", "pipeline_run_id"], + "properties": { + "parser_name": {"type": "string"}, + "parser_version": {"type": "string"}, + "pipeline_run_id": {"type": "string"} + }, + "additionalProperties": false + } }, - "extraction_method": { "type": "string" }, - "validation_status": { "type": "string" }, - "schema_version": { "type": "string" }, - "created_at": { "type": "string" }, - "source": { - "type": "object", - "required": ["source_id", "source_url", "retrieved_at", "artifact_sha256"], - "properties": { - "source_id": { "type": "string" }, - "source_url": { "type": "string" }, - "retrieved_at": { "type": "string" }, - "publication_date": { "type": ["string", "null"] }, - "artifact_sha256": { "type": "string", "pattern": "^[a-fA-F0-9]{64}$" }, - "artifact_path": { "type": ["string", "null"] } - }, - "additionalProperties": false - }, - "provenance": { - "type": "object", - "required": ["parser_name", "parser_version", "pipeline_run_id"], - "properties": { - "parser_name": { "type": "string" }, - "parser_version": { "type": "string" }, - "pipeline_run_id": { "type": "string" } - }, - "additionalProperties": false - } - }, - "additionalProperties": false + "additionalProperties": false } diff --git a/src/mesa_legal_data/pipeline.py b/src/mesa_legal_data/pipeline.py index bd644d6..a6cb162 100644 --- a/src/mesa_legal_data/pipeline.py +++ b/src/mesa_legal_data/pipeline.py @@ -429,8 +429,10 @@ def process_artifact_pipeline( "id": c_id, "record_type": "citation", "source_record_id": leg_record["id"], + "source_version_id": version_id, "target_legislation_id": c.target_legislation_id, "target_article_id": c.target_article_id, + "target_version_id": None, "raw_text": c.raw_text, "citation_status": c.citation_status, "relation_hint": c.relation_hint, @@ -487,8 +489,10 @@ def process_artifact_pipeline( "id": c_id, "record_type": "citation", "source_record_id": dec_id, + "source_version_id": version_id, "target_legislation_id": c.target_legislation_id, "target_article_id": c.target_article_id, + "target_version_id": None, "raw_text": c.raw_text, "citation_status": c.citation_status, "relation_hint": c.relation_hint, diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py index faee7cb..65f8db2 100644 --- a/src/mesa_legal_data/publisher/engine.py +++ b/src/mesa_legal_data/publisher/engine.py @@ -202,6 +202,7 @@ def build_delivery_plan( v_id = v_info["version_id"] canonical_text = "" records: list[dict[str, Any]] = [] + citations: list[dict[str, Any]] = [] document_identity: dict[str, Any] = {} if release_id: packaged_records = v_info["packaged_records"] @@ -245,6 +246,8 @@ def build_delivery_plan( "content": item.get("text") or "", } ) + elif record_type == "citation": + citations.append(item) if not canonical_text.strip(): conn.close() raise MesaClientError(f"Release {release_id} version {v_id} has no canonical document text") @@ -308,6 +311,8 @@ def build_delivery_plan( "content": item.get("text") or "", } ) + elif record_type == "citation": + citations.append(item) if not canonical_text.strip(): raise ValueError(f"Version {v_id} has no authoritative canonical document text") except (OSError, ValueError, json.JSONDecodeError): @@ -330,6 +335,38 @@ def build_delivery_plan( document_metadata=document_identity, ) + # Canonical citation edges are release-owned records, not text inferred + # by the publisher. Carry only citations whose authoritative source span + # overlaps this chunk. Target versions stay null until a corpus-aware or + # human resolver verifies the exact legal version. + for chunk in chunks: + chunk_citations: list[dict[str, Any]] = [] + for citation in citations: + span = citation.get("source_span") or {} + start = span.get("char_start") + end = span.get("char_end") + if not isinstance(start, int) or not isinstance(end, int) or start >= end: + continue + if start >= chunk.char_end or end <= chunk.char_start: + continue + chunk_citations.append( + { + "citation_id": citation.get("id"), + "source_record_id": citation.get("source_record_id"), + "source_version_id": citation.get("source_version_id") or v_id, + "target_legislation_id": citation.get("target_legislation_id"), + "target_article_id": citation.get("target_article_id"), + "target_version_id": citation.get("target_version_id"), + "citation_status": citation.get("citation_status"), + "relation_hint": citation.get("relation_hint"), + "char_start": start, + "char_end": end, + "fully_contained": start >= chunk.char_start and end <= chunk.char_end, + } + ) + if chunk_citations: + chunk.metadata["citations"] = chunk_citations + provenance = conn.execute( """SELECT v.revision_number, v.supersedes_version_id, a.source_url, a.source_id, a.sha256 FROM versions v JOIN artifacts a ON a.artifact_id = v.artifact_id diff --git a/tests/integration/test_master_mvp_closure.py b/tests/integration/test_master_mvp_closure.py index c20e671..1db753b 100644 --- a/tests/integration/test_master_mvp_closure.py +++ b/tests/integration/test_master_mvp_closure.py @@ -108,7 +108,7 @@ def test_final_panel_confirmation_binds_exact_release_manifest_and_target(tmp_pa art_v1 = _helper_create_law_artifact( tmp_path, doc_id, - "

Panel V1

Madde 1- Frozen.

", + "

Panel V1

Madde 1- 6098 sayılı Kanunun 1/A maddesi değiştirilmiştir.

", "art-panel-final-v1", pub_date="2026-01-01", ) @@ -198,6 +198,22 @@ def fake_submit(conn, *, operation_type, requested_by, input_dict): assert authorization_details["scope"]["estimated_chunks"] > 0 frozen_chunks, _ = build_delivery_plan(release_id=captured["release_id"]) assert {chunk.version_id for chunk, _ in frozen_chunks} == {v1["version_id"]} + citation_metadata = [citation for chunk, _ in frozen_chunks for citation in chunk.metadata.get("citations", [])] + assert citation_metadata == [ + { + "citation_id": citation_metadata[0]["citation_id"], + "source_record_id": doc_id, + "source_version_id": v1["version_id"], + "target_legislation_id": "tr:legislation:law:6098", + "target_article_id": "tr:legislation:law:6098:article:1-a", + "target_version_id": None, + "citation_status": "EXTRACTED", + "relation_hint": "amends", + "char_start": citation_metadata[0]["char_start"], + "char_end": citation_metadata[0]["char_end"], + "fully_contained": True, + } + ] def test_master_a_chronology_current_version_selection(tmp_path, monkeypatch): From c7952165fe4a8c039810b317ceb5551bae5958a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 16:02:57 +0300 Subject: [PATCH 7/8] docs: publish comprehensive legal data repair audit --- docs/BUILD_STATE.json | 27 ++++--- .../00_BASELINE_AND_PRIOR_FINDINGS.md | 28 +++++++ .../01_ARCHITECTURE_AND_CONTRACT_MAP.md | 34 +++++++++ .../2026-10-11-round-01/02_DEFECT_REGISTER.md | 21 ++++++ .../03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md | 17 +++++ ...LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md | 25 +++++++ .../05_RELEASE_APPROVAL_AND_IMMUTABILITY.md | 17 +++++ .../06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md | 18 +++++ ...RELATIONAL_CITATIONS_AND_REPRESENTATION.md | 15 ++++ .../08_REGRESSION_AND_ADVERSARIAL_RESULTS.md | 63 ++++++++++++++++ .../09_SCALE_AND_OPERATIONS_READINESS.md | 9 +++ .../10_CROSS_REPO_HANDOFF.md | 24 ++++++ .../11_FINAL_READINESS_VERDICT.md | 30 ++++++++ .../data_lineage_matrix.csv | 13 ++++ .../2026-10-11-round-01/defect_register.json | 74 +++++++++++++++++++ .../native_contract_matrix.csv | 15 ++++ .../source_quality_summary.csv | 10 +++ .../2026-10-11-round-01/test_execution.json | 60 +++++++++++++++ 18 files changed, 490 insertions(+), 10 deletions(-) create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv create mode 100644 reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json diff --git a/docs/BUILD_STATE.json b/docs/BUILD_STATE.json index 108472a..f3d49b4 100644 --- a/docs/BUILD_STATE.json +++ b/docs/BUILD_STATE.json @@ -3,7 +3,7 @@ "scope": "feature_freeze", "pilot_ready": true, "external_mesa_ready": false, - "final_audit_status": "master_closed", + "final_audit_status": "comprehensive_repair_verified_contract_only", "current_task": "CLOSED", "completed": [ "FREEZE-001", @@ -41,23 +41,30 @@ "PROMPT-5-FINAL-ADVERSARIAL-AUDIT", "MASTER-MVP-CLOSURE", "POST-MASTER-INDEPENDENT-VERIFICATION", - "P1-RECORD-INSTANCE-NOT-NULL-ENFORCED" + "P1-RECORD-INSTANCE-NOT-NULL-ENFORCED", + "LEGAL-IDENTITY-AND-LOSSLESS-CHUNK-REPAIR", + "AUTHORITATIVE-TEMPORAL-VERSION-REPAIR", + "NATIVE-PUBLISH-AUTHORIZATION-BINDING", + "VERSIONED-CITATION-NATIVE-HANDOFF", + "COMPREHENSIVE-REPAIR-ROUND-2026-10-11" ], - "last_test_command": "uv run pytest -ra", - "last_test_result": "passed (360 passed: 356 non-scale + 4 scale in 72.83s)", + "last_test_command": ".venv/bin/pytest -ra --durations=20 --tb=short", + "last_test_result": "passed (413 passed, 1 cross-repo test skipped in 78.95s)", "verification_checks": { - "ruff_format": "passed (212 Python files)", + "ruff_format": "passed (217 files)", "ruff_check": "passed", - "mypy": "passed (77 source files)", + "mypy": "passed (78 source files)", "pip_audit": "passed (no known vulnerabilities)", - "pytest_not_scale": "passed (356 passed)", - "pytest_scale": "passed (4 passed)", + "pytest_not_scale": "passed (409 passed, 1 cross-repo skipped, 4 scale deselected)", + "pytest_scale": "passed (4 passed; peak RSS 171784 KiB)", "live_mesa_contract": "LIVE_MESA_NOT_VERIFIED" }, "blockers": [ - "Live MESA HTTP routes, schemas, mutation semantics, and server-side idempotency remain unverified because no endpoint or source access is available." + "Live MESA HTTP routes, schemas, mutation semantics, server-side idempotency and SQL/vector/graph projection remain unverified because no endpoint or MESA source checkout is available.", + "Official Profile B TEST was not run and no certification result is claimed.", + "10-20 GB target capacity is not proven; only bounded synthetic scale tests were run." ], "polish_status": "completed", "polish_current_task": "CLOSED", - "updated_at": "2026-08-30T23:30:00Z" + "updated_at": "2026-10-11T12:45:00Z" } diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md new file mode 100644 index 0000000..285c16f --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/00_BASELINE_AND_PRIOR_FINDINGS.md @@ -0,0 +1,28 @@ +# Başlangıç Durumu ve Önceki Bulgular + +## Doğrulanmış başlangıç + +- Depo: `MESA_Data` (`/storage/data-law/mesa-legal-data`) +- Başlangıç SHA: `38c3fa5d3240cc944e8da611926fd220b7df145c` +- Başlangıç dalı: `main`; çalışma ağacı temizdi. +- Yetkili onarım dalı: `fix/mesa-data-legal-integrity-native-publish`. Yerel/uzak geçmişte birleştirilmemiş ve açıkça devam edilmesi gereken bir onarım dalı bulunmadığı için, talimatta izin verilen adla başlangıç SHA'sından oluşturuldu. +- Denetlenen uygulama ucu: `dfc752a386b9f1287ee42256aa4013b24309ac6c` (rapor ve durum dosyası commit'i hariç). +- Ortam: Linux, Python 3.13.12, pytest 9.1.1, AnyIO 4.15.1, Starlette 1.5.1, FastAPI 0.141.1. + +Başlangıçta `reports/` dizini yoktu; dolayısıyla bu dizin altında okunacak ya da korunacak eski rapor bulunmadı. `docs/` altındaki 11 mevcut durum/kullanım/UX belgesi okundu. Bunlar hukuki doğruluk veya canlı MESA sertifikasyonu kanıtı olarak kabul edilmedi. `docs/BUILD_STATE.json` içindeki “360 test geçti” ve `LIVE_MESA_NOT_VERIFIED` kayıtları yalnız tarihsel ipucu olarak ele alındı. + +## Önceki iddia uzlaştırma matrisi + +| İddia/kaynak | HEAD incelemesi | Kanıt | Sonuç | +|---|---|---|---| +| `docs/BUILD_STATE.json`: MVP kapanmış, testler yeşil | Test sayısı artmıştı; yeşil testler hukuki kimlik uçlarını kapsamıyordu | Ön-fix 6 yeni regresyonun 6'sı başarısız | `CONFIRMED_GAP` | +| Aynı dosya: canlı MESA doğrulanmadı | MESA çalışma kopyası, `MESA_SOURCE_ROOT` ve yetkili DEV uç noktası yok | Çapraz-depo testinin tek skip nedeni | `EXTERNAL` | +| UX kapanış raporları: panel akışları tamam | Panel/operasyon testleri mevcut ve son suite'te geçti | 413/413 test | `ALREADY_FIXED` | +| Prompt'taki tarihsel Profile B sayıları | Resmî TEST/evidence bu depoda çalıştırılmadı ve değiştirilmedi | Bu turda Profile B komutu yok | `UNVERIFIED_EVIDENCE` | +| “Plan raporu” olarak anılan `MESA_Data_Kapsamli_Denetim_Raporu.md` | Çalışma ağacında yok | `find docs`, `find reports` | `UNAVAILABLE` | + +## Başlangıçta yeniden üretilen kusurlar + +Yeni `tests/unit/test_comprehensive_legal_integrity.py` taslağındaki 6 testin tamamı onarım öncesinde kırmızıydı: mükerrer/harf ekli madde kimliği, yapısal chunk kaybı/örtüşme, chunk yasal üst kimliği, sentaktik atfın yanlış `RESOLVED` olması, edinim zamanının hukuk tarihi yapılması ve etkinlik aralığının atılması. Testler daha sonra kimlik ve zamansal dosyalara ayrıldı; son durumda tamamı yeşildir. + +İlk tam onarım-sonrası çalıştırmada 25 test, sentetik releasable fixture'ların hiçbir hukuk tarihi beyan etmemesi nedeniyle beklenen biçimde kapalı kaldı. Gate zayıflatılmadı; yalnız bu sentetik fixture'lara açık `publication_date` eklendi. İlk scale turundaki aynı nedenli 3 hata da aynı yöntemle düzeltildi. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md new file mode 100644 index 0000000..e8876a6 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/01_ARCHITECTURE_AND_CONTRACT_MAP.md @@ -0,0 +1,34 @@ +# Mimari ve Sözleşme Haritası + +## Gerçek veri hattı + +`resmî/manual kaynak → raw immutable dosya + SHA-256 → artifact/catalog → MIME/encoding çıkarımı → normalize canonical text → legislation/decision parser → version + article + citation JSONL → quality/privacy/validation → insan/otomatik onay → frozen release + manifest + release-index → verified publish authorization → SourceChunk → MESA v4 session/insert/mutation ledger` + +Kritik sahiplik sınırı: raw, canonical, kalite, release ve HTTP producer davranışı MESA_Data'ya aittir. MESA'nın SQL/vector/graph tüketici uygulaması ile MESA_E2E_Certification bu çalışma alanında yoktur ve değiştirilmemiştir. + +## Alan matrisi + +| Aşama | Kimlik/provenance alanları | Koruma | +|---|---|---| +| Artifact | `artifact_id`, `document_id`, `source_id`, `source_url`, `retrieved_at`, `sha256`, `raw_path`, `metadata_json` | dosya SHA ve immutable artifact collision | +| Version | `version_id`, `snapshot_date`, `effective_from/to`, `canonical_sha256`, parser/schema sürümü, onay/kalite | hukuk tarihi edinim zamanından türetilmez | +| Article | kanun ID, version ID, `article_number`, `article_kind`, exact `source_span`, etkinlik | `GEÇİCİ/EK/MÜKERRER/ordinary` ayrımı | +| Citation | source record/version, target law/article/version (doğrulanmamışsa null), exact span, status, relation hint | sentaks tek başına `RESOLVED` yapmaz | +| Release | manifest SHA, payload SHA, release-index'te version/document/record bağları | unmanifested/symlink/tamper reddi | +| Native payload | session/dataset/document/revision/chunk, content=evidence_span, revision, metadata, body idempotency | yalnız terminal `COMMITTED` başarıdır | + +## Mevcut MESA producer sözleşmesi + +- Sağlık: yapılandırılabilir, varsayılan `/health` +- Oturum: `POST /v4/sessions/start` +- Yazma: `POST /v4/memory/insert` +- Mutasyon: `GET /v4/mutations/{mutation_id}` +- Oturum kapatma: `POST /v4/sessions/{session_id}/end` +- Kimlik doğrulama: ortamdan `X-API-Key`; sır depoya/ledger'a yazılmaz. +- Kapsam: tenant/workspace/dataset/agent; hedef yapılandırma SHA ile insan onayına bağlanır. + +Bu rota ve alanlar MESA_Data producer testleriyle doğrulandı. Gerçek MESA consumer kaynağı ve DEV runtime erişilemediği için consumer uyumu `CONTRACT_ONLY` durumundadır. + +## Temsil ayrımı + +Canonical metin ve SHA release'in otoritesidir. `SourceChunk` MESA'ya taşınan türetilmiş retrieval temsilidir; içerik canonical span'dan gelir, 4.096 karakter evidence sınırına göre bölünür ve canonical dosyayı yeniden yazmaz. Embedding modeli/representation-version ve gerçek re-embed davranışı bu depoda tüketici tarafı olarak doğrulanamamıştır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md new file mode 100644 index 0000000..b96e838 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/02_DEFECT_REGISTER.md @@ -0,0 +1,21 @@ +# Kusur Kaydı + +| ID | Öncelik | Yeniden üretim / kök neden | Onarım | Commit / durum | +|---|---|---|---|---| +| MD-P0-001 | P0 | `MÜKERRER MADDE` ve `MADDE 1/A` parser desenine girmiyor, kimlik çöküyordu | Parser 1.1.0 ve tür-duyarlı ID | `0a5cd5c`, kapalı | +| MD-P0-002 | P0 | Makaleler arasındaki yapısal canonical aralıklar yayın chunk'larından düşüyor; örtüşme kabul ediliyordu | Gap chunk, span sınır/örtüşme reddi, çağıran kayıtlarını mutasyonsuz işleme, parent metadata | `0a5cd5c`, kapalı | +| MD-P0-003 | P0 | `last_modified/retrieved_at`, authoritative hukuk tarihi yerine kullanılıyordu | `unknown-date` + REVIEW + açık error issue; release dışlama | `9500542`, kapalı | +| MD-P0-004 | P0 | `effective_from/to` atılıyor; gelecek sürüm aktif sürümü seçebiliyor; çelişkili bounded aralık sessizdi | Aralık saklama/doğrulama, as-of sıralama, overlap REVIEW, publish filtreleri | `9500542`, kapalı | +| MD-P0-005 | P0 | Parser'ın yalnız fallback “Madde N” üretmesi kalite PASS olabiliyordu | Tümü header-only ise BLOCK, kısmi ise REVIEW | `9500542`, kapalı | +| MD-P0-006 | P0 | Native publish job'u gerçek aktörü kaybetti; operasyon girdisinde manifest ve scope yoktu | Actor + exact manifest + target hash + frozen scope ve audit event | `17ce8f8`, kapalı | +| MD-P1-001 | P1 | Sentaktik bilinen kanun numarası corpus/version kanıtı olmadan `RESOLVED` oluyordu; harfli/türlü madde hedefleri kayıptı | `EXTRACTED`, doğru article identity; target version null kalır | `0a5cd5c`, kapalı/fail-closed | +| MD-P1-002 | P1 | Citation canonical release'te vardı ama native chunk metadata'sına taşınmıyordu | Source/target version alanları ve span-overlap citation metadata | `dfc752a`, kapalı | + +Satır örnekleri mevcut checkout içindir: parser `src/mesa_legal_data/parsers/legislation.py:31`; chunk guard `src/mesa_legal_data/publisher/chunker.py:93`; hukuk tarihi `src/mesa_legal_data/pipeline.py:223`; current selection `src/mesa_legal_data/catalog.py:244`; header-only gate `src/mesa_legal_data/quality.py:238`; native authorization `src/mesa_legal_data/web/api.py:2244`; citation handoff `src/mesa_legal_data/publisher/engine.py:338`. + +## Dış/kanıtsız kalanlar + +- EXTERNAL-P0-001: gerçek MESA tüketici rotası, schema, terminal mutation ve server-side idempotency canlı doğrulanmadı. Sahip: MESA/runtime. +- EXTERNAL-P1-001: corpus-aware citation target-version resolver ve gerçek graph projection tüketimi yok. MESA_Data yanlış çözüm üretmez; doğrulanmış target version olmadan `EXTRACTED` kalır. Sahip: MESA_Data + MESA entegrasyon turu. +- EXTERNAL-P1-002: embedding/FTS representation-version ile gerçek reindex/re-embed geçişi ölçülmedi. Sahip: MESA consumer/runtime. +- CAPACITY-P2-001: 10–20 GB hedef ölçek çalıştırılmadı. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md new file mode 100644 index 0000000..54ba8a0 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/03_LEGAL_SOURCE_ENCODING_AND_COVERAGE.md @@ -0,0 +1,17 @@ +# Hukuki Kaynak, Encoding ve Kapsama + +## Doğrulananlar + +- Raw artifact SHA uyuşmazlığı pipeline/release'i durdurur; canonical satır SHA'ları release sırasında tekrar okunur. +- UTF-8 ve cp1254 fixture'ları geçti (`tests/unit/test_encoding_cp1254.py`, `tests/integration/test_pipeline_encoding.py`). Null byte/ağır replacement loss BLOCK, mojibake anomalisi REVIEW üretir. +- Sıfır makale legislation için BLOCK; düşük coverage ve büyük açıklanamayan boşluklar REVIEW/BLOCK üretir. +- Tüm madde kayıtları sadece parser fallback başlığından ibaretse PASS değildir. +- Article span'ları canonical koordinatlarında sınır ve içerik tutarlılığıyla denetlenir; publisher ayrıca out-of-range ve overlap'i reddeder. + +## Ölçülen sentetik denetim özeti + +Bu tur resmî corpus veya Profile B TEST kullanmadı. Yeni bir küçük DEV fixture setinde 5 genişletilmiş madde kimliği (`ordinary`, `EK`, `GEÇİCİ`, `MÜKERRER`, `1/A`), 1 yapısal gap örneği, 1 overlap saldırısı, 1 header-only kaynak ve 2 etkinlik aralığı senaryosu doğrulandı. Sonuçlar `source_quality_summary.csv` içindedir; bu sayılar üretim corpus toplamı değildir. + +## Destek/kısıt + +Kaynak politikaları `config/sources.yaml` ve MIME/URL güvenlik katmanıyla uygulanır. Erişilemeyen veya hukuken doğrulanmamış kaynak için başarı iddiası yoktur. Eksik orta sayfa gibi fiziksel belge bütünlüğü, kaynağın beklenen sayfa/madde sayısı bilinmiyorsa otomatik olarak ispatlanamaz; mevcut sistem coverage/gap sinyali üretir ve insan incelemesi gerekir. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md new file mode 100644 index 0000000..b7425ef --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/04_LAW_ARTICLE_CHUNK_AND_TEMPORAL_IDENTITY.md @@ -0,0 +1,25 @@ +# Kanun, Madde, Chunk ve Zamansal Kimlik + +## Kimlik + +Madde parser'ı artık `MÜKERRER MADDE`, `EK MADDE`, `GEÇİCİ MADDE`, ordinary `MADDE` ve `1/A` benzeri ekli numaraları ayırır. ID örnekleri: + +- `...:article:1` +- `...:article:ek-1` +- `...:article:gecici-1` +- `...:article:mukerrer-1` +- `...:article:1-a` + +Aynı “Madde 1”, kanun ID'si parent olduğundan 6098 ve 6102 arasında çakışmaz. Chunk metadata'sı law ID/number/title, version kind/date/aralık, article ID/number/kind ve exact char koordinatını taşır. Başlık tekrar etmese bile parent kimlik korunur. + +## Span ve reconstruction + +Publisher, article span'larını sıralar; negatif/out-of-range/boş/örtüşen span'ı reddeder. Anlamlı preamble, bölüm/kısım gap'i ve annex/trailing metin ayrı chunk olur. Uzun metin 4.096 karakter MESA evidence sınırında deterministik bölünür; mevcut testler parçaların birleşiminin canonical metne eşit olduğunu doğrular. Yalnız whitespace gap'leri retrieval için ayrı junk chunk yapılmaz; hukuki anlam taşıyan gap'ler korunur. + +## Zamansal seçim + +Authoritative tarih sırası: kaynak metadata'sındaki publication/source/decision/snapshot ve beyan edilmiş effective interval. `retrieved_at` ile `last_modified` hukuki tarih yerine geçmez. Seçim rank'i bugün yürürlükte > süresi bitmiş tarihsel > gelecekte yürürlüğe girecek > tarihi bilinmeyen şeklindedir; böylece gelecek-effective sürüm aktif sürümü erken değiştirmez. + +İki farklı içerik açık ve bounded etkinlik aralıklarıyla çakışıyorsa `VERSION_EFFECTIVE_INTERVAL_AMBIGUITY` açılır, version/records pending ve kalite REVIEW yapılır. Aynı authoritative tarihte farklı içerik ve explicit supersession yoksa mevcut current seçimi korunur ve otomatik onay geri alınır. + +Kanıt: `tests/unit/test_comprehensive_legal_identity.py` 8/8; `tests/unit/test_comprehensive_temporal_integrity.py` 6/6; tarihsel chronology kontrolleri tam suite içinde geçti. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md new file mode 100644 index 0000000..4da7825 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/05_RELEASE_APPROVAL_AND_IMMUTABILITY.md @@ -0,0 +1,17 @@ +# Release, Onay ve Değişmezlik + +Release builder yalnız current, valid, privacy-clean/approved, quality PASS, version-approved ve record-approved üyeleri seçer. Yeni temporal filtre, gelecekte başlamamış veya sona ermiş explicit aralıkları release dışında bırakır. REVIEW/BLOCK veya açık candidate-scoped error/blocker issue release'e giremez. + +## H1 eşdeğeri insan onayı + +Panel `prepare` adımında exact frozen release oluşturur ve doğrular; kullanıcıya `release_id`, `manifest_sha256`, `target_config_sha256`, hedef scope ve sayımlar döner. `publish` aynı manifest ve hedef hash'i yeniden karşılaştırır. Onarım sonrası authorization audit olayı gerçek actor, manifest, target hash, delivery/operation ID ve frozen scope'u saklar. Stale veya değiştirilmiş manifest/hedef 409 ile kapanır. + +## Değişmezlik ve negatif durumlar + +- Manifestte olmayan dosya, symlink/path traversal, payload/index hash veya count değişikliği reddedilir. +- Release-index, record → version → document üyeliğini dondurur; canlı katalog sonradan değişse bile plan frozen sürümden kurulur. +- Release revoked olduğunda devam eden delivery yeni item göndermez. +- HTTP 200/202 başarı sayılmaz; yalnız açık terminal `COMMITTED` ledger başarısıdır. +- Timeout/poll penceresi `AWAITING_MUTATION` olarak korunur; retry önce aynı remote mutation'ı uzlaştırır ve aynı idempotency key'i kullanır. + +Kanıt: `tests/integration/test_master_mvp_closure.py`, `test_release_tamper.py`, `test_release_atomicity.py`, `test_mesa_v4_publisher_e2e.py` ve final 413-test koşusu. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md new file mode 100644 index 0000000..19d1221 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/06_MESA_NATIVE_PUBLISH_AND_IDEMPOTENCY.md @@ -0,0 +1,18 @@ +# MESA Native Publish ve Idempotency + +## Producer tarafından doğrulanan sözleşme + +`MesaClient` yapılandırılabilir v4 session/insert/mutation/end rotalarını kullanır. Payload şu çekirdek bağları taşır: `session_id`, `dataset_id`, `document_id`, `revision_id`, `chunk_id`, `content`, `evidence_span`, `revision_number`, ordinal/finalize/supersedes, metadata ve body içi `idempotency_key`. API anahtarı yalnız environment'tan okunur. + +Idempotency anahtarı tenant/workspace/dataset/document/version/chunk/content hash üzerinden deterministiktir. Ledger item gönderilmeden önce yazılır. Yanıt kaybı ve crash-window testleri aynı key ve varsa aynı mutation ID ile uzlaştırır; cross-release aynı mantıksal chunk `SKIPPED` olur. + +Durum normalizasyonu yalnız bilinen açık state'leri kabul eder. 200/201/202 içinde state veya gerekli mutation ID yoksa FAIL'dir. `QUEUED/PROCESSING` poll edilir; pencere biterse `AWAITING_MUTATION`; `COMMITTED`, `FAILED`, `REJECTED` terminaldir. 401/403, 429/503, bilinmeyen endpoint/state ve target güvenlik kontrolleri fail-closed test edilmiştir. + +## Kanıt düzeyi + +- Mock/stateful adapter: doğrulandı (`tests/integration/test_mesa_v4_publisher_e2e.py`, 3/3; post-master publisher kontrolleri). +- Frozen release, actor/manifest/target/scope bağlama: doğrulandı. +- Gerçek MESA source/runtime: yok. +- Canlı mutation ID, gerçek SQL/vector/graph projection ve restart-sonrası server idempotency: `UNVERIFIED_ENV`. + +Sonuç: `NATIVE_MESA_INTEGRATION=CONTRACT_ONLY`. Mock başarı canlı entegrasyon başarısı olarak raporlanmamıştır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md new file mode 100644 index 0000000..5c37e77 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/07_RELATIONAL_CITATIONS_AND_REPRESENTATION.md @@ -0,0 +1,15 @@ +# İlişkisel Atıflar ve Temsil + +## Atıf doğruluğu + +Parser numbered law ve alias referanslarını exact Unicode span ile çıkarır. `amends`, `repeals`, `adds` fiilleri relation hint üretir. `1/A` article suffix olarak korunur; `27/2` ve `25/II` article 27/25 içindeki alt bölüm olarak parent article ID'ye bağlanır. `EK/GEÇİCİ/MÜKERRER` türleri farklı hedef kimliği üretir. + +Sentaktik hedef hiçbir zaman corpus varlığı veya doğru tarihsel version kanıtı değildir. Bu nedenle bilinen kanunlar da `EXTRACTED`, belirsiz genel atıflar `UNRESOLVED` kalır. Yeni kayıtlar `source_version_id` ve nullable `target_version_id` taşır; doğrulanmamış target version uydurulmaz. + +Native plan artık release-owned citation kaydını, exact source span ile örtüşen chunk'ın metadata'sına taşır. Alanlar citation/source record/source version/target law/article/version/status/relation/span ve `fully_contained` bilgisidir. Publisher metinden yeniden graph edge tahmin etmez. + +## Kalan tüketici bağımlılığı + +MESA graph projection kodu yoktur. Metadata'nın gerçek graph edge'e dönüştürülmesi, target version corpus resolver'ı ve version-as-of graph sorgusu canlı consumer ile doğrulanmalıdır. Bu nedenle kabul kapısı G producer tarafında fail-closed `VERIFIED_PASS`, gerçek graph consumption için `UNVERIFIED_ENV` olarak ayrılmıştır. + +Canonical metin immutable kalır; retrieval chunk ve gelecekteki embedding/FTS representation türetilmiştir. Bu depoda embedding model/version alanının tüketici tarafından nasıl kullanıldığı görülemediğinden gerçek reindex/re-embed `EMBEDDING_DEPENDENCY_BLOCKED` durumundadır. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md new file mode 100644 index 0000000..964e500 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/08_REGRESSION_AND_ADVERSARIAL_RESULTS.md @@ -0,0 +1,63 @@ +# Regresyon ve Adversarial Sonuçlar + +## Son koşular + +| Komut | Sonuç | +|---|---| +| `.venv/bin/pytest -ra --durations=20 --tb=short` | 413 passed, 1 skipped, 78.95 s | +| `.venv/bin/pytest -m "not scale" -ra --durations=20 --tb=short` | 409 passed, 1 skipped, 4 deselected, 33.45 s | +| `/usr/bin/time -v .venv/bin/pytest -m scale ...` | 4 passed, 1 skipped, 409 deselected, 46.53 s, peak RSS 171,784 KiB | +| `.venv/bin/ruff format --check .` | 217 files formatted | +| `.venv/bin/ruff check .` | PASS | +| `.venv/bin/mypy src` | 78 source files, 0 issue | +| `.venv/bin/pip-audit` | bilinen açık yok; yerel paket PyPI'da olmadığı için skip | +| `git diff --check` | PASS | + +Tek skip: `tests/integration/test_mesa_v4_vector_cross_repo.py`; `MESA_SOURCE_ROOT` ayarlı değil ve sibling MESA checkout bulunmadı. TestClient koşuları managed filesystem sandbox dışında çalıştırıldı; sandbox AF_UNIX wakeup yazmasını EPERM ile engelliyordu. Aynı minimal probe dışarıda 200 döndü; bağımlılık lock'u değiştirilmedi. + +## 38 maddelik negatif matris + +| # | Sonuç | Kanıt/Not | +|---|---|---| +| 1 | PASS | raw/canonical SHA ve tamper testleri | +| 2 | PASS | zero article/coverage/header-only gate | +| 3 | PASS | cp1254/UTF-8 testleri | +| 4 | PASS | mojibake/replacement quality testleri | +| 5 | PASS | parent law ID dahil article ID | +| 6 | PASS | header-only BLOCK | +| 7 | PASS | chunk parent metadata testi | +| 8 | PASS | ordinary/EK/GEÇİCİ/MÜKERRER ayrımı | +| 9 | PASS | overlap/mixed span redde kapalı | +| 10 | PASS | long split/reconstruction | +| 11 | PASS | overlap duplicate üretmez | +| 12 | PASS | fetched_at hukuk tarihi değildir | +| 13 | PASS | out-of-order historical import | +| 14 | PASS | effective interval/current filtreleri | +| 15 | PASS | bounded overlap REVIEW | +| 16 | PASS | content hash/version/manifest kimliği | +| 17 | PASS | deterministik hash/chunk/release | +| 18 | PASS | approval/quality/release seçimi | +| 19 | PASS | manifest+target+actor+scope yeniden doğrulama | +| 20 | PASS | manifest tamper/unmanifested/order bağları | +| 21 | PASS | schema/response validation | +| 22 | PASS | unknown route/contract FAIL | +| 23 | PASS | 200/202 tek başına COMMITTED değil | +| 24 | PASS | AWAITING_MUTATION reconcile | +| 25 | CONTRACT_ONLY | body idempotency producer testi; gerçek consumer yok | +| 26 | PASS | cross-release dedup/repeat publish | +| 27 | PASS | response-loss/ledger retry | +| 28 | PASS | auth ve target-scope fail-closed | +| 29 | PASS | finite retry/poll | +| 30 | PASS | ambiguous citation UNRESOLVED | +| 31 | PASS | target version doğrulanmadan RESOLVED değil | +| 32 | PASS | out-of-range span redde kapalı | +| 33 | PASS | chunking canonical/raw'u değiştirmez | +| 34 | UNVERIFIED_ENV | consumer representation version/re-embed yok | +| 35 | PASS | rapor canlı başarı iddia etmiyor | +| 36 | PASS | yalnız sentetik DEV fixture; resmî TEST çalıştırılmadı | +| 37 | PASS | release path/symlink güvenlik testleri | +| 38 | PASS | pipeline/release atomicity ve rollback | + +## Önce/sonra + +İlk hedefli koşu 6/6 kırmızıydı; ilgili hedefli son koşu 27/27 yeşil oldu. İlk full onarım koşusundaki 25 hata ve ilk scale koşusundaki 3 hata, tarih beyan etmeyen sentetik release fixture'larını gösterdi; gate düşürülmeden fixture metadata'sı düzeltildi. Nihai full koşu 413/413 geçti. Resmî Recall@5/MRR karşılaştırması yapılmadı. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md new file mode 100644 index 0000000..9d499ff --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/09_SCALE_AND_OPERATIONS_READINESS.md @@ -0,0 +1,9 @@ +# Ölçek ve Operasyon Hazırlığı + +Scale işaretli 4 testin tamamı geçti. Ölçülen birleşik koşu 46.53 saniye duvar zamanı, 171,784 KiB maksimum RSS ve 735,504 filesystem output block kullandı. En yavaş testler: 10.000 article hardening 21.44 s, bulk review streaming 13.55 s, operations scale 10.35 s. + +Mevcut mimari canonical JSONL'yi streaming işler; release seçimini geçici SQLite spool ile yapar ve canonical part dosyasını tek geçişte okuyan testlere sahiptir. Approval streaming, bounded fetchmany ve operasyon recovery testleri geçti. Immutable canonical core'dan derived staging/projection yeniden kurulabilir. + +Bu ölçüm küçük sentetik ölçektir. 10–20 GB artifact/canonical corpus, gerçek disk baskısı, backup/restore süresi, MESA vector/graph rebuild ve eşzamanlı operasyon davranışı çalıştırılmadı. Sonuç: `CAPACITY_NOT_PROVEN_AT_TARGET_SCALE`. + +Önerilen sonraki ölçüm, resmî TEST'ten bağımsız sabit DEV corpus ile 1x/5x/20x koşularında wall time, peak RSS, disk amplification, release spool boyutu ve restart recovery ölçmektir. Üretim verisi üzerinde destructive stress önerilmez. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md new file mode 100644 index 0000000..3175b09 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/10_CROSS_REPO_HANDOFF.md @@ -0,0 +1,24 @@ +# Çapraz Depo Teslimi + +## Erişim durumu + +`/storage` altında sibling `MESA` veya `MESA_E2E_Certification` checkout bulunmadı; `MESA_SOURCE_ROOT` ayarlı değildi. Bu iki depo read-only bile incelenemedi ve hiçbir dosyaları değiştirilmedi. Resmî Profile B evidence/TEST verisine dokunulmadı. + +## MESA sahibine doğrulatılacak sözleşme + +1. Session start request: tenant/workspace, `dataset_ids`, agent; 201 ve nonempty `session_id`. +2. Memory insert: document/revision/chunk identity, source_ref, content/evidence_span ≤4096, revision/finalize/supersedes, metadata, body `idempotency_key`. +3. Mutation response: açık state; queued/processing ise nonempty mutation ID; status route ve session yaşam döngüsü. +4. Server-side duplicate key semantiği, response-loss/restart sonrası aynı mutation'a dönüş. +5. Citation metadata'nın graph projection'a typed/versioned edge olarak nasıl alındığı; `EXTRACTED`/null target version'ın graph'a kesin edge yapılmaması. +6. Representation/model version değişiminde gerçek reindex/re-embed ve canonical provenance korunumu. + +## E2E sahibine önerilen takip + +- Yeni producer branch SHA'sını MESA ile aynı DEV scope'ta checkout edin. +- İzole DEV release'i native akışla yayınlayın; gerçek mutation ID'leri terminale kadar kaydedin. +- SQL canonical, vector metadata ve graph edge'lerinde document/version/article/span/hash provenance'ını kontrol edin. +- Aynı release'i tekrar ve process restart sonrası gönderip logical duplicate oluşmadığını kanıtlayın. +- Bu sonuçları yeni bir DEV evidence run olarak saklayın; mevcut Profile B TEST sonucunu yeniden etiketlemeyin. + +Çapraz-depo testi, `MESA_SOURCE_ROOT` sağlandığında çalıştırılmaya hazırdır; bu turdaki tek skip budur. diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md new file mode 100644 index 0000000..0c59bd0 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md @@ -0,0 +1,30 @@ +# Nihai Hazırlık Kararı + +## Karar + +`MESA_DATA_REPAIR_VERIFIED_PUSHED` hedeflenmiştir: tüm doğrulanmış MESA_Data-sahipli P0/P1 kusurları kapatılmış ve yerel testleri geçmiştir; bu rapor commit'i sonrasında yetkili dal push edilecek ve kesin final SHA teslim mesajında verilecektir. Bu karar `PROFILE_B_PASS` veya `PRODUCTION_READY` değildir. + +Denetlenen uygulama SHA: `dfc752a386b9f1287ee42256aa4013b24309ac6c`. + +## Kabul kapıları + +| Kapı | Sınıf | Kanıt | +|---|---|---| +| A raw→canonical izlenebilirlik | VERIFIED_PASS | SHA, source/provenance, release-index/tamper testleri | +| B doğru law/article/chunk/version | VERIFIED_PASS | yeni identity/chunker testleri | +| C current/historical zaman | VERIFIED_PASS | unknown/future/out-of-order/overlap testleri | +| D gate bypass yok | VERIFIED_PASS | quality/approval/release/adversarial suite | +| E gerçek MESA API uyumu | UNVERIFIED_ENV | producer contract testli; consumer checkout/runtime yok | +| F idempotency | VERIFIED_PASS | stateful mock/crash/replay testleri; canlı DEV kısmı UNVERIFIED_ENV | +| G citation/graph | VERIFIED_PASS | producer typed/versioned/span evidence ve fail-closed status; gerçek graph consumer UNVERIFIED_ENV | +| H canonical/retrieval ayrımı | VERIFIED_PASS | raw/canonical immutable, chunk türetilmiş; gerçek re-embed UNVERIFIED_ENV | +| I negatif/regresyon | VERIFIED_PASS | 413 passed, 38 maddelik matris | +| J kalan bağımlılıklar açık | VERIFIED_PASS | MESA/E2E/embedding/corpus/scale açıkça listeli | + +## Kalan engeller + +MESA_Data içinde doğrulanmış açık P0/P1 yoktur. Dış engeller: gerçek MESA DEV runtime ve source yok; gerçek graph/vector/SQL tüketimi yok; embedding provider yok; resmî corpus/TEST yetkili çalıştırması yok; 10–20 GB kapasite ölçülmedi. Güvenli sonraki adım `10_CROSS_REPO_HANDOFF.md` içindeki bounded DEV entegrasyonudur. + +`PROFILE_B_CERTIFICATION: NOT_RUN_NOT_CLAIMED` + +`CAPACITY_10_20_GB: NOT_PROVEN` diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv new file mode 100644 index 0000000..6f5630c --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/data_lineage_matrix.csv @@ -0,0 +1,13 @@ +stage,producer_field,consumer_field,verification,provenance_hash +raw,artifacts.sha256,canonical.source.artifact_sha256,PASS,SHA-256 +raw,artifacts.raw_path,canonical.source.artifact_path,PASS,SHA-256 +version,versions.version_id,canonical.version.version_id,PASS,canonical_sha256 +article,article.id,chunk.metadata.article_id,PASS,content_hash +article,article.article_kind,chunk.metadata.article_kind,PASS,content_hash +article,article.source_span,chunk.char_start/char_end,PASS,content_hash +temporal,version.effective_from/to,chunk.metadata.effective_from/to,PASS,manifest_sha256 +citation,citation.source_version_id,chunk.metadata.citations[].source_version_id,PASS,manifest_sha256 +citation,citation.target_version_id,chunk.metadata.citations[].target_version_id,PASS_NULL_UNTIL_VERIFIED,manifest_sha256 +release,release-index.version/document/record,publisher frozen membership,PASS,manifest_sha256 +payload,chunk.content_hash,metadata.mesa_data_content_hash,PASS,idempotency_key +payload,release.manifest_sha256,delivery.release_manifest_sha256,PASS,manifest_sha256 diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json new file mode 100644 index 0000000..eded1e9 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/defect_register.json @@ -0,0 +1,74 @@ +[ + { + "id": "MD-P0-001", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_legal_identity.py::test_article_parser_preserves_extended_turkish_article_identity", + "fix_sha": "0a5cd5c", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P0-002", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_legal_identity.py::test_chunk_plan_is_lossless_and_rejects_overlapping_article_identity", + "fix_sha": "0a5cd5c", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P0-003", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_temporal_integrity.py::test_acquisition_time_is_not_used_as_legal_snapshot_date", + "fix_sha": "9500542", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P0-004", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_temporal_integrity.py future/overlap/invalid interval tests", + "fix_sha": "9500542", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P0-005", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_temporal_integrity.py::test_header_only_legislation_cannot_pass_quality_gate", + "fix_sha": "9500542", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P0-006", + "severity": "P0", + "owner": "MESA_Data", + "proof": "tests/integration/test_master_mvp_closure.py::test_final_panel_confirmation_binds_exact_release_manifest_and_target", + "fix_sha": "17ce8f8", + "verification_state": "VERIFIED_FIXED" + }, + { + "id": "MD-P1-001", + "severity": "P1", + "owner": "MESA_Data", + "proof": "tests/unit/test_comprehensive_legal_identity.py citation cases", + "fix_sha": "0a5cd5c", + "verification_state": "VERIFIED_FIXED_FAIL_CLOSED" + }, + { + "id": "MD-P1-002", + "severity": "P1", + "owner": "MESA_Data", + "proof": "tests/integration/test_master_mvp_closure.py native citation metadata assertion", + "fix_sha": "dfc752a", + "verification_state": "VERIFIED_FIXED_PRODUCER" + }, + { + "id": "EXTERNAL-P0-001", + "severity": "P0_EXTERNAL", + "owner": "MESA/runtime", + "proof": "MESA_SOURCE_ROOT unset; cross-repo test skipped", + "fix_sha": null, + "verification_state": "UNVERIFIED_ENV" + } +] diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv new file mode 100644 index 0000000..158f446 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/native_contract_matrix.csv @@ -0,0 +1,15 @@ +producer_field,consumer_field,expected,observed,status +session.tenant_id,session tenant scope,nonempty,target settings emitted,CONTRACT_ONLY +session.workspace_id,session workspace scope,nonempty,target settings emitted,CONTRACT_ONLY +session.dataset_ids,session dataset scope,one configured dataset,list emitted,CONTRACT_ONLY +insert.document_id,canonical document identity,stable law/decision ID,emitted and mock asserted,PASS_MOCK +insert.revision_id,legal version identity,exact frozen version,emitted and mock asserted,PASS_MOCK +insert.chunk_id,chunk identity,deterministic within version,emitted and replay tested,PASS_MOCK +insert.content/evidence_span,retrieval evidence,identical canonical slice <=4096,validated,PASS_MOCK +insert.metadata.article_id,article identity,parent law plus kind/number,validated,PASS_MOCK +insert.metadata.citations,graph evidence,typed span/version candidate,validated producer side,PASS_MOCK +insert.idempotency_key,server idempotency,stable body field,stable key tested consumer unknown,CONTRACT_ONLY +response.mutation_id,remote mutation identity,required for nonterminal,fail closed if absent,PASS_MOCK +response.state,ledger terminal state,explicit COMMITTED only,normalization tested,PASS_MOCK +GET /v4/mutations/{id},reconciliation,terminal poll after async/timeout,stateful mock tested,CONTRACT_ONLY +server projections,SQL/vector/graph provenance,document/version/chunk/span/hash,no runtime available,UNVERIFIED_ENV diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv new file mode 100644 index 0000000..d350ddf --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/source_quality_summary.csv @@ -0,0 +1,10 @@ +fixture,source_type,units_expected,units_observed,quality_result,warning_or_blocker,scope +extended_article_identity,synthetic_text,5,5,PASS,,DEV_ONLY +structural_gap,synthetic_text,1_gap,1_gap,PASS,,DEV_ONLY +overlapping_spans,synthetic_text,REJECT,REJECT,BLOCK,span_overlap,DEV_ONLY +missing_legal_date,synthetic_html,REVIEW,REVIEW,REVIEW,LEGAL_DATE_MISSING,DEV_ONLY +invalid_effective_interval,synthetic_html,REVIEW,REVIEW,REVIEW,EFFECTIVE_INTERVAL_INVALID,DEV_ONLY +overlapping_effective_intervals,synthetic_html,2_review,2_review,REVIEW,VERSION_EFFECTIVE_INTERVAL_AMBIGUITY,DEV_ONLY +header_only_legislation,synthetic_html,BLOCK,BLOCK,BLOCK,article_body_presence,DEV_ONLY +cp1254_encoding,synthetic_bytes,2_tests,2_pass,PASS,,DEV_ONLY +official_profile_b,official_test,NOT_RUN,NOT_RUN,UNVERIFIED,protected evidence not accessed,EXTERNAL diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json new file mode 100644 index 0000000..4923247 --- /dev/null +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/test_execution.json @@ -0,0 +1,60 @@ +[ + { + "command": ".venv/bin/pytest tests/unit/test_comprehensive_legal_integrity.py -q", + "exit_code": 1, + "executed": 6, + "passed": 0, + "failed": 6, + "skipped": 0, + "evidence": "pre-fix reproduction; tests later split into comprehensive_legal_identity and comprehensive_temporal_integrity" + }, + { + "command": ".venv/bin/pytest -m \"not scale\" -ra --durations=20 --tb=short", + "exit_code": 0, + "executed": 409, + "passed": 409, + "failed": 0, + "skipped": 1, + "deselected": 4, + "evidence": "terminal run 2026-10-11; cross-repo skip only" + }, + { + "command": "/usr/bin/time -v .venv/bin/pytest -m scale -ra --durations=20 --tb=short", + "exit_code": 0, + "executed": 4, + "passed": 4, + "failed": 0, + "skipped": 1, + "deselected": 409, + "wall_seconds": 46.53, + "peak_rss_kib": 171784, + "evidence": "bounded synthetic scale; not 10-20 GB" + }, + { + "command": ".venv/bin/pytest -ra --durations=20 --tb=short", + "exit_code": 0, + "executed": 413, + "passed": 413, + "failed": 0, + "skipped": 1, + "evidence": "final full suite; skip requires MESA_SOURCE_ROOT" + }, + { + "command": ".venv/bin/ruff format --check . && .venv/bin/ruff check . && .venv/bin/mypy src", + "exit_code": 0, + "executed": 3, + "passed": 3, + "failed": 0, + "skipped": 0, + "evidence": "217 formatted files; lint pass; 78 typed source files" + }, + { + "command": ".venv/bin/pip-audit", + "exit_code": 0, + "executed": 1, + "passed": 1, + "failed": 0, + "skipped": 1, + "evidence": "no known vulnerabilities; local mesa-legal-data package not on PyPI" + } +] From 9f5c9d5ff005d3f9b9e381267188692ba386dd98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yasin=20B=C3=BCy=C3=BCktepe?= Date: Sun, 11 Oct 2026 16:04:42 +0300 Subject: [PATCH 8/8] docs: record verified GitHub branch delivery --- .../2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md index 0c59bd0..5e255e5 100644 --- a/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md +++ b/reports/mesa-data-comprehensive-repair/2026-10-11-round-01/11_FINAL_READINESS_VERDICT.md @@ -2,7 +2,7 @@ ## Karar -`MESA_DATA_REPAIR_VERIFIED_PUSHED` hedeflenmiştir: tüm doğrulanmış MESA_Data-sahipli P0/P1 kusurları kapatılmış ve yerel testleri geçmiştir; bu rapor commit'i sonrasında yetkili dal push edilecek ve kesin final SHA teslim mesajında verilecektir. Bu karar `PROFILE_B_PASS` veya `PRODUCTION_READY` değildir. +`MESA_DATA_REPAIR_VERIFIED_PUSHED`: tüm doğrulanmış MESA_Data-sahipli P0/P1 kusurları kapatılmış, yerel doğrulamaları geçmiş ve yetkili `fix/mesa-data-legal-integrity-native-publish` dalına gönderilmiştir. Commit kimliği kendi içeriğinde öz-referanslı olamayacağı için kesin terminal SHA teslim mesajında verilir. Bu karar `PROFILE_B_PASS` veya `PRODUCTION_READY` değildir. Denetlenen uygulama SHA: `dfc752a386b9f1287ee42256aa4013b24309ac6c`.