fix(dla): inference plurals + overlap evidence, tightened pattern heuristics, composite-FK grouping

apps/dla/config/default.yaml

@@ -4,7 +4,21 @@
 
 thresholds:
   name_match_min_score: 0.85
+  # Relationship inference — value-overlap evidence (D10/D11).
+  # A computed overlap ratio >= value_overlap_min_ratio passes; <=
+  # value_overlap_failed_max_ratio is negative evidence (demotes the inferred
+  # relationship to Weak, recorded as `value_overlap_failed`); in between is
+  # inconclusive (neutral). An overlap that could not be computed is neutral.
   value_overlap_min_ratio: 0.5
+  value_overlap_failed_max_ratio: 0.05
+  # A passing overlap only *upgrades* confidence when it is selective:
+  # at least value_overlap_min_distinct distinct FK-side sample values, and
+  # the overlapped values must not form a dense integer surrogate range
+  # bounded by value_overlap_dense_int_max (serial ids 1..N overlap any other
+  # small serial column by construction). Non-selective overlaps are recorded
+  # as `value_overlap_low_selectivity` and stay neutral.
+  value_overlap_min_distinct: 10
+  value_overlap_dense_int_max: 100
   high_null_rate: 0.5
   high_null_rate_critical: 0.9
   sample_budget_rows: 10000

apps/dla/config/schemas/bundle-schema.json

@@ -1,7 +1,7 @@
 {
   "$schema": "https://json-schema.org/draft/2020-12/schema",
   "title": "DLA bundle artifact",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "$defs": {
     "ArtifactType": {
       "description": "Every bundle artifact's `artifact_type` value.",
@@ -2295,6 +2295,18 @@
           },
           "title": "Signals",
           "type": "array"
+        },
+        "composite_group": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Composite Group"
         }
       },
       "required": [

apps/dla/src/dla/bundle/schema.py

@@ -134,6 +134,17 @@ class RelationshipPayload(CommonFields):
     to_column_ref: str
     relationship_type: Literal["declared_fk", "inferred_fk", "inferred_join_key"]
     signals: list[str] = Field(default_factory=list)
+    composite_group: str | None = None
+    """Groups the column pairs of one multi-column (composite) foreign key.
+
+    A composite FK (e.g. `ledger(fiscal_year, fiscal_month) ->
+    fiscal_periods(fiscal_year, fiscal_month)`) is persisted as one
+    relationship artifact per column pair; every member carries the same
+    `composite_group` id (derived deterministically from the declared
+    constraint name, falling back to the sorted source-column set) so
+    downstream consumers can reassemble the composite join. `None` for
+    ordinary single-column relationships. Additive since schema 1.1.0 —
+    bundles written before this field remain valid."""
 
 
 class IndexPayload(CommonFields):
@@ -430,7 +441,9 @@ class RecommendationPayload(CommonFields):
 # Single source of truth for the bundle contract version. The published JSON
 # Schema (`bundle-schema.json`) and every manifest carry this exact string; a
 # parity test (T183) pins them equal so the contract can never silently drift.
-SCHEMA_VERSION = "1.0.0"
+# 1.1.0: additive `composite_group` on relationship artifacts (D13) — backward
+# compatible, 1.0.0 bundles remain valid.
+SCHEMA_VERSION = "1.1.0"
 
 
 class BundleManifest(BaseModel):

apps/dla/src/dla/config/models.py

@@ -90,6 +90,23 @@ class ThresholdsConfig(BaseModel):
     # Discovery (M1) thresholds.
     name_match_min_score: float = 0.85
     value_overlap_min_ratio: float = 0.5
+    """Overlap ratio (|FK-sample ∩ PK-sample| / |FK-sample|) at/above which the
+    value-overlap check *passes* (subject to the selectivity guards below)."""
+    value_overlap_failed_max_ratio: float = 0.05
+    """A *computed* overlap ratio at/below this is negative evidence (D11):
+    the FK-side values do not exist on the PK side, so the relationship is
+    demoted to Weak and `value_overlap_failed` is recorded in `signals`.
+    Ratios between this and `value_overlap_min_ratio` are inconclusive
+    (neutral). An overlap that could not be computed at all is also neutral."""
+    value_overlap_min_distinct: int = 10
+    """Minimum distinct values in the FK-side sample for a passing overlap to
+    count as corroboration (D10). Below this the overlap is recorded as
+    `value_overlap_low_selectivity` and does not upgrade confidence."""
+    value_overlap_dense_int_max: int = 100
+    """A passing overlap whose overlapped values form a (nearly) dense integer
+    range bounded by this ceiling is treated as a small surrogate-id range
+    (ids 1..N) and does not upgrade confidence (D10) — any two small serial
+    columns overlap by construction, so the signal carries no information."""
 
     # Profiling (M2) thresholds.
     sample_budget_rows: int = 10000

apps/dla/src/dla/discovery/engine.py

@@ -29,6 +29,7 @@
     IntrospectionResult,
     RawColumn,
     RawIndex,
+    RawRelationship,
     RawTable,
     SourceConnector,
 )
@@ -156,11 +157,52 @@ def _column_body(table: RawTable, col: RawColumn) -> str:
     )
 
 
-def _relationship_body(rel_id: str, confidence: str, signals: list[str]) -> str:
-    return (
+def _relationship_body(
+    rel_id: str, confidence: str, signals: list[str], composite_group: str | None = None
+) -> str:
+    body = (
         f"# {rel_id}\n\n"
         f"Confidence: **{confidence}**. Signals: {', '.join(signals) or '(none)'}.\n"
     )
+    if composite_group:
+        body += f"Part of composite foreign key `{composite_group}`.\n"
+    return body
+
+
+def _rel_key(rel: RawRelationship) -> tuple[str, str, str, str]:
+    return (rel.from_table, rel.from_column, rel.to_table, rel.to_column)
+
+
+def _composite_groups(
+    rels: list[RawRelationship],
+) -> dict[tuple[str, str, str, str], str]:
+    """Map relationship key -> composite-group id for every declared
+    relationship that is one column pair of a multi-column FK (D13).
+
+    Column pairs belong to the same composite FK when they share a source
+    table, target table, and constraint name (a SQL composite FK is one named
+    constraint, so its per-column halves arrive with an identical `name`).
+    The group id is deterministic — `fkgroup:<from_table>:<constraint_name>`
+    — so re-runs always produce the same id (idempotency, FR-016; the sorted
+    column set would work equally, the constraint name is simply more
+    readable). Relationships without a
+    constraint name are never grouped: without the name two independent
+    single-column FKs onto the same table would be indistinguishable from a
+    composite, and inventing a composite would be worse than flattening one.
+    """
+    by_constraint: dict[tuple[str, str, str], list[RawRelationship]] = {}
+    for rel in rels:
+        if rel.name:
+            by_constraint.setdefault((rel.from_table, rel.to_table, rel.name), []).append(rel)
+
+    groups: dict[tuple[str, str, str, str], str] = {}
+    for (from_table, _to_table, name), members in by_constraint.items():
+        if len(members) < 2:
+            continue  # single-column FK — no compositeness to preserve
+        group_id = f"fkgroup:{from_table}:{name}"
+        for m in members:
+            groups[_rel_key(m)] = group_id
+    return groups
 
 
 def _index_body(index: RawIndex) -> str:
@@ -241,12 +283,15 @@ def discover(
                     else:
                         report.columns_written += 1
 
-        # Declared FKs.
+        # Declared FKs. Multi-column FKs arrive as one RawRelationship per
+        # column pair; `composite_group` re-links the pairs (D13).
+        composite_groups = _composite_groups(intro.declared_relationships)
         for rel in intro.declared_relationships:
             rel_id = _relationship_artifact_id(
                 cfg.source.source_id, rel.from_table, rel.from_column, rel.to_table, rel.to_column
             )
             tag = tag_declared()
+            composite_group = composite_groups.get(_rel_key(rel))
             payload = RelationshipPayload(
                 artifact_id=rel_id,
                 source_id=cfg.source.source_id,
@@ -259,8 +304,13 @@ def discover(
                 to_column_ref=_column_artifact_id(cfg.source.source_id, rel.to_table, rel.to_column),
                 relationship_type="declared_fk",
                 signals=tag.signals,
+                composite_group=composite_group,
+            )
+            res = write_artifact(
+                bundle_root,
+                payload,
+                body=_relationship_body(rel_id, tag.confidence, tag.signals, composite_group),
             )
-            res = write_artifact(bundle_root, payload, body=_relationship_body(rel_id, tag.confidence, tag.signals))
             report.write_results.append(res)
             if not res.skipped_to_preserve_sme:
                 report.relationships_written += 1

apps/dla/src/dla/discovery/relationships.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass
+from typing import Any
 
 from dla.config.models import ThresholdsConfig
 from dla.connectors.base import (
@@ -22,7 +23,7 @@
     RawRelationship,
     SourceConnector,
 )
-from dla.discovery.tagger import ConfidenceTag, tag_inferred
+from dla.discovery.tagger import ConfidenceTag, OverlapEvidence, tag_inferred
 
 
 @dataclass(frozen=True)
@@ -44,14 +45,95 @@ def _types_compatible(a: RawColumn, b: RawColumn) -> bool:
     }
 
 
+def _singular_candidates(plural: str) -> set[str]:
+    """Deterministic singular forms of a table basename (D9).
+
+    A small fixed rule set — deliberately not a stemming library:
+      - `categories` -> `category`  (-ies -> -y)
+      - `statuses`   -> `status`    (-ses -> -s, i.e. strip the -es)
+      - `orders`     -> `order`     (strip a trailing -s)
+    The name itself is always a candidate (already-singular table names).
+    """
+    candidates = {plural}
+    if plural.endswith("ies") and len(plural) > 3:
+        candidates.add(plural[:-3] + "y")
+    if plural.endswith("ses") and len(plural) > 3:
+        candidates.add(plural[:-2])
+    if plural.endswith("s") and not plural.endswith("ss") and len(plural) > 1:
+        candidates.add(plural[:-1])
+    return candidates
+
+
 def _name_match(column: RawColumn, target_table: str) -> bool:
-    """`customer_id` matches table `customers` (singular suffix `s`)."""
+    """`customer_id` matches table `customers`; `category_id` matches
+    `categories`; `status_id` matches `statuses` (see `_singular_candidates`)."""
     target_base = _basename(target_table).lower()
     col_name = column.name.lower()
-    if col_name == f"{target_base}_id":
-        return True
-    # singular form (strip a trailing 's')
-    return bool(target_base.endswith("s") and col_name == f"{target_base[:-1]}_id")
+    return any(col_name == f"{cand}_id" for cand in _singular_candidates(target_base))
+
+
+def _is_dense_int_range(values: set[Any], *, dense_int_max: int) -> bool:
+    """True when `values` is (nearly) a dense integer range whose ceiling is
+    small enough to be a surrogate-id range (D10).
+
+    Such ranges overlap *any* other small serial column by construction, so a
+    high overlap ratio over them carries no information.
+    """
+    ints: list[int] = []
+    for v in values:
+        if isinstance(v, bool):
+            return False
+        if isinstance(v, int):
+            ints.append(v)
+            continue
+        try:
+            ints.append(int(str(v)))
+        except (TypeError, ValueError):
+            return False
+    if not ints:
+        return False
+    lo, hi = min(ints), max(ints)
+    if lo < 0 or hi > dense_int_max:
+        return False
+    span = hi - lo + 1
+    return len(set(ints)) / span >= 0.9
+
+
+def _evaluate_overlap(
+    sample_fk: list[Any],
+    sample_pk: list[Any],
+    *,
+    thresholds: ThresholdsConfig,
+) -> OverlapEvidence:
+    """Classify the value-overlap evidence between an FK-side sample and the
+    candidate PK-side sample. See `tagger.OverlapEvidence` for the semantics.
+    """
+    if not sample_fk or not sample_pk:
+        return OverlapEvidence.UNKNOWN
+
+    set_fk = {repr(v) for v in sample_fk}
+    set_pk = {repr(v) for v in sample_pk}
+    overlap_keys = set_fk & set_pk
+    ratio = len(overlap_keys) / max(len(set_fk), 1)
+
+    # Computed and (near) zero: the FK-side values do not exist on the PK
+    # side. Negative evidence (D11) — distinct from "not computable" above.
+    if ratio <= thresholds.value_overlap_failed_max_ratio:
+        return OverlapEvidence.FAILED
+
+    if ratio < thresholds.value_overlap_min_ratio:
+        return OverlapEvidence.UNKNOWN  # inconclusive: neither corroborates nor demotes
+
+    # The ratio passed — but is the overlapped value set selective enough to
+    # mean anything (D10)? Low-cardinality samples and dense small-integer
+    # surrogate ranges match any similar serial column by construction.
+    if len(set_fk) < thresholds.value_overlap_min_distinct:
+        return OverlapEvidence.LOW_SELECTIVITY
+    overlap_values = {v for v in sample_fk if repr(v) in overlap_keys}
+    if _is_dense_int_range(overlap_values, dense_int_max=thresholds.value_overlap_dense_int_max):
+        return OverlapEvidence.LOW_SELECTIVITY
+
+    return OverlapEvidence.SUPPORTED
 
 
 def infer_relationships(
@@ -67,7 +149,6 @@ def infer_relationships(
 
     # Index PK columns by table for quick lookup.
     pk_index: dict[str, RawColumn] = {}
-    {t.name: t for t in intro.tables}
     for table in intro.tables:
         if len(table.pk_columns) == 1:
             pk_name = table.pk_columns[0]
@@ -88,15 +169,11 @@ def infer_relationships(
                 if not _name_match(col, target_name):
                     continue
                 type_match = _types_compatible(col, target_pk)
-                value_overlap = False
+                overlap = OverlapEvidence.UNKNOWN
                 if connector is not None and type_match:
                     sample_a = connector.sample_column(table.name, col.name, 50)
                     sample_b = connector.sample_column(target_name, target_pk.name, 200)
-                    if sample_a and sample_b:
-                        set_a = {repr(v) for v in sample_a}
-                        set_b = {repr(v) for v in sample_b}
-                        overlap = len(set_a & set_b) / max(len(set_a), 1)
-                        value_overlap = overlap >= thresholds.value_overlap_min_ratio
+                    overlap = _evaluate_overlap(sample_a, sample_b, thresholds=thresholds)
                 results.append(
                     InferredRelationship(
                         relationship=RawRelationship(
@@ -108,7 +185,7 @@ def infer_relationships(
                         tag=tag_inferred(
                             name_match=True,
                             type_match=type_match,
-                            value_overlap=value_overlap,
+                            overlap=overlap,
                         ),
                     )
                 )