Merge pull request #1 from deesatzed/port/fractal-retriever-plugin

Port FractRAG multi-scale retrieval as RetrieverPlugin

Author: deesatzed

README.md

@@ -61,6 +61,24 @@ benchmark model-backed embeddings such as `intfloat/e5-base-v2`, BGE, Jina,
 Nomic, OpenAI, or Voyage adapters, then plug the winning backend into
 `DenseVectorRetriever`.
 
+
+## Fractal multi-scale retrieval (opt-in)
+
+CAM-RAG can use fractLrag-style three-level indexing (sentence / paragraph /
+document) with derivative signals between levels. It is **not** on by default.
+
+```python
+from cam_rag.rag.spec import RAGAppSpec
+from cam_rag.retrieval.fractal import FractalRetrieverPlugin, HashEmbedding
+
+plugin = FractalRetrieverPlugin(backend=HashEmbedding(dim=64))
+spec = RAGAppSpec(name="my-app", use_pipeline=True, retriever_plugins=[plugin])
+```
+
+`HashEmbedding` is deterministic and needs no GPU. Optional
+`SentenceTransformerEmbedding` backends (e.g. BGE-M3) require
+`sentence-transformers`.
+
 ## Current Status
 
 Alpha platform scaffold with working document-folder ingestion, chunking,

pyproject.toml

@@ -14,6 +14,8 @@ authors = [
 ]
 dependencies = [
     "pydantic>=2.0",
+    "numpy>=1.21.0",
+    "pyyaml>=6.0",
 ]
 
 [project.optional-dependencies]

src/cam_rag/retrieval/fractal/__init__.py

@@ -0,0 +1,47 @@
+"""Fractal multi-scale retrieval with derivative signals between levels.
+
+Ported from fractLrag (deesatzed/fractLrag) into CAM-RAG as an optional
+``RetrieverPlugin``.  The fractal index is **not** enabled by default —
+callers must pass ``FractalRetrieverPlugin`` in ``RAGAppSpec.retriever_plugins``.
+
+Documents are indexed at three self-similar levels (sentence / paragraph /
+document).  First- and second-order derivative signals between levels act as
+scoring bonuses.  ``retrieve_adaptive`` classifies the query type and routes
+to flat, reranked, or RRF retrieval.
+
+Hash embeddings work out of the box so tests do not need sentence-transformers
+or a GPU.  Real semantic backends (e.g. BGE-M3) are optional.
+"""
+
+from cam_rag.retrieval.fractal.core import (
+    EmbeddingBackend,
+    HashEmbedding,
+    SentenceTransformerEmbedding,
+    normalize,
+)
+from cam_rag.retrieval.fractal.engine import FractalRAG, IndexEntry
+from cam_rag.retrieval.fractal.plugin import FractalRetrieverPlugin
+from cam_rag.retrieval.fractal.profile import DocumentProfile
+from cam_rag.retrieval.fractal.query import (
+    classify_query_type,
+    extract_domain_hints,
+    get_type_weights,
+)
+from cam_rag.retrieval.fractal.storage import DimensionMismatchError, load, save
+
+__all__ = [
+    "DimensionMismatchError",
+    "DocumentProfile",
+    "EmbeddingBackend",
+    "FractalRAG",
+    "FractalRetrieverPlugin",
+    "HashEmbedding",
+    "IndexEntry",
+    "SentenceTransformerEmbedding",
+    "classify_query_type",
+    "extract_domain_hints",
+    "get_type_weights",
+    "load",
+    "normalize",
+    "save",
+]

src/cam_rag/retrieval/fractal/core.py

@@ -0,0 +1,110 @@
+"""
+Shared primitives for Fractal Latent RAG.
+
+All embedding, normalization, and chunking logic lives here — ONE copy.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Protocol, runtime_checkable
+
+import numpy as np
+
+
+@runtime_checkable
+class EmbeddingBackend(Protocol):
+    """Protocol for pluggable embedding backends."""
+    def embed(self, text: str) -> np.ndarray: ...
+    @property
+    def dim(self) -> int: ...
+
+
+class HashEmbedding:
+    """Deterministic MD5-seeded random vectors. For testing and reproducibility only.
+    These vectors have NO semantic meaning — cosine similarity is random noise."""
+
+    def __init__(self, dim: int = 64):
+        self._dim = dim
+
+    def embed(self, text: str) -> np.ndarray:
+        seed = int(hashlib.md5(text.encode('utf-8')).hexdigest(), 16) % (2**32)
+        rng = np.random.default_rng(seed)
+        vec = rng.standard_normal(self._dim).astype(np.float32)
+        return vec / np.linalg.norm(vec)
+
+    @property
+    def dim(self) -> int:
+        return self._dim
+
+
+class SentenceTransformerEmbedding:
+    """Real semantic embeddings via sentence-transformers.
+    Default model: BAAI/bge-m3 (1024 dim, multi-granularity native).
+    """
+
+    def __init__(self, model_name: str = "BAAI/bge-m3"):
+        from sentence_transformers import SentenceTransformer
+        self._model = SentenceTransformer(model_name)
+        self._dim = self._model.get_sentence_embedding_dimension()
+
+    def embed(self, text: str) -> np.ndarray:
+        return self._model.encode(text, normalize_embeddings=True).astype(np.float32)
+
+    @property
+    def dim(self) -> int:
+        return self._dim
+
+
+def normalize(vec: np.ndarray) -> np.ndarray:
+    """Safe L2 normalization. Returns zero vector on zero-norm input."""
+    norm = np.linalg.norm(vec)
+    return vec / norm if norm > 0 else vec
+
+
+def chunk_fractal(text: str) -> tuple[list[str], list[str], str]:
+    """Fractal chunking: sentences (L0) -> paragraphs (L1) -> full doc (L2).
+
+    Sentence splitting uses regex boundaries that handle decimals, percentages,
+    abbreviations, and statistical notation common in scientific text.
+    Fragments shorter than 30 chars are merged into the preceding sentence.
+    """
+    import re
+
+    # Split on period/!/? followed by whitespace and an uppercase letter.
+    # This avoids splitting on decimals (0.79), abbreviations (e.g.), and
+    # statistical notation (P = 0.001).
+    raw = re.split(r'(?<=[.!?])\s+(?=[A-Z])', text)
+
+    # Clean and filter
+    sentences = [s.strip() for s in raw if s.strip()]
+
+    # Merge short fragments (< 30 chars) into previous sentence
+    if len(sentences) > 1:
+        merged = [sentences[0]]
+        for s in sentences[1:]:
+            if len(s) < 30 and merged:
+                merged[-1] = merged[-1] + ' ' + s
+            else:
+                merged.append(s)
+        sentences = merged
+
+    # Ensure non-empty
+    if not sentences:
+        sentences = [text] if text.strip() else []
+
+    paragraphs = [p.strip() for p in text.split('\n\n') if p.strip()] or [text]
+
+    # Virtual paragraph chunking for single-paragraph docs
+    # When \n\n split produces only 1 paragraph but the doc has 4+ sentences,
+    # group sentences into virtual paragraphs of ~3 sentences each.
+    # This prevents L1 from being degenerate (identical to L2).
+    if len(paragraphs) == 1 and len(sentences) >= 4:
+        chunk_size = 3
+        virtual_paras = []
+        for i in range(0, len(sentences), chunk_size):
+            group = sentences[i:i + chunk_size]
+            virtual_paras.append(' '.join(group))
+        paragraphs = virtual_paras
+
+    return sentences, paragraphs, text