diff --git a/docs/_ext/agentgrep_fastmcp.py b/docs/_ext/agentgrep_fastmcp.py
index 9760c91f0..475b3497e 100644
--- a/docs/_ext/agentgrep_fastmcp.py
+++ b/docs/_ext/agentgrep_fastmcp.py
@@ -18,8 +18,10 @@
CatalogAgentSelector,
DbStatusModel,
FindToolResponse,
+ InsightsListResponse,
SearchScopeName,
SearchToolResponse,
+ SuggestionsListResponse,
)
from agentgrep.mcp._library import SearchEffortName
from agentgrep.mcp.models import (
@@ -483,6 +485,42 @@ async def db_status(
t.cast(t.Any, db_status).__fastmcp__ = types.SimpleNamespace(
name="db_status",
title="DB Status",
- tags=READONLY_TAGS | {"db"},
+ tags=READONLY_TAGS | {"db", "insights"},
+ annotations=None,
+)
+
+
+async def insights_list(
+ db_path: t.Annotated[
+ str | None,
+ Field(default=None, description="Optional agentgrep db path."),
+ ] = None,
+) -> InsightsListResponse:
+ """List persisted deterministic insight artifacts."""
+ raise NotImplementedError(DOCS_ONLY_MESSAGE)
+
+
+t.cast(t.Any, insights_list).__fastmcp__ = types.SimpleNamespace(
+ name="insights_list",
+ title="Insights List",
+ tags=READONLY_TAGS | {"insights"},
+ annotations=None,
+)
+
+
+async def suggestions_list(
+ db_path: t.Annotated[
+ str | None,
+ Field(default=None, description="Optional agentgrep db path."),
+ ] = None,
+) -> SuggestionsListResponse:
+ """List persisted review-only instruction suggestions."""
+ raise NotImplementedError(DOCS_ONLY_MESSAGE)
+
+
+t.cast(t.Any, suggestions_list).__fastmcp__ = types.SimpleNamespace(
+ name="suggestions_list",
+ title="Suggestions List",
+ tags=READONLY_TAGS | {"insights", "suggestions"},
annotations=None,
)
diff --git a/docs/cli/db/sync.md b/docs/cli/db/sync.md
index b19945e6a..76b75b903 100644
--- a/docs/cli/db/sync.md
+++ b/docs/cli/db/sync.md
@@ -35,6 +35,13 @@ Force a full refresh even when sources look unchanged:
$ agentgrep db sync --force
```
+Build deterministic insight features during sync instead of deferring
+them:
+
+```console
+$ agentgrep db sync --features inline
+```
+
Show progress even when writing structured output:
```console
@@ -72,6 +79,14 @@ history files stop answering cached searches. Narrowed syncs
(`--agent`, `--scope`, `--limit-sources`) never prune, because they do
not observe the full catalog.
+## Features
+
+The default `--features defer` mode writes source rows, normalized
+records, and the FTS5 cache immediately, but leaves expensive
+similarity features for the insights pipeline. This keeps
+`agentgrep db sync` focused on cache freshness. Use `--features inline`
+when you want the feature table populated as part of sync itself.
+
## Command
```{eval-rst}
diff --git a/docs/cli/index.md b/docs/cli/index.md
index 9de5bb1e0..d502c8321 100644
--- a/docs/cli/index.md
+++ b/docs/cli/index.md
@@ -58,6 +58,18 @@ Open the interactive Textual explorer command surface.
Sync and inspect the persistent DB index.
:::
+:::{grid-item-card} agentgrep insights
+:link: insights/index
+:link-type: doc
+Run and inspect deterministic similarity and omission analysis.
+:::
+
+:::{grid-item-card} agentgrep suggestions
+:link: suggestions/index
+:link-type: doc
+List, inspect, and render review-only instruction suggestions.
+:::
+
:::{grid-item-card} API Reference
:link: reference
:link-type: doc
@@ -180,5 +192,7 @@ search
find
ui
db/index
+insights/index
+suggestions/index
reference
```
diff --git a/docs/cli/insights/analyze.md b/docs/cli/insights/analyze.md
new file mode 100644
index 000000000..cfd4d9d00
--- /dev/null
+++ b/docs/cli/insights/analyze.md
@@ -0,0 +1,49 @@
+(cli-insights-analyze)=
+
+# agentgrep insights analyze
+
+Analyze deterministic insight jobs against the DB index. Insight
+analysis persists evidence artifacts for later listing and review.
+
+## Examples
+
+Analyze every insight family:
+
+```console
+$ agentgrep insights analyze
+```
+
+Analyze only similarity evidence:
+
+```console
+$ agentgrep insights analyze --kind similarity
+```
+
+Analyze omissions for an instruction file:
+
+```console
+$ agentgrep insights analyze \
+ --kind omissions \
+ --target AGENTS.md
+```
+
+## Progress
+
+Text-mode analysis shows stderr progress by default. In an interactive
+terminal, press Enter on a blank line to stop before the next insight
+step and return partial analysis counters. The active insight step
+finishes before the command exits.
+
+Progress output always goes to stderr. JSON and NDJSON stdout stay
+machine-readable even when progress is forced with `--progress always`.
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: insights analyze
+ :nodescription:
+```
diff --git a/docs/cli/insights/explain.md b/docs/cli/insights/explain.md
new file mode 100644
index 000000000..6f570cdec
--- /dev/null
+++ b/docs/cli/insights/explain.md
@@ -0,0 +1,30 @@
+(cli-insights-explain)=
+
+# agentgrep insights explain
+
+Show persisted insight counters for the selected agentgrep database.
+
+## Examples
+
+Explain insight counts:
+
+```console
+$ agentgrep insights explain
+```
+
+Emit structured JSON:
+
+```console
+$ agentgrep insights explain --json
+```
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: insights explain
+ :nodescription:
+```
diff --git a/docs/cli/insights/index.md b/docs/cli/insights/index.md
new file mode 100644
index 000000000..c00ee5227
--- /dev/null
+++ b/docs/cli/insights/index.md
@@ -0,0 +1,35 @@
+(cli-insights)=
+
+# agentgrep insights
+
+The `agentgrep insights` command group analyzes and inspects deterministic
+similarity and omission analysis over a DB index. The CLI
+pages document command flags; the feature guide lives in
+{ref}`insights`.
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: insights
+ :nosubcommands:
+ :nodescription:
+```
+
+Choose a subcommand for details:
+
+- {ref}`cli-insights-analyze` - analyze similarity and omission evidence
+- {ref}`cli-insights-list` - list persisted insight artifacts
+- {ref}`cli-insights-explain` - show persisted insight counters
+
+```{toctree}
+:maxdepth: 1
+:hidden:
+
+analyze
+list
+explain
+```
diff --git a/docs/cli/insights/list.md b/docs/cli/insights/list.md
new file mode 100644
index 000000000..a9d2635ec
--- /dev/null
+++ b/docs/cli/insights/list.md
@@ -0,0 +1,46 @@
+(cli-insights-list)=
+
+# agentgrep insights list
+
+List a bounded page of persisted similarity edges and omission
+findings. By default this prints a terminal summary with sampled rows;
+use `--json` or `--ndjson` when stdout must be machine-readable. This
+command does not run new analysis. Use `agentgrep insights explain` for
+cheap counts without returning row samples.
+
+## Examples
+
+List a small persisted-insight sample:
+
+```console
+$ agentgrep insights list
+```
+
+Change the per-kind row limit:
+
+```console
+$ agentgrep insights list --limit 10
+```
+
+List only omission findings as JSON:
+
+```console
+$ agentgrep insights list --kind omissions --json
+```
+
+Read from a non-default agentgrep database:
+
+```console
+$ agentgrep insights list --db .tmp/agentgrep.sqlite
+```
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: insights list
+ :nodescription:
+```
diff --git a/docs/cli/reference.md b/docs/cli/reference.md
index 9bab00131..ed8a4c494 100644
--- a/docs/cli/reference.md
+++ b/docs/cli/reference.md
@@ -27,6 +27,14 @@ CLI argument types, serialization helpers, and command entry points.
.. autoclass:: agentgrep.DbArgs
:members:
:no-undoc-members:
+
+.. autoclass:: agentgrep.InsightsArgs
+ :members:
+ :no-undoc-members:
+
+.. autoclass:: agentgrep.SuggestionsArgs
+ :members:
+ :no-undoc-members:
```
## Serialization
@@ -46,5 +54,7 @@ CLI argument types, serialization helpers, and command entry points.
.. autofunction:: agentgrep.run_find_command
.. autofunction:: agentgrep.run_ui_command
.. autofunction:: agentgrep.run_db_command
+.. autofunction:: agentgrep.run_insights_command
+.. autofunction:: agentgrep.run_suggestions_command
.. autofunction:: agentgrep.main
```
diff --git a/docs/cli/suggestions/index.md b/docs/cli/suggestions/index.md
new file mode 100644
index 000000000..641044561
--- /dev/null
+++ b/docs/cli/suggestions/index.md
@@ -0,0 +1,35 @@
+(cli-suggestions)=
+
+# agentgrep suggestions
+
+The `agentgrep suggestions` command group lists and renders
+review-only instruction suggestions derived from omission findings.
+The CLI pages document command flags; the suggestion workflow is
+explained in {ref}`insights-suggestions`.
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: suggestions
+ :nosubcommands:
+ :nodescription:
+```
+
+Choose a subcommand for details:
+
+- {ref}`cli-suggestions-list` - list or create persisted suggestions
+- {ref}`cli-suggestions-show` - inspect one suggestion as structured output
+- {ref}`cli-suggestions-render` - render one suggestion as review text
+
+```{toctree}
+:maxdepth: 1
+:hidden:
+
+list
+show
+render
+```
diff --git a/docs/cli/suggestions/list.md b/docs/cli/suggestions/list.md
new file mode 100644
index 000000000..89c8855a7
--- /dev/null
+++ b/docs/cli/suggestions/list.md
@@ -0,0 +1,47 @@
+(cli-suggestions-list)=
+
+# agentgrep suggestions list
+
+List persisted suggestion artifacts. By default this prints a terminal
+summary; use `--json` or `--ndjson` when stdout must be
+machine-readable. When `--target` is provided, agentgrep first creates
+review-only suggestions from open omission findings for that target.
+
+## Examples
+
+List existing suggestions:
+
+```console
+$ agentgrep suggestions list
+```
+
+Create suggestions for `AGENTS.md` and emit JSON:
+
+```console
+$ agentgrep suggestions list \
+ --target AGENTS.md \
+ --json
+```
+
+Use a non-default agentgrep database:
+
+```console
+$ agentgrep suggestions list --db .tmp/agentgrep.sqlite
+```
+
+Return only the most confident suggestion:
+
+```console
+$ agentgrep suggestions list --limit 1
+```
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: suggestions list
+ :nodescription:
+```
diff --git a/docs/cli/suggestions/render.md b/docs/cli/suggestions/render.md
new file mode 100644
index 000000000..36b2aa49e
--- /dev/null
+++ b/docs/cli/suggestions/render.md
@@ -0,0 +1,34 @@
+(cli-suggestions-render)=
+
+# agentgrep suggestions render
+
+Render one suggestion as review text that can be copied into a patch
+or review note. Rendering does not edit `AGENTS.md`, create skills, or
+reload an agent session.
+
+## Examples
+
+Render one suggestion:
+
+```console
+$ agentgrep suggestions render demo-id
+```
+
+Read from a non-default agentgrep database:
+
+```console
+$ agentgrep suggestions render \
+ --db .tmp/agentgrep.sqlite \
+ demo-id
+```
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: suggestions render
+ :nodescription:
+```
diff --git a/docs/cli/suggestions/show.md b/docs/cli/suggestions/show.md
new file mode 100644
index 000000000..dea9d495a
--- /dev/null
+++ b/docs/cli/suggestions/show.md
@@ -0,0 +1,31 @@
+(cli-suggestions-show)=
+
+# agentgrep suggestions show
+
+Show one persisted suggestion artifact. Use this for structured
+review data, especially with `--json`.
+
+## Examples
+
+Show one suggestion:
+
+```console
+$ agentgrep suggestions show demo-id
+```
+
+Emit one suggestion as JSON:
+
+```console
+$ agentgrep suggestions show demo-id --json
+```
+
+## Command
+
+```{eval-rst}
+.. argparse::
+ :module: agentgrep
+ :func: build_docs_parser
+ :prog: agentgrep
+ :path: suggestions show
+ :nodescription:
+```
diff --git a/docs/dev/adr/0017-agentic-insights-engine.md b/docs/dev/adr/0017-agentic-insights-engine.md
new file mode 100644
index 000000000..30ab060ef
--- /dev/null
+++ b/docs/dev/adr/0017-agentic-insights-engine.md
@@ -0,0 +1,201 @@
+(adr-agentic-insights-engine)=
+
+# ADR 0017: Agentic insights engine
+
+## Status
+
+Accepted.
+
+Initial implementation landed with a deterministic `InsightEngine` over the
+SQLite DB store. It records duplicate variant edges, omission
+findings, insight runs, clusters, and evidence rows without adding LanceDB as
+a required dependency.
+
+## Context
+
+The DB index from {ref}`adr-persistent-agentic-db-index`
+normalizes agent history into a durable local read model. That is necessary
+but not enough for insight workflows. Similarity, variants, missing pieces,
+meaningful omissions, and ranked evidence are interpretations over records,
+not record storage itself.
+
+agentgrep already has lightweight ranking and near-duplicate logic for search
+results, but global insight generation needs a different shape. It must avoid
+pairwise comparison over the entire history, preserve evidence, and separate
+deterministic candidate generation from optional LLM judgment.
+
+Prior systems point to the same direction:
+
+- LanceDB combines vector search, full-text search, scalar filtering, hybrid
+ execution, and reranking in one table-oriented retrieval API. That is the
+ useful model for optional semantic insight storage, not for the required
+ DB cache:
+ [query API](https://github.com/lancedb/lancedb/blob/v0.30.0/rust/lancedb/src/query.rs),
+ [hybrid query helpers](https://github.com/lancedb/lancedb/blob/v0.30.0/rust/lancedb/src/query/hybrid.rs),
+ [RRF reranker](https://github.com/lancedb/lancedb/blob/v0.30.0/rust/lancedb/src/rerankers/rrf.rs),
+ and [index builders](https://github.com/lancedb/lancedb/blob/v0.30.0/rust/lancedb/src/index.rs).
+- Lance's index formats show why semantic, scalar, and full-text structures
+ should remain index artifacts over row identifiers instead of redefining the
+ base record model:
+ [index overview](https://github.com/lance-format/lance/blob/v7.0.0/docs/src/format/index/index.md),
+ [vector indices](https://github.com/lance-format/lance/blob/v7.0.0/docs/src/format/index/vector/index.md),
+ and [FTS indices](https://github.com/lance-format/lance/blob/v7.0.0/docs/src/format/index/scalar/fts.md).
+- Chroma's embedded mode is the closest shape to agentgrep's: one local
+ SQLite file carries the system catalog, record metadata, and an FTS5
+ full-text index, while vector indexes live beside it as per-collection
+ segment artifacts a local segment manager opens on demand. Chroma's own
+ 0.4 release consolidated onto SQLite for exactly the reasons ADR 0005
+ chose it — fewer moving parts and robust local full-text search — which
+ makes its segment layout the reference for attaching an optional vector
+ segment without changing the SQLite-first base:
+ [SQLite metadata and FTS segment](https://github.com/chroma-core/chroma/blob/1.5.9/chromadb/segment/impl/metadata/sqlite.py),
+ [local persistent HNSW segment](https://github.com/chroma-core/chroma/blob/1.5.9/chromadb/segment/impl/vector/local_persistent_hnsw.py),
+ [local segment manager](https://github.com/chroma-core/chroma/blob/1.5.9/rust/segment/src/local_segment_manager.rs),
+ and the [Chroma 0.4 storage consolidation note](https://www.trychroma.com/blog/chroma_0.4.0).
+- Lucene and Tantivy keep candidate generation, scoring, and result gathering as
+ query-time behaviors over immutable index state:
+ [Lucene IndexSearcher](https://github.com/apache/lucene/blob/releases/lucene/9.12.3/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java),
+ [Lucene TopDocs](https://github.com/apache/lucene/blob/releases/lucene/9.12.3/lucene/core/src/java/org/apache/lucene/search/TopDocs.java),
+ [Tantivy query trait](https://github.com/quickwit-oss/tantivy/blob/0.26.1/src/query/query.rs),
+ and [Tantivy collector module](https://github.com/quickwit-oss/tantivy/blob/0.26.1/src/collector/mod.rs).
+- DataFusion's session and planner boundaries are useful for explainable
+ insight execution: logical intent is lowered into a physical plan, and
+ runtime state stays separate:
+ [session state](https://github.com/apache/datafusion/blob/53.1.0/datafusion/core/src/execution/session_state.rs),
+ [physical planner](https://github.com/apache/datafusion/blob/53.1.0/datafusion/core/src/physical_planner.rs),
+ and [execution plan](https://github.com/apache/datafusion/blob/53.1.0/datafusion/physical-plan/src/execution_plan.rs).
+
+agentgrep does not need to become a vector database or a general workflow
+engine. The useful pattern is narrower: deterministic retrieval builds small
+candidate sets, then optional judgment operates only on evidence packs.
+
+## Decision
+
+agentgrep will introduce an insights engine above the persistent DB
+index.
+
+The insights engine writes separate derived artifacts. It must not mutate the
+DB index's normalized source and record rows. It must be possible to delete
+and recompute insight artifacts without rebuilding the DB index.
+
+The default insights implementation is deterministic and local. Optional
+semantic retrieval may attach behind a typed store boundary, and Chroma's
+embedded segment shape — vector artifacts keyed by record id living beside
+the same SQLite substrate — is the preferred integration model, with
+LanceDB's table-oriented hybrid retrieval as the alternative. Neither is a
+required dependency for normal agentgrep installation, import, CLI search,
+or MCP search.
+
+The first insight categories are:
+
+1. **Similarity clusters**: groups of prompts, conversations, instructions,
+ or agent guidance with shared intent or text shape.
+2. **Variant edges**: typed relationships such as exact duplicate, near
+ duplicate, same intent in a different project, same project but different
+ issue, toolchain variant, and instruction variant.
+3. **Omission findings**: evidence that a meaningful recurring instruction or
+ pattern is absent from a target project or instruction file.
+4. **Evidence packs**: bounded, reviewable source records and feature scores
+ used by a human or LLM judge.
+5. **Insight runs**: provenance for algorithm versions, optional model
+ versions, thresholds, inputs, and generated artifacts.
+
+## Interfaces
+
+Names below describe intended internal contracts. They are not public APIs
+until implemented and documented.
+
+`InsightStore`
+: Stores insight runs, feature rows, clusters, variant edges, omission
+ findings, and evidence packs. The default store uses SQLite. Optional
+ semantic stores may attach by stable DB record id.
+
+`FeatureExtractor`
+: Builds deterministic features from normalized records: exact hashes,
+ normalized hashes, token shingles, SimHash, MinHash signatures, token
+ counts, path/project hints, agent/store hints, and quality flags.
+
+`CandidateGenerator`
+: Produces bounded candidate pairs or candidate groups from independent
+ signals: FTS5/BM25, metadata filters, hash equality, SimHash distance,
+ MinHash overlap, and optional embedding nearest neighbors.
+
+`VariantClassifier`
+: Assigns relationship types to candidate pairs using deterministic feature
+ agreement before any LLM judgment is considered.
+
+`OmissionDetector`
+: Compares recurring cluster evidence with a target project, AGENTS.md file,
+ or skill corpus to find meaningful missing pieces.
+
+`InsightJudge`
+: Optional judging boundary. It receives a small evidence pack and returns a
+ structured judgment with confidence and rationale. It does not mutate files.
+
+## Similarity and confidence rules
+
+The insights engine must generate candidates before ranking or judging them.
+It must not run unbounded pairwise comparison across the entire DB index.
+
+Candidate signals are intentionally independent:
+
+- exact and normalized hashes for duplicates;
+- SimHash distance for near duplicates;
+- MinHash or shingled Jaccard for prompt-template variants;
+- FTS5/BM25 for lexical candidates;
+- metadata agreement for agent, project, session, role, time, and toolchain;
+- optional embedding distance for semantic similarity.
+
+RapidFuzz remains useful for small candidate reranking. It is not the global
+similarity engine.
+
+High confidence requires either a direct proof, such as an exact normalized
+hash match, or agreement across multiple independent signals. A semantic match
+alone is not enough for a high-confidence variant edge or omission finding.
+
+Omission detection is conservative. A missing piece is meaningful only when:
+
+- the source pattern recurs in neighboring or comparable projects;
+- the target project has matching context, tooling, or workflow;
+- the target instruction surface lacks the pattern;
+- the absence is not explained by different tooling or project scope;
+- the evidence pack is small enough for review.
+
+## Consequences
+
+### Positive
+
+- Similarity and omission workflows become reproducible and inspectable.
+- LLM calls, when configured, judge evidence instead of searching the whole
+ local history.
+- LanceDB can be valuable without becoming a required dependency.
+- Future central/local contrast workflows can reuse the same insight artifacts.
+
+### Tradeoffs
+
+- The project gains another derived-data lifecycle beyond the DB
+ cache.
+- Thresholds and feature versions must be recorded so old insight runs remain
+ explainable.
+- Optional embedding backends introduce model/version drift that deterministic
+ tests cannot fully cover.
+
+### Risks
+
+False positives: prompts can look similar while solving different issues. The
+mitigation is typed variant edges plus metadata-aware confidence.
+
+False omissions: a repeated instruction can be absent for a good reason. The
+mitigation is conservative omission rules and human review before suggestions.
+
+Opaque model judgment: an LLM could overstate weak evidence. The mitigation is
+to make deterministic signals and source evidence primary, and to record the
+LLM output as a judgment artifact rather than a fact.
+
+## Final position
+
+The insights engine is a derived, explainable analysis layer over the
+DB index. It finds candidates deterministically, records evidence and
+provenance, and uses optional semantic or LLM components only behind explicit
+boundaries. Its outputs feed suggestion workflows, but it does not edit
+project instructions or skills.
diff --git a/docs/dev/adr/0018-suggestion-skills-and-agent-instruction-changes.md b/docs/dev/adr/0018-suggestion-skills-and-agent-instruction-changes.md
new file mode 100644
index 000000000..d72da1edc
--- /dev/null
+++ b/docs/dev/adr/0018-suggestion-skills-and-agent-instruction-changes.md
@@ -0,0 +1,152 @@
+(adr-suggestion-skills-agent-instruction-changes)=
+
+# ADR 0018: Suggestion skills and agent instruction changes
+
+## Status
+
+Accepted.
+
+Initial implementation landed with a review-only `SuggestionEngine`, persisted
+suggestion artifacts, CLI rendering commands, and read-only MCP listing tools.
+Suggestions do not edit instruction files or call an LLM automatically.
+
+## Context
+
+The insights engine from {ref}`adr-agentic-insights-engine` can identify
+similar prompts, variants, and meaningful omissions. Those findings are useful
+only if they become safe, reviewable changes to project instruction surfaces:
+AGENTS.md files, skills, local agent guidance, or future agent-specific
+configuration.
+
+That is a separate architectural concern from DB indexing and insight
+generation. Suggesting a change has user-facing consequences. It can affect
+future agent behavior, cross-project conventions, and local trust boundaries.
+agentgrep search must remain a read-only local evidence surface unless a user
+explicitly invokes a suggestion workflow.
+
+Prior systems point to the same direction:
+
+- ADR 0004 keeps query intent, planning, execution, and result sinks separate.
+ Suggestion skills should follow the same boundary: they query evidence
+ through a backend contract and render suggestions as outputs, not hidden side
+ effects: {ref}`adr-headless-query-planning-non-blocking-execution`.
+- LangGraph models long-running agent work as explicit runs, checkpoints,
+ statuses, interrupts, and commands. The useful pattern is not the framework
+ itself, but the explicit pause/resume and human-review control surface:
+ [SDK run and interrupt schema](https://github.com/langchain-ai/langgraph/blob/1.1.3/libs/sdk-py/langgraph_sdk/schema.py)
+ and [Postgres checkpointer setup](https://github.com/langchain-ai/langgraph/blob/1.1.3/libs/checkpoint-postgres/README.md).
+- Chroma's separation of system metadata, log state, and execution is a
+ useful reminder that suggestion state should be another materialized artifact
+ instead of being mixed into raw DB records:
+ [sysdb mixin](https://github.com/chroma-core/chroma/blob/1.5.9/chromadb/db/mixins/sysdb.py)
+ and [log service](https://github.com/chroma-core/chroma/blob/1.5.9/chromadb/logservice/logservice.py).
+
+The practical rule is simple: an LLM may call agentgrep through a CLI or MCP
+tool only when that tool is available and selected. agentgrep itself must not
+silently call an LLM during normal search.
+
+## Decision
+
+agentgrep will treat suggestion skills as review workflows over insight
+artifacts.
+
+A suggestion skill may query insight outputs, collect evidence packs, call an
+LLM judge when explicitly configured, and produce a structured suggestion. It
+must not directly edit AGENTS.md, create skills, update project guidance, or
+change agent configuration without an explicit patch/apply step owned by the
+caller.
+
+Suggested changes take effect only after they are accepted and written to the
+relevant instruction surface. Existing agent sessions may need a restart,
+reload, or explicit context refresh before they observe the changed AGENTS.md
+or skill. New sessions should see the accepted files through their normal
+instruction-loading behavior.
+
+## Interfaces
+
+Names below describe intended internal contracts. They are not public APIs
+until implemented and documented.
+
+`SuggestionQuery`
+: User intent for a suggestion run. It names the target project or instruction
+ surface, the requested suggestion type, evidence limits, and whether optional
+ LLM judging is allowed.
+
+`SuggestionSkill`
+: Headless workflow that queries insight artifacts and emits suggestion
+ events. It does not scan raw history directly unless the insights engine
+ requests a DB refresh.
+
+`SuggestionArtifact`
+: Stored output with target path or scope, suggested change summary, evidence
+ ids, confidence, rationale, model/tool provenance, and review state.
+
+`SuggestionPatch`
+: Optional patch representation generated from an accepted artifact. It is a
+ separate artifact so review can happen before mutation.
+
+`InstructionSurface`
+: Typed target for AGENTS.md, skill files, or future instruction stores. It
+ records reload expectations and safety constraints for the target.
+
+## Suggestion rules
+
+Suggestion workflows must preserve the read-only default:
+
+- Normal `agentgrep search`, `agentgrep find`, and MCP search tools do not call
+ LLMs.
+- An explicit future command, such as `agentgrep insights suggest`, may call an
+ LLM only when configured and consented.
+- LLM input is a bounded evidence pack, not the whole local history.
+- LLM output is a judgment or draft suggestion, not an automatic file write.
+- Suggestions must record evidence ids and confidence, not just prose.
+
+Instruction changes require review:
+
+1. Query insights for relevant clusters, variants, and omission findings.
+2. Build a bounded evidence pack.
+3. Optionally ask an LLM judge to classify the evidence and draft wording.
+4. Store a `SuggestionArtifact`.
+5. Present the suggestion to the caller.
+6. Create a patch only after explicit acceptance.
+7. Let the caller run the normal verification and commit workflow.
+
+## Consequences
+
+### Positive
+
+- Suggestion behavior is auditable and does not alter future agent behavior
+ invisibly.
+- LLM usage is explicit, bounded, and tied to stored evidence.
+- The same suggestion artifacts can support CLI, MCP, TUI, and future frontend
+ surfaces.
+- AGENTS.md and skill changes remain normal repo changes that can be reviewed,
+ tested, committed, or rejected.
+
+### Tradeoffs
+
+- The workflow has more steps than direct auto-editing.
+- Existing sessions may not observe accepted instruction changes immediately.
+- Suggestion quality depends on insight quality and evidence selection.
+
+### Risks
+
+Instruction overfitting: a suggestion could encode a local habit that should
+not become general guidance. The mitigation is evidence review and target
+context matching before patch generation.
+
+Hidden LLM dependence: suggestion workflows could make local search feel like
+it requires a model. The mitigation is to keep normal search LLM-free and make
+LLM judging opt-in.
+
+Stale reload assumptions: different agent tools load AGENTS.md and skills at
+different times. The mitigation is to state reload expectations in the
+suggestion artifact rather than pretending all tools apply changes
+immediately.
+
+## Final position
+
+Suggestion skills are consumers of insight artifacts. They produce reviewable
+recommendations and optional patches, but they do not silently call LLMs during
+search and do not directly change AGENTS.md, skills, or agent behavior without
+explicit acceptance.
diff --git a/docs/dev/adr/index.md b/docs/dev/adr/index.md
index 362f6f84e..9f4f3cb10 100644
--- a/docs/dev/adr/index.md
+++ b/docs/dev/adr/index.md
@@ -24,6 +24,8 @@ multiple adapters or public payloads.
0014-result-order-limit-and-streaming-merge
0015-persistent-agentic-db-index
0016-cache-schema-versioning-and-rebuilds
+0017-agentic-insights-engine
+0018-suggestion-skills-and-agent-instruction-changes
0020-progressive-deep-search
0021-prompt-guided-conversation-routing
```
diff --git a/docs/dev/db-index.md b/docs/dev/db-index.md
index d65f0d422..3b1a0af22 100644
--- a/docs/dev/db-index.md
+++ b/docs/dev/db-index.md
@@ -10,8 +10,13 @@ artifacts, not the original agent history.
## Roles
-The agentgrep db has one job: cache search results that can be served
-without rescanning source stores.
+The agentgrep db supports three jobs:
+
+- cache search results that can be served without rescanning source
+ stores
+- provide stable record ids and normalized text features for insight
+ runs
+- persist insight and suggestion artifacts with provenance
Search commands default to `--cache auto`, which uses the DB index
only when it can answer the query. Use `--no-cache` to force the live
@@ -30,8 +35,11 @@ records split into a search read-model: a narrow `records_search`
table (identity, sort, session, and hash columns), a `record_details`
table with the text/title/role/model/metadata payload, and a
content-full trigram FTS5 table that owns the casefolded haystack.
+The artifact tables — deterministic features, variant edges,
+omission findings, and suggestions — sit beside the read-model.
This keeps the default backend local, transactional, and inspectable
-while letting search touch dense pages.
+while letting search touch dense pages and leaving semantic backends
+such as LanceDB optional for later insight work.
Limited searches run a keyset probe: lean columns ordered by
`(COALESCE(timestamp,''), agent, path, rowid) DESC` in windows of
@@ -44,9 +52,13 @@ Unlimited searches reuse the same lean fetch and deterministic order.
The probe phases appear in profiles as `records.probe_fts` /
`records.probe_scan` / `records.hydrate` statement samples.
-Repeated syncs consult `source_state` fingerprints before opening
-record iterators, so unchanged source files are skipped unless the
-caller uses `--force`.
+The cache-fast sync path treats deterministic features as derived
+secondary state. `agentgrep db sync` defaults to `--features defer`,
+which writes the source ledger, normalized records, and FTS index while
+leaving feature rows for insight runs to refresh in batches. Repeated
+syncs consult `source_state` fingerprints before opening record
+iterators, so unchanged source files are skipped unless the caller uses
+`--force`.
## Commands
diff --git a/docs/dev/index.md b/docs/dev/index.md
index dc90c66a7..f6e03aece 100644
--- a/docs/dev/index.md
+++ b/docs/dev/index.md
@@ -22,7 +22,7 @@ On-disk store layouts for Codex, Claude Code, Cursor, Gemini CLI, Antigravity, G
:::{grid-item-card} DB index
:link: db-index
:link-type: doc
-SQLite cache and sync planner for the persistent DB index.
+SQLite cache, sync planner, and insight artifact substrate.
:::
:::{grid-item-card} Architecture decisions
diff --git a/docs/index.md b/docs/index.md
index d2b5e65bc..3708338a0 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -63,6 +63,12 @@ Search and find from the terminal. Pipe `--json` / `--ndjson` for scripts and ag
Interactive Textual explorer for browsing prompt and conversation records.
:::
+:::{grid-item-card} Insights
+:link: insights/index
+:link-type: doc
+Compare indexed records, detect omissions, and review suggestions.
+:::
+
:::{grid-item-card} MCP
:link: mcp/index
:link-type: doc
@@ -109,6 +115,13 @@ List the stores, session files, and SQLite databases that agentgrep can read.
{tool}`find`
+### Insights
+
+Compare indexed records, list persisted omissions, and review suggested
+instruction changes.
+
+{ref}`insights`
+
### MCP guidance
Use prompts for common agent workflows:
@@ -122,6 +135,7 @@ getting-started/index
cli/index
tui/index
library/index
+insights/index
mcp/index
backends/index
dev/index
diff --git a/docs/insights/index.md b/docs/insights/index.md
new file mode 100644
index 000000000..c954169cb
--- /dev/null
+++ b/docs/insights/index.md
@@ -0,0 +1,95 @@
+(insights)=
+
+# Insights
+
+Insights compare the normalized records already present in the
+DB index. They are deterministic local analysis steps for
+finding similar prompts, duplicated instruction families, variants,
+and meaningful omissions.
+
+The agentgrep database is required because insight runs need stable
+record ids, normalized text hashes, metadata, and persisted evidence.
+Run a DB sync before running insights:
+
+```console
+$ agentgrep db sync
+```
+
+`agentgrep db sync` defers expensive feature rows by default so the
+cache refresh path stays fast. Insight runs refresh any missing
+deterministic features before they compare records, so the default sync
+mode is sufficient for later similarity and omission analysis.
+
+Analyze similarity and omission evidence:
+
+```console
+$ agentgrep insights analyze
+```
+
+List a bounded human summary of stored evidence:
+
+```console
+$ agentgrep insights list --limit 10
+```
+
+Emit the same evidence page as JSON:
+
+```console
+$ agentgrep insights list --limit 10 --json
+```
+
+Get cheap persisted-insight counts without returning evidence rows:
+
+```console
+$ agentgrep insights explain --json
+```
+
+## Similarity
+
+Similarity insights group records that share deterministic signals:
+exact normalized text, lexical overlap, and metadata that makes the
+relationship meaningful. The current implementation persists variant
+edges so later tools can inspect why two records were considered
+related.
+
+Use similarity mode when you want to find repeated prompts,
+copy-pasted instruction fragments, or near-equivalent prompt families
+across agent stores:
+
+```console
+$ agentgrep insights analyze --kind similarity
+```
+
+## Omissions
+
+Omission insights compare indexed instructions against a target
+surface, such as `AGENTS.md`. A finding means the indexed DB
+contains a recurring instruction-like record that is absent from the
+target text.
+
+Run omission detection for one target:
+
+```console
+$ agentgrep insights analyze \
+ --kind omissions \
+ --target AGENTS.md
+```
+
+Omission findings are evidence, not edits. They can later feed
+review-only suggestions; see {ref}`insights-suggestions`.
+
+## LLM boundary
+
+Normal search and insight commands do not silently call an LLM. An LLM
+can call agentgrep only when a user or client gives it access to the
+CLI or MCP server and it chooses, or is instructed, to use that tool.
+
+Future LLM-assisted judgement should run as an explicit command or MCP
+tool over a small evidence pack. The output should be a persisted
+suggestion artifact with provenance, confidence, and review state.
+
+```{toctree}
+:hidden:
+
+suggestions
+```
diff --git a/docs/insights/suggestions.md b/docs/insights/suggestions.md
new file mode 100644
index 000000000..99ab7cdac
--- /dev/null
+++ b/docs/insights/suggestions.md
@@ -0,0 +1,43 @@
+(insights-suggestions)=
+
+# Suggestions
+
+Suggestions are review-only artifacts derived from omission findings.
+They are designed to help a person decide whether to create or edit
+`AGENTS.md` content or a skill. They do not modify files, create
+skills, call an LLM, or reload an agent session by themselves.
+
+Create or list suggestions for a target:
+
+```console
+$ agentgrep suggestions list \
+ --target AGENTS.md \
+ --json
+```
+
+Render one suggestion for review:
+
+```console
+$ agentgrep suggestions render demo-id
+```
+
+## When changes take effect
+
+A suggested instruction change takes effect only after a patch is
+accepted and the relevant agent reloads context. Existing sessions may
+need a restart or explicit reload. New sessions normally pick up the
+changed `AGENTS.md` or skill file through their normal startup context
+loading.
+
+## Review state
+
+Each suggestion carries a confidence score, rationale, target path,
+body, and reload note. Treat those fields as an evidence pack for
+human review, not as authority to apply the change automatically.
+
+See the command reference for exact flags:
+
+- {ref}`cli-suggestions`
+- {ref}`cli-suggestions-list`
+- {ref}`cli-suggestions-show`
+- {ref}`cli-suggestions-render`
diff --git a/docs/library/reference.md b/docs/library/reference.md
index 4015b4af4..164bdf090 100644
--- a/docs/library/reference.md
+++ b/docs/library/reference.md
@@ -222,7 +222,7 @@ explicitly close the async generator, for example with
.. autofunction:: agentgrep.store_catalog.gemini_project_hash
```
-## DB
+## DB and insights
```{eval-rst}
.. autofunction:: agentgrep.db.default_db_path
@@ -240,4 +240,22 @@ explicitly close the async generator, for example with
.. autoclass:: agentgrep.db.DbRuntime
:members:
+
+.. autoclass:: agentgrep.insights.InsightRunResult
+ :members:
+
+.. autoclass:: agentgrep.insights.VariantEdge
+ :members:
+
+.. autoclass:: agentgrep.insights.OmissionFinding
+ :members:
+
+.. autoclass:: agentgrep.insights.InsightEngine
+ :members:
+
+.. autoclass:: agentgrep.suggestions.SuggestionArtifact
+ :members:
+
+.. autoclass:: agentgrep.suggestions.SuggestionEngine
+ :members:
```
diff --git a/docs/mcp/index.md b/docs/mcp/index.md
index f5f519bff..9d6b4a199 100644
--- a/docs/mcp/index.md
+++ b/docs/mcp/index.md
@@ -59,9 +59,13 @@ Payload models, server factory, and MCP helpers.
{tool}`find`
-## DB
+## DB and insights
db_status
+·
+insights_list
+·
+suggestions_list
```{toctree}
diff --git a/src/agentgrep/__init__.py b/src/agentgrep/__init__.py
index deb1249a6..f450241dc 100644
--- a/src/agentgrep/__init__.py
+++ b/src/agentgrep/__init__.py
@@ -86,6 +86,7 @@
FIND_DESCRIPTION,
GREP_DESCRIPTION,
INLINE_CODE_RE,
+ INSIGHTS_DESCRIPTION,
MARKUP_HIGHLIGHT_ROLES,
MARKUP_TOKEN_RE,
QUERY_BOOLEAN_KEYWORDS,
@@ -94,6 +95,7 @@
QUERY_TOKEN_RE,
SEARCH_DESCRIPTION,
SHELL_TOKEN_RE,
+ SUGGESTIONS_DESCRIPTION,
UI_DESCRIPTION,
AnsiColors,
ContentFormat,
@@ -511,6 +513,10 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
return run_ui_command(parsed)
if isinstance(parsed, DbArgs):
return run_db_command(parsed)
+ if isinstance(parsed, InsightsArgs):
+ return run_insights_command(parsed)
+ if isinstance(parsed, SuggestionsArgs):
+ return run_suggestions_command(parsed)
assert isinstance(parsed, FindArgs)
return run_find_command(parsed)
except KeyboardInterrupt:
@@ -573,9 +579,11 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
FindPatternMode,
FindTypeFilter,
GrepArgs,
+ InsightsArgs,
ParserBundle,
PatternMode,
SearchArgs,
+ SuggestionsArgs,
UIArgs,
add_common_agent_options,
add_output_mode_options,
@@ -597,7 +605,9 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
run_db_command,
run_find_command,
run_grep_command,
+ run_insights_command,
run_search_command,
+ run_suggestions_command,
run_ui_command,
serialize_find_record,
serialize_grep_record,
@@ -621,6 +631,7 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
"FIND_DESCRIPTION",
"GREP_DESCRIPTION",
"INLINE_CODE_RE",
+ "INSIGHTS_DESCRIPTION",
"ITER_SOURCE_RECORD_ADAPTERS",
"JSON_FILE_SUFFIXES",
"MARKUP_HIGHLIGHT_ROLES",
@@ -636,6 +647,7 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
"SCHEMA_VERSION",
"SEARCH_DESCRIPTION",
"SHELL_TOKEN_RE",
+ "SUGGESTIONS_DESCRIPTION",
"UI_DESCRIPTION",
"USER_ROLES",
"AgentGrepHelpFormatter",
@@ -668,6 +680,7 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
"GrepArgs",
"GrepStyle",
"HelpTheme",
+ "InsightsArgs",
"JSONScalar",
"JSONValue",
"KeyValueRow",
@@ -730,6 +743,7 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
"StreamingRecordsBatch",
"StreamingSearchFinished",
"StreamingSearchProgress",
+ "SuggestionsArgs",
"SummaryRow",
"TextualAppModule",
"TextualBindingModule",
@@ -893,10 +907,12 @@ def main(argv: cabc.Sequence[str] | None = None) -> int:
"run_find_command",
"run_find_query",
"run_grep_command",
+ "run_insights_command",
"run_readonly_command",
"run_search_command",
"run_search_query",
"run_search_result",
+ "run_suggestions_command",
"run_ui",
"run_ui_command",
"search_record_sort_key",
diff --git a/src/agentgrep/_text.py b/src/agentgrep/_text.py
index 0ce67b097..af1a0bac2 100644
--- a/src/agentgrep/_text.py
+++ b/src/agentgrep/_text.py
@@ -39,6 +39,7 @@
"FIND_DESCRIPTION",
"GREP_DESCRIPTION",
"INLINE_CODE_RE",
+ "INSIGHTS_DESCRIPTION",
"MARKUP_HIGHLIGHT_ROLES",
"MARKUP_TOKEN_RE",
"QUERY_BOOLEAN_KEYWORDS",
@@ -47,6 +48,7 @@
"QUERY_TOKEN_RE",
"SEARCH_DESCRIPTION",
"SHELL_TOKEN_RE",
+ "SUGGESTIONS_DESCRIPTION",
"UI_DESCRIPTION",
"AnsiColors",
"ContentFormat",
@@ -384,6 +386,22 @@ def build_description(
"agentgrep db explain",
),
),
+ (
+ "insights",
+ (
+ "agentgrep insights",
+ "agentgrep insights analyze --kind similarity",
+ "agentgrep insights list --limit 10 --json",
+ ),
+ ),
+ (
+ "suggestions",
+ (
+ "agentgrep suggestions",
+ "agentgrep suggestions list --target AGENTS.md",
+ "agentgrep suggestions render ",
+ ),
+ ),
),
)
FIND_DESCRIPTION = build_description(
@@ -531,6 +549,48 @@ def build_description(
)
+INSIGHTS_DESCRIPTION = build_description(
+ """
+ Run and inspect deterministic similarity, variant, and omission
+ analysis over the DB index. Insights generate persisted
+ evidence artifacts and do not call an LLM by default.
+ """,
+ (
+ (
+ "insights",
+ (
+ "agentgrep insights",
+ "agentgrep insights analyze --kind similarity",
+ "agentgrep insights analyze --kind omissions --target AGENTS.md",
+ "agentgrep insights list --limit 10 --json",
+ "agentgrep insights explain",
+ ),
+ ),
+ ),
+)
+
+
+SUGGESTIONS_DESCRIPTION = build_description(
+ """
+ List, inspect, and render review-only instruction suggestions derived
+ from omission findings. Suggestions never edit AGENTS.md or skills
+ automatically; they take effect only after a human accepts a patch
+ and the relevant agent reloads context.
+ """,
+ (
+ (
+ "suggestions",
+ (
+ "agentgrep suggestions",
+ "agentgrep suggestions list --target AGENTS.md --json",
+ "agentgrep suggestions show --json",
+ "agentgrep suggestions render ",
+ ),
+ ),
+ ),
+)
+
+
class PrivatePath(PrivatePathBase):
"""Path subclass that hides the user's home directory in textual output."""
diff --git a/src/agentgrep/cli/parser.py b/src/agentgrep/cli/parser.py
index 171b509cb..6d11fbe88 100644
--- a/src/agentgrep/cli/parser.py
+++ b/src/agentgrep/cli/parser.py
@@ -31,7 +31,9 @@
DB_DESCRIPTION,
FIND_DESCRIPTION,
GREP_DESCRIPTION,
+ INSIGHTS_DESCRIPTION,
SEARCH_DESCRIPTION,
+ SUGGESTIONS_DESCRIPTION,
UI_DESCRIPTION,
)
from agentgrep.cli.help_theme import create_themed_formatter
@@ -120,7 +122,13 @@ def _normalize_args_conversation_limit(
DbAction = t.Literal["sync", "status", "explain"]
+DbFeatureMode = t.Literal["defer", "inline"]
+InsightsAction = t.Literal["analyze", "list", "explain"]
+InsightsKind = t.Literal["similarity", "omissions", "all"]
+SuggestionsAction = t.Literal["list", "show", "render"]
+DEFAULT_INSIGHTS_LIST_LIMIT = 50
+DEFAULT_SUGGESTIONS_LIST_LIMIT = 50
__all__ = [
"CaseMode",
@@ -129,9 +137,11 @@ def _normalize_args_conversation_limit(
"FindPatternMode",
"FindTypeFilter",
"GrepArgs",
+ "InsightsArgs",
"ParserBundle",
"PatternMode",
"SearchArgs",
+ "SuggestionsArgs",
"UIArgs",
"add_cache_options",
"add_common_agent_options",
@@ -482,9 +492,37 @@ class DbArgs:
color_mode: ColorMode
progress_mode: ProgressMode
limit_sources: int | None = None
+ features_mode: DbFeatureMode = "defer"
force: bool = False
+@dataclasses.dataclass(slots=True)
+class InsightsArgs:
+ """Typed arguments for ``agentgrep insights`` subcommands."""
+
+ action: InsightsAction
+ db_path: str | None
+ kind: InsightsKind
+ target: str | None
+ output_mode: OutputMode
+ color_mode: ColorMode = "auto"
+ progress_mode: ProgressMode = "never"
+ limit: int = DEFAULT_INSIGHTS_LIST_LIMIT
+
+
+@dataclasses.dataclass(slots=True)
+class SuggestionsArgs:
+ """Typed arguments for ``agentgrep suggestions`` subcommands."""
+
+ action: SuggestionsAction
+ db_path: str | None
+ suggestion_id: str | None
+ target: str | None
+ output_mode: OutputMode
+ color_mode: ColorMode = "auto"
+ limit: int = DEFAULT_SUGGESTIONS_LIST_LIMIT
+
+
@dataclasses.dataclass(slots=True)
class ParserBundle:
"""CLI parsers used for root and subcommand help.
@@ -509,6 +547,8 @@ class ParserBundle:
grep_parser: argparse.ArgumentParser
search_parser: argparse.ArgumentParser
db_parser: argparse.ArgumentParser
+ insights_parser: argparse.ArgumentParser
+ suggestions_parser: argparse.ArgumentParser
class _VersionAction(argparse.Action):
@@ -1063,6 +1103,12 @@ def create_parser(
metavar="N",
help="Limit the number of sources synced",
)
+ _ = db_sync_parser.add_argument(
+ "--features",
+ choices=["defer", "inline"],
+ default="defer",
+ help="Feature generation mode: defer expensive features or build inline",
+ )
_ = db_sync_parser.add_argument(
"--force",
action="store_true",
@@ -1101,12 +1147,117 @@ def create_parser(
_ = db_explain_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
add_output_mode_options(db_explain_parser, allow_ui=False)
+ insights_parser = subparsers.add_parser(
+ "insights",
+ help="Run and inspect deterministic agentic-data insights",
+ description=INSIGHTS_DESCRIPTION,
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ insights_subparsers = insights_parser.add_subparsers(dest="insights_action")
+ insights_analyze_parser = insights_subparsers.add_parser(
+ "analyze",
+ help="Analyze deterministic insights over the agentgrep db",
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ _ = insights_analyze_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
+ _ = insights_analyze_parser.add_argument(
+ "--kind",
+ choices=["similarity", "omissions", "all"],
+ default="all",
+ help="Insight family to analyze",
+ )
+ _ = insights_analyze_parser.add_argument("--target", help="Target file for omission insights")
+ _ = insights_analyze_parser.add_argument(
+ "--progress",
+ choices=["auto", "always", "never"],
+ default="auto",
+ help="Show insight analysis progress on stderr",
+ )
+ _ = insights_analyze_parser.add_argument(
+ "--no-progress",
+ dest="progress",
+ action="store_const",
+ const="never",
+ help="Silence the stderr progress spinner (alias for --progress=never)",
+ )
+ add_output_mode_options(insights_analyze_parser, allow_ui=False)
+
+ insights_list_parser = insights_subparsers.add_parser(
+ "list",
+ help="List persisted insights",
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ _ = insights_list_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
+ _ = insights_list_parser.add_argument(
+ "--kind",
+ choices=["similarity", "omissions", "all"],
+ default="all",
+ help="Insight family to list",
+ )
+ _ = insights_list_parser.add_argument(
+ "--limit",
+ type=int,
+ default=DEFAULT_INSIGHTS_LIST_LIMIT,
+ help=f"Maximum rows to return per insight family (default: {DEFAULT_INSIGHTS_LIST_LIMIT})",
+ )
+ add_output_mode_options(insights_list_parser, allow_ui=False)
+
+ insights_explain_parser = insights_subparsers.add_parser(
+ "explain",
+ help="Explain persisted insight counters",
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ _ = insights_explain_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
+ add_output_mode_options(insights_explain_parser, allow_ui=False)
+
+ suggestions_parser = subparsers.add_parser(
+ "suggestions",
+ help="Render review-only instruction suggestions",
+ description=SUGGESTIONS_DESCRIPTION,
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ suggestions_subparsers = suggestions_parser.add_subparsers(
+ dest="suggestions_action",
+ )
+ suggestions_list_parser = suggestions_subparsers.add_parser(
+ "list",
+ help="List persisted suggestions",
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ _ = suggestions_list_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
+ _ = suggestions_list_parser.add_argument("--target", help="Create suggestions for a target")
+ _ = suggestions_list_parser.add_argument(
+ "--limit",
+ type=int,
+ default=DEFAULT_SUGGESTIONS_LIST_LIMIT,
+ help=f"Maximum suggestions to return (default: {DEFAULT_SUGGESTIONS_LIST_LIMIT})",
+ )
+ add_output_mode_options(suggestions_list_parser, allow_ui=False)
+ for action in ("show", "render"):
+ suggestion_parser = suggestions_subparsers.add_parser(
+ action,
+ help=f"{action.title()} one persisted suggestion",
+ formatter_class=formatter_class,
+ color=color_mode != "never",
+ )
+ _ = suggestion_parser.add_argument("suggestion_id")
+ _ = suggestion_parser.add_argument("--db", dest="db_path", help="agentgrep db path")
+ add_output_mode_options(suggestion_parser, allow_ui=False)
+
return ParserBundle(
parser=parser,
find_parser=find_parser,
grep_parser=grep_parser,
search_parser=search_parser,
db_parser=db_parser,
+ insights_parser=insights_parser,
+ suggestions_parser=suggestions_parser,
)
@@ -1513,7 +1664,7 @@ def _check_for_mangled_field_predicate(
def parse_args(
argv: cabc.Sequence[str] | None = None,
-) -> DbArgs | FindArgs | GrepArgs | SearchArgs | UIArgs | None:
+) -> DbArgs | FindArgs | GrepArgs | InsightsArgs | SearchArgs | SuggestionsArgs | UIArgs | None:
"""Parse CLI arguments into typed dataclasses."""
color_mode = normalize_color_mode(argv)
effective_argv = list(argv) if argv is not None else list(sys.argv[1:])
@@ -1544,6 +1695,20 @@ def parse_args(
return None
return _build_db_args(namespace, color_mode=color_mode, bundle=bundle)
+ if command == "insights":
+ if getattr(namespace, "insights_action", None) is None:
+ with configured_color_environment(color_mode):
+ bundle.insights_parser.print_help()
+ return None
+ return _build_insights_args(namespace, color_mode=color_mode, bundle=bundle)
+
+ if command == "suggestions":
+ if getattr(namespace, "suggestions_action", None) is None:
+ with configured_color_environment(color_mode):
+ bundle.suggestions_parser.print_help()
+ return None
+ return _build_suggestions_args(namespace, color_mode=color_mode, bundle=bundle)
+
agents = parse_agents(t.cast("list[str]", namespace.agent))
output_mode = parse_output_mode(namespace)
@@ -1654,10 +1819,63 @@ def _build_db_args(
color_mode=color_mode,
progress_mode=t.cast("ProgressMode", getattr(namespace, "progress", "never")),
limit_sources=limit_sources,
+ features_mode=t.cast("DbFeatureMode", getattr(namespace, "features", "defer")),
force=t.cast("bool", getattr(namespace, "force", False)),
)
+def _build_insights_args(
+ namespace: argparse.Namespace,
+ *,
+ color_mode: ColorMode,
+ bundle: ParserBundle,
+) -> InsightsArgs:
+ """Build :class:`InsightsArgs` from a parsed argparse namespace."""
+ action = t.cast("InsightsAction", namespace.insights_action)
+ kind = t.cast("InsightsKind", getattr(namespace, "kind", "all"))
+ target = t.cast("str | None", getattr(namespace, "target", None))
+ if action == "analyze" and kind == "omissions" and target is None:
+ with configured_color_environment(color_mode):
+ bundle.insights_parser.error("--target is required for omission insight analysis")
+ limit = t.cast("int", getattr(namespace, "limit", DEFAULT_INSIGHTS_LIST_LIMIT))
+ if action == "list" and limit < 1:
+ with configured_color_environment(color_mode):
+ bundle.insights_parser.error("--limit must be greater than 0")
+ return InsightsArgs(
+ action=action,
+ db_path=t.cast("str | None", getattr(namespace, "db_path", None)),
+ kind=kind,
+ target=target,
+ output_mode=parse_output_mode(namespace),
+ color_mode=color_mode,
+ progress_mode=t.cast("ProgressMode", getattr(namespace, "progress", "never")),
+ limit=limit,
+ )
+
+
+def _build_suggestions_args(
+ namespace: argparse.Namespace,
+ *,
+ color_mode: ColorMode,
+ bundle: ParserBundle,
+) -> SuggestionsArgs:
+ """Build :class:`SuggestionsArgs` from a parsed argparse namespace."""
+ action = t.cast("SuggestionsAction", namespace.suggestions_action)
+ limit = t.cast("int", getattr(namespace, "limit", DEFAULT_SUGGESTIONS_LIST_LIMIT))
+ if action == "list" and limit < 1:
+ with configured_color_environment(color_mode):
+ bundle.suggestions_parser.error("--limit must be greater than 0")
+ return SuggestionsArgs(
+ action=action,
+ db_path=t.cast("str | None", getattr(namespace, "db_path", None)),
+ suggestion_id=t.cast("str | None", getattr(namespace, "suggestion_id", None)),
+ target=t.cast("str | None", getattr(namespace, "target", None)),
+ output_mode=parse_output_mode(namespace),
+ color_mode=color_mode,
+ limit=limit,
+ )
+
+
def _build_grep_args(
namespace: argparse.Namespace,
*,
diff --git a/src/agentgrep/cli/render.py b/src/agentgrep/cli/render.py
index 3c4d7b831..c8f814de4 100644
--- a/src/agentgrep/cli/render.py
+++ b/src/agentgrep/cli/render.py
@@ -37,7 +37,16 @@
_visible_width,
format_display_path,
)
-from agentgrep.cli.parser import DbArgs, FindArgs, GrepArgs, SearchArgs, UIArgs
+from agentgrep.cli.parser import (
+ DbArgs,
+ FindArgs,
+ GrepArgs,
+ InsightsArgs,
+ InsightsKind,
+ SearchArgs,
+ SuggestionsArgs,
+ UIArgs,
+)
from agentgrep.cli.renderers import (
GrepSummary,
_compile_search_patterns,
@@ -109,7 +118,9 @@
"run_db_command",
"run_find_command",
"run_grep_command",
+ "run_insights_command",
"run_search_command",
+ "run_suggestions_command",
"run_ui_command",
"serialize_find_record",
"serialize_grep_record",
@@ -997,6 +1008,35 @@ def _format_structured_text(payload: object, *, colors: AnsiColors) -> str:
return _format_db_explain_text(payload, colors=colors)
if _has_attributes(payload, ("db_path", "schema_version", "sources", "records")):
return _format_db_status_text(payload, colors=colors)
+ if _is_suggestions_list_payload(payload):
+ return _format_suggestions_page_text(
+ t.cast("cabc.Mapping[str, object]", payload),
+ colors=colors,
+ )
+ if _is_insights_list_payload(payload):
+ return _format_insights_list_text(
+ t.cast("cabc.Mapping[str, object]", payload),
+ colors=colors,
+ )
+ if _is_insights_explain_payload(payload):
+ return _format_insights_explain_text(
+ t.cast("cabc.Mapping[str, object]", payload),
+ colors=colors,
+ )
+ if _is_suggestion_collection(payload):
+ return _format_suggestions_list_text(
+ t.cast("cabc.Sequence[object]", payload),
+ colors=colors,
+ )
+ if _is_suggestion(payload):
+ return _format_suggestion_text(payload, colors=colors)
+ if _is_insight_result_collection(payload):
+ return _format_insights_analyze_text(
+ t.cast("cabc.Sequence[object]", payload),
+ colors=colors,
+ )
+ if _is_insight_result(payload):
+ return _format_insights_analyze_text((payload,), colors=colors)
return _format_generic_structured_text(payload, colors=colors)
@@ -1005,12 +1045,75 @@ def _has_attributes(payload: object, names: cabc.Sequence[str]) -> bool:
return all(hasattr(payload, name) for name in names)
+def _is_suggestion(payload: object) -> bool:
+ """Return whether ``payload`` looks like a suggestion artifact."""
+ return _has_attributes(payload, ("suggestion_id", "target_path", "title", "confidence"))
+
+
+def _is_suggestion_collection(payload: object) -> bool:
+ """Return whether ``payload`` is a sequence of suggestion artifacts."""
+ return (
+ isinstance(payload, cabc.Sequence)
+ and not isinstance(payload, (str, bytes, bytearray))
+ and all(_is_suggestion(item) for item in payload)
+ )
+
+
+def _is_insight_result(payload: object) -> bool:
+ """Return whether ``payload`` looks like one insight analyze result."""
+ return _has_attributes(payload, ("run_id", "kind")) and (
+ hasattr(payload, "variant_edges") or hasattr(payload, "omission_findings")
+ )
+
+
+def _is_insight_result_collection(payload: object) -> bool:
+ """Return whether ``payload`` is a sequence of insight analyze results."""
+ return (
+ isinstance(payload, cabc.Sequence)
+ and not isinstance(payload, (str, bytes, bytearray))
+ and all(_is_insight_result(item) for item in payload)
+ )
+
+
+def _is_suggestions_list_payload(payload: object) -> bool:
+ """Return whether ``payload`` is the bounded suggestions list shape."""
+ if not isinstance(payload, cabc.Mapping):
+ return False
+ mapping = t.cast("cabc.Mapping[str, object]", payload)
+ return isinstance(mapping.get("suggestions"), cabc.Mapping)
+
+
+def _is_insights_list_payload(payload: object) -> bool:
+ """Return whether ``payload`` is the persisted-insights list shape."""
+ if not isinstance(payload, cabc.Mapping):
+ return False
+ mapping = t.cast("cabc.Mapping[str, object]", payload)
+ return any(
+ isinstance(mapping.get(key), cabc.Mapping) for key in ("variant_edges", "omission_findings")
+ )
+
+
+def _is_insights_explain_payload(payload: object) -> bool:
+ """Return whether ``payload`` is the persisted-insights counter shape."""
+ if not isinstance(payload, cabc.Mapping):
+ return False
+ mapping = t.cast("cabc.Mapping[str, object]", payload)
+ keys = set(mapping)
+ return keys <= {"variant_edges", "omission_findings"} and bool(keys)
+
+
def _format_db_status_text(payload: object, *, colors: AnsiColors) -> str:
"""Return human-readable DB status text."""
db_path = _attribute_or_mapping_value(payload, "db_path", "")
schema_version = _attribute_or_mapping_value(payload, "schema_version", "")
sources = _as_int_value(_attribute_or_mapping_value(payload, "sources", 0))
records = _as_int_value(_attribute_or_mapping_value(payload, "records", 0))
+ features = _as_int_value(_attribute_or_mapping_value(payload, "features", 0))
+ variant_edges = _as_int_value(_attribute_or_mapping_value(payload, "variant_edges", 0))
+ omission_findings = _as_int_value(
+ _attribute_or_mapping_value(payload, "omission_findings", 0),
+ )
+ suggestions = _as_int_value(_attribute_or_mapping_value(payload, "suggestions", 0))
lines = [
colors.heading("DB status"),
f"{colors.muted('Path')} | {colors.path(str(db_path))}",
@@ -1019,6 +1122,14 @@ def _format_db_status_text(payload: object, *, colors: AnsiColors) -> str:
(
colors.warning(format_db_source_count(sources)),
colors.warning(_format_count(records, "record")),
+ colors.warning(_format_count(features, "feature")),
+ ),
+ ),
+ " | ".join(
+ (
+ colors.warning(format_insights_variant_count(variant_edges)),
+ colors.warning(format_insights_omission_count(omission_findings)),
+ colors.warning(_format_count(suggestions, "suggestion")),
),
),
]
@@ -1099,12 +1210,18 @@ def _format_db_sync_result_text(payload: object, *, colors: AnsiColors) -> str:
),
),
]
+ optional = []
skipped = _as_int_value(_attribute_or_mapping_value(payload, "sources_skipped", 0))
+ deferred = _as_int_value(_attribute_or_mapping_value(payload, "features_deferred", 0))
if skipped:
- lines.append(colors.warning(format_db_skipped_count(skipped)))
+ optional.append(colors.warning(format_db_skipped_count(skipped)))
pruned = _as_int_value(_attribute_or_mapping_value(payload, "sources_pruned", 0))
if pruned:
- lines.append(colors.warning(format_db_pruned_count(pruned)))
+ optional.append(colors.warning(format_db_pruned_count(pruned)))
+ if deferred:
+ optional.append(colors.warning(format_db_deferred_count(deferred)))
+ if optional:
+ lines.append(" | ".join(optional))
return "\n".join(lines)
@@ -1157,6 +1274,52 @@ def _format_count(count: int, singular: str, plural: str | None = None) -> str:
return f"{count} {label}"
+def _as_int(payload: cabc.Mapping[str, object], key: str) -> int:
+ """Return one mapping value as an integer count."""
+ return _as_int_value(payload.get(key, 0))
+
+
+def _format_confidence(value: object) -> str:
+ """Return a compact confidence value.
+
+ Examples
+ --------
+ >>> _format_confidence(0.825)
+ '0.82'
+ >>> _format_confidence("1")
+ '1.00'
+ >>> _format_confidence("high")
+ 'high'
+ """
+ if isinstance(value, str | int | float):
+ try:
+ return f"{float(value):.2f}"
+ except ValueError:
+ return str(value)
+ return str(value)
+
+
+def _short_identifier(value: str, *, width: int = 16) -> str:
+ """Return a compact identifier for terminal summaries.
+
+ Examples
+ --------
+ >>> _short_identifier("edge-1")
+ 'edge-1'
+ >>> _short_identifier("0123456789abcdef0123", width=8)
+ '01234567...'
+ """
+ return value if len(value) <= width else f"{value[:width]}..."
+
+
+def _short_text(value: str, width: int) -> str:
+ """Return text shortened to ``width`` display characters."""
+ compact = " ".join(value.split())
+ if len(compact) <= width:
+ return compact
+ return f"{compact[: max(0, width - 3)]}..."
+
+
def _format_scalar(value: object) -> str:
"""Return one scalar-ish value without Python dataclass reprs."""
if dataclasses.is_dataclass(value) and not isinstance(value, type):
@@ -1794,6 +1957,20 @@ def format_db_pruned_count(count: int) -> str:
return f"{count} {suffix}"
+def format_db_deferred_count(count: int) -> str:
+ """Return a human-readable deferred-feature count.
+
+ Examples
+ --------
+ >>> format_db_deferred_count(1)
+ '1 feature deferred'
+ >>> format_db_deferred_count(5)
+ '5 features deferred'
+ """
+ suffix = "feature deferred" if count == 1 else "features deferred"
+ return f"{count} {suffix}"
+
+
def _format_optional_db_sync_counts(result: SyncResult, *, colors: SearchColors) -> str:
"""Return optional DB sync counters prefixed for inline summaries."""
parts: list[str] = []
@@ -1920,6 +2097,10 @@ def _run_db_status_command(args: DbArgs) -> int:
schema_version=SCHEMA_VERSION,
sources=0,
records=0,
+ features=0,
+ variant_edges=0,
+ omission_findings=0,
+ suggestions=0,
)
_print_json_or_text(payload, output_mode=args.output_mode, color_mode=args.color_mode)
return 0
@@ -2002,6 +2183,7 @@ def _run_db_command_with_runtime(args: DbArgs, runtime: DbRuntime) -> int:
sources,
control=control,
progress=progress,
+ features_mode=args.features_mode,
force=args.force,
coverage=coverage,
prune_missing=prune_missing,
@@ -2017,3 +2199,1135 @@ def _run_db_command_with_runtime(args: DbArgs, runtime: DbRuntime) -> int:
progress.close()
_print_json_or_text(result, output_mode=args.output_mode, color_mode=args.color_mode)
return 0
+
+
+def _format_insights_explain_text(
+ payload: cabc.Mapping[str, object],
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return human-readable persisted insight counters."""
+ return "\n".join(
+ (
+ colors.heading("Insights"),
+ " | ".join(
+ (
+ colors.warning(
+ format_insights_variant_count(_as_int(payload, "variant_edges")),
+ ),
+ colors.warning(
+ format_insights_omission_count(_as_int(payload, "omission_findings")),
+ ),
+ ),
+ ),
+ ),
+ )
+
+
+def _format_suggestions_page_text(
+ payload: cabc.Mapping[str, object],
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return human-readable bounded suggestion list text."""
+ lines = [colors.heading("Suggestions")]
+ limit = _as_int(payload, "limit")
+ lines.append(colors.muted(f"limit {limit}"))
+ family = payload.get("suggestions")
+ if isinstance(family, cabc.Mapping):
+ lines.extend(
+ _format_insight_family_lines(
+ "Suggestions",
+ "suggestion",
+ t.cast("cabc.Mapping[str, object]", family),
+ colors=colors,
+ item_formatter=_format_suggestion_summary_line_for_family,
+ ),
+ )
+ return "\n".join(lines)
+
+
+def _format_suggestion_summary_line_for_family(
+ item: object,
+ colors: AnsiColors,
+) -> str:
+ """Adapt the suggestion row formatter to the family-lines signature."""
+ return _format_suggestion_summary_line(item, colors=colors)
+
+
+def _format_insights_list_text(
+ payload: cabc.Mapping[str, object],
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return human-readable persisted insight list text."""
+ lines = [colors.heading("Insights")]
+ if "limit" in payload:
+ lines.append(f"{colors.muted('Page')} | limit {colors.warning(str(payload['limit']))}")
+ variant_edges = payload.get("variant_edges")
+ if isinstance(variant_edges, cabc.Mapping):
+ lines.extend(
+ _format_insight_family_lines(
+ "Variant edges",
+ "variant edge",
+ t.cast("cabc.Mapping[str, object]", variant_edges),
+ colors=colors,
+ item_formatter=_format_variant_edge_line,
+ ),
+ )
+ omission_findings = payload.get("omission_findings")
+ if isinstance(omission_findings, cabc.Mapping):
+ lines.extend(
+ _format_insight_family_lines(
+ "Omission findings",
+ "omission finding",
+ t.cast("cabc.Mapping[str, object]", omission_findings),
+ colors=colors,
+ item_formatter=_format_omission_finding_line,
+ ),
+ )
+ return "\n".join(lines)
+
+
+def _format_insight_family_lines(
+ heading: str,
+ item_name: str,
+ payload: cabc.Mapping[str, object],
+ *,
+ colors: AnsiColors,
+ item_formatter: cabc.Callable[[object, AnsiColors], str],
+) -> list[str]:
+ """Return lines for one insights list family."""
+ total = _as_int(payload, "total")
+ returned = _as_int(payload, "returned")
+ truncated = bool(payload.get("truncated", False))
+ suffix = item_name if total == 1 else f"{item_name}s"
+ state = colors.warning("truncated") if truncated else colors.success("complete")
+ lines = [
+ (f"{colors.heading(heading)} | {colors.warning(f'{returned}/{total} {suffix}')} | {state}"),
+ ]
+ items = payload.get("items", ())
+ if not isinstance(items, cabc.Sequence) or isinstance(items, (str, bytes, bytearray)):
+ return lines
+ rows = t.cast("cabc.Sequence[object]", items)
+ shown = 0
+ for item in rows[:_HUMAN_SAMPLE_LIMIT]:
+ lines.append(f" {item_formatter(item, colors)}")
+ shown += 1
+ remaining = len(rows) - shown
+ if remaining > 0:
+ lines.append(colors.dim(f" ... {remaining} more returned; use --json for full rows"))
+ return lines
+
+
+def _format_variant_edge_line(item: object, colors: AnsiColors) -> str:
+ """Return one human-readable variant edge row."""
+ edge_id = _attribute_or_mapping_value(item, "edge_id", "")
+ variant_type = _attribute_or_mapping_value(item, "variant_type", "")
+ confidence = _attribute_or_mapping_value(item, "confidence", 0.0)
+ explanation = _attribute_or_mapping_value(item, "explanation", "")
+ return " | ".join(
+ (
+ colors.accent(_short_identifier(str(edge_id))),
+ colors.warning(str(variant_type)),
+ colors.muted(f"confidence {_format_confidence(confidence)}"),
+ _short_text(str(explanation), 88),
+ ),
+ )
+
+
+def _format_omission_finding_line(item: object, colors: AnsiColors) -> str:
+ """Return one human-readable omission finding row."""
+ finding_id = _attribute_or_mapping_value(item, "finding_id", "")
+ target_path = _attribute_or_mapping_value(item, "target_path", "")
+ confidence = _attribute_or_mapping_value(item, "confidence", 0.0)
+ rationale = _attribute_or_mapping_value(item, "rationale", "")
+ return " | ".join(
+ (
+ colors.accent(_short_identifier(str(finding_id))),
+ colors.path(str(target_path)),
+ colors.muted(f"confidence {_format_confidence(confidence)}"),
+ _short_text(str(rationale), 88),
+ ),
+ )
+
+
+def _format_insights_analyze_text(
+ results: cabc.Sequence[object],
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return human-readable insights analyze result text."""
+ aggregate = _empty_insights_analyze_progress_result()
+ for result in results:
+ aggregate = _add_insight_result(aggregate, result)
+ lines = [
+ colors.heading("Insights analyze"),
+ " | ".join(
+ (
+ colors.warning(format_insights_run_count(aggregate.runs_analyzed)),
+ colors.warning(format_insights_feature_count(aggregate.features_refreshed)),
+ colors.warning(format_insights_cluster_count(aggregate.clusters)),
+ colors.warning(format_insights_variant_count(aggregate.variant_edges)),
+ colors.warning(format_insights_omission_count(aggregate.omission_findings)),
+ ),
+ ),
+ ]
+ for result in results[:_HUMAN_SAMPLE_LIMIT]:
+ kind = _attribute_or_mapping_value(result, "kind", "insight")
+ run_id = _attribute_or_mapping_value(result, "run_id", "")
+ lines.append(f" {colors.accent(str(kind))} | {colors.muted(str(run_id))}")
+ remaining = len(results) - _HUMAN_SAMPLE_LIMIT
+ if remaining > 0:
+ lines.append(colors.dim(f" ... {remaining} more runs"))
+ return "\n".join(lines)
+
+
+def _format_suggestions_list_text(
+ suggestions: cabc.Sequence[object],
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return human-readable suggestion list text."""
+ count = len(suggestions)
+ lines = [
+ colors.heading("Suggestions"),
+ colors.warning(_format_count(count, "suggestion")),
+ ]
+ lines.extend(
+ f" {_format_suggestion_summary_line(suggestion, colors=colors)}"
+ for suggestion in suggestions[:_HUMAN_SAMPLE_LIMIT]
+ )
+ remaining = count - _HUMAN_SAMPLE_LIMIT
+ if remaining > 0:
+ lines.append(colors.dim(f" ... {remaining} more suggestions; use --json for full rows"))
+ return "\n".join(lines)
+
+
+def _format_suggestion_text(payload: object, *, colors: AnsiColors) -> str:
+ """Return human-readable suggestion detail text."""
+ rationale = _attribute_or_mapping_value(payload, "rationale", "")
+ reload_note = _attribute_or_mapping_value(payload, "reload_note", "")
+ body = _attribute_or_mapping_value(payload, "body", "")
+ return "\n".join(
+ (
+ colors.heading("Suggestion"),
+ _format_suggestion_summary_line(payload, colors=colors),
+ f"{colors.muted('Rationale')} | {rationale}",
+ f"{colors.muted('Reload')} | {reload_note}",
+ "",
+ str(body),
+ ),
+ )
+
+
+def _format_suggestion_summary_line(
+ suggestion: object,
+ *,
+ colors: AnsiColors,
+) -> str:
+ """Return one human-readable suggestion row."""
+ suggestion_id = str(_attribute_or_mapping_value(suggestion, "suggestion_id", ""))
+ target_path = str(_attribute_or_mapping_value(suggestion, "target_path", ""))
+ surface_kind = str(_attribute_or_mapping_value(suggestion, "surface_kind", ""))
+ status = str(_attribute_or_mapping_value(suggestion, "status", ""))
+ confidence = _attribute_or_mapping_value(suggestion, "confidence", 0.0)
+ title = str(_attribute_or_mapping_value(suggestion, "title", ""))
+ return " | ".join(
+ (
+ colors.accent(_short_identifier(suggestion_id)),
+ colors.path(target_path),
+ colors.muted(surface_kind),
+ colors.success(status),
+ colors.warning(_format_confidence(confidence)),
+ title,
+ ),
+ )
+
+
+@dataclasses.dataclass(frozen=True)
+class InsightsAnalyzeProgressResult:
+ """Aggregate counters for one insights analyze command."""
+
+ runs_analyzed: int
+ features_refreshed: int
+ clusters: int
+ variant_edges: int
+ omission_findings: int
+
+
+@dataclasses.dataclass(frozen=True)
+class InsightsAnalyzeProgressSnapshot:
+ """Immutable view of insights analyze progress state for one render pass."""
+
+ phase: str
+ current: int | None
+ total: int | None
+ detail: str | None
+ activity: str | None
+ activity_detail: str | None
+ result: InsightsAnalyzeProgressResult
+ elapsed: float
+
+
+class ConsoleInsightsAnalyzeProgress:
+ """Human progress reporter for insights analyze operations."""
+
+ _SPINNER_FRAMES: t.ClassVar[str] = "⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏"
+
+ def __init__(
+ self,
+ *,
+ enabled: bool,
+ stream: t.TextIO | None = None,
+ tty: bool | None = None,
+ color_mode: ColorMode = "auto",
+ refresh_interval: float = 0.1,
+ heartbeat_interval: float = 10.0,
+ answer_now_hint: bool = False,
+ ) -> None:
+ self._enabled = enabled
+ self._stream = stream if stream is not None else sys.stderr
+ self._tty = (
+ tty if tty is not None else bool(getattr(self._stream, "isatty", lambda: False)())
+ )
+ self._colors = AnsiColors.for_stream(color_mode, self._stream)
+ self._refresh_interval = refresh_interval
+ self._heartbeat_interval = heartbeat_interval
+ self._answer_now_hint = answer_now_hint
+ self._lock = threading.Lock()
+ self._stop_event = threading.Event()
+ self._thread: threading.Thread | None = None
+ self._started_at: float | None = None
+ self._last_heartbeat_at: float | None = None
+ self._last_line_len = 0
+ self._phase = "analyzing"
+ self._detail: str | None = None
+ self._activity: str | None = None
+ self._activity_detail: str | None = None
+ self._current: int | None = None
+ self._total: int | None = None
+ self._result = InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=0,
+ variant_edges=0,
+ omission_findings=0,
+ )
+ self._finished = False
+ self._last_tty_lines = 0
+
+ def start(self, total_steps: int) -> None:
+ """Begin insight analysis progress."""
+ if not self._enabled:
+ return
+ started_now = self._ensure_started(
+ "analyzing",
+ current=0,
+ total=total_steps,
+ detail=f"{total_steps} steps",
+ )
+ if started_now:
+ if self._tty:
+ self._ensure_tty_thread()
+ else:
+ self._emit_line(self._start_line())
+
+ def step_started(
+ self,
+ index: int,
+ total: int,
+ step: str,
+ result: InsightsAnalyzeProgressResult,
+ ) -> None:
+ """Report that one insight analysis step is starting."""
+ if not self._enabled:
+ return
+ self._update_result(result)
+ self.set_status(
+ "analyzing",
+ current=index,
+ total=total,
+ detail=step,
+ )
+ self.set_activity("preparing insight step", detail=step)
+
+ def step_finished(
+ self,
+ index: int,
+ total: int,
+ step: str,
+ result: InsightsAnalyzeProgressResult,
+ ) -> None:
+ """Report that one insight analysis step has finished."""
+ if not self._enabled:
+ return
+ self._update_result(result)
+ self.set_status(
+ "analyzing",
+ current=index,
+ total=total,
+ detail=f"{step} complete",
+ )
+ self.set_activity("completed insight step", detail=step)
+
+ def set_status(
+ self,
+ phase: str,
+ *,
+ current: int | None = None,
+ total: int | None = None,
+ detail: str | None = None,
+ ) -> None:
+ """Update the current progress status."""
+ if not self._enabled:
+ return
+ with self._lock:
+ self._phase = phase
+ self._current = current
+ self._total = total
+ self._detail = detail
+ self._emit_heartbeat_if_due()
+
+ def set_activity(self, activity: str, *, detail: str | None = None) -> None:
+ """Update the current backend activity."""
+ if not self._enabled:
+ return
+ with self._lock:
+ self._activity = activity
+ self._activity_detail = detail
+ self._emit_heartbeat_if_due()
+
+ def finish(self, result: InsightsAnalyzeProgressResult) -> None:
+ """Finish progress reporting after a complete analysis."""
+ if not self._enabled:
+ return
+ self._update_result(result)
+ with self._lock:
+ self._phase = "complete"
+ self._finished = True
+ if self._tty:
+ self._stop_tty_thread()
+ self._clear_tty_line()
+ return
+ self._emit_line(self._finish_line(result))
+
+ def exiting_early(self, result: InsightsAnalyzeProgressResult) -> None:
+ """Finish progress reporting after cooperative early exit."""
+ if not self._enabled:
+ return
+ self._update_result(result)
+ with self._lock:
+ self._phase = "exiting early"
+ self._finished = True
+ line = self._exiting_early_line(result)
+ if self._tty:
+ self._stop_tty_thread()
+ self._write_tty_line(line)
+ return
+ self._emit_line(line)
+
+ def interrupt(self) -> None:
+ """Stop progress rendering while preserving the current status."""
+ if not self._enabled:
+ return
+ if self._tty:
+ self._stop_tty_thread()
+ self._write_tty_summary_line()
+ return
+ self._emit_line(self._summary())
+
+ def close(self) -> None:
+ """Stop any active progress renderer."""
+ if not self._enabled:
+ return
+ if self._tty:
+ self._stop_tty_thread()
+ if not self._finished:
+ self._clear_tty_line()
+
+ def _ensure_started(
+ self,
+ phase: str,
+ *,
+ current: int | None = None,
+ total: int | None = None,
+ detail: str | None = None,
+ ) -> bool:
+ now = time.monotonic()
+ with self._lock:
+ already_started = self._started_at is not None
+ if not already_started:
+ self._started_at = now
+ self._last_heartbeat_at = now
+ self._finished = False
+ self._phase = phase
+ self._current = current
+ self._total = total
+ self._detail = detail
+ if not already_started and self._tty:
+ self._ensure_tty_thread()
+ return not already_started
+
+ def _update_result(self, result: InsightsAnalyzeProgressResult) -> None:
+ with self._lock:
+ self._result = result
+
+ def _ensure_tty_thread(self) -> None:
+ if self._thread is not None and self._thread.is_alive():
+ return
+ self._stop_event.clear()
+ self._thread = threading.Thread(
+ target=self._tty_loop,
+ daemon=True,
+ name="agentgrep-insights-analyze-progress",
+ )
+ self._thread.start()
+
+ def _stop_tty_thread(self) -> None:
+ self._stop_event.set()
+ thread = self._thread
+ self._thread = None
+ if thread is not None:
+ thread.join(timeout=1.0)
+
+ def _tty_loop(self) -> None:
+ frames = itertools.cycle(self._SPINNER_FRAMES)
+ while not self._stop_event.is_set():
+ self._render_tty(next(frames))
+ self._stop_event.wait(self._refresh_interval)
+
+ def _render_tty(self, frame: str) -> None:
+ frame_text = self._colors.info(frame)
+ terminal_width = self._terminal_width()
+ summary_lines = list(self._summary_lines(max_width=terminal_width))
+ if summary_lines:
+ first_width = max(1, terminal_width - _visible_width(frame_text) - 1)
+ summary_lines[0] = _hard_truncate_ansi(summary_lines[0], first_width)
+ summary_lines[0] = f"{frame_text} {summary_lines[0]}"
+ else:
+ summary_lines = [frame_text]
+ with self._lock:
+ try:
+ self._clear_tty_block_locked()
+ self._stream.write("\n".join(summary_lines))
+ self._stream.flush()
+ self._last_line_len = len(summary_lines[-1])
+ self._last_tty_lines = len(summary_lines)
+ except OSError, ValueError:
+ pass
+
+ def _clear_tty_line(self) -> None:
+ with self._lock:
+ self._clear_tty_block_locked()
+ self._last_line_len = 0
+ self._last_tty_lines = 0
+
+ def _clear_tty_block_locked(self) -> None:
+ if self._last_tty_lines <= 0:
+ return
+ if self._last_tty_lines > 1:
+ self._stream.write(f"\033[{self._last_tty_lines - 1}A")
+ self._stream.write("\r")
+ for index in range(self._last_tty_lines):
+ self._stream.write("\033[2K")
+ if index < self._last_tty_lines - 1:
+ self._stream.write("\033[1B")
+ if self._last_tty_lines > 1:
+ self._stream.write(f"\033[{self._last_tty_lines - 1}A")
+ self._stream.write("\r")
+
+ def _write_tty_summary_line(self) -> None:
+ lines = self._summary_lines(max_width=self._terminal_width())
+ self._write_tty_lines(lines)
+
+ def _write_tty_line(self, line: str) -> None:
+ self._write_tty_lines((line,))
+
+ def _write_tty_lines(self, lines: tuple[str, ...]) -> None:
+ with self._lock:
+ try:
+ self._clear_tty_block_locked()
+ self._stream.write("\n".join(lines) + "\n")
+ self._stream.flush()
+ except OSError, ValueError:
+ pass
+ self._last_line_len = 0
+ self._last_tty_lines = 0
+
+ def _emit_heartbeat_if_due(self) -> None:
+ if not self._enabled or self._tty:
+ return
+ with self._lock:
+ last = self._last_heartbeat_at
+ if last is None:
+ return
+ now = time.monotonic()
+ if now - last < self._heartbeat_interval:
+ return
+ elapsed = self._elapsed_seconds()
+ self._emit_line(self._heartbeat_line(elapsed))
+ with self._lock:
+ self._last_heartbeat_at = now
+
+ def _emit_line(self, line: str) -> None:
+ try:
+ self._stream.write(line + "\n")
+ self._stream.flush()
+ except OSError, ValueError:
+ pass
+
+ def _summary(self, *, max_width: int | None = None) -> str:
+ return " / ".join(self._summary_lines(max_width=max_width))
+
+ def _summary_lines(self, *, max_width: int | None = None) -> tuple[str, ...]:
+ return format_insights_analyze_progress_lines(
+ self._snapshot(),
+ colors=self._colors,
+ answer_now_hint=self._answer_now_hint,
+ max_width=max_width,
+ )
+
+ def _terminal_width(self) -> int:
+ try:
+ columns = os.get_terminal_size(self._stream.fileno()).columns
+ except AttributeError, OSError, TypeError, ValueError:
+ columns = 0
+ if columns >= 20:
+ return columns
+ return max(20, shutil.get_terminal_size(fallback=(80, 24)).columns)
+
+ def _snapshot(self) -> InsightsAnalyzeProgressSnapshot:
+ elapsed = self._elapsed_seconds()
+ with self._lock:
+ return InsightsAnalyzeProgressSnapshot(
+ phase=self._phase,
+ current=self._current,
+ total=self._total,
+ detail=self._detail,
+ activity=self._activity,
+ activity_detail=self._activity_detail,
+ result=self._result,
+ elapsed=elapsed,
+ )
+
+ def _start_line(self) -> str:
+ return f"{self._colors.heading('Insights analyze')} {self._colors.muted('starting')}"
+
+ def _heartbeat_line(self, elapsed: float) -> str:
+ prefix = f"{self._colors.muted('...')} {self._colors.heading('still analyzing')}"
+ elapsed_text = self._colors.muted(f"{elapsed:.0f}s elapsed")
+ return f"{prefix}: {self._summary()} ({elapsed_text})"
+
+ def _finish_line(self, result: InsightsAnalyzeProgressResult) -> str:
+ return (
+ f"{self._colors.success('Analyze complete:')} "
+ f"{self._colors.warning(format_insights_run_count(result.runs_analyzed))}, "
+ f"{self._colors.warning(format_insights_feature_count(result.features_refreshed))}, "
+ f"{self._colors.warning(format_insights_cluster_count(result.clusters))}, "
+ f"{self._colors.warning(format_insights_variant_count(result.variant_edges))}, "
+ f"{self._colors.warning(format_insights_omission_count(result.omission_findings))} "
+ f"({self._colors.muted(f'{self._elapsed_seconds():.1f}s elapsed')})"
+ )
+
+ def _exiting_early_line(self, result: InsightsAnalyzeProgressResult) -> str:
+ feature_count = format_insights_feature_count(result.features_refreshed)
+ parts = [
+ (
+ f"{self._colors.success('Exiting early:')} "
+ f"{self._colors.warning(format_insights_run_count(result.runs_analyzed))}, "
+ f"{self._colors.warning(feature_count)}, "
+ f"{self._colors.warning(format_insights_variant_count(result.variant_edges))}"
+ ),
+ ]
+ if self._answer_now_hint:
+ parts.append(self._colors.white("[Press enter, exit early]"))
+ return " | ".join(parts)
+
+ def _elapsed_seconds(self) -> float:
+ with self._lock:
+ started = self._started_at
+ if started is None:
+ return 0.0
+ return time.monotonic() - started
+
+
+def format_insights_run_count(count: int) -> str:
+ """Return a human-readable insight-run count."""
+ suffix = "run analyzed" if count == 1 else "runs analyzed"
+ return f"{count} {suffix}"
+
+
+def format_insights_feature_count(count: int) -> str:
+ """Return a human-readable refreshed-feature count."""
+ suffix = "feature refreshed" if count == 1 else "features refreshed"
+ return f"{count} {suffix}"
+
+
+def format_insights_cluster_count(count: int) -> str:
+ """Return a human-readable cluster count."""
+ suffix = "cluster" if count == 1 else "clusters"
+ return f"{count} {suffix}"
+
+
+def format_insights_variant_count(count: int) -> str:
+ """Return a human-readable variant-edge count."""
+ suffix = "variant edge" if count == 1 else "variant edges"
+ return f"{count} {suffix}"
+
+
+def format_insights_omission_count(count: int) -> str:
+ """Return a human-readable omission-finding count."""
+ suffix = "omission finding" if count == 1 else "omission findings"
+ return f"{count} {suffix}"
+
+
+def format_insights_analyze_progress_line(
+ snapshot: InsightsAnalyzeProgressSnapshot,
+ *,
+ colors: SearchColors,
+ answer_now_hint: bool = False,
+ max_width: int | None = None,
+) -> str:
+ """Format the single-line insights analyze progress summary."""
+ return " / ".join(
+ format_insights_analyze_progress_lines(
+ snapshot,
+ colors=colors,
+ answer_now_hint=answer_now_hint,
+ max_width=max_width,
+ ),
+ )
+
+
+def format_insights_analyze_progress_lines(
+ snapshot: InsightsAnalyzeProgressSnapshot,
+ *,
+ colors: SearchColors,
+ answer_now_hint: bool = False,
+ max_width: int | None = None,
+) -> tuple[str, ...]:
+ """Format multi-line insights analyze progress."""
+ lines = _format_insights_analyze_progress_lines(
+ snapshot,
+ colors=colors,
+ answer_now_hint=answer_now_hint,
+ include_detail=True,
+ )
+ if max_width is None:
+ return lines
+ return tuple(_hard_truncate_ansi(line, max_width) for line in lines)
+
+
+def _format_insights_analyze_progress_lines(
+ snapshot: InsightsAnalyzeProgressSnapshot,
+ *,
+ colors: SearchColors,
+ answer_now_hint: bool,
+ include_detail: bool,
+) -> tuple[str, ...]:
+ """Build one insights analyze progress-line variant."""
+ label_part = colors.heading("Insights analyze")
+ detail_part = colors.muted(snapshot.detail) if include_detail and snapshot.detail else None
+ if snapshot.current is not None and snapshot.total is not None:
+ count = colors.warning(f"{snapshot.current}/{snapshot.total}")
+ status_part = f"{colors.heading(snapshot.phase)} {count} {colors.muted('steps')}"
+ elif include_detail and snapshot.detail:
+ status_part = f"{colors.heading(snapshot.phase)} {colors.muted(snapshot.detail)}"
+ detail_part = None
+ else:
+ status_part = colors.heading(snapshot.phase)
+ result = snapshot.result
+ status_parts = [
+ label_part,
+ status_part,
+ ]
+ if detail_part:
+ status_parts.append(detail_part)
+ status_parts.append(colors.muted(f"{snapshot.elapsed:.1f}s"))
+ if answer_now_hint:
+ status_parts.append(colors.white("[Press enter, exit early]"))
+ lines = [" | ".join(status_parts)]
+ if snapshot.activity is not None:
+ activity_parts = [
+ colors.heading("Doing"),
+ colors.warning(snapshot.activity),
+ ]
+ if include_detail and snapshot.activity_detail:
+ activity_parts.append(colors.muted(snapshot.activity_detail))
+ lines.append(" | ".join(activity_parts))
+ if _has_insights_analyze_progress_result(result):
+ lines.append(
+ " | ".join(
+ (
+ colors.heading("Results"),
+ colors.warning(format_insights_run_count(result.runs_analyzed)),
+ colors.warning(format_insights_feature_count(result.features_refreshed)),
+ colors.warning(format_insights_cluster_count(result.clusters)),
+ colors.warning(format_insights_variant_count(result.variant_edges)),
+ colors.warning(format_insights_omission_count(result.omission_findings)),
+ ),
+ ),
+ )
+ return tuple(lines)
+
+
+def _has_insights_analyze_progress_result(result: InsightsAnalyzeProgressResult) -> bool:
+ """Return whether aggregate insight counters carry completed work."""
+ return (
+ result.runs_analyzed > 0
+ or result.features_refreshed > 0
+ or result.clusters > 0
+ or result.variant_edges > 0
+ or result.omission_findings > 0
+ )
+
+
+def _empty_insights_analyze_progress_result() -> InsightsAnalyzeProgressResult:
+ """Return zeroed aggregate counters for insights analyze progress."""
+ return InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=0,
+ variant_edges=0,
+ omission_findings=0,
+ )
+
+
+def _insight_counter(result: object, name: str) -> int:
+ """Read an integer counter from dataclass or mapping insight results."""
+ value: object
+ if dataclasses.is_dataclass(result) and not isinstance(result, type):
+ value = getattr(result, name, 0)
+ elif isinstance(result, cabc.Mapping):
+ mapping = t.cast("cabc.Mapping[str, object]", result)
+ value = mapping.get(name, 0)
+ else:
+ return 0
+ if isinstance(value, bool):
+ return int(value)
+ if isinstance(value, int):
+ return value
+ if isinstance(value, float):
+ return int(value)
+ if isinstance(value, str):
+ try:
+ return int(value)
+ except ValueError:
+ return 0
+ return 0
+
+
+def _add_insight_result(
+ aggregate: InsightsAnalyzeProgressResult,
+ result: object,
+) -> InsightsAnalyzeProgressResult:
+ """Add one insight result into aggregate analyze counters."""
+ return InsightsAnalyzeProgressResult(
+ runs_analyzed=aggregate.runs_analyzed + 1,
+ features_refreshed=aggregate.features_refreshed
+ + _insight_counter(result, "features_refreshed"),
+ clusters=aggregate.clusters + _insight_counter(result, "clusters"),
+ variant_edges=aggregate.variant_edges + _insight_counter(result, "variant_edges"),
+ omission_findings=aggregate.omission_findings
+ + _insight_counter(result, "omission_findings"),
+ )
+
+
+def _insights_analyze_payload(results: list[object]) -> object:
+ """Return the public analyze payload for one or multiple insight results."""
+ return results[0] if len(results) == 1 else results
+
+
+def _insights_list_payload(
+ engine: object,
+ *,
+ kind: InsightsKind,
+ limit: int,
+) -> dict[str, object]:
+ """Return a bounded persisted-insights listing payload."""
+ payload: dict[str, object] = {"limit": limit}
+ typed = t.cast("t.Any", engine)
+ if kind in {"similarity", "all"}:
+ total = int(typed.count_variant_edges())
+ items = typed.list_variant_edges(limit=limit)
+ payload["variant_edges"] = {
+ "total": total,
+ "returned": len(items),
+ "truncated": total > len(items),
+ "items": items,
+ }
+ if kind in {"omissions", "all"}:
+ total = int(typed.count_omission_findings())
+ items = typed.list_omission_findings(limit=limit)
+ payload["omission_findings"] = {
+ "total": total,
+ "returned": len(items),
+ "truncated": total > len(items),
+ "items": items,
+ }
+ return payload
+
+
+def _empty_insights_list_payload(*, kind: InsightsKind, limit: int) -> dict[str, object]:
+ """Return the zero-rows listing payload for a missing cache."""
+ payload: dict[str, object] = {"limit": limit}
+ empty: dict[str, object] = {
+ "total": 0,
+ "returned": 0,
+ "truncated": False,
+ "items": [],
+ }
+ if kind in {"similarity", "all"}:
+ payload["variant_edges"] = dict(empty)
+ if kind in {"omissions", "all"}:
+ payload["omission_findings"] = dict(empty)
+ return payload
+
+
+def _insights_explain_payload(engine: object) -> dict[str, int]:
+ """Return cheap persisted-insight counters."""
+ typed = t.cast("t.Any", engine)
+ return {
+ "variant_edges": int(typed.count_variant_edges()),
+ "omission_findings": int(typed.count_omission_findings()),
+ }
+
+
+def _run_readonly_db_action(
+ db_path: str | None,
+ action: cabc.Callable[[DbRuntime], int],
+ *,
+ on_missing: cabc.Callable[[], int],
+) -> int:
+ """Run a read action against a read-only DB open, without cache writes.
+
+ Mirrors the db status path: a missing cache reports an empty payload
+ without creating the file, and a foreign file fails cleanly.
+ """
+ import sqlite3
+
+ from agentgrep.db import DbRuntime, default_db_path
+
+ path = default_db_path() if db_path is None else pathlib.Path(db_path).expanduser()
+ if not path.exists():
+ return on_missing()
+ try:
+ with DbRuntime.open_readonly(path) as runtime:
+ return action(runtime)
+ except sqlite3.DatabaseError:
+ print(f"agentgrep: not an agentgrep database: {path}", file=sys.stderr)
+ return 1
+
+
+def run_insights_command(args: InsightsArgs) -> int:
+ """Execute ``agentgrep insights`` subcommands."""
+ if args.action in {"list", "explain"}:
+
+ def list_action(runtime: DbRuntime) -> int:
+ return _run_insights_command_with_runtime(args, runtime)
+
+ def empty_listing() -> int:
+ payload: object
+ if args.action == "explain":
+ payload = {"variant_edges": 0, "omission_findings": 0}
+ else:
+ payload = _empty_insights_list_payload(kind=args.kind, limit=args.limit)
+ _print_json_or_text(
+ payload,
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+
+ return _run_readonly_db_action(args.db_path, list_action, on_missing=empty_listing)
+ runtime = _open_db_runtime(args.db_path)
+ try:
+ return _run_insights_command_with_runtime(args, runtime)
+ finally:
+ runtime.close()
+
+
+def _run_insights_command_with_runtime(args: InsightsArgs, runtime: DbRuntime) -> int:
+ """Execute one insights action against an already-open runtime."""
+ from agentgrep.insights import InsightEngine
+
+ engine = InsightEngine(runtime.store)
+ if args.action == "analyze":
+ control = SearchControl()
+ human_output = args.output_mode == "text"
+ progress_enabled = args.progress_mode == "always" or (
+ args.progress_mode == "auto" and human_output
+ )
+ answer_now_enabled = (
+ progress_enabled
+ and human_output
+ and bool(getattr(sys.stdin, "isatty", lambda: False)())
+ and bool(getattr(sys.stderr, "isatty", lambda: False)())
+ )
+ progress = (
+ ConsoleInsightsAnalyzeProgress(
+ enabled=True,
+ color_mode=args.color_mode,
+ answer_now_hint=answer_now_enabled,
+ )
+ if progress_enabled
+ else None
+ )
+ listener = AnswerNowInputListener(control) if answer_now_enabled else None
+ results: list[object] = []
+ aggregate = _empty_insights_analyze_progress_result()
+ jobs: list[tuple[str, cabc.Callable[[], object]]] = []
+ if args.kind in {"similarity", "all"}:
+ jobs.append(
+ (
+ "similarity",
+ lambda: engine.run_similarity(control=control, progress=progress),
+ ),
+ )
+ if args.kind in {"omissions", "all"} and args.target is not None:
+ target_path = pathlib.Path(args.target)
+ target_text = target_path.read_text(encoding="utf-8")
+ jobs.append(
+ (
+ "omissions",
+ lambda: engine.run_omissions(
+ target_path=target_path,
+ target_text=target_text,
+ control=control,
+ progress=progress,
+ ),
+ ),
+ )
+ if listener is not None:
+ listener.start()
+ if progress is not None:
+ progress.start(len(jobs))
+ exited_early = False
+ try:
+ for index, (label, job) in enumerate(jobs, start=1):
+ if control.answer_now_requested():
+ exited_early = True
+ if progress is not None:
+ progress.exiting_early(aggregate)
+ break
+ if progress is not None:
+ progress.step_started(index, len(jobs), label, aggregate)
+ result = job()
+ results.append(result)
+ aggregate = _add_insight_result(aggregate, result)
+ if progress is not None:
+ progress.step_finished(index, len(jobs), label, aggregate)
+ if control.answer_now_requested():
+ exited_early = True
+ if progress is not None:
+ progress.exiting_early(aggregate)
+ break
+ if not exited_early and progress is not None:
+ progress.finish(aggregate)
+ except KeyboardInterrupt:
+ if progress is not None:
+ progress.interrupt()
+ raise
+ finally:
+ if listener is not None:
+ listener.stop()
+ if progress is not None:
+ progress.close()
+ _print_json_or_text(
+ _insights_analyze_payload(results),
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+ if args.action == "explain":
+ _print_json_or_text(
+ _insights_explain_payload(engine),
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+ _print_json_or_text(
+ _insights_list_payload(engine, kind=args.kind, limit=args.limit),
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+
+
+def run_suggestions_command(args: SuggestionsArgs) -> int:
+ """Execute ``agentgrep suggestions`` subcommands."""
+ writes = args.action == "list" and args.target is not None
+ if not writes:
+
+ def read_action(runtime: DbRuntime) -> int:
+ return _run_suggestions_command_with_runtime(args, runtime)
+
+ def empty_result() -> int:
+ if args.action == "list":
+ _print_json_or_text(
+ {
+ "limit": args.limit,
+ "suggestions": {
+ "total": 0,
+ "returned": 0,
+ "truncated": False,
+ "items": [],
+ },
+ },
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+ return 1
+
+ return _run_readonly_db_action(args.db_path, read_action, on_missing=empty_result)
+ runtime = _open_db_runtime(args.db_path)
+ try:
+ return _run_suggestions_command_with_runtime(args, runtime)
+ finally:
+ runtime.close()
+
+
+def _run_suggestions_command_with_runtime(args: SuggestionsArgs, runtime: DbRuntime) -> int:
+ """Execute one suggestions action against an already-open runtime."""
+ from agentgrep.suggestions import SuggestionEngine
+
+ engine = SuggestionEngine(runtime.store)
+ if args.action == "list":
+ if args.target is not None:
+ _ = engine.create_from_omissions(target_path=pathlib.Path(args.target))
+ total = engine.count_suggestions()
+ items = engine.list_suggestions(limit=args.limit)
+ _print_json_or_text(
+ {
+ "limit": args.limit,
+ "suggestions": {
+ "total": total,
+ "returned": len(items),
+ "truncated": total > len(items),
+ "items": items,
+ },
+ },
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
+ if args.suggestion_id is None:
+ msg = "suggestion id is required"
+ raise SystemExit(msg)
+ if args.action == "render" and args.output_mode != "json":
+ rendered = engine.render_suggestion(args.suggestion_id)
+ if rendered is None:
+ return 1
+ print(rendered)
+ return 0
+ suggestion = engine.get_suggestion(args.suggestion_id)
+ if suggestion is None:
+ return 1
+ _print_json_or_text(
+ suggestion,
+ output_mode=args.output_mode,
+ color_mode=args.color_mode,
+ )
+ return 0
diff --git a/src/agentgrep/db.py b/src/agentgrep/db.py
index 91769aeca..237c36110 100644
--- a/src/agentgrep/db.py
+++ b/src/agentgrep/db.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import collections.abc as cabc
+import concurrent.futures
import dataclasses
import datetime
import hashlib
@@ -42,6 +43,7 @@
logger = logging.getLogger(__name__)
CacheMode = t.Literal["auto", "require", "off"]
+FeatureMode = t.Literal["defer", "inline"]
SCHEMA_VERSION = 1
DEFAULT_DB_FILENAME = "agentgrep.sqlite"
@@ -49,6 +51,14 @@
#: limit-50 searches at every measured term frequency.
_PROBE_WINDOW_FLOOR = 200
_TOKEN_RE = re.compile(r"[a-z0-9_./:-]+")
+_FEATURE_PROGRESS_INTERVAL = 1024
+
+
+class FeatureRefreshProgress(t.Protocol):
+ """Progress sink for deterministic feature-refresh phases."""
+
+ def set_activity(self, activity: str, *, detail: str | None = None) -> None:
+ """Report the current feature-refresh activity."""
@dataclasses.dataclass(frozen=True, slots=True)
@@ -59,6 +69,10 @@ class DbStatus:
schema_version: int
sources: int
records: int
+ features: int
+ variant_edges: int
+ omission_findings: int
+ suggestions: int
@dataclasses.dataclass(frozen=True, slots=True)
@@ -91,6 +105,7 @@ class SyncResult:
records_removed: int
sources_skipped: int = 0
sources_pruned: int = 0
+ features_deferred: int = 0
@dataclasses.dataclass(frozen=True, slots=True)
@@ -117,6 +132,48 @@ class DbRecordRow:
record: SearchRecord
+@dataclasses.dataclass(frozen=True, slots=True)
+class DbSimilarityRow:
+ """One record id plus its normalized text hash for similarity analysis."""
+
+ record_id: str
+ normalized_hash: str
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class DbFeatureRow:
+ """One record's persisted MinHash sketch plus its text for near-dup analysis."""
+
+ record_id: str
+ normalized_hash: str
+ minhash_signature: tuple[str, ...]
+ text: str
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class _FeatureInput:
+ """Minimal pickleable input for deterministic feature building."""
+
+ record_id: str
+ text: str
+ normalized_hash: str
+ timestamp: str | None
+ updated_at: str
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class _FeatureValues:
+ """Feature row values ready for SQLite insertion."""
+
+ record_id: str
+ normalized_hash: str
+ simhash_hex: str
+ minhash_json: str
+ token_count: int
+ quality_flags_json: str
+ updated_at: str
+
+
class DbQueryUnsupportedError(RuntimeError):
"""Raised when a query cannot be answered from the DB index."""
@@ -451,7 +508,7 @@ def _sql_explain_enabled() -> bool:
class DbStore:
- """SQLite-backed store for the persistent DB index."""
+ """SQLite-backed store for DB, insight, and suggestion artifacts."""
def __init__(self, db_path: pathlib.Path, *, readonly: bool = False) -> None:
self.db_path = db_path
@@ -656,6 +713,14 @@ def _migrate(self) -> None:
self._executescript(
"schema.drop",
"""
+ DROP TABLE IF EXISTS suggestion_evidence;
+ DROP TABLE IF EXISTS suggestions;
+ DROP TABLE IF EXISTS omission_findings;
+ DROP TABLE IF EXISTS variant_edges;
+ DROP TABLE IF EXISTS cluster_members;
+ DROP TABLE IF EXISTS clusters;
+ DROP TABLE IF EXISTS insight_runs;
+ DROP TABLE IF EXISTS record_features;
DROP TABLE IF EXISTS record_text_fts;
DROP TABLE IF EXISTS source_state;
DROP TABLE IF EXISTS record_details;
@@ -732,6 +797,101 @@ def _migrate(self) -> None:
CREATE INDEX IF NOT EXISTS idx_records_search_source_id
ON records_search(source_id);
+ CREATE TABLE IF NOT EXISTS record_features (
+ record_id TEXT PRIMARY KEY
+ REFERENCES records_search(record_id) ON DELETE CASCADE,
+ normalized_hash TEXT NOT NULL,
+ simhash_hex TEXT NOT NULL,
+ minhash_json TEXT NOT NULL,
+ token_count INTEGER NOT NULL,
+ quality_flags_json TEXT NOT NULL,
+ updated_at TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS insight_runs (
+ run_id TEXT PRIMARY KEY,
+ kind TEXT NOT NULL,
+ started_at TEXT NOT NULL,
+ finished_at TEXT NOT NULL,
+ status TEXT NOT NULL,
+ algorithm_version TEXT NOT NULL,
+ input_json TEXT NOT NULL,
+ counters_json TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS clusters (
+ cluster_id TEXT PRIMARY KEY,
+ run_id TEXT NOT NULL REFERENCES insight_runs(run_id) ON DELETE CASCADE,
+ kind TEXT NOT NULL,
+ label TEXT NOT NULL,
+ centroid_record_id TEXT,
+ confidence REAL NOT NULL,
+ evidence_json TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS cluster_members (
+ cluster_id TEXT NOT NULL REFERENCES clusters(cluster_id) ON DELETE CASCADE,
+ record_id TEXT NOT NULL REFERENCES records_search(record_id) ON DELETE CASCADE,
+ score REAL NOT NULL,
+ signals_json TEXT NOT NULL,
+ PRIMARY KEY(cluster_id, record_id)
+ );
+
+ CREATE TABLE IF NOT EXISTS variant_edges (
+ edge_id TEXT PRIMARY KEY,
+ run_id TEXT NOT NULL REFERENCES insight_runs(run_id) ON DELETE CASCADE,
+ left_record_id TEXT NOT NULL
+ REFERENCES records_search(record_id) ON DELETE CASCADE,
+ right_record_id TEXT NOT NULL
+ REFERENCES records_search(record_id) ON DELETE CASCADE,
+ variant_type TEXT NOT NULL,
+ confidence REAL NOT NULL,
+ signals_json TEXT NOT NULL,
+ explanation TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS omission_findings (
+ finding_id TEXT PRIMARY KEY,
+ run_id TEXT NOT NULL REFERENCES insight_runs(run_id) ON DELETE CASCADE,
+ target_path TEXT NOT NULL,
+ cluster_id TEXT,
+ representative_record_id TEXT NOT NULL
+ REFERENCES records_search(record_id) ON DELETE CASCADE,
+ confidence REAL NOT NULL,
+ status TEXT NOT NULL,
+ evidence_json TEXT NOT NULL,
+ rationale TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS suggestions (
+ suggestion_id TEXT PRIMARY KEY,
+ run_id TEXT NOT NULL REFERENCES insight_runs(run_id) ON DELETE CASCADE,
+ target_path TEXT NOT NULL,
+ surface_kind TEXT NOT NULL,
+ title TEXT NOT NULL,
+ body TEXT NOT NULL,
+ confidence REAL NOT NULL,
+ status TEXT NOT NULL,
+ rationale TEXT NOT NULL,
+ reload_note TEXT NOT NULL,
+ created_at TEXT NOT NULL
+ );
+
+ CREATE TABLE IF NOT EXISTS suggestion_evidence (
+ suggestion_id TEXT NOT NULL
+ REFERENCES suggestions(suggestion_id) ON DELETE CASCADE,
+ record_id TEXT NOT NULL REFERENCES records_search(record_id) ON DELETE CASCADE,
+ evidence_role TEXT NOT NULL,
+ score REAL NOT NULL,
+ signals_json TEXT NOT NULL,
+ PRIMARY KEY(suggestion_id, record_id, evidence_role)
+ );
+
+ CREATE INDEX IF NOT EXISTS idx_variant_edges_confidence_edge_id
+ ON variant_edges(confidence DESC, edge_id);
+
+ CREATE INDEX IF NOT EXISTS idx_suggestions_confidence_suggestion_id
+ ON suggestions(confidence DESC, suggestion_id);
""",
)
_ = self._execute(
@@ -748,6 +908,10 @@ def status(self) -> DbStatus:
schema_version=SCHEMA_VERSION,
sources=self._count("sources"),
records=self._count("records_search"),
+ features=self._count("record_features"),
+ variant_edges=self._count("variant_edges"),
+ omission_findings=self._count("omission_findings"),
+ suggestions=self._count("suggestions"),
)
finally:
self._flush_sql_samples()
@@ -862,13 +1026,15 @@ def replace_source_records(
self,
source: SourceHandle,
records: cabc.Iterable[SearchRecord],
- ) -> tuple[int, int]:
+ *,
+ features_mode: FeatureMode = "defer",
+ ) -> tuple[int, int, int]:
"""Replace every indexed record for ``source``.
Returns
-------
- tuple[int, int]
- ``(records_indexed, records_removed)``.
+ tuple[int, int, int]
+ ``(records_indexed, records_removed, features_deferred)``.
"""
source_id = source_id_for(source)
now = _now_iso()
@@ -879,6 +1045,7 @@ def replace_source_records(
removed = self._remove_source_records(source_id)
indexed = 0
seen_record_ids: dict[str, int] = {}
+ features_deferred = 0
for record in record_list:
raw_hash = text_hash(record.text)
normalized_text = normalize_record_text(record.text)
@@ -901,9 +1068,13 @@ def replace_source_records(
record_id=record_id,
now=now,
raw_hash=raw_hash,
+ normalized_text=normalized_text,
normalized_hash=normalized_hash,
+ features_mode=features_mode,
)
indexed += 1
+ if features_mode == "defer":
+ features_deferred += 1
_ = self._execute(
"source_state.upsert",
"""
@@ -915,7 +1086,7 @@ def replace_source_records(
""",
(source_id, source.mtime_ns, fingerprint, now),
)
- return indexed, removed
+ return indexed, removed, features_deferred
def source_is_current(self, source: SourceHandle) -> bool:
"""Return whether ``source`` has an up-to-date successful sync state.
@@ -1050,9 +1221,11 @@ def _insert_record(
record_id: str,
now: str,
raw_hash: str,
+ normalized_text: str,
normalized_hash: str,
+ features_mode: FeatureMode,
) -> str:
- """Insert one normalized record across the search/details/FTS surfaces."""
+ """Insert one normalized record across the search, details, FTS, and feature surfaces."""
haystack = build_record_match_surface(record, "haystack").casefold()
cursor = self._execute(
"records_search.insert",
@@ -1105,6 +1278,20 @@ def _insert_record(
"INSERT INTO record_text_fts(rowid, haystack) VALUES(?, ?)",
(rowid, haystack),
)
+ if features_mode == "defer":
+ return record_id
+ self._insert_feature_values(
+ _feature_values_for_input(
+ _FeatureInput(
+ record_id=record_id,
+ text=record.text,
+ normalized_hash=normalized_hash,
+ timestamp=record.timestamp,
+ updated_at=now,
+ ),
+ normalized_text=normalized_text,
+ ),
+ )
return record_id
def _scope_catalog(self) -> tuple[frozenset[str], frozenset[tuple[str, str]]]:
@@ -1133,6 +1320,87 @@ def _scope_catalog(self) -> tuple[frozenset[str], frozenset[tuple[str, str]]]:
conversation_pairs.add((store, adapter_id))
return frozenset(prompt_history_agents), frozenset(conversation_pairs)
+ def _insert_feature_values(self, values: _FeatureValues) -> None:
+ """Insert one precomputed feature row."""
+ self.connection.execute(
+ """
+ INSERT INTO record_features(
+ record_id, normalized_hash, simhash_hex, minhash_json,
+ token_count, quality_flags_json, updated_at
+ )
+ VALUES(?, ?, ?, ?, ?, ?, ?)
+ """,
+ (
+ values.record_id,
+ values.normalized_hash,
+ values.simhash_hex,
+ values.minhash_json,
+ values.token_count,
+ values.quality_flags_json,
+ values.updated_at,
+ ),
+ )
+
+ def refresh_missing_features(
+ self,
+ *,
+ limit: int | None = None,
+ workers: int | None = None,
+ progress: FeatureRefreshProgress | None = None,
+ ) -> int:
+ """Build deterministic feature rows missing from deferred syncs."""
+ if progress is not None:
+ progress.set_activity(
+ "checking feature cache",
+ detail="querying records missing deterministic features",
+ )
+ sql = """
+ SELECT r.record_id, d.text, r.normalized_text_hash, r.timestamp
+ FROM records_search r
+ JOIN record_details d ON d.rowid = r.rowid
+ LEFT JOIN record_features f ON f.record_id = r.record_id
+ WHERE f.record_id IS NULL
+ ORDER BY r.rowid
+ """
+ params: tuple[object, ...] = ()
+ if limit is not None:
+ sql += " LIMIT ?"
+ params = (limit,)
+ rows = self.connection.execute(sql, params).fetchall()
+ if not rows:
+ if progress is not None:
+ progress.set_activity(
+ "checking feature cache",
+ detail="feature cache already complete",
+ )
+ return 0
+ now = _now_iso()
+ inputs = tuple(
+ _FeatureInput(
+ record_id=str(row["record_id"]),
+ text=str(row["text"]),
+ normalized_hash=str(row["normalized_text_hash"]),
+ timestamp=t.cast("str | None", row["timestamp"]),
+ updated_at=now,
+ )
+ for row in rows
+ )
+ values = _build_feature_values_batch(inputs, workers=workers, progress=progress)
+ if progress is not None:
+ progress.set_activity(
+ "writing feature cache",
+ detail=_format_feature_write_progress(0, len(values)),
+ )
+ with self.connection:
+ for done, item in enumerate(values, start=1):
+ self._insert_feature_values(item)
+ if progress is not None and _should_report_feature_progress(done, len(values)):
+ progress.set_activity(
+ "writing feature cache",
+ detail=_format_feature_write_progress(done, len(values)),
+ )
+ return len(values)
+
def search_records(self, query: SearchQuery) -> list[SearchRecord]:
"""Return SearchRecord objects matching ``query`` from SQLite/FTS."""
try:
@@ -1409,6 +1677,51 @@ def iter_record_rows(self) -> tuple[DbRecordRow, ...]:
for row in rows
)
+ def iter_similarity_rows(self) -> tuple[DbSimilarityRow, ...]:
+ """Return record ids with precomputed normalized hashes for similarity."""
+ rows = self.connection.execute(
+ """
+ SELECT record_id, normalized_text_hash
+ FROM records_search
+ ORDER BY rowid
+ """,
+ ).fetchall()
+ return tuple(
+ DbSimilarityRow(
+ record_id=str(row["record_id"]),
+ normalized_hash=str(row["normalized_text_hash"]),
+ )
+ for row in rows
+ )
+
+ def iter_feature_rows(self) -> tuple[DbFeatureRow, ...]:
+ """Return records with persisted MinHash sketches and their text.
+
+ Only records that already have a ``record_features`` row are
+ returned; after :meth:`refresh_missing_features` that is every
+ indexed record. The MinHash signature is parsed from its stored
+ JSON list of hex strings so callers can bucket records into LSH
+ bands without recomputing sketches.
+ """
+ rows = self.connection.execute(
+ """
+ SELECT r.record_id, f.normalized_hash, f.minhash_json, d.text
+ FROM records_search r
+ JOIN record_features f ON f.record_id = r.record_id
+ JOIN record_details d ON d.rowid = r.rowid
+ ORDER BY r.rowid
+ """,
+ ).fetchall()
+ return tuple(
+ DbFeatureRow(
+ record_id=str(row["record_id"]),
+ normalized_hash=str(row["normalized_hash"]),
+ minhash_signature=_parse_minhash_signature(str(row["minhash_json"])),
+ text=str(row["text"]),
+ )
+ for row in rows
+ )
+
def get_record_row(self, record_id: str) -> DbRecordRow | None:
"""Return one indexed record row by id."""
rows = self._query(
@@ -1483,6 +1796,7 @@ def sync_records(
*,
control: SearchControl | None = None,
progress: DbSyncProgress | None = None,
+ features_mode: FeatureMode = "defer",
force: bool = False,
coverage: SyncCoverage | None = None,
prune_missing: bool = False,
@@ -1503,6 +1817,7 @@ def sync_records(
batches,
control=control,
progress=progress,
+ features_mode=features_mode,
force=force,
coverage=coverage,
prune_missing=prune_missing,
@@ -1516,6 +1831,7 @@ def _sync_records(
*,
control: SearchControl | None = None,
progress: DbSyncProgress | None = None,
+ features_mode: FeatureMode = "defer",
force: bool = False,
coverage: SyncCoverage | None = None,
prune_missing: bool = False,
@@ -1536,6 +1852,7 @@ def _sync_records(
source,
records,
result=result,
+ features_mode=features_mode,
force=force,
)
return self._finish_complete_sync(
@@ -1558,6 +1875,7 @@ def _sync_records(
source,
records,
result=result,
+ features_mode=features_mode,
force=force,
)
progress.source_finished(index, total, source, indexed, removed, result)
@@ -1605,6 +1923,7 @@ def _sync_one_source(
records: cabc.Iterable[SearchRecord],
*,
result: SyncResult,
+ features_mode: FeatureMode,
force: bool,
) -> tuple[SyncResult, int, int]:
"""Sync one source and return updated counters plus source deltas."""
@@ -1615,17 +1934,23 @@ def _sync_one_source(
records_indexed=result.records_indexed,
records_removed=result.records_removed,
sources_skipped=result.sources_skipped + 1,
+ features_deferred=result.features_deferred,
),
0,
0,
)
- indexed, removed = self.store.replace_source_records(source, records)
+ indexed, removed, deferred = self.store.replace_source_records(
+ source,
+ records,
+ features_mode=features_mode,
+ )
return (
SyncResult(
sources_synced=result.sources_synced + 1,
records_indexed=result.records_indexed + indexed,
records_removed=result.records_removed + removed,
sources_skipped=result.sources_skipped,
+ features_deferred=result.features_deferred + deferred,
),
indexed,
removed,
@@ -1637,6 +1962,7 @@ def sync_sources(
*,
control: SearchControl | None = None,
progress: DbSyncProgress | None = None,
+ features_mode: FeatureMode = "defer",
force: bool = False,
coverage: SyncCoverage | None = None,
prune_missing: bool = False,
@@ -1646,6 +1972,7 @@ def sync_sources(
((source, iter_source_records(source)) for source in sources),
control=control,
progress=progress,
+ features_mode=features_mode,
force=force,
coverage=coverage,
prune_missing=prune_missing,
@@ -1654,3 +1981,196 @@ def sync_sources(
def search_records(self, query: SearchQuery) -> list[SearchRecord]:
"""Search the DB index."""
return self.store.search_records(query)
+
+
+def simhash_hex(text: str) -> str:
+ """Return a deterministic 64-bit SimHash as fixed-width hex.
+
+ Examples
+ --------
+ >>> simhash_hex("Run ruff check.")
+ 'a33bc7c107168285'
+ >>> simhash_hex("")
+ '0000000000000000'
+ """
+ return _simhash_hex_from_tokens(token_set(text))
+
+
+def _simhash_hex_from_tokens(tokens: cabc.Iterable[str]) -> str:
+ """Return a deterministic 64-bit SimHash for pre-tokenized text."""
+ weights = [0] * 64
+ for token in tokens:
+ digest = int(hashlib.blake2b(token.encode("utf-8"), digest_size=8).hexdigest(), 16)
+ for bit in range(64):
+ weights[bit] += 1 if digest & (1 << bit) else -1
+ value = 0
+ for bit, weight in enumerate(weights):
+ if weight > 0:
+ value |= 1 << bit
+ return f"{value:016x}"
+
+
+def minhash_signature(text: str, *, size: int = 16) -> list[str]:
+ """Return a small deterministic MinHash-style signature.
+
+ Examples
+ --------
+ >>> minhash_signature("ruff check", size=4)[0]
+ '7cd7bdec93a8d7d1'
+ >>> minhash_signature("", size=4)
+ []
+ """
+ return _minhash_signature_from_tokens(token_set(text), size=size)
+
+
+def _minhash_signature_from_tokens(tokens: cabc.Iterable[str], *, size: int = 16) -> list[str]:
+ """Return a small deterministic MinHash-style signature for tokens."""
+ sorted_tokens = sorted(tokens)
+ if not sorted_tokens:
+ return []
+ signature: list[str] = []
+ for index in range(size):
+ minimum = min(
+ hashlib.blake2b(
+ f"{index}\0{token}".encode(),
+ digest_size=8,
+ ).hexdigest()
+ for token in sorted_tokens
+ )
+ signature.append(minimum)
+ return signature
+
+
+def _parse_minhash_signature(minhash_json: str) -> tuple[str, ...]:
+ """Return the MinHash signature stored as a JSON list of hex strings.
+
+ Examples
+ --------
+ >>> _parse_minhash_signature('["ab","cd"]')
+ ('ab', 'cd')
+ >>> _parse_minhash_signature('[]')
+ ()
+ >>> _parse_minhash_signature('not json')
+ ()
+ """
+ try:
+ parsed = json.loads(minhash_json)
+ except json.JSONDecodeError:
+ return ()
+ if not isinstance(parsed, list):
+ return ()
+ return tuple(str(item) for item in parsed)
+
+
+def _quality_flags_for_text(text: str, *, timestamp: str | None) -> dict[str, object]:
+ """Return lightweight quality/noise flags from scalar record fields."""
+ stripped = text.strip()
+ return {
+ "empty": not stripped,
+ "short": len(stripped) < 12,
+ "has_timestamp": timestamp is not None,
+ }
+
+
+def quality_flags(record: SearchRecord) -> dict[str, object]:
+ """Return lightweight quality/noise flags for a record."""
+ return _quality_flags_for_text(record.text, timestamp=record.timestamp)
+
+
+def _feature_values_for_input(
+ item: _FeatureInput,
+ *,
+ normalized_text: str | None = None,
+) -> _FeatureValues:
+ """Build feature values for one record without touching SQLite."""
+ effective_normalized_text = (
+ normalize_record_text(item.text) if normalized_text is None else normalized_text
+ )
+ tokens = frozenset(effective_normalized_text.split())
+ return _FeatureValues(
+ record_id=item.record_id,
+ normalized_hash=item.normalized_hash,
+ simhash_hex=_simhash_hex_from_tokens(tokens),
+ minhash_json=_json_dumps(_minhash_signature_from_tokens(tokens)),
+ token_count=len(tokens),
+ quality_flags_json=_json_dumps(
+ _quality_flags_for_text(item.text, timestamp=item.timestamp),
+ ),
+ updated_at=item.updated_at,
+ )
+
+
+def _build_feature_values_batch(
+ inputs: tuple[_FeatureInput, ...],
+ *,
+ workers: int | None,
+ progress: FeatureRefreshProgress | None = None,
+) -> tuple[_FeatureValues, ...]:
+ """Build feature rows, using processes only for sufficiently large batches."""
+ if not inputs:
+ return ()
+ worker_count = workers
+ if worker_count is None:
+ worker_count = min(4, os.cpu_count() or 1)
+ if worker_count <= 1 or len(inputs) < 512:
+ return _build_feature_values_inline(inputs, progress=progress)
+ if progress is not None:
+ progress.set_activity(
+ "building feature signatures",
+ detail=_format_feature_build_progress(0, len(inputs), workers=worker_count),
+ )
+ values: list[_FeatureValues] = []
+ with concurrent.futures.ProcessPoolExecutor(max_workers=worker_count) as executor:
+ for done, value in enumerate(
+ executor.map(_feature_values_for_input, inputs, chunksize=128),
+ start=1,
+ ):
+ values.append(value)
+ if progress is not None and _should_report_feature_progress(done, len(inputs)):
+ progress.set_activity(
+ "building feature signatures",
+ detail=_format_feature_build_progress(done, len(inputs), workers=worker_count),
+ )
+ return tuple(values)
+
+
+def _build_feature_values_inline(
+ inputs: tuple[_FeatureInput, ...],
+ *,
+ progress: FeatureRefreshProgress | None,
+) -> tuple[_FeatureValues, ...]:
+ """Build feature rows in-process while reporting bounded progress."""
+ if progress is not None:
+ progress.set_activity(
+ "building feature signatures",
+ detail=_format_feature_build_progress(0, len(inputs), workers=1),
+ )
+ values: list[_FeatureValues] = []
+ for done, item in enumerate(inputs, start=1):
+ values.append(_feature_values_for_input(item))
+ if progress is not None and _should_report_feature_progress(done, len(inputs)):
+ progress.set_activity(
+ "building feature signatures",
+ detail=_format_feature_build_progress(done, len(inputs), workers=1),
+ )
+ return tuple(values)
+
+
+def _should_report_feature_progress(done: int, total: int) -> bool:
+ """Return whether feature progress should emit for this count."""
+ if done == total:
+ return True
+ interval = min(_FEATURE_PROGRESS_INTERVAL, max(1, total // 20))
+ return done % interval == 0
+
+
+def _format_feature_build_progress(done: int, total: int, *, workers: int) -> str:
+ """Return a compact feature-build progress detail."""
+ percent = 100.0 if total == 0 else done / total * 100
+ return f"{done:,}/{total:,} rows built ({percent:.1f}%, {workers:,}w)"
+
+
+def _format_feature_write_progress(done: int, total: int) -> str:
+ """Return a compact feature-cache write progress detail."""
+ percent = 100.0 if total == 0 else done / total * 100
+ return f"{done:,}/{total:,} rows written ({percent:.1f}%)"
diff --git a/src/agentgrep/insights.py b/src/agentgrep/insights.py
new file mode 100644
index 000000000..9e110316b
--- /dev/null
+++ b/src/agentgrep/insights.py
@@ -0,0 +1,654 @@
+"""Deterministic insights engine for indexed agent data."""
+
+from __future__ import annotations
+
+import collections
+import dataclasses
+import itertools
+import json
+import pathlib
+import typing as t
+
+from agentgrep.db import (
+ DbFeatureRow,
+ DbStore,
+ normalize_record_text,
+ text_hash,
+ token_set,
+)
+
+if t.TYPE_CHECKING:
+ from agentgrep.progress import SearchControl
+
+_INSIGHT_PROGRESS_INTERVAL = 1024
+
+#: Minimum exact token-set Jaccard for a verified near-duplicate pair.
+NEAR_DUP_THRESHOLD = 0.6
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class InsightRunResult:
+ """Counters returned by an insight run."""
+
+ run_id: str
+ kind: str
+ clusters: int = 0
+ variant_edges: int = 0
+ omission_findings: int = 0
+ features_refreshed: int = 0
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class VariantEdge:
+ """One deterministic similarity or variant relationship."""
+
+ edge_id: str
+ run_id: str
+ left_record_id: str
+ right_record_id: str
+ variant_type: str
+ confidence: float
+ explanation: str
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class OmissionFinding:
+ """One meaningful omission candidate for a target instruction surface."""
+
+ finding_id: str
+ run_id: str
+ target_path: pathlib.Path
+ representative_record_id: str
+ confidence: float
+ rationale: str
+
+
+class InsightAnalyzeProgress(t.Protocol):
+ """Progress sink for deterministic insight-analysis phases."""
+
+ def set_activity(self, activity: str, *, detail: str | None = None) -> None:
+ """Report the current insight-analysis activity."""
+
+
+def _run_id(kind: str, payload: str) -> str:
+ """Return a deterministic run id for repeatable local insight runs."""
+ return text_hash(f"{kind}\0{payload}")[:24]
+
+
+def _edge_id(left: str, right: str, variant_type: str) -> str:
+ """Return a stable edge id independent of pair order."""
+ ordered = "\0".join(sorted((left, right)))
+ return text_hash(f"{variant_type}\0{ordered}")[:32]
+
+
+def _jaccard(left: frozenset[str], right: frozenset[str]) -> float:
+ """Return Jaccard similarity for two token sets."""
+ if not left and not right:
+ return 1.0
+ if not left or not right:
+ return 0.0
+ return len(left & right) / len(left | right)
+
+
+def _should_report_insight_progress(done: int, total: int) -> bool:
+ """Return whether an insight loop should emit a progress update."""
+ if done == total:
+ return True
+ interval = min(_INSIGHT_PROGRESS_INTERVAL, max(1, total // 20))
+ return done % interval == 0
+
+
+def _format_similarity_write_progress(
+ *,
+ clusters_done: int,
+ clusters_total: int,
+ edges_done: int,
+ edges_total: int,
+) -> str:
+ """Return compact similarity artifact write progress."""
+ return f"{clusters_done:,}/{clusters_total:,} clusters, {edges_done:,}/{edges_total:,} edges"
+
+
+class _UnionFind:
+ """Minimal dict-based union-find with path compression and union by rank."""
+
+ def __init__(self) -> None:
+ self._parent: dict[str, str] = {}
+ self._rank: dict[str, int] = {}
+
+ def add(self, item: str) -> None:
+ """Register ``item`` as its own singleton set if unseen."""
+ if item not in self._parent:
+ self._parent[item] = item
+ self._rank[item] = 0
+
+ def find(self, item: str) -> str:
+ """Return the representative root for ``item`` with path compression."""
+ root = item
+ while self._parent[root] != root:
+ root = self._parent[root]
+ while self._parent[item] != root:
+ self._parent[item], item = root, self._parent[item]
+ return root
+
+ def union(self, left: str, right: str) -> None:
+ """Merge the sets containing ``left`` and ``right``."""
+ left_root = self.find(left)
+ right_root = self.find(right)
+ if left_root == right_root:
+ return
+ if self._rank[left_root] < self._rank[right_root]:
+ left_root, right_root = right_root, left_root
+ self._parent[right_root] = left_root
+ if self._rank[left_root] == self._rank[right_root]:
+ self._rank[left_root] += 1
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class _NearCluster:
+ """One near-duplicate cluster with its deterministic id and confidence."""
+
+ cluster_id: str
+ members: tuple[str, ...]
+ member_scores: dict[str, float]
+ confidence: float
+
+
+def _near_duplicate_pairs(
+ feature_rows: tuple[DbFeatureRow, ...],
+) -> list[tuple[str, str, float]]:
+ """Return sorted near-duplicate pairs above :data:`NEAR_DUP_THRESHOLD`.
+
+ Records are first collapsed to one representative per normalized
+ hash (the lexicographically smallest record id) so exact duplicates
+ are never re-reported as near-duplicates. Representatives are then
+ bucketed into MinHash LSH bands — one band per signature position —
+ and any two records sharing a bucket become a candidate pair. Each
+ unique candidate pair is verified with an exact token-set Jaccard;
+ pairs with an empty token set on either side are skipped.
+ """
+ representatives: dict[str, DbFeatureRow] = {}
+ for row in feature_rows:
+ existing = representatives.get(row.normalized_hash)
+ if existing is None or row.record_id < existing.record_id:
+ representatives[row.normalized_hash] = row
+ tokens_by_id = {row.record_id: token_set(row.text) for row in representatives.values()}
+ buckets: dict[tuple[int, str], list[str]] = collections.defaultdict(list)
+ for row in representatives.values():
+ for band_index, band_hash in enumerate(row.minhash_signature):
+ buckets[(band_index, band_hash)].append(row.record_id)
+ candidate_pairs: set[tuple[str, str]] = set()
+ for bucket in buckets.values():
+ if len(bucket) < 2:
+ continue
+ for left, right in itertools.combinations(sorted(set(bucket)), 2):
+ candidate_pairs.add((left, right))
+ near_pairs: list[tuple[str, str, float]] = []
+ for left, right in sorted(candidate_pairs):
+ left_tokens = tokens_by_id[left]
+ right_tokens = tokens_by_id[right]
+ if not left_tokens or not right_tokens:
+ continue
+ score = _jaccard(left_tokens, right_tokens)
+ if score >= NEAR_DUP_THRESHOLD:
+ near_pairs.append((left, right, score))
+ return near_pairs
+
+
+def _cluster_near_pairs(
+ near_pairs: list[tuple[str, str, float]],
+) -> list[_NearCluster]:
+ """Group verified near-duplicate pairs into clusters via union-find.
+
+ Cluster confidence is the minimum pairwise Jaccard among the
+ cluster's near-duplicate edges (the weakest link that still cleared
+ the threshold). Members and clusters are ordered by record id so the
+ resulting cluster ids are reproducible across runs.
+ """
+ if not near_pairs:
+ return []
+ union_find = _UnionFind()
+ for left, right, _score in near_pairs:
+ union_find.add(left)
+ union_find.add(right)
+ union_find.union(left, right)
+ nodes = sorted({node for left, right, _score in near_pairs for node in (left, right)})
+ members_by_root: dict[str, list[str]] = collections.defaultdict(list)
+ for node in nodes:
+ members_by_root[union_find.find(node)].append(node)
+ min_score_by_root: dict[str, float] = {}
+ member_scores_by_root: dict[str, dict[str, float]] = collections.defaultdict(dict)
+ for left, right, score in near_pairs:
+ root = union_find.find(left)
+ current = min_score_by_root.get(root)
+ if current is None or score < current:
+ min_score_by_root[root] = score
+ scores = member_scores_by_root[root]
+ scores[left] = max(scores.get(left, 0.0), score)
+ scores[right] = max(scores.get(right, 0.0), score)
+ clusters: list[_NearCluster] = []
+ for root, members in members_by_root.items():
+ ordered = tuple(members)
+ cluster_id = text_hash("near_cluster\0" + "\0".join(ordered))[:32]
+ clusters.append(
+ _NearCluster(
+ cluster_id=cluster_id,
+ members=ordered,
+ member_scores=member_scores_by_root[root],
+ confidence=min_score_by_root[root],
+ ),
+ )
+ clusters.sort(key=lambda cluster: cluster.members[0])
+ return clusters
+
+
+class InsightEngine:
+ """Generate deterministic clusters, variants, and omission findings."""
+
+ def __init__(self, store: DbStore) -> None:
+ self.store = store
+
+ def run_similarity(
+ self,
+ *,
+ control: SearchControl | None = None,
+ progress: InsightAnalyzeProgress | None = None,
+ ) -> InsightRunResult:
+ """Detect exact and near-duplicate prompt variants."""
+ if control is not None and control.answer_now_requested():
+ return InsightRunResult(run_id=_run_id("similarity", "cancelled"), kind="similarity")
+ features_refreshed = self.store.refresh_missing_features(progress=progress)
+ if control is not None and control.answer_now_requested():
+ return InsightRunResult(
+ run_id=_run_id("similarity", "cancelled"),
+ kind="similarity",
+ features_refreshed=features_refreshed,
+ )
+ if progress is not None:
+ progress.set_activity(
+ "loading similarity rows",
+ detail="reading normalized record hashes",
+ )
+ rows = self.store.iter_similarity_rows()
+ run_id = _run_id("similarity", ",".join(row.record_id for row in rows))
+ now = "deterministic"
+ if progress is not None:
+ progress.set_activity(
+ "grouping duplicate prompts",
+ detail=f"{len(rows):,} normalized records",
+ )
+ groups: dict[str, list[str]] = collections.defaultdict(list)
+ for row in rows:
+ groups[row.normalized_hash].append(row.record_id)
+ exact_candidate_groups = sum(1 for record_ids in groups.values() if len(record_ids) >= 2)
+ exact_variant_edges = sum(
+ len(record_ids) * (len(record_ids) - 1) // 2
+ for record_ids in groups.values()
+ if len(record_ids) >= 2
+ )
+ if progress is not None:
+ progress.set_activity(
+ "grouping duplicate prompts",
+ detail="bucketing minhash sketches for near-duplicates",
+ )
+ near_pairs = _near_duplicate_pairs(self.store.iter_feature_rows())
+ near_clusters = _cluster_near_pairs(near_pairs)
+ clusters_total = exact_candidate_groups + len(near_clusters)
+ edges_total = exact_variant_edges + len(near_pairs)
+ if progress is not None:
+ progress.set_activity(
+ "writing similarity artifacts",
+ detail=_format_similarity_write_progress(
+ clusters_done=0,
+ clusters_total=clusters_total,
+ edges_done=0,
+ edges_total=edges_total,
+ ),
+ )
+ edge_count = 0
+ cluster_count = 0
+ with self.store.connection:
+ self._record_run(run_id, "similarity", now, {"records": len(rows)})
+ for normalized_hash, record_ids in groups.items():
+ if len(record_ids) < 2:
+ continue
+ cluster_count += 1
+ cluster_id = text_hash(f"cluster\0{normalized_hash}")[:32]
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO clusters(
+ cluster_id, run_id, kind, label, centroid_record_id,
+ confidence, evidence_json
+ )
+ VALUES(?, ?, 'duplicate_prompt', ?, ?, 1.0, ?)
+ """,
+ (
+ cluster_id,
+ run_id,
+ "exact duplicate prompt family",
+ record_ids[0],
+ '{"signal":"normalized_hash"}',
+ ),
+ )
+ for record_id in record_ids:
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO cluster_members(
+ cluster_id, record_id, score, signals_json
+ )
+ VALUES(?, ?, 1.0, ?)
+ """,
+ (cluster_id, record_id, '{"normalized_hash":1.0}'),
+ )
+ for left, right in itertools.combinations(record_ids, 2):
+ edge_id = _edge_id(left, right, "exact_duplicate")
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO variant_edges(
+ edge_id, run_id, left_record_id, right_record_id,
+ variant_type, confidence, signals_json, explanation
+ )
+ VALUES(?, ?, ?, ?, 'exact_duplicate', 1.0, ?, ?)
+ """,
+ (
+ edge_id,
+ run_id,
+ left,
+ right,
+ '{"normalized_hash":1.0}',
+ "normalized prompt text is identical",
+ ),
+ )
+ edge_count += 1
+ if progress is not None and _should_report_insight_progress(
+ edge_count,
+ edges_total,
+ ):
+ progress.set_activity(
+ "writing similarity artifacts",
+ detail=_format_similarity_write_progress(
+ clusters_done=cluster_count,
+ clusters_total=clusters_total,
+ edges_done=edge_count,
+ edges_total=edges_total,
+ ),
+ )
+ for near_cluster in near_clusters:
+ cluster_count += 1
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO clusters(
+ cluster_id, run_id, kind, label, centroid_record_id,
+ confidence, evidence_json
+ )
+ VALUES(?, ?, 'near_duplicate_prompt', ?, ?, ?, ?)
+ """,
+ (
+ near_cluster.cluster_id,
+ run_id,
+ "near-duplicate prompt family",
+ near_cluster.members[0],
+ near_cluster.confidence,
+ json.dumps(
+ {"signal": "minhash_lsh+jaccard", "threshold": NEAR_DUP_THRESHOLD},
+ sort_keys=True,
+ ),
+ ),
+ )
+ for member in near_cluster.members:
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO cluster_members(
+ cluster_id, record_id, score, signals_json
+ )
+ VALUES(?, ?, ?, '{"signal":"minhash_lsh"}')
+ """,
+ (
+ near_cluster.cluster_id,
+ member,
+ near_cluster.member_scores[member],
+ ),
+ )
+ for left, right, jaccard in near_pairs:
+ edge_id = _edge_id(left, right, "near_duplicate")
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO variant_edges(
+ edge_id, run_id, left_record_id, right_record_id,
+ variant_type, confidence, signals_json, explanation
+ )
+ VALUES(?, ?, ?, ?, 'near_duplicate', ?, ?, ?)
+ """,
+ (
+ edge_id,
+ run_id,
+ left,
+ right,
+ jaccard,
+ json.dumps({"signal": "minhash_lsh", "jaccard": jaccard}, sort_keys=True),
+ "normalized prompt token sets overlap heavily",
+ ),
+ )
+ edge_count += 1
+ if progress is not None and _should_report_insight_progress(
+ edge_count,
+ edges_total,
+ ):
+ progress.set_activity(
+ "writing similarity artifacts",
+ detail=_format_similarity_write_progress(
+ clusters_done=cluster_count,
+ clusters_total=clusters_total,
+ edges_done=edge_count,
+ edges_total=edges_total,
+ ),
+ )
+ if progress is not None:
+ progress.set_activity(
+ "writing similarity artifacts",
+ detail=_format_similarity_write_progress(
+ clusters_done=cluster_count,
+ clusters_total=clusters_total,
+ edges_done=edge_count,
+ edges_total=edges_total,
+ ),
+ )
+ return InsightRunResult(
+ run_id=run_id,
+ kind="similarity",
+ clusters=cluster_count,
+ variant_edges=edge_count,
+ features_refreshed=features_refreshed,
+ )
+
+ def run_omissions(
+ self,
+ *,
+ target_path: pathlib.Path,
+ target_text: str,
+ control: SearchControl | None = None,
+ progress: InsightAnalyzeProgress | None = None,
+ ) -> InsightRunResult:
+ """Detect indexed instructions absent from a target file."""
+ if control is not None and control.answer_now_requested():
+ return InsightRunResult(run_id=_run_id("omissions", "cancelled"), kind="omissions")
+ features_refreshed = self.store.refresh_missing_features(progress=progress)
+ if control is not None and control.answer_now_requested():
+ return InsightRunResult(
+ run_id=_run_id("omissions", "cancelled"),
+ kind="omissions",
+ features_refreshed=features_refreshed,
+ )
+ if progress is not None:
+ progress.set_activity(
+ "loading indexed records",
+ detail="reading prompt and instruction text",
+ )
+ rows = self.store.iter_record_rows()
+ run_id = _run_id("omissions", f"{target_path}\0{target_text}\0{len(rows)}")
+ if progress is not None:
+ progress.set_activity(
+ "normalizing target text",
+ detail=str(target_path),
+ )
+ target_normalized = normalize_record_text(target_text)
+ target_tokens = token_set(target_text)
+ finding_count = 0
+ if progress is not None:
+ progress.set_activity(
+ "comparing omission candidates",
+ detail=f"{len(rows):,} indexed records",
+ )
+ with self.store.connection:
+ self._record_run(
+ run_id,
+ "omissions",
+ "deterministic",
+ {"target_path": str(target_path), "records": len(rows)},
+ )
+ for row in rows:
+ record_normalized = normalize_record_text(row.record.text)
+ if not record_normalized or record_normalized in target_normalized:
+ continue
+ record_tokens = token_set(row.record.text)
+ if len(record_tokens) < 3:
+ continue
+ overlap = _jaccard(record_tokens, target_tokens)
+ confidence = max(0.72, min(0.95, 0.72 + overlap / 4))
+ finding_id = text_hash(
+ f"omission\0{target_path}\0{row.record_id}\0{run_id}",
+ )[:32]
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO omission_findings(
+ finding_id, run_id, target_path, cluster_id,
+ representative_record_id, confidence, status,
+ evidence_json, rationale
+ )
+ VALUES(?, ?, ?, NULL, ?, ?, 'open', ?, ?)
+ """,
+ (
+ finding_id,
+ run_id,
+ str(target_path),
+ row.record_id,
+ confidence,
+ '{"signal":"absent_instruction"}',
+ "indexed instruction is absent from the target surface",
+ ),
+ )
+ finding_count += 1
+ if progress is not None:
+ progress.set_activity(
+ "writing omission findings",
+ detail=f"{finding_count:,} findings",
+ )
+ return InsightRunResult(
+ run_id=run_id,
+ kind="omissions",
+ omission_findings=finding_count,
+ features_refreshed=features_refreshed,
+ )
+
+ def count_variant_edges(self) -> int:
+ """Return the number of persisted variant edges."""
+ row = self.store.connection.execute(
+ "SELECT COUNT(*) AS count FROM variant_edges",
+ ).fetchone()
+ return int(row["count"])
+
+ def list_variant_edges(self, *, limit: int | None = None) -> list[VariantEdge]:
+ """Return persisted variant edges."""
+ sql = """
+ SELECT edge_id, run_id, left_record_id, right_record_id,
+ variant_type, confidence, explanation
+ FROM variant_edges
+ ORDER BY confidence DESC, edge_id
+ """
+ params: tuple[object, ...] = ()
+ if limit is not None:
+ sql += " LIMIT ?"
+ params = (limit,)
+ rows = self.store.connection.execute(sql, params).fetchall()
+ return [
+ VariantEdge(
+ edge_id=str(row["edge_id"]),
+ run_id=str(row["run_id"]),
+ left_record_id=str(row["left_record_id"]),
+ right_record_id=str(row["right_record_id"]),
+ variant_type=str(row["variant_type"]),
+ confidence=float(row["confidence"]),
+ explanation=str(row["explanation"]),
+ )
+ for row in rows
+ ]
+
+ def list_omission_findings(
+ self,
+ *,
+ target_path: pathlib.Path | None = None,
+ limit: int | None = None,
+ ) -> list[OmissionFinding]:
+ """Return persisted omission findings."""
+ params: list[object] = []
+ if target_path is None:
+ sql = """
+ SELECT finding_id, run_id, target_path, representative_record_id,
+ confidence, rationale
+ FROM omission_findings
+ ORDER BY confidence DESC, finding_id
+ """
+ else:
+ sql = """
+ SELECT finding_id, run_id, target_path, representative_record_id,
+ confidence, rationale
+ FROM omission_findings
+ WHERE target_path = ?
+ ORDER BY confidence DESC, finding_id
+ """
+ params.append(str(target_path))
+ if limit is not None:
+ sql += " LIMIT ?"
+ params.append(limit)
+ rows = self.store.connection.execute(sql, tuple(params)).fetchall()
+ return [
+ OmissionFinding(
+ finding_id=str(row["finding_id"]),
+ run_id=str(row["run_id"]),
+ target_path=pathlib.Path(str(row["target_path"])),
+ representative_record_id=str(row["representative_record_id"]),
+ confidence=float(row["confidence"]),
+ rationale=str(row["rationale"]),
+ )
+ for row in rows
+ ]
+
+ def count_omission_findings(
+ self,
+ *,
+ target_path: pathlib.Path | None = None,
+ ) -> int:
+ """Return the number of persisted omission findings."""
+ if target_path is None:
+ row = self.store.connection.execute(
+ "SELECT COUNT(*) AS count FROM omission_findings",
+ ).fetchone()
+ else:
+ row = self.store.connection.execute(
+ "SELECT COUNT(*) AS count FROM omission_findings WHERE target_path = ?",
+ (str(target_path),),
+ ).fetchone()
+ return int(row["count"])
+
+ def _record_run(self, run_id: str, kind: str, now: str, counters: dict[str, object]) -> None:
+ """Upsert one insight run row."""
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO insight_runs(
+ run_id, kind, started_at, finished_at, status,
+ algorithm_version, input_json, counters_json
+ )
+ VALUES(?, ?, ?, ?, 'ok', 'deterministic-v1', '{}', ?)
+ """,
+ (run_id, kind, now, now, json.dumps(counters, sort_keys=True)),
+ )
diff --git a/src/agentgrep/mcp/__init__.py b/src/agentgrep/mcp/__init__.py
index c5fc2e9c2..505b026f3 100644
--- a/src/agentgrep/mcp/__init__.py
+++ b/src/agentgrep/mcp/__init__.py
@@ -49,6 +49,7 @@
FindRequestModel,
FindToolResponse,
GetStoreDescriptorRequest,
+ InsightsListResponse,
InspectResultRequest,
InspectResultResponse,
InspectSampleRequest,
@@ -59,6 +60,7 @@
ListStoresResponse,
NextActionModel,
NormalizedSearchRequestModel,
+ OmissionFindingModel,
PageInfoModel,
RecentSessionsRequest,
RecentSessionsResponse,
@@ -76,8 +78,11 @@
SourceRecordModel,
SourceVersionDetectionModel,
StoreDescriptorModel,
+ SuggestionArtifactModel,
+ SuggestionsListResponse,
ValidateQueryRequest,
ValidateQueryResponse,
+ VariantEdgeModel,
)
from agentgrep.mcp.resources import build_capabilities, list_source_models
from agentgrep.mcp.server import build_mcp_server, main
@@ -106,6 +111,7 @@
"FindRequestModel",
"FindToolResponse",
"GetStoreDescriptorRequest",
+ "InsightsListResponse",
"InspectResultRequest",
"InspectResultResponse",
"InspectSampleRequest",
@@ -116,6 +122,7 @@
"ListStoresResponse",
"NextActionModel",
"NormalizedSearchRequestModel",
+ "OmissionFindingModel",
"PageInfoModel",
"RecentSessionsRequest",
"RecentSessionsResponse",
@@ -137,8 +144,11 @@
"SourceRecordModel",
"SourceVersionDetectionModel",
"StoreDescriptorModel",
+ "SuggestionArtifactModel",
+ "SuggestionsListResponse",
"ValidateQueryRequest",
"ValidateQueryResponse",
+ "VariantEdgeModel",
"agentgrep",
"build_capabilities",
"build_mcp_server",
diff --git a/src/agentgrep/mcp/models.py b/src/agentgrep/mcp/models.py
index e6e641caa..0cf6c2064 100644
--- a/src/agentgrep/mcp/models.py
+++ b/src/agentgrep/mcp/models.py
@@ -218,6 +218,71 @@ class DbStatusModel(AgentGrepModel):
db_schema_version: int
sources: int
records: int
+ features: int
+ variant_edges: int
+ omission_findings: int
+ suggestions: int
+
+
+class VariantEdgeModel(AgentGrepModel):
+ """Persisted variant edge payload."""
+
+ edge_id: str
+ run_id: str
+ left_record_id: str
+ right_record_id: str
+ variant_type: str
+ confidence: float
+ explanation: str
+
+
+class OmissionFindingModel(AgentGrepModel):
+ """Persisted omission finding payload."""
+
+ finding_id: str
+ run_id: str
+ target_path: str
+ representative_record_id: str
+ confidence: float
+ rationale: str
+
+
+class SuggestionArtifactModel(AgentGrepModel):
+ """Persisted review-only suggestion payload."""
+
+ suggestion_id: str
+ run_id: str
+ target_path: str
+ surface_kind: str
+ title: str
+ body: str
+ confidence: float
+ status: str
+ rationale: str
+ reload_note: str
+
+
+class InsightsListResponse(AgentGrepModel):
+ """Structured response for persisted insights."""
+
+ schema_version: str = agentgrep.SCHEMA_VERSION
+ limit: int
+ variant_edges_total: int
+ variant_edges_truncated: bool
+ variant_edges: list[VariantEdgeModel]
+ omission_findings_total: int
+ omission_findings_truncated: bool
+ omission_findings: list[OmissionFindingModel]
+
+
+class SuggestionsListResponse(AgentGrepModel):
+ """Structured response for persisted suggestions."""
+
+ schema_version: str = agentgrep.SCHEMA_VERSION
+ limit: int
+ suggestions_total: int
+ suggestions_truncated: bool
+ suggestions: list[SuggestionArtifactModel]
class ResultStatsModel(AgentGrepModel):
diff --git a/src/agentgrep/mcp/resources.py b/src/agentgrep/mcp/resources.py
index f537590d3..d1d668558 100644
--- a/src/agentgrep/mcp/resources.py
+++ b/src/agentgrep/mcp/resources.py
@@ -92,6 +92,8 @@ def build_capabilities() -> CapabilitiesModel:
"inspect_result",
"validate_query",
"db_status",
+ "insights_list",
+ "suggestions_list",
],
resources=[
"agentgrep://capabilities",
diff --git a/src/agentgrep/mcp/tools/__init__.py b/src/agentgrep/mcp/tools/__init__.py
index 21e5e06e4..e79b961ab 100644
--- a/src/agentgrep/mcp/tools/__init__.py
+++ b/src/agentgrep/mcp/tools/__init__.py
@@ -17,6 +17,7 @@ def register_tools(mcp: FastMCP, *, runtime: SearchRuntime | None = None) -> Non
db_tools,
diagnostic_tools,
discovery_tools,
+ insight_tools,
search_tools,
)
@@ -25,3 +26,4 @@ def register_tools(mcp: FastMCP, *, runtime: SearchRuntime | None = None) -> Non
catalog_tools.register(mcp)
diagnostic_tools.register(mcp)
db_tools.register(mcp)
+ insight_tools.register(mcp)
diff --git a/src/agentgrep/mcp/tools/db_tools.py b/src/agentgrep/mcp/tools/db_tools.py
index c0acbc38a..42b827013 100644
--- a/src/agentgrep/mcp/tools/db_tools.py
+++ b/src/agentgrep/mcp/tools/db_tools.py
@@ -36,6 +36,10 @@ def _db_status_sync(db_path: str | None) -> DbStatusModel:
db_schema_version=0,
sources=0,
records=0,
+ features=0,
+ variant_edges=0,
+ omission_findings=0,
+ suggestions=0,
)
try:
with DbRuntime.open_readonly(path) as runtime:
@@ -46,12 +50,20 @@ def _db_status_sync(db_path: str | None) -> DbStatusModel:
db_schema_version=0,
sources=0,
records=0,
+ features=0,
+ variant_edges=0,
+ omission_findings=0,
+ suggestions=0,
)
return DbStatusModel(
db_path=str(status.db_path),
db_schema_version=status.schema_version,
sources=status.sources,
records=status.records,
+ features=status.features,
+ variant_edges=status.variant_edges,
+ omission_findings=status.omission_findings,
+ suggestions=status.suggestions,
)
diff --git a/src/agentgrep/mcp/tools/insight_tools.py b/src/agentgrep/mcp/tools/insight_tools.py
new file mode 100644
index 000000000..0dccf0e2a
--- /dev/null
+++ b/src/agentgrep/mcp/tools/insight_tools.py
@@ -0,0 +1,210 @@
+"""Read-only MCP tools for DB and insight artifacts."""
+
+from __future__ import annotations
+
+import asyncio
+import functools
+import pathlib
+import typing as t
+
+from pydantic import Field
+
+from agentgrep.mcp._library import READONLY_TAGS
+from agentgrep.mcp.models import (
+ InsightsListResponse,
+ OmissionFindingModel,
+ SuggestionArtifactModel,
+ SuggestionsListResponse,
+ VariantEdgeModel,
+)
+
+if t.TYPE_CHECKING:
+ from fastmcp import FastMCP
+
+DEFAULT_INSIGHTS_LIST_LIMIT = 50
+DEFAULT_SUGGESTIONS_LIST_LIMIT = 50
+
+
+def _selected_db_path(db_path: str | None) -> pathlib.Path:
+ """Return the selected agentgrep db path without creating it."""
+ from agentgrep.db import default_db_path
+
+ if db_path is not None:
+ return pathlib.Path(db_path).expanduser()
+ return default_db_path()
+
+
+def _empty_insights_response(limit: int) -> InsightsListResponse:
+ """Return the empty insights payload for missing or unreadable caches."""
+ return InsightsListResponse(
+ limit=limit,
+ variant_edges_total=0,
+ variant_edges_truncated=False,
+ variant_edges=[],
+ omission_findings_total=0,
+ omission_findings_truncated=False,
+ omission_findings=[],
+ )
+
+
+def _empty_suggestions_response(limit: int) -> SuggestionsListResponse:
+ """Return the empty suggestions payload for missing or unreadable caches."""
+ return SuggestionsListResponse(
+ limit=limit,
+ suggestions_total=0,
+ suggestions_truncated=False,
+ suggestions=[],
+ )
+
+
+def _insights_list_sync(
+ db_path: str | None,
+ *,
+ limit: int = DEFAULT_INSIGHTS_LIST_LIMIT,
+) -> InsightsListResponse:
+ """Return persisted insight artifacts without running new insights."""
+ import sqlite3
+
+ from agentgrep.db import DbRuntime
+ from agentgrep.insights import InsightEngine
+
+ path = _selected_db_path(db_path)
+ if not path.exists():
+ return _empty_insights_response(limit)
+ try:
+ with DbRuntime.open_readonly(path) as runtime:
+ engine = InsightEngine(runtime.store)
+ variant_edges_total = engine.count_variant_edges()
+ omission_findings_total = engine.count_omission_findings()
+ variant_edges = engine.list_variant_edges(limit=limit)
+ omission_findings = engine.list_omission_findings(limit=limit)
+ except sqlite3.DatabaseError:
+ # A foreign or corrupt file gets the same empty payload as a
+ # missing one — matching db_status and the CLI read surfaces.
+ return _empty_insights_response(limit)
+ return InsightsListResponse(
+ limit=limit,
+ variant_edges_total=variant_edges_total,
+ variant_edges_truncated=variant_edges_total > len(variant_edges),
+ variant_edges=[
+ VariantEdgeModel(
+ edge_id=edge.edge_id,
+ run_id=edge.run_id,
+ left_record_id=edge.left_record_id,
+ right_record_id=edge.right_record_id,
+ variant_type=edge.variant_type,
+ confidence=edge.confidence,
+ explanation=edge.explanation,
+ )
+ for edge in variant_edges
+ ],
+ omission_findings_total=omission_findings_total,
+ omission_findings_truncated=omission_findings_total > len(omission_findings),
+ omission_findings=[
+ OmissionFindingModel(
+ finding_id=finding.finding_id,
+ run_id=finding.run_id,
+ target_path=str(finding.target_path),
+ representative_record_id=finding.representative_record_id,
+ confidence=finding.confidence,
+ rationale=finding.rationale,
+ )
+ for finding in omission_findings
+ ],
+ )
+
+
+def _suggestions_list_sync(
+ db_path: str | None,
+ *,
+ limit: int = DEFAULT_SUGGESTIONS_LIST_LIMIT,
+) -> SuggestionsListResponse:
+ """Return a bounded page of persisted review-only suggestions."""
+ import sqlite3
+
+ from agentgrep.db import DbRuntime
+ from agentgrep.suggestions import SuggestionEngine
+
+ path = _selected_db_path(db_path)
+ if not path.exists():
+ return _empty_suggestions_response(limit)
+ try:
+ with DbRuntime.open_readonly(path) as runtime:
+ engine = SuggestionEngine(runtime.store)
+ suggestions_total = engine.count_suggestions()
+ suggestions = engine.list_suggestions(limit=limit)
+ except sqlite3.DatabaseError:
+ # Same empty payload as a missing file; see _insights_list_sync.
+ return _empty_suggestions_response(limit)
+ return SuggestionsListResponse(
+ limit=limit,
+ suggestions_total=suggestions_total,
+ suggestions_truncated=suggestions_total > len(suggestions),
+ suggestions=[
+ SuggestionArtifactModel(
+ suggestion_id=suggestion.suggestion_id,
+ run_id=suggestion.run_id,
+ target_path=str(suggestion.target_path),
+ surface_kind=suggestion.surface_kind,
+ title=suggestion.title,
+ body=suggestion.body,
+ confidence=suggestion.confidence,
+ status=suggestion.status,
+ rationale=suggestion.rationale,
+ reload_note=suggestion.reload_note,
+ )
+ for suggestion in suggestions
+ ],
+ )
+
+
+def register(mcp: FastMCP) -> None:
+ """Register read-only insight and suggestion tools."""
+
+ @mcp.tool(
+ name="insights_list",
+ tags=READONLY_TAGS | {"insights"},
+ description="List persisted deterministic insight artifacts.",
+ )
+ async def insights_list_tool(
+ db_path: t.Annotated[
+ str | None,
+ Field(default=None, description="Optional agentgrep db path."),
+ ] = None,
+ limit: t.Annotated[
+ int,
+ Field(
+ default=DEFAULT_INSIGHTS_LIST_LIMIT,
+ ge=1,
+ description="Maximum rows to return per insight family.",
+ ),
+ ] = DEFAULT_INSIGHTS_LIST_LIMIT,
+ ) -> InsightsListResponse:
+ return await asyncio.to_thread(_insights_list_sync, db_path, limit=limit)
+
+ _ = insights_list_tool
+
+ @mcp.tool(
+ name="suggestions_list",
+ tags=READONLY_TAGS | {"insights", "suggestions"},
+ description="List persisted review-only instruction suggestions.",
+ )
+ async def suggestions_list_tool(
+ db_path: t.Annotated[
+ str | None,
+ Field(default=None, description="Optional agentgrep db path."),
+ ] = None,
+ limit: t.Annotated[
+ int,
+ Field(
+ default=DEFAULT_SUGGESTIONS_LIST_LIMIT,
+ ge=1,
+ description="Maximum suggestions to return.",
+ ),
+ ] = DEFAULT_SUGGESTIONS_LIST_LIMIT,
+ ) -> SuggestionsListResponse:
+ return await asyncio.to_thread(
+ functools.partial(_suggestions_list_sync, db_path, limit=limit),
+ )
+
+ _ = suggestions_list_tool
diff --git a/src/agentgrep/suggestions.py b/src/agentgrep/suggestions.py
new file mode 100644
index 000000000..1fc9e62fc
--- /dev/null
+++ b/src/agentgrep/suggestions.py
@@ -0,0 +1,166 @@
+"""Review-only suggestion artifacts built from insight findings."""
+
+from __future__ import annotations
+
+import dataclasses
+import pathlib
+import sqlite3
+
+from agentgrep.db import DbStore, text_hash
+from agentgrep.insights import InsightEngine
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class SuggestionArtifact:
+ """A reviewable instruction or skill suggestion."""
+
+ suggestion_id: str
+ run_id: str
+ target_path: pathlib.Path
+ surface_kind: str
+ title: str
+ body: str
+ confidence: float
+ status: str
+ rationale: str
+ reload_note: str
+
+
+class SuggestionEngine:
+ """Create and read review-only suggestion artifacts."""
+
+ def __init__(self, store: DbStore) -> None:
+ self.store = store
+
+ def create_from_omissions(self, *, target_path: pathlib.Path) -> list[SuggestionArtifact]:
+ """Create suggestions from open omission findings for ``target_path``."""
+ insights = InsightEngine(self.store)
+ findings = insights.list_omission_findings(target_path=target_path)
+ suggestions: list[SuggestionArtifact] = []
+ with self.store.connection:
+ for finding in findings:
+ row = self.store.get_record_row(finding.representative_record_id)
+ if row is None:
+ continue
+ suggestion_id = text_hash(
+ f"suggestion\0{finding.finding_id}\0{target_path}",
+ )[:32]
+ title = "Add missing agent instruction"
+ body = row.record.text.strip()
+ reload_note = (
+ "This suggestion takes effect only after the patch is accepted "
+ "and the relevant agent session reloads or restarts."
+ )
+ artifact = SuggestionArtifact(
+ suggestion_id=suggestion_id,
+ run_id=finding.run_id,
+ target_path=target_path,
+ surface_kind="agents_md",
+ title=title,
+ body=body,
+ confidence=finding.confidence,
+ status="proposed",
+ rationale=finding.rationale,
+ reload_note=reload_note,
+ )
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO suggestions(
+ suggestion_id, run_id, target_path, surface_kind,
+ title, body, confidence, status, rationale,
+ reload_note, created_at
+ )
+ VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'deterministic')
+ """,
+ (
+ artifact.suggestion_id,
+ artifact.run_id,
+ str(artifact.target_path),
+ artifact.surface_kind,
+ artifact.title,
+ artifact.body,
+ artifact.confidence,
+ artifact.status,
+ artifact.rationale,
+ artifact.reload_note,
+ ),
+ )
+ self.store.connection.execute(
+ """
+ INSERT OR REPLACE INTO suggestion_evidence(
+ suggestion_id, record_id, evidence_role, score, signals_json
+ )
+ VALUES(?, ?, 'representative', ?, ?)
+ """,
+ (
+ artifact.suggestion_id,
+ row.record_id,
+ artifact.confidence,
+ '{"signal":"omission_representative"}',
+ ),
+ )
+ suggestions.append(artifact)
+ return suggestions
+
+ def count_suggestions(self) -> int:
+ """Return the persisted suggestion count."""
+ row = self.store.connection.execute(
+ "SELECT COUNT(*) AS count FROM suggestions",
+ ).fetchone()
+ return int(row["count"]) if row is not None else 0
+
+ def list_suggestions(self, *, limit: int | None = None) -> list[SuggestionArtifact]:
+ """Return persisted suggestions, optionally bounded to ``limit`` rows."""
+ sql = """
+ SELECT suggestion_id, run_id, target_path, surface_kind, title, body,
+ confidence, status, rationale, reload_note
+ FROM suggestions
+ ORDER BY confidence DESC, suggestion_id
+ """
+ params: tuple[object, ...] = ()
+ if limit is not None:
+ sql += " LIMIT ?"
+ params = (limit,)
+ rows = self.store.connection.execute(sql, params).fetchall()
+ return [self._row_to_artifact(row) for row in rows]
+
+ def get_suggestion(self, suggestion_id: str) -> SuggestionArtifact | None:
+ """Return one suggestion by id."""
+ row = self.store.connection.execute(
+ """
+ SELECT suggestion_id, run_id, target_path, surface_kind, title, body,
+ confidence, status, rationale, reload_note
+ FROM suggestions
+ WHERE suggestion_id = ?
+ """,
+ (suggestion_id,),
+ ).fetchone()
+ return None if row is None else self._row_to_artifact(row)
+
+ def render_suggestion(self, suggestion_id: str) -> str | None:
+ """Render one suggestion as reviewable text."""
+ artifact = self.get_suggestion(suggestion_id)
+ if artifact is None:
+ return None
+ return (
+ f"{artifact.title}\n\n"
+ f"Target: {artifact.target_path}\n"
+ f"Confidence: {artifact.confidence:.2f}\n\n"
+ f"{artifact.body}\n\n"
+ f"{artifact.reload_note}"
+ )
+
+ def _row_to_artifact(self, row: sqlite3.Row) -> SuggestionArtifact:
+ """Convert one SQLite row to a suggestion artifact."""
+ return SuggestionArtifact(
+ suggestion_id=str(row["suggestion_id"]),
+ run_id=str(row["run_id"]),
+ target_path=pathlib.Path(str(row["target_path"])),
+ surface_kind=str(row["surface_kind"]),
+ title=str(row["title"]),
+ body=str(row["body"]),
+ confidence=float(row["confidence"]),
+ status=str(row["status"]),
+ rationale=str(row["rationale"]),
+ reload_note=str(row["reload_note"]),
+ )
diff --git a/src/pytest_documentation/sandbox.py b/src/pytest_documentation/sandbox.py
index b60306507..63f0478ab 100644
--- a/src/pytest_documentation/sandbox.py
+++ b/src/pytest_documentation/sandbox.py
@@ -735,9 +735,13 @@ def _accepts_data_dependent_empty_result(script: str) -> bool:
"agentgrep search",
"agentgrep grep",
"agentgrep find",
+ "agentgrep insights",
+ "agentgrep suggestions",
"uv run agentgrep search",
"uv run agentgrep grep",
"uv run agentgrep find",
+ "uv run agentgrep insights",
+ "uv run agentgrep suggestions",
)
)
diff --git a/tests/test_cache_cli.py b/tests/test_cache_cli.py
index a36c3d5de..b2169db71 100644
--- a/tests/test_cache_cli.py
+++ b/tests/test_cache_cli.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
+import os
import pathlib
import sqlite3
import typing as t
@@ -19,6 +20,8 @@
SyncCoverage,
SyncResult,
)
+from agentgrep.insights import OmissionFinding, VariantEdge
+from agentgrep.suggestions import SuggestionArtifact
if t.TYPE_CHECKING:
import collections.abc as cabc
@@ -73,9 +76,18 @@ class DbSyncModeFlagCase(t.NamedTuple):
test_id: str
argv: tuple[str, ...]
+ expected_features_mode: t.Literal["defer", "inline"]
expected_force: bool
+class InsightsAnalyzeFlagCase(t.NamedTuple):
+ """Named case for insights analyze progress flag parsing."""
+
+ test_id: str
+ argv: tuple[str, ...]
+ expected_progress_mode: agentgrep.ProgressMode
+
+
CACHE_FLAG_CASES: tuple[CacheFlagCase, ...] = (
CacheFlagCase("search-default-auto", ("search", "ruff"), "auto"),
CacheFlagCase("search-no-cache-off", ("search", "--no-cache", "ruff"), "off"),
@@ -116,18 +128,45 @@ class DbSyncModeFlagCase(t.NamedTuple):
DB_SYNC_MODE_FLAG_CASES: tuple[DbSyncModeFlagCase, ...] = (
DbSyncModeFlagCase(
- test_id="default-skip-current",
+ test_id="default-defer-skip-current",
argv=("db", "sync"),
+ expected_features_mode="defer",
+ expected_force=False,
+ ),
+ DbSyncModeFlagCase(
+ test_id="inline-features",
+ argv=("db", "sync", "--features", "inline"),
+ expected_features_mode="inline",
expected_force=False,
),
DbSyncModeFlagCase(
test_id="force-resync",
argv=("db", "sync", "--force"),
+ expected_features_mode="defer",
expected_force=True,
),
)
+INSIGHTS_ANALYZE_FLAG_CASES: tuple[InsightsAnalyzeFlagCase, ...] = (
+ InsightsAnalyzeFlagCase(
+ test_id="default-auto",
+ argv=("insights", "analyze"),
+ expected_progress_mode="auto",
+ ),
+ InsightsAnalyzeFlagCase(
+ test_id="explicit-never",
+ argv=("insights", "analyze", "--progress", "never"),
+ expected_progress_mode="never",
+ ),
+ InsightsAnalyzeFlagCase(
+ test_id="no-progress-alias",
+ argv=("insights", "analyze", "--no-progress"),
+ expected_progress_mode="never",
+ ),
+)
+
+
COMMAND_GROUP_HELP_CASES: tuple[CommandGroupHelpCase, ...] = (
CommandGroupHelpCase(
test_id="db",
@@ -135,6 +174,18 @@ class DbSyncModeFlagCase(t.NamedTuple):
expected_usage="usage: agentgrep db",
expected_examples_heading="db examples:",
),
+ CommandGroupHelpCase(
+ test_id="insights",
+ argv=("insights",),
+ expected_usage="usage: agentgrep insights",
+ expected_examples_heading="insights examples:",
+ ),
+ CommandGroupHelpCase(
+ test_id="suggestions",
+ argv=("suggestions",),
+ expected_usage="usage: agentgrep suggestions",
+ expected_examples_heading="suggestions examples:",
+ ),
)
@@ -150,6 +201,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"records_removed": 0,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
},
),
),
@@ -164,6 +216,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"records_removed": 0,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
},
),
),
@@ -181,6 +234,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"records_removed": 0,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
},
{
"sources_synced": 3,
@@ -188,6 +242,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"records_removed": 1,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
},
),
),
@@ -208,6 +263,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"records_removed": 0,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
},
],
},
@@ -224,12 +280,17 @@ class DbSyncModeFlagCase(t.NamedTuple):
schema_version=1,
sources=2,
records=3,
+ features=4,
+ variant_edges=5,
+ omission_findings=6,
+ suggestions=7,
),
expected_contains=(
"DB status",
"/tmp/agentgrep.sqlite",
"2 sources",
"3 records",
+ "5 variant edges",
),
expected_not_contains=("DbStatus(", "{", "'db_path'"),
),
@@ -240,6 +301,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
records_indexed=3,
records_removed=1,
sources_skipped=4,
+ features_deferred=5,
),
expected_contains=(
"DB sync",
@@ -247,6 +309,7 @@ class DbSyncModeFlagCase(t.NamedTuple):
"3 records indexed",
"1 record removed",
"4 sources skipped",
+ "5 features deferred",
),
expected_not_contains=("SyncResult(", "{", "'sources_synced'"),
),
@@ -301,6 +364,79 @@ class DbSyncModeFlagCase(t.NamedTuple):
expected_contains=("DB explain", "Coverage", "not recorded"),
expected_not_contains=("DbExplain(",),
),
+ StructuredTextOutputCase(
+ test_id="insights-list",
+ payload={
+ "limit": 10,
+ "variant_edges": {
+ "total": 100,
+ "returned": 1,
+ "truncated": True,
+ "items": [
+ VariantEdge(
+ edge_id="edge-1",
+ run_id="run-1",
+ left_record_id="left-record",
+ right_record_id="right-record",
+ variant_type="exact_duplicate",
+ confidence=1.0,
+ explanation="normalized prompt text is identical",
+ ),
+ ],
+ },
+ "omission_findings": {
+ "total": 2,
+ "returned": 1,
+ "truncated": True,
+ "items": [
+ OmissionFinding(
+ finding_id="finding-1",
+ run_id="run-1",
+ target_path=pathlib.Path("AGENTS.md"),
+ representative_record_id="record-1",
+ confidence=0.82,
+ rationale="neighboring projects repeat this instruction",
+ ),
+ ],
+ },
+ },
+ expected_contains=(
+ "Insights",
+ "limit 10",
+ "1/100 variant edges",
+ "edge-1",
+ "exact_duplicate",
+ "1/2 omission findings",
+ "finding-1",
+ ),
+ expected_not_contains=("VariantEdge(", "OmissionFinding(", "{", "'variant_edges'"),
+ ),
+ StructuredTextOutputCase(
+ test_id="suggestions-list",
+ payload=[
+ SuggestionArtifact(
+ suggestion_id="suggestion-1",
+ run_id="run-1",
+ target_path=pathlib.Path("AGENTS.md"),
+ surface_kind="agents_md",
+ title="Add missing agent instruction",
+ body="Run ruff check before committing.",
+ confidence=0.92,
+ status="proposed",
+ rationale="repeated nearby instruction",
+ reload_note="Reload the agent session.",
+ ),
+ ],
+ expected_contains=(
+ "Suggestions",
+ "suggestion-1",
+ "AGENTS.md",
+ "0.92",
+ "proposed",
+ "Add missing agent instruction",
+ ),
+ expected_not_contains=("SuggestionArtifact(", "{", "'suggestion_id'"),
+ ),
)
@@ -355,9 +491,24 @@ def test_db_sync_parses_cache_refresh_modes(case: DbSyncModeFlagCase) -> None:
parsed = agentgrep.parse_args(case.argv)
assert isinstance(parsed, agentgrep.DbArgs)
+ assert parsed.features_mode == case.expected_features_mode
assert parsed.force is case.expected_force
+@pytest.mark.parametrize(
+ "case",
+ INSIGHTS_ANALYZE_FLAG_CASES,
+ ids=[case.test_id for case in INSIGHTS_ANALYZE_FLAG_CASES],
+)
+def test_insights_analyze_parses_progress_modes(case: InsightsAnalyzeFlagCase) -> None:
+ """Insights analyze exposes the same progress controls as DB sync."""
+ parsed = agentgrep.parse_args(case.argv)
+
+ assert isinstance(parsed, agentgrep.InsightsArgs)
+ assert parsed.action == "analyze"
+ assert parsed.progress_mode == case.expected_progress_mode
+
+
@pytest.mark.parametrize(
"case",
COMMAND_GROUP_HELP_CASES,
@@ -367,7 +518,7 @@ def test_command_groups_without_actions_print_help_directory(
case: CommandGroupHelpCase,
capsys: pytest.CaptureFixture[str],
) -> None:
- """DB command groups act as help directories."""
+ """DB/insight command groups act as help directories."""
parsed = agentgrep.parse_args(case.argv)
captured = capsys.readouterr()
@@ -404,7 +555,7 @@ def test_small_structured_commands_emit_machine_readable_output(
case: StructuredOutputCase,
capsys: pytest.CaptureFixture[str],
) -> None:
- """DB helpers honor JSON and NDJSON."""
+ """DB, insight, and suggestion helpers honor JSON and NDJSON."""
render._print_json_or_text(case.payload, output_mode=case.output_mode)
captured = capsys.readouterr()
@@ -476,6 +627,7 @@ def test_db_sync_forced_progress_keeps_json_stdout_clean(
class RuntimeStub:
"""Runtime stub that exercises the sync progress protocol."""
+ features_mode: t.Literal["defer", "inline"] | None = None
force: bool | None = None
closed: bool = False
coverage: SyncCoverage | None = None
@@ -491,10 +643,12 @@ def sync_sources(
*,
control: agentgrep.SearchControl | None = None,
progress: DbSyncProgress | None = None,
+ features_mode: t.Literal["defer", "inline"] = "defer",
force: bool = False,
coverage: SyncCoverage | None = None,
prune_missing: bool = False,
) -> SyncResult:
+ self.features_mode = features_mode
self.force = force
self.coverage = coverage
self.prune_missing = prune_missing
@@ -543,6 +697,7 @@ def discover_sources_for_search(
output_mode="json",
color_mode="never",
progress_mode="always",
+ features_mode="inline",
force=True,
)
@@ -556,7 +711,9 @@ def discover_sources_for_search(
"records_removed": 0,
"sources_skipped": 0,
"sources_pruned": 0,
+ "features_deferred": 0,
}
+ assert runtime_stub.features_mode == "inline"
assert runtime_stub.force is True
assert runtime_stub.closed is True
assert "DB sync" in captured.err
@@ -613,6 +770,50 @@ def getvalue(self) -> str:
return "".join(self._parts)
+class _TinyTerminalBuffer(_StringBuffer):
+ """TTY stream stub with a file descriptor for terminal-size probing."""
+
+ def fileno(self) -> int:
+ """Return a harmless descriptor number for monkeypatched size probes."""
+ return 1
+
+
+def test_insights_progress_tiny_tty_width_uses_readable_fallback(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Insights progress remains readable when a PTY reports zero columns."""
+ monkeypatch.setattr(render.os, "get_terminal_size", lambda _fd: os.terminal_size((0, 24)))
+ monkeypatch.setattr(
+ render.shutil,
+ "get_terminal_size",
+ lambda *, fallback: os.terminal_size((88, 24)),
+ )
+ buffer = _TinyTerminalBuffer()
+ progress = render.ConsoleInsightsAnalyzeProgress(
+ enabled=True,
+ stream=t.cast("t.TextIO", buffer),
+ tty=True,
+ color_mode="never",
+ refresh_interval=60.0,
+ )
+ result = render.InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=0,
+ variant_edges=0,
+ omission_findings=0,
+ )
+
+ progress.start(1)
+ progress.step_started(1, 1, "similarity", result)
+ progress.set_activity("refreshing feature cache", detail="15,084 missing feature rows")
+ progress.interrupt()
+
+ output = buffer.getvalue()
+ assert "Insights analyze" in output
+ assert "refreshing feature cache" in output
+
+
def test_collection_is_not_a_command(
capsys: pytest.CaptureFixture[str],
) -> None:
@@ -626,12 +827,592 @@ def test_collection_is_not_a_command(
assert "db" in captured.err
+def test_insights_analyze_command_parses_target_path() -> None:
+ """Insights commands expose deterministic analysis configuration."""
+ parsed = agentgrep.parse_args(
+ ("insights", "analyze", "--target", "AGENTS.md", "--kind", "omissions"),
+ )
+
+ assert isinstance(parsed, agentgrep.InsightsArgs)
+ assert parsed.action == "analyze"
+ assert parsed.kind == "omissions"
+ assert parsed.target == "AGENTS.md"
+
+
+def test_insights_list_parses_default_and_explicit_limit() -> None:
+ """Insights list defaults to a bounded CLI page."""
+ default = agentgrep.parse_args(("insights", "list"))
+ explicit = agentgrep.parse_args(("insights", "list", "--limit", "12"))
+
+ assert isinstance(default, agentgrep.InsightsArgs)
+ assert default.action == "list"
+ assert default.limit == 50
+ assert isinstance(explicit, agentgrep.InsightsArgs)
+ assert explicit.limit == 12
+
+
+def test_insights_list_rejects_non_positive_limit(
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights list limit must stay positive."""
+ with pytest.raises(SystemExit) as exc_info:
+ _ = agentgrep.parse_args(("insights", "list", "--limit", "0"))
+
+ captured = capsys.readouterr()
+ assert exc_info.value.code == 2
+ assert "--limit" in captured.err
+
+
def test_command_group_actions_keep_default_behavior() -> None:
"""No-arg directory help does not remove existing action defaults."""
db = agentgrep.parse_args(("db", "sync"))
+ insights = agentgrep.parse_args(("insights", "analyze"))
+ suggestions = agentgrep.parse_args(("suggestions", "list"))
assert isinstance(db, agentgrep.DbArgs)
assert db.action == "sync"
+ assert isinstance(insights, agentgrep.InsightsArgs)
+ assert insights.action == "analyze"
+ assert insights.kind == "all"
+ assert isinstance(suggestions, agentgrep.SuggestionsArgs)
+ assert suggestions.action == "list"
+
+
+def test_suggestions_show_without_identifier_still_errors() -> None:
+ """Only command groups become directories; required action operands remain strict."""
+ with pytest.raises(SystemExit) as exc_info:
+ _ = agentgrep.parse_args(("suggestions", "show"))
+
+ assert exc_info.value.code == 2
+
+
+def test_insights_analyze_omissions_requires_target(
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Omission analysis needs an explicit target instruction surface."""
+ with pytest.raises(SystemExit) as exc_info:
+ _ = agentgrep.parse_args(("insights", "analyze", "--kind", "omissions"))
+
+ captured = capsys.readouterr()
+ assert exc_info.value.code == 2
+ assert "--target" in captured.err
+
+
+def test_insights_run_is_not_an_action(
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """``run`` is rejected as an unknown insights action."""
+ with pytest.raises(SystemExit) as exc_info:
+ _ = agentgrep.parse_args(("insights", "run"))
+
+ captured = capsys.readouterr()
+ assert exc_info.value.code == 2
+ assert "invalid choice: 'run'" in captured.err
+ assert "analyze" in captured.err
+
+
+def test_insights_analyze_forced_progress_keeps_json_stdout_clean(
+ tmp_path: pathlib.Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Forced insights progress writes status to stderr, never JSON stdout."""
+
+ class RuntimeStub:
+ """Runtime stub with a store attribute for the insight engine."""
+
+ store = object()
+
+ def close(self) -> None:
+ """Accept the command's close call."""
+
+ class EngineStub:
+ """Insight engine stub that exercises analyze progress."""
+
+ def __init__(self, _store: object) -> None:
+ self._store = _store
+
+ def run_similarity(
+ self,
+ *,
+ control: agentgrep.SearchControl | None = None,
+ progress: object | None = None,
+ ) -> object:
+ _ = (control, progress)
+ return {"kind": "similarity", "variant_edges": 2}
+
+ import agentgrep.insights as insights_module
+
+ monkeypatch.setattr(render, "_open_db_runtime", lambda _path: RuntimeStub())
+ monkeypatch.setattr(insights_module, "InsightEngine", EngineStub)
+ args = agentgrep.InsightsArgs(
+ action="analyze",
+ db_path=None,
+ kind="similarity",
+ target=None,
+ output_mode="json",
+ color_mode="never",
+ progress_mode="always",
+ )
+
+ exit_code = render.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ assert exit_code == 0
+ assert json.loads(captured.out) == {"kind": "similarity", "variant_edges": 2}
+ assert "Insights analyze" in captured.err
+ assert "Analyze complete:" in captured.err
+
+
+def test_insights_list_uses_limited_pages_and_count_metadata(
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights list fetches a bounded page plus SQL-backed totals."""
+
+ class RuntimeStub:
+ """Runtime stub with a store attribute for the insight engine."""
+
+ store = object()
+
+ def close(self) -> None:
+ """Accept the command's close call."""
+
+ def __enter__(self) -> t.Self:
+ """Return the stub for context-managed reads."""
+ return self
+
+ def __exit__(self, *exc_info: object) -> None:
+ """Close on context exit."""
+ self.close()
+
+ class EngineStub:
+ """Insight engine stub that records list limits."""
+
+ variant_limit: int | None = None
+ omission_limit: int | None = None
+
+ def __init__(self, _store: object) -> None:
+ self._store = _store
+
+ def count_variant_edges(self) -> int:
+ """Return the full persisted edge count."""
+ return 100
+
+ def count_omission_findings(self) -> int:
+ """Return the full persisted omission count."""
+ return 2
+
+ def list_variant_edges(self, *, limit: int | None = None) -> list[dict[str, object]]:
+ """Return one bounded edge sample."""
+ type(self).variant_limit = limit
+ return [{"edge_id": "edge-1"}]
+
+ def list_omission_findings(
+ self,
+ *,
+ limit: int | None = None,
+ ) -> list[dict[str, object]]:
+ """Return one bounded omission sample."""
+ type(self).omission_limit = limit
+ return [{"finding_id": "finding-1"}]
+
+ import agentgrep.db as agentgrep_db
+ import agentgrep.insights as insights_module
+
+ db_file = pathlib.Path(os.environ["AGENTGREP_DB"])
+ db_file.touch()
+ monkeypatch.setattr(
+ agentgrep_db.DbRuntime,
+ "open_readonly",
+ classmethod(lambda _cls, _path=None: RuntimeStub()),
+ )
+ monkeypatch.setattr(insights_module, "InsightEngine", EngineStub)
+ args = agentgrep.InsightsArgs(
+ action="list",
+ db_path=None,
+ kind="all",
+ target=None,
+ output_mode="json",
+ color_mode="never",
+ limit=7,
+ )
+
+ exit_code = render.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ payload = json.loads(captured.out)
+ assert exit_code == 0
+ assert EngineStub.variant_limit == 7
+ assert EngineStub.omission_limit == 7
+ assert payload["variant_edges"]["total"] == 100
+ assert payload["variant_edges"]["returned"] == 1
+ assert payload["variant_edges"]["truncated"] is True
+ assert payload["omission_findings"]["total"] == 2
+
+
+def test_insights_list_default_output_is_human_summary(
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights list defaults to terminal text, not Python repr or JSON."""
+
+ class RuntimeStub:
+ """Runtime stub with a store attribute for the insight engine."""
+
+ store = object()
+
+ def close(self) -> None:
+ """Accept the command's close call."""
+
+ def __enter__(self) -> t.Self:
+ """Return the stub for context-managed reads."""
+ return self
+
+ def __exit__(self, *exc_info: object) -> None:
+ """Close on context exit."""
+ self.close()
+
+ class EngineStub:
+ """Insight engine stub with one persisted edge sample."""
+
+ def __init__(self, _store: object) -> None:
+ self._store = _store
+
+ def count_variant_edges(self) -> int:
+ """Return the full persisted edge count."""
+ return 100
+
+ def count_omission_findings(self) -> int:
+ """Return the full persisted omission count."""
+ return 0
+
+ def list_variant_edges(self, *, limit: int | None = None) -> list[VariantEdge]:
+ """Return one bounded edge sample."""
+ assert limit == 7
+ return [
+ VariantEdge(
+ edge_id="edge-1",
+ run_id="run-1",
+ left_record_id="left-record",
+ right_record_id="right-record",
+ variant_type="exact_duplicate",
+ confidence=1.0,
+ explanation="normalized prompt text is identical",
+ ),
+ ]
+
+ def list_omission_findings(
+ self,
+ *,
+ limit: int | None = None,
+ ) -> list[OmissionFinding]:
+ """Return no omission samples."""
+ assert limit == 7
+ return []
+
+ import agentgrep.db as agentgrep_db
+ import agentgrep.insights as insights_module
+
+ db_file = pathlib.Path(os.environ["AGENTGREP_DB"])
+ db_file.touch()
+ monkeypatch.setattr(
+ agentgrep_db.DbRuntime,
+ "open_readonly",
+ classmethod(lambda _cls, _path=None: RuntimeStub()),
+ )
+ monkeypatch.setattr(insights_module, "InsightEngine", EngineStub)
+ args = agentgrep.InsightsArgs(
+ action="list",
+ db_path=None,
+ kind="all",
+ target=None,
+ output_mode="text",
+ color_mode="never",
+ limit=7,
+ )
+
+ exit_code = render.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ assert exit_code == 0
+ assert captured.err == ""
+ assert "Insights" in captured.out
+ assert "limit 7" in captured.out
+ assert "1/100 variant edges" in captured.out
+ assert "edge-1" in captured.out
+ assert not captured.out.lstrip().startswith(("{", "["))
+ assert "VariantEdge(" not in captured.out
+
+
+def test_insights_explain_uses_counts_without_listing_rows(
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights explain is a cheap summary, not a row dump."""
+
+ class RuntimeStub:
+ """Runtime stub with a store attribute for the insight engine."""
+
+ store = object()
+
+ def close(self) -> None:
+ """Accept the command's close call."""
+
+ def __enter__(self) -> t.Self:
+ """Return the stub for context-managed reads."""
+ return self
+
+ def __exit__(self, *exc_info: object) -> None:
+ """Close on context exit."""
+ self.close()
+
+ class EngineStub:
+ """Insight engine stub that rejects unbounded list calls."""
+
+ def __init__(self, _store: object) -> None:
+ self._store = _store
+
+ def count_variant_edges(self) -> int:
+ """Return the full persisted edge count."""
+ return 100
+
+ def count_omission_findings(self) -> int:
+ """Return the full persisted omission count."""
+ return 2
+
+ def list_variant_edges(self, *, limit: int | None = None) -> list[object]:
+ """Reject accidental row listing."""
+ _ = limit
+ msg = "explain should not list variant edges"
+ raise AssertionError(msg)
+
+ def list_omission_findings(self, *, limit: int | None = None) -> list[object]:
+ """Reject accidental row listing."""
+ _ = limit
+ msg = "explain should not list omission findings"
+ raise AssertionError(msg)
+
+ import agentgrep.db as agentgrep_db
+ import agentgrep.insights as insights_module
+
+ db_file = pathlib.Path(os.environ["AGENTGREP_DB"])
+ db_file.touch()
+ monkeypatch.setattr(
+ agentgrep_db.DbRuntime,
+ "open_readonly",
+ classmethod(lambda _cls, _path=None: RuntimeStub()),
+ )
+ monkeypatch.setattr(insights_module, "InsightEngine", EngineStub)
+ args = agentgrep.InsightsArgs(
+ action="explain",
+ db_path=None,
+ kind="all",
+ target=None,
+ output_mode="json",
+ color_mode="never",
+ )
+
+ exit_code = render.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ assert exit_code == 0
+ assert json.loads(captured.out) == {
+ "variant_edges": 100,
+ "omission_findings": 2,
+ }
+
+
+def test_insights_analyze_can_exit_early_between_steps(
+ tmp_path: pathlib.Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights analyze honors answer-now control before the next insight step."""
+ target = tmp_path / "AGENTS.md"
+ target.write_text("Run pytest before committing.\n", encoding="utf-8")
+
+ class RuntimeStub:
+ """Runtime stub with a store attribute for the insight engine."""
+
+ store = object()
+
+ def close(self) -> None:
+ """Accept the command's close call."""
+
+ class EngineStub:
+ """Insight engine stub that requests early exit after similarity."""
+
+ def __init__(self, _store: object) -> None:
+ self._store = _store
+
+ def run_similarity(
+ self,
+ *,
+ control: agentgrep.SearchControl | None = None,
+ progress: object | None = None,
+ ) -> object:
+ _ = progress
+ assert control is not None
+ control.request_answer_now()
+ return {"kind": "similarity", "variant_edges": 1}
+
+ def run_omissions(
+ self,
+ *,
+ target_path: pathlib.Path,
+ target_text: str,
+ control: agentgrep.SearchControl | None = None,
+ progress: object | None = None,
+ ) -> object:
+ _ = (target_path, target_text, control, progress)
+ msg = "omissions should not run after early exit"
+ raise AssertionError(msg)
+
+ import agentgrep.insights as insights_module
+
+ monkeypatch.setattr(render, "_open_db_runtime", lambda _path: RuntimeStub())
+ monkeypatch.setattr(insights_module, "InsightEngine", EngineStub)
+ args = agentgrep.InsightsArgs(
+ action="analyze",
+ db_path=None,
+ kind="all",
+ target=str(target),
+ output_mode="json",
+ color_mode="never",
+ progress_mode="always",
+ )
+
+ exit_code = render.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ assert exit_code == 0
+ assert json.loads(captured.out) == {"kind": "similarity", "variant_edges": 1}
+ assert "Exiting early:" in captured.err
+
+
+def test_insights_analyze_tty_progress_renders_exit_hint() -> None:
+ """TTY insights progress mirrors DB sync hint and color semantics."""
+ buffer = _StringBuffer()
+ progress = render.ConsoleInsightsAnalyzeProgress(
+ enabled=True,
+ stream=t.cast("t.TextIO", buffer),
+ tty=True,
+ color_mode="always",
+ refresh_interval=60.0,
+ answer_now_hint=True,
+ )
+ result = render.InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=0,
+ variant_edges=0,
+ omission_findings=0,
+ )
+
+ progress.start(1)
+ progress.step_started(1, 1, "similarity", result)
+ progress.exiting_early(result)
+
+ output = buffer.getvalue()
+ assert "\x1b[" in output
+ assert "[Press enter, exit early]" in output
+ assert "Exiting early:" in output
+
+
+def test_insights_analyze_progress_lines_show_activity_without_empty_results() -> None:
+ """Insights progress does not show zero aggregate counters as live results."""
+ colors = agentgrep.AnsiColors(enabled=False)
+ snapshot = render.InsightsAnalyzeProgressSnapshot(
+ phase="analyzing",
+ current=1,
+ total=1,
+ detail="similarity",
+ activity="refreshing feature cache",
+ activity_detail="15,084 missing feature rows",
+ result=render.InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=0,
+ variant_edges=0,
+ omission_findings=0,
+ ),
+ elapsed=16.6,
+ )
+
+ lines = render.format_insights_analyze_progress_lines(
+ snapshot,
+ colors=colors,
+ answer_now_hint=True,
+ )
+
+ assert lines == (
+ "Insights analyze | analyzing 1/1 steps | similarity | 16.6s | [Press enter, exit early]",
+ "Doing | refreshing feature cache | 15,084 missing feature rows",
+ )
+
+
+def test_insights_analyze_progress_lines_show_nonzero_results() -> None:
+ """Insights progress keeps aggregate counters once a step has produced output."""
+ colors = agentgrep.AnsiColors(enabled=False)
+ snapshot = render.InsightsAnalyzeProgressSnapshot(
+ phase="analyzing",
+ current=2,
+ total=2,
+ detail="omissions",
+ activity="comparing omission candidates",
+ activity_detail="100 indexed records",
+ result=render.InsightsAnalyzeProgressResult(
+ runs_analyzed=1,
+ features_refreshed=3,
+ clusters=2,
+ variant_edges=4,
+ omission_findings=0,
+ ),
+ elapsed=5.5,
+ )
+
+ lines = render.format_insights_analyze_progress_lines(snapshot, colors=colors)
+
+ assert lines == (
+ "Insights analyze | analyzing 2/2 steps | omissions | 5.5s",
+ "Doing | comparing omission candidates | 100 indexed records",
+ (
+ "Results | 1 run analyzed | 3 features refreshed | 2 clusters | "
+ "4 variant edges | 0 omission findings"
+ ),
+ )
+
+
+def test_insights_analyze_progress_lines_preserve_step_detail_when_results_truncate() -> None:
+ """Wide result counters do not hide the current insight step."""
+ colors = agentgrep.AnsiColors(enabled=False)
+ snapshot = render.InsightsAnalyzeProgressSnapshot(
+ phase="analyzing",
+ current=1,
+ total=1,
+ detail="similarity",
+ activity="writing similarity artifacts",
+ activity_detail="658 duplicate prompt families",
+ result=render.InsightsAnalyzeProgressResult(
+ runs_analyzed=0,
+ features_refreshed=0,
+ clusters=658,
+ variant_edges=5584,
+ omission_findings=0,
+ ),
+ elapsed=16.6,
+ )
+
+ lines = render.format_insights_analyze_progress_lines(
+ snapshot,
+ colors=colors,
+ max_width=72,
+ )
+
+ assert "similarity" in lines[0]
+ assert "writing similarity artifacts" in lines[1]
+ assert lines[2].endswith("…")
def test_grep_cache_require_unsupported_query_exits_without_traceback(
@@ -690,6 +1471,16 @@ def close(self) -> None:
assert "Traceback" not in captured.err
+def test_suggestions_render_command_parses_identifier() -> None:
+ """Suggestions commands retrieve stored review artifacts by id."""
+ parsed = agentgrep.parse_args(("suggestions", "render", "suggestion-1", "--json"))
+
+ assert isinstance(parsed, agentgrep.SuggestionsArgs)
+ assert parsed.action == "render"
+ assert parsed.suggestion_id == "suggestion-1"
+ assert parsed.output_mode == "json"
+
+
def test_db_command_closes_runtime_on_exit(
tmp_path: pathlib.Path,
monkeypatch: pytest.MonkeyPatch,
@@ -1214,9 +2005,10 @@ def sync_sources(
force: bool = False,
coverage: SyncCoverage | None = None,
prune_missing: bool = False,
+ features_mode: str = "defer",
) -> SyncResult:
"""Record coverage and pruning, returning zero counters."""
- del sources, control, progress, force
+ del sources, control, progress, force, features_mode
self.coverage = coverage
self.prune_missing = prune_missing
return SyncResult(sources_synced=0, records_indexed=0, records_removed=0)
@@ -1368,3 +2160,202 @@ def test_db_sync_prunes_only_on_full_syncs(
assert exit_code == 0
assert stub.prune_missing is case.expected_prune
+
+
+def test_insights_command_closes_runtime_on_exit(
+ tmp_path: pathlib.Path,
+ monkeypatch: pytest.MonkeyPatch,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights read actions open read-only and close their connection."""
+ import agentgrep.db as agentgrep_db
+
+ db_path = tmp_path / "agentgrep.sqlite"
+ agentgrep_db.DbRuntime.open(db_path).close()
+ opened: list[agentgrep_db.DbRuntime] = []
+ real_open_readonly = agentgrep_db.DbRuntime.open_readonly
+
+ def capturing_open_readonly(
+ db_path: pathlib.Path | str | None = None,
+ ) -> agentgrep_db.DbRuntime:
+ runtime = real_open_readonly(db_path)
+ opened.append(runtime)
+ return runtime
+
+ monkeypatch.setattr(agentgrep_db.DbRuntime, "open_readonly", capturing_open_readonly)
+ args = agentgrep.InsightsArgs(
+ action="explain",
+ db_path=str(db_path),
+ kind="all",
+ target=None,
+ output_mode="json",
+ )
+
+ exit_code = agentgrep.run_insights_command(args)
+
+ _ = capsys.readouterr()
+ assert exit_code == 0
+ assert len(opened) == 1
+ with pytest.raises(sqlite3.ProgrammingError):
+ _ = opened[0].store.connection.execute("SELECT 1")
+
+
+def test_insights_list_on_missing_db_reports_empty_without_creating(
+ tmp_path: pathlib.Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Insights list on a missing cache reports zeros and creates nothing."""
+ db_path = tmp_path / "missing.sqlite"
+ args = agentgrep.InsightsArgs(
+ action="list",
+ db_path=str(db_path),
+ kind="all",
+ target=None,
+ output_mode="json",
+ )
+
+ exit_code = agentgrep.run_insights_command(args)
+
+ captured = capsys.readouterr()
+ payload = json.loads(captured.out)
+ assert exit_code == 0
+ assert payload["variant_edges"]["total"] == 0
+ assert payload["omission_findings"]["truncated"] is False
+ assert not db_path.exists()
+
+
+def test_suggestions_list_on_missing_db_reports_empty_without_creating(
+ tmp_path: pathlib.Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """Suggestions list on a missing cache reports zero rows, creates nothing."""
+ db_path = tmp_path / "missing.sqlite"
+ args = agentgrep.SuggestionsArgs(
+ action="list",
+ db_path=str(db_path),
+ suggestion_id=None,
+ target=None,
+ output_mode="json",
+ )
+
+ exit_code = agentgrep.run_suggestions_command(args)
+
+ captured = capsys.readouterr()
+ payload = json.loads(captured.out)
+ assert exit_code == 0
+ assert payload["suggestions"]["total"] == 0
+ assert payload["suggestions"]["truncated"] is False
+ assert not db_path.exists()
+
+
+class SuggestionsLimitFlagCase(t.NamedTuple):
+ """Named case for suggestions list limit parsing."""
+
+ test_id: str
+ argv: tuple[str, ...]
+ expected_limit: int
+
+
+SUGGESTIONS_LIMIT_FLAG_CASES: tuple[SuggestionsLimitFlagCase, ...] = (
+ SuggestionsLimitFlagCase(
+ test_id="default-limit",
+ argv=("suggestions", "list"),
+ expected_limit=50,
+ ),
+ SuggestionsLimitFlagCase(
+ test_id="explicit-limit",
+ argv=("suggestions", "list", "--limit", "5"),
+ expected_limit=5,
+ ),
+)
+
+
+@pytest.mark.parametrize(
+ "case",
+ SUGGESTIONS_LIMIT_FLAG_CASES,
+ ids=[case.test_id for case in SUGGESTIONS_LIMIT_FLAG_CASES],
+)
+def test_suggestions_list_parses_limit(case: SuggestionsLimitFlagCase) -> None:
+ """Suggestions list exposes a bounded page size."""
+ parsed = agentgrep.parse_args(case.argv)
+
+ assert isinstance(parsed, agentgrep.SuggestionsArgs)
+ assert parsed.limit == case.expected_limit
+
+
+def test_suggestions_list_rejects_non_positive_limit(
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """A non-positive suggestions limit is a parse-time error."""
+ with pytest.raises(SystemExit) as exc_info:
+ _ = agentgrep.parse_args(("suggestions", "list", "--limit", "0"))
+
+ captured = capsys.readouterr()
+ assert exc_info.value.code == 2
+ assert "--limit must be greater than 0" in captured.err
+
+
+def test_suggestions_list_returns_bounded_page_with_totals(
+ tmp_path: pathlib.Path,
+ capsys: pytest.CaptureFixture[str],
+) -> None:
+ """The suggestions list payload carries totals and truncation."""
+ import agentgrep.db as agentgrep_db
+ from agentgrep.insights import InsightEngine
+ from agentgrep.suggestions import SuggestionEngine
+
+ db_path = tmp_path / "agentgrep.sqlite"
+ source_path = tmp_path / "session.jsonl"
+ target_path = tmp_path / "AGENTS.md"
+ source_path.write_text("{}", encoding="utf-8")
+ target_path.write_text("Run pytest before committing.\n", encoding="utf-8")
+ runtime = agentgrep_db.DbRuntime.open(db_path)
+ source = agentgrep.SourceHandle(
+ agent="codex",
+ store="codex.sessions",
+ adapter_id="codex.sessions_jsonl.v1",
+ path=source_path,
+ path_kind="session_file",
+ source_kind="jsonl",
+ search_root=source_path.parent,
+ mtime_ns=source_path.stat().st_mtime_ns,
+ )
+ records = tuple(
+ agentgrep.SearchRecord(
+ kind="prompt",
+ agent=source.agent,
+ store=source.store,
+ adapter_id=source.adapter_id,
+ path=source.path,
+ text=text,
+ timestamp="2026-06-05T12:00:00Z",
+ session_id=f"session-{index}",
+ )
+ for index, text in enumerate(
+ ("Run ruff check before committing.", "Run ty check before committing."),
+ )
+ )
+ _ = runtime.sync_records(((source, records),), features_mode="inline")
+ insights = InsightEngine(runtime.store)
+ _ = insights.run_omissions(target_path=target_path, target_text=target_path.read_text())
+ created = SuggestionEngine(runtime.store).create_from_omissions(target_path=target_path)
+ runtime.close()
+ assert len(created) >= 2
+ args = agentgrep.SuggestionsArgs(
+ action="list",
+ db_path=str(db_path),
+ suggestion_id=None,
+ target=None,
+ output_mode="json",
+ limit=1,
+ )
+
+ exit_code = agentgrep.run_suggestions_command(args)
+
+ captured = capsys.readouterr()
+ payload = json.loads(captured.out)
+ assert exit_code == 0
+ assert payload["limit"] == 1
+ assert payload["suggestions"]["returned"] == 1
+ assert payload["suggestions"]["total"] >= 2
+ assert payload["suggestions"]["truncated"] is True
diff --git a/tests/test_db_index.py b/tests/test_db_index.py
index c81e69984..5b22deebc 100644
--- a/tests/test_db_index.py
+++ b/tests/test_db_index.py
@@ -1,4 +1,4 @@
-"""Tests for the persistent DB index layer."""
+"""Tests for the persistent DB, insights, and suggestions layers."""
from __future__ import annotations
@@ -11,6 +11,8 @@
import agentgrep
from agentgrep.db import DbRuntime, DbStatus, DbSyncProgress, SyncCoverage, SyncResult
+from agentgrep.insights import InsightEngine, VariantEdge
+from agentgrep.suggestions import SuggestionArtifact, SuggestionEngine
class CachedSearchCase(t.NamedTuple):
@@ -29,6 +31,15 @@ class DuplicateRecordCase(t.NamedTuple):
expected_records: int
+class SyncFeatureModeCase(t.NamedTuple):
+ """Named case for feature-generation behavior during DB sync."""
+
+ test_id: str
+ features_mode: t.Literal["defer", "inline"]
+ expected_features: int
+ expected_deferred: int
+
+
class StopAfterFirstSourceProgress:
"""Progress stub that requests early exit after one source transaction."""
@@ -77,6 +88,17 @@ def exiting_early(self, result: SyncResult) -> None:
self.early_result = result
+class InsightActivityProgress:
+ """Progress stub that records insight engine activity labels."""
+
+ def __init__(self) -> None:
+ self.activities: list[tuple[str, str | None]] = []
+
+ def set_activity(self, activity: str, *, detail: str | None = None) -> None:
+ """Capture one current-work update."""
+ self.activities.append((activity, detail))
+
+
CACHED_SEARCH_CASES: tuple[CachedSearchCase, ...] = (
CachedSearchCase(
test_id="require-uses-index",
@@ -103,6 +125,22 @@ def exiting_early(self, result: SyncResult) -> None:
)
+SYNC_FEATURE_MODE_CASES: tuple[SyncFeatureModeCase, ...] = (
+ SyncFeatureModeCase(
+ test_id="default-defer",
+ features_mode="defer",
+ expected_features=0,
+ expected_deferred=2,
+ ),
+ SyncFeatureModeCase(
+ test_id="inline-features",
+ features_mode="inline",
+ expected_features=2,
+ expected_deferred=0,
+ ),
+)
+
+
def _source(
path: pathlib.Path,
*,
@@ -203,6 +241,42 @@ def test_db_runtime_syncs_records_and_serves_fts_results(
]
+@pytest.mark.parametrize(
+ "case",
+ SYNC_FEATURE_MODE_CASES,
+ ids=[case.test_id for case in SYNC_FEATURE_MODE_CASES],
+)
+def test_db_runtime_sync_feature_modes(
+ case: SyncFeatureModeCase,
+ tmp_path: pathlib.Path,
+) -> None:
+ """DB sync can defer expensive feature generation without breaking FTS."""
+ source_path = tmp_path / "session.jsonl"
+ source_path.write_text("ruff", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+
+ result = runtime.sync_records(
+ (
+ (
+ source,
+ (
+ _record(source, "Run ruff check before committing."),
+ _record(source, "Run pytest for the focused suite."),
+ ),
+ ),
+ ),
+ features_mode=case.features_mode,
+ )
+
+ assert result.records_indexed == 2
+ assert result.features_deferred == case.expected_deferred
+ assert runtime.status().features == case.expected_features
+ assert [record.text for record in runtime.search_records(_query("ruff"))] == [
+ "Run ruff check before committing.",
+ ]
+
+
def test_db_runtime_sync_skips_unchanged_sources_without_reading_records(
tmp_path: pathlib.Path,
) -> None:
@@ -227,6 +301,7 @@ def records_that_should_not_be_read() -> t.Iterator[agentgrep.SearchRecord]:
records_indexed=0,
records_removed=0,
sources_skipped=1,
+ features_deferred=0,
)
assert runtime.status().records == 1
@@ -332,6 +407,7 @@ def test_db_runtime_sync_can_exit_early_between_sources(
sources_synced=1,
records_indexed=1,
records_removed=0,
+ features_deferred=1,
)
assert progress.started_total == 2
assert progress.finished_sources == ["first.jsonl"]
@@ -394,6 +470,422 @@ def iter_source_records(_source: agentgrep.SourceHandle) -> tuple[agentgrep.Sear
assert [record.text for record in records] == list(case.expected_texts)
+def test_insight_engine_records_duplicate_variant_edges(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Similarity insights persist deterministic variant edges with confidence."""
+ first_path = tmp_path / "one.jsonl"
+ second_path = tmp_path / "two.jsonl"
+ first_path.write_text("{}", encoding="utf-8")
+ second_path.write_text("{}", encoding="utf-8")
+ first = _source(first_path)
+ second = _source(second_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (first, (_record(first, "Run ruff check before committing.", session_id="one"),)),
+ (second, (_record(second, "run ruff check before committing", session_id="two"),)),
+ ),
+ )
+
+ result = InsightEngine(runtime.store).run_similarity()
+ edges = InsightEngine(runtime.store).list_variant_edges()
+
+ assert result.variant_edges == 1
+ assert runtime.status().features == 2
+ assert len(edges) == 1
+ assert isinstance(edges[0], VariantEdge)
+ assert edges[0].variant_type == "exact_duplicate"
+ assert edges[0].confidence == pytest.approx(1.0)
+
+
+def test_insight_engine_counts_and_limits_variant_edges(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Variant edge listing can be paged without materializing every row."""
+ source_path = tmp_path / "source.jsonl"
+ source_path.write_text("{}", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (
+ source,
+ tuple(
+ _record(
+ source,
+ "Run ruff check before committing.",
+ session_id=f"duplicate-{index}",
+ )
+ for index in range(4)
+ ),
+ ),
+ ),
+ )
+ _ = InsightEngine(runtime.store).run_similarity()
+ engine = InsightEngine(runtime.store)
+
+ edges = engine.list_variant_edges(limit=2)
+
+ assert engine.count_variant_edges() == 6
+ assert len(edges) == 2
+
+
+def test_variant_edge_listing_uses_confidence_order_index(
+ tmp_path: pathlib.Path,
+) -> None:
+ """The default variant-edge page can stop at the requested limit."""
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+
+ rows = runtime.store.connection.execute(
+ """
+ EXPLAIN QUERY PLAN
+ SELECT edge_id, run_id, left_record_id, right_record_id,
+ variant_type, confidence, explanation
+ FROM variant_edges
+ ORDER BY confidence DESC, edge_id
+ LIMIT 1
+ """,
+ ).fetchall()
+ plan = "\n".join(str(row["detail"]) for row in rows)
+
+ assert "idx_variant_edges_confidence_edge_id" in plan
+ assert "USE TEMP B-TREE" not in plan
+
+
+def test_insight_engine_reports_similarity_activity(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Similarity analysis reports the current backend phase."""
+ first_path = tmp_path / "one.jsonl"
+ second_path = tmp_path / "two.jsonl"
+ first_path.write_text("{}", encoding="utf-8")
+ second_path.write_text("{}", encoding="utf-8")
+ first = _source(first_path)
+ second = _source(second_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (first, (_record(first, "Run ruff check before committing.", session_id="one"),)),
+ (second, (_record(second, "run ruff check before committing", session_id="two"),)),
+ ),
+ )
+ progress = InsightActivityProgress()
+
+ _ = InsightEngine(runtime.store).run_similarity(progress=progress)
+
+ labels = [activity for activity, _detail in progress.activities]
+ first_seen_labels = tuple(dict.fromkeys(labels))
+ assert first_seen_labels == (
+ "checking feature cache",
+ "building feature signatures",
+ "writing feature cache",
+ "loading similarity rows",
+ "grouping duplicate prompts",
+ "writing similarity artifacts",
+ )
+
+
+def test_feature_refresh_reports_incremental_build_and_write_progress(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Feature refresh reports row-level progress inside long phases."""
+ source_path = tmp_path / "source.jsonl"
+ source_path.write_text("{}", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (
+ source,
+ (
+ _record(source, "Run ruff check before committing.", session_id="one"),
+ _record(source, "Run ty check before committing.", session_id="two"),
+ _record(source, "Build docs before committing.", session_id="three"),
+ ),
+ ),
+ ),
+ )
+ progress = InsightActivityProgress()
+
+ refreshed = runtime.store.refresh_missing_features(workers=1, progress=progress)
+
+ assert refreshed == 3
+ build_details = [
+ detail
+ for activity, detail in progress.activities
+ if activity == "building feature signatures"
+ ]
+ write_details = [
+ detail for activity, detail in progress.activities if activity == "writing feature cache"
+ ]
+ assert build_details[-1] == "3/3 rows built (100.0%, 1w)"
+ assert write_details[-1] == "3/3 rows written (100.0%)"
+
+
+def test_similarity_analysis_reports_artifact_write_progress(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Similarity analysis reports cluster and edge write counters."""
+ source_path = tmp_path / "source.jsonl"
+ source_path.write_text("{}", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (
+ source,
+ tuple(
+ _record(
+ source,
+ "Run ruff check before committing.",
+ session_id=f"duplicate-{index}",
+ )
+ for index in range(4)
+ ),
+ ),
+ ),
+ )
+ progress = InsightActivityProgress()
+
+ result = InsightEngine(runtime.store).run_similarity(progress=progress)
+
+ write_details = [
+ detail
+ for activity, detail in progress.activities
+ if activity == "writing similarity artifacts"
+ ]
+ assert result.variant_edges == 6
+ assert write_details[-1] == "1/1 clusters, 6/6 edges"
+
+
+def _sync_prompts(
+ tmp_path: pathlib.Path,
+ texts: tuple[str, ...],
+) -> DbRuntime:
+ """Sync one synthetic prompt per text into a fresh DB runtime."""
+ source_path = tmp_path / "source.jsonl"
+ source_path.write_text("{}", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (
+ source,
+ tuple(
+ _record(source, text, session_id=f"session-{index}")
+ for index, text in enumerate(texts)
+ ),
+ ),
+ ),
+ )
+ return runtime
+
+
+def _near_duplicate_edges(runtime: DbRuntime) -> list[tuple[frozenset[str], float]]:
+ """Return near-duplicate edges as ``({left_text, right_text}, confidence)``."""
+ rows = runtime.store.connection.execute(
+ """
+ SELECT dl.text AS left_text, dr.text AS right_text, v.confidence AS confidence
+ FROM variant_edges v
+ JOIN records_search rl ON rl.record_id = v.left_record_id
+ JOIN record_details dl ON dl.rowid = rl.rowid
+ JOIN records_search rr ON rr.record_id = v.right_record_id
+ JOIN record_details dr ON dr.rowid = rr.rowid
+ WHERE v.variant_type = 'near_duplicate'
+ """,
+ ).fetchall()
+ return [
+ (frozenset({str(row["left_text"]), str(row["right_text"])}), float(row["confidence"]))
+ for row in rows
+ ]
+
+
+def _near_duplicate_cluster_member_texts(runtime: DbRuntime) -> list[frozenset[str]]:
+ """Return the record texts grouped in each near-duplicate cluster."""
+ rows = runtime.store.connection.execute(
+ """
+ SELECT c.cluster_id AS cluster_id, d.text AS text
+ FROM clusters c
+ JOIN cluster_members m ON m.cluster_id = c.cluster_id
+ JOIN records_search r ON r.record_id = m.record_id
+ JOIN record_details d ON d.rowid = r.rowid
+ WHERE c.kind = 'near_duplicate_prompt'
+ ORDER BY c.cluster_id, d.text
+ """,
+ ).fetchall()
+ by_cluster: dict[str, set[str]] = {}
+ for row in rows:
+ by_cluster.setdefault(str(row["cluster_id"]), set()).add(str(row["text"]))
+ return [frozenset(texts) for texts in by_cluster.values()]
+
+
+def test_insight_engine_links_near_duplicate_prompts(tmp_path: pathlib.Path) -> None:
+ """Slightly different prompts become one near-duplicate edge and cluster."""
+ left = "run ruff check across the repo"
+ right = "run ruff check across the whole repo"
+ runtime = _sync_prompts(tmp_path, (left, right))
+
+ result = InsightEngine(runtime.store).run_similarity()
+ edges = _near_duplicate_edges(runtime)
+ clusters = _near_duplicate_cluster_member_texts(runtime)
+
+ assert result.variant_edges == 1
+ assert result.clusters == 1
+ assert len(edges) == 1
+ pair, confidence = edges[0]
+ assert pair == frozenset({left, right})
+ # Confidence is the exact token-set Jaccard: 6 shared / 7 union tokens.
+ assert confidence == pytest.approx(6 / 7)
+ assert clusters == [frozenset({left, right})]
+
+
+def test_insight_engine_keeps_exact_and_near_duplicates_separate(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Exact duplicates stay exact-only and are not re-reported as near."""
+ exact = "run ruff check across the repo"
+ variant = "run ruff check across the whole repo"
+ runtime = _sync_prompts(tmp_path, (exact, exact, variant))
+
+ result = InsightEngine(runtime.store).run_similarity()
+ exact_edges = runtime.store.connection.execute(
+ "SELECT confidence FROM variant_edges WHERE variant_type = 'exact_duplicate'",
+ ).fetchall()
+ near_edges = _near_duplicate_edges(runtime)
+
+ assert len(exact_edges) == 1
+ assert float(exact_edges[0]["confidence"]) == pytest.approx(1.0)
+ # The two identical prompts collapse to one representative, so their
+ # pair is never emitted as a near-duplicate edge.
+ assert frozenset({exact}) not in {pair for pair, _confidence in near_edges}
+ # The representative still links to the distinct near-duplicate variant.
+ assert near_edges == [(frozenset({exact, variant}), pytest.approx(6 / 7))]
+ assert result.variant_edges == 2
+
+
+def test_insight_engine_ignores_unrelated_prompts_for_near_duplicates(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Prompts below the Jaccard threshold produce no near-duplicate edge."""
+ runtime = _sync_prompts(
+ tmp_path,
+ (
+ "run ruff check across the repo",
+ "deploy the kubernetes cluster to staging tonight",
+ ),
+ )
+
+ result = InsightEngine(runtime.store).run_similarity()
+
+ assert result.variant_edges == 0
+ assert result.clusters == 0
+ assert _near_duplicate_edges(runtime) == []
+
+
+def test_insight_engine_clusters_transitive_near_duplicates(
+ tmp_path: pathlib.Path,
+) -> None:
+ """A~B and B~C above threshold land in one cluster via union-find."""
+ a = "run ruff check across the repo before every commit"
+ b = "run ruff check across the repo"
+ c = "run ruff check across the repo inside docker containers"
+ runtime = _sync_prompts(tmp_path, (a, b, c))
+
+ result = InsightEngine(runtime.store).run_similarity()
+ edges = _near_duplicate_edges(runtime)
+ clusters = _near_duplicate_cluster_member_texts(runtime)
+
+ assert result.clusters == 1
+ assert clusters == [frozenset({a, b, c})]
+ # A~C (Jaccard 0.5) is below threshold, so only the two chain edges exist.
+ assert {pair for pair, _confidence in edges} == {
+ frozenset({a, b}),
+ frozenset({b, c}),
+ }
+ assert frozenset({a, c}) not in {pair for pair, _confidence in edges}
+ cluster_confidence = runtime.store.connection.execute(
+ "SELECT confidence FROM clusters WHERE kind = 'near_duplicate_prompt'",
+ ).fetchone()
+ # Cluster confidence is the weakest-link (minimum) pairwise Jaccard.
+ assert float(cluster_confidence["confidence"]) == pytest.approx(6 / 9)
+
+
+def test_near_duplicate_similarity_run_is_deterministic(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Re-running similarity yields identical near-duplicate ids and counts."""
+ runtime = _sync_prompts(
+ tmp_path,
+ (
+ "run ruff check across the repo before every commit",
+ "run ruff check across the repo",
+ "run ruff check across the repo inside docker containers",
+ ),
+ )
+
+ def snapshot() -> tuple[frozenset[str], frozenset[str]]:
+ edge_ids = frozenset(
+ str(row["edge_id"])
+ for row in runtime.store.connection.execute(
+ "SELECT edge_id FROM variant_edges WHERE variant_type = 'near_duplicate'",
+ )
+ )
+ cluster_ids = frozenset(
+ str(row["cluster_id"])
+ for row in runtime.store.connection.execute(
+ "SELECT cluster_id FROM clusters WHERE kind = 'near_duplicate_prompt'",
+ )
+ )
+ return edge_ids, cluster_ids
+
+ first_result = InsightEngine(runtime.store).run_similarity()
+ first_snapshot = snapshot()
+ second_result = InsightEngine(runtime.store).run_similarity()
+ second_snapshot = snapshot()
+
+ assert first_snapshot == second_snapshot
+ assert first_result.run_id == second_result.run_id
+ assert first_result.variant_edges == second_result.variant_edges == 2
+ assert first_result.clusters == second_result.clusters == 1
+
+
+def test_suggestion_engine_renders_review_only_instruction_suggestion(
+ tmp_path: pathlib.Path,
+) -> None:
+ """Omission suggestions are persisted artifacts and do not edit the target file."""
+ source_path = tmp_path / "source.jsonl"
+ target_path = tmp_path / "AGENTS.md"
+ source_path.write_text("{}", encoding="utf-8")
+ target_path.write_text("Run pytest before committing.\n", encoding="utf-8")
+ source = _source(source_path)
+ runtime = DbRuntime.open(tmp_path / "agentgrep.sqlite")
+ _ = runtime.sync_records(
+ (
+ (
+ source,
+ (
+ _record(
+ source,
+ "Run ruff check before committing.",
+ session_id="instruction-source",
+ ),
+ ),
+ ),
+ ),
+ )
+ insights = InsightEngine(runtime.store)
+ _ = insights.run_omissions(target_path=target_path, target_text=target_path.read_text())
+
+ suggestions = SuggestionEngine(runtime.store).create_from_omissions(target_path=target_path)
+
+ assert len(suggestions) == 1
+ assert isinstance(suggestions[0], SuggestionArtifact)
+ assert "Run ruff check before committing." in suggestions[0].body
+ assert "reload" in suggestions[0].reload_note.casefold()
+ assert target_path.read_text(encoding="utf-8") == "Run pytest before committing.\n"
+
+
def test_resync_keeps_fts_index_consistent(tmp_path: pathlib.Path) -> None:
"""Re-syncing a source removes old FTS rows with their stored values.
@@ -606,7 +1098,15 @@ def test_schema_version_mismatch_rebuilds_cache(tmp_path: pathlib.Path) -> None:
runtime = DbRuntime.open(db_path)
_ = runtime.sync_records(
((source, (_record(source, "Run ruff check before committing."),)),),
+ features_mode="inline",
)
+ _ = InsightEngine(runtime.store).run_similarity()
+ fresh_tables = {
+ str(row["name"])
+ for row in runtime.store.connection.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'table'",
+ )
+ }
with runtime.store.connection:
_ = runtime.store.connection.execute(
"UPDATE meta SET value = '999' WHERE key = 'schema_version'",
@@ -615,8 +1115,28 @@ def test_schema_version_mismatch_rebuilds_cache(tmp_path: pathlib.Path) -> None:
reopened = DbRuntime.open(db_path)
- assert reopened.status().records == 0
- assert reopened.status().sources == 0
+ status = reopened.status()
+ assert status.records == 0
+ assert status.sources == 0
+ assert status.features == 0
+ assert status.variant_edges == 0
+ assert status.omission_findings == 0
+ assert status.suggestions == 0
+ rebuilt_tables = {
+ str(row["name"])
+ for row in reopened.store.connection.execute(
+ "SELECT name FROM sqlite_master WHERE type = 'table'",
+ )
+ }
+ assert rebuilt_tables == fresh_tables
+ # Tables without a cascade path to records keep rows if the drop
+ # list misses them; count them directly since status() does not.
+ for table in ("insight_runs", "clusters"):
+ row = reopened.store.connection.execute(
+ f"SELECT COUNT(*) AS count FROM {table}",
+ ).fetchone()
+ assert row is not None
+ assert int(row["count"]) == 0, table
class CacheDedupeCase(t.NamedTuple):