Implement a new MCP tool and API that exposes similar dup...

application/handler/indexing/ordering_test.go

@@ -67,6 +67,10 @@ func (e *emptyEmbeddingStore) Exists(_ context.Context, _ ...repository.Option)
 	return false, nil
 }
 
+func (e *emptyEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) {
+	return nil, nil
+}
+
 func (e *emptyEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error {
 	return nil
 }

application/service/duplicates.go

@@ -0,0 +1,137 @@
+package service
+
+import (
+	"context"
+	"math"
+	"sort"
+
+	"github.com/rs/zerolog"
+
+	"github.com/helixml/kodit/domain/search"
+)
+
+// maxEmbeddingsForDuplicate is the maximum number of embeddings that will be
+// compared pairwise. Larger sets are truncated to avoid O(N²) timeouts.
+const maxEmbeddingsForDuplicate = 5000
+
+// DuplicatePair holds two snippet IDs and their pairwise cosine similarity.
+type DuplicatePair struct {
+	SnippetIDA string
+	SnippetIDB string
+	Similarity float64
+}
+
+// DuplicateSearch finds semantically duplicated code snippets using pairwise
+// embedding comparison.
+type DuplicateSearch struct {
+	codeVectorStore search.EmbeddingStore
+	logger          zerolog.Logger
+}
+
+// NewDuplicateSearch creates a new DuplicateSearch service.
+func NewDuplicateSearch(codeVectorStore search.EmbeddingStore, logger zerolog.Logger) *DuplicateSearch {
+	return &DuplicateSearch{
+		codeVectorStore: codeVectorStore,
+		logger:          logger,
+	}
+}
+
+// FindDuplicates returns snippet pairs whose code embeddings have cosine
+// similarity ≥ threshold. Pairs are sorted by similarity descending and
+// capped at limit. The bool return value is true when the embedding set was
+// truncated to maxEmbeddingsForDuplicate.
+//
+// Returns empty slice (not an error) when the store is nil or has no data.
+func (s *DuplicateSearch) FindDuplicates(
+	ctx context.Context,
+	repoIDs []int64,
+	threshold float64,
+	limit int,
+) ([]DuplicatePair, bool, error) {
+	if s.codeVectorStore == nil {
+		return nil, false, nil
+	}
+
+	filters := search.NewFilters(search.WithSourceRepos(repoIDs))
+	embeddings, err := s.codeVectorStore.FindAll(ctx, filters)
+	if err != nil {
+		return nil, false, err
+	}
+
+	if len(embeddings) == 0 {
+		return nil, false, nil
+	}
+
+	truncated := false
+	if len(embeddings) > maxEmbeddingsForDuplicate {
+		s.logger.Warn().
+			Int("total", len(embeddings)).
+			Int("cap", maxEmbeddingsForDuplicate).
+			Msg("truncating embeddings for duplicate detection")
+		embeddings = embeddings[:maxEmbeddingsForDuplicate]
+		truncated = true
+	}
+
+	// Normalize all vectors to unit length once, then pairwise similarity = dot product.
+	type unitVec struct {
+		snippetID string
+		v         []float64
+	}
+	normalized := make([]unitVec, 0, len(embeddings))
+	for _, emb := range embeddings {
+		vec := emb.Vector()
+		mag := magnitude(vec)
+		if mag == 0 {
+			continue // skip zero vectors
+		}
+		unit := make([]float64, len(vec))
+		for i, x := range vec {
+			unit[i] = x / mag
+		}
+		normalized = append(normalized, unitVec{snippetID: emb.SnippetID(), v: unit})
+	}
+
+	// Triangle scan: compare each pair (i, j) where j > i.
+	var pairs []DuplicatePair
+	for i := 0; i < len(normalized); i++ {
+		for j := i + 1; j < len(normalized); j++ {
+			sim := dotProduct(normalized[i].v, normalized[j].v)
+			if sim >= threshold {
+				pairs = append(pairs, DuplicatePair{
+					SnippetIDA: normalized[i].snippetID,
+					SnippetIDB: normalized[j].snippetID,
+					Similarity: sim,
+				})
+			}
+		}
+	}
+
+	// Sort by similarity descending.
+	sort.Slice(pairs, func(i, j int) bool {
+		return pairs[i].Similarity > pairs[j].Similarity
+	})
+
+	if limit > 0 && len(pairs) > limit {
+		pairs = pairs[:limit]
+	}
+
+	return pairs, truncated, nil
+}
+
+// magnitude returns the Euclidean magnitude of a vector.
+func magnitude(v []float64) float64 {
+	var sum float64
+	for _, x := range v {
+		sum += x * x
+	}
+	return math.Sqrt(sum)
+}
+
+// dotProduct returns the dot product of two vectors (assumed equal length).
+func dotProduct(a, b []float64) float64 {
+	var sum float64
+	for i := range a {
+		sum += a[i] * b[i]
+	}
+	return sum
+}

application/service/duplicates_test.go

@@ -0,0 +1,228 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"github.com/rs/zerolog"
+
+	"github.com/helixml/kodit/domain/repository"
+	"github.com/helixml/kodit/domain/search"
+)
+
+// dupEmbeddingStore is a fake EmbeddingStore for duplicate detection tests.
+type dupEmbeddingStore struct {
+	embeddings []search.Embedding
+	err        error
+}
+
+func (d *dupEmbeddingStore) SaveAll(_ context.Context, _ []search.Embedding) error { return nil }
+func (d *dupEmbeddingStore) Find(_ context.Context, _ ...repository.Option) ([]search.Embedding, error) {
+	return nil, nil
+}
+func (d *dupEmbeddingStore) Search(_ context.Context, _ ...repository.Option) ([]search.Result, error) {
+	return nil, nil
+}
+func (d *dupEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) {
+	return false, nil
+}
+func (d *dupEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) {
+	if d.err != nil {
+		return nil, d.err
+	}
+	return d.embeddings, nil
+}
+func (d *dupEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil }
+
+func newDupService(embeddings []search.Embedding) *DuplicateSearch {
+	store := &dupEmbeddingStore{embeddings: embeddings}
+	return NewDuplicateSearch(store, zerolog.Nop())
+}
+
+func TestFindDuplicates_NoStore_ReturnsEmpty(t *testing.T) {
+	svc := NewDuplicateSearch(nil, zerolog.Nop())
+	pairs, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if truncated {
+		t.Error("want truncated=false, got true")
+	}
+	if len(pairs) != 0 {
+		t.Errorf("want 0 pairs, got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_EmptyEmbeddings_ReturnsEmpty(t *testing.T) {
+	svc := newDupService(nil)
+	pairs, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if truncated {
+		t.Error("want truncated=false, got true")
+	}
+	if len(pairs) != 0 {
+		t.Errorf("want 0 pairs, got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_SingleEmbedding_ReturnsEmpty(t *testing.T) {
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", []float64{1, 0, 0}),
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 0 {
+		t.Errorf("want 0 pairs, got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_IdenticalVectors_ReturnsSimilarityOne(t *testing.T) {
+	vec := []float64{1.0, 0.5, 0.3}
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", vec),
+		search.NewEmbedding("2", vec),
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 1 {
+		t.Fatalf("want 1 pair, got %d", len(pairs))
+	}
+	if pairs[0].Similarity < 0.999 {
+		t.Errorf("want similarity ~1.0, got %f", pairs[0].Similarity)
+	}
+	// Verify snippet IDs
+	gotA, gotB := pairs[0].SnippetIDA, pairs[0].SnippetIDB
+	if !((gotA == "1" && gotB == "2") || (gotA == "2" && gotB == "1")) {
+		t.Errorf("unexpected snippet IDs: A=%s B=%s", gotA, gotB)
+	}
+}
+
+func TestFindDuplicates_DissimilarVectors_NotReturned(t *testing.T) {
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", []float64{1, 0, 0}),
+		search.NewEmbedding("2", []float64{0, 1, 0}), // orthogonal: cosine similarity = 0
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 0 {
+		t.Errorf("want 0 pairs, got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_ThresholdBoundary_Inclusive(t *testing.T) {
+	// Two vectors with known cosine similarity
+	// cos([1,1,0], [1,0,0]) = 1/sqrt(2) ≈ 0.7071
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", []float64{1, 1, 0}),
+		search.NewEmbedding("2", []float64{1, 0, 0}),
+	}
+	svc := newDupService(embeddings)
+
+	// Threshold just below the similarity → should be returned
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.70, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 1 {
+		t.Errorf("threshold below sim: want 1 pair, got %d", len(pairs))
+	}
+
+	// Threshold above the similarity → should not be returned
+	pairs, _, err = svc.FindDuplicates(context.Background(), []int64{1}, 0.80, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 0 {
+		t.Errorf("threshold above sim: want 0 pairs, got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_LimitCap(t *testing.T) {
+	// 4 embeddings → up to 6 pairs; we set limit=3
+	vec := []float64{1, 0, 0}
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", vec),
+		search.NewEmbedding("2", vec),
+		search.NewEmbedding("3", vec),
+		search.NewEmbedding("4", vec),
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 3)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) != 3 {
+		t.Errorf("want 3 pairs (capped), got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_SortedBySimilarityDescending(t *testing.T) {
+	// 3 embeddings: pair (1,2) is very similar, pair (1,3) is less similar
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", []float64{1.0, 0.0}),
+		search.NewEmbedding("2", []float64{0.99, 0.01}), // close to "1"
+		search.NewEmbedding("3", []float64{0.5, 0.5}),   // farther from "1"
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.0, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if len(pairs) < 2 {
+		t.Fatalf("want at least 2 pairs, got %d", len(pairs))
+	}
+	for i := 1; i < len(pairs); i++ {
+		if pairs[i].Similarity > pairs[i-1].Similarity {
+			t.Errorf("pairs not sorted: pairs[%d].Similarity=%f > pairs[%d].Similarity=%f",
+				i, pairs[i].Similarity, i-1, pairs[i-1].Similarity)
+		}
+	}
+}
+
+func TestFindDuplicates_ZeroVector_Skipped(t *testing.T) {
+	// Zero vectors should not produce NaN similarities and should be skipped
+	embeddings := []search.Embedding{
+		search.NewEmbedding("1", []float64{0, 0, 0}),
+		search.NewEmbedding("2", []float64{1, 0, 0}),
+	}
+	svc := newDupService(embeddings)
+	pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.0, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	// Zero-magnitude vector should be skipped, no pairs returned
+	if len(pairs) != 0 {
+		t.Errorf("want 0 pairs (zero vector skipped), got %d", len(pairs))
+	}
+}
+
+func TestFindDuplicates_Truncated_WhenExceedsMax(t *testing.T) {
+	// Create more embeddings than maxEmbeddingsForDuplicate
+	n := maxEmbeddingsForDuplicate + 1
+	embeddings := make([]search.Embedding, n)
+	for i := range embeddings {
+		embeddings[i] = search.NewEmbedding(
+			string(rune('A'+i%26))+string(rune('a'+i%26)),
+			[]float64{float64(i), 1, 0},
+		)
+	}
+	svc := newDupService(embeddings)
+	_, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if !truncated {
+		t.Error("want truncated=true when N > maxEmbeddingsForDuplicate")
+	}
+}

application/service/enrichment_test.go

@@ -46,6 +46,9 @@ func (r *recordingEmbeddingStore) Search(_ context.Context, _ ...repository.Opti
 func (r *recordingEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) {
 	return false, nil
 }
+func (r *recordingEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) {
+	return nil, nil
+}
 func (r *recordingEmbeddingStore) DeleteBy(_ context.Context, opts ...repository.Option) error {
 	r.deleteCalled = true
 	r.deleteOpts = opts

application/service/search_test.go

@@ -53,6 +53,9 @@ func (f fakeEmbeddingStore) Search(_ context.Context, _ ...repository.Option) ([
 func (f fakeEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) {
 	return false, nil
 }
+func (f fakeEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) {
+	return nil, nil
+}
 func (f fakeEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil }
 
 // fakeBM25Store implements search.BM25Store for testing.