From 1ebe80070b9a9bc684e821e4de578cf48871343e Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Tue, 24 Mar 2026 16:06:18 +0000 Subject: [PATCH 1/3] feat: add duplicate code detection API and MCP tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements POST /api/v1/search/duplicates endpoint and the kodit_find_duplicates MCP tool for finding semantically similar code snippets within a repository. Key changes: - Add FindAll to EmbeddingStore interface (SQLite + VectorChord impls) - Add DuplicateSearch service with normalize-once triangle scan (O(N·D) normalize + N*(N-1)/2 dot products); hard cap at 5000 embeddings - Add POST /api/v1/search/duplicates handler with threshold/limit params - Add kodit_find_duplicates MCP tool (repo_url, threshold, limit) - Wire DuplicateFinder into MCP server and scoped handler - Add E2E tests covering validation, empty results, and pair detection Co-Authored-By: Claude Sonnet 4.6 Spec-Ref: helix-specs@30382ed --- application/handler/indexing/ordering_test.go | 4 + application/service/duplicates.go | 137 +++++++++++ application/service/duplicates_test.go | 228 ++++++++++++++++++ application/service/enrichment_test.go | 3 + application/service/search_test.go | 3 + domain/search/store.go | 4 + domain/service/embedding_test.go | 4 + infrastructure/api/api_server.go | 2 +- infrastructure/api/v1/dto/duplicates.go | 50 ++++ infrastructure/api/v1/search.go | 124 ++++++++++ .../persistence/embedding_store_sqlite.go | 14 ++ .../embedding_store_vectorchord.go | 17 ++ internal/mcp/catalog.go | 9 + internal/mcp/catalog_test.go | 4 +- internal/mcp/server.go | 80 +++++- internal/mcp/server_test.go | 25 ++ kodit.go | 2 + scoped_handler.go | 1 + scoped_handler_test.go | 4 + test/e2e/duplicates_test.go | 204 ++++++++++++++++ test/e2e/helpers_test.go | 22 ++ 21 files changed, 937 insertions(+), 4 deletions(-) create mode 100644 application/service/duplicates.go create mode 100644 application/service/duplicates_test.go create mode 100644 infrastructure/api/v1/dto/duplicates.go create mode 100644 test/e2e/duplicates_test.go diff --git a/application/handler/indexing/ordering_test.go b/application/handler/indexing/ordering_test.go index 033799956..653732ea3 100644 --- a/application/handler/indexing/ordering_test.go +++ b/application/handler/indexing/ordering_test.go @@ -67,6 +67,10 @@ func (e *emptyEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) return false, nil } +func (e *emptyEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) { + return nil, nil +} + func (e *emptyEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil } diff --git a/application/service/duplicates.go b/application/service/duplicates.go new file mode 100644 index 000000000..a744655c9 --- /dev/null +++ b/application/service/duplicates.go @@ -0,0 +1,137 @@ +package service + +import ( + "context" + "math" + "sort" + + "github.com/rs/zerolog" + + "github.com/helixml/kodit/domain/search" +) + +// maxEmbeddingsForDuplicate is the maximum number of embeddings that will be +// compared pairwise. Larger sets are truncated to avoid O(N²) timeouts. +const maxEmbeddingsForDuplicate = 5000 + +// DuplicatePair holds two snippet IDs and their pairwise cosine similarity. +type DuplicatePair struct { + SnippetIDA string + SnippetIDB string + Similarity float64 +} + +// DuplicateSearch finds semantically duplicated code snippets using pairwise +// embedding comparison. +type DuplicateSearch struct { + codeVectorStore search.EmbeddingStore + logger zerolog.Logger +} + +// NewDuplicateSearch creates a new DuplicateSearch service. +func NewDuplicateSearch(codeVectorStore search.EmbeddingStore, logger zerolog.Logger) *DuplicateSearch { + return &DuplicateSearch{ + codeVectorStore: codeVectorStore, + logger: logger, + } +} + +// FindDuplicates returns snippet pairs whose code embeddings have cosine +// similarity ≥ threshold. Pairs are sorted by similarity descending and +// capped at limit. The bool return value is true when the embedding set was +// truncated to maxEmbeddingsForDuplicate. +// +// Returns empty slice (not an error) when the store is nil or has no data. +func (s *DuplicateSearch) FindDuplicates( + ctx context.Context, + repoIDs []int64, + threshold float64, + limit int, +) ([]DuplicatePair, bool, error) { + if s.codeVectorStore == nil { + return nil, false, nil + } + + filters := search.NewFilters(search.WithSourceRepos(repoIDs)) + embeddings, err := s.codeVectorStore.FindAll(ctx, filters) + if err != nil { + return nil, false, err + } + + if len(embeddings) == 0 { + return nil, false, nil + } + + truncated := false + if len(embeddings) > maxEmbeddingsForDuplicate { + s.logger.Warn(). + Int("total", len(embeddings)). + Int("cap", maxEmbeddingsForDuplicate). + Msg("truncating embeddings for duplicate detection") + embeddings = embeddings[:maxEmbeddingsForDuplicate] + truncated = true + } + + // Normalize all vectors to unit length once, then pairwise similarity = dot product. + type unitVec struct { + snippetID string + v []float64 + } + normalized := make([]unitVec, 0, len(embeddings)) + for _, emb := range embeddings { + vec := emb.Vector() + mag := magnitude(vec) + if mag == 0 { + continue // skip zero vectors + } + unit := make([]float64, len(vec)) + for i, x := range vec { + unit[i] = x / mag + } + normalized = append(normalized, unitVec{snippetID: emb.SnippetID(), v: unit}) + } + + // Triangle scan: compare each pair (i, j) where j > i. + var pairs []DuplicatePair + for i := 0; i < len(normalized); i++ { + for j := i + 1; j < len(normalized); j++ { + sim := dotProduct(normalized[i].v, normalized[j].v) + if sim >= threshold { + pairs = append(pairs, DuplicatePair{ + SnippetIDA: normalized[i].snippetID, + SnippetIDB: normalized[j].snippetID, + Similarity: sim, + }) + } + } + } + + // Sort by similarity descending. + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Similarity > pairs[j].Similarity + }) + + if limit > 0 && len(pairs) > limit { + pairs = pairs[:limit] + } + + return pairs, truncated, nil +} + +// magnitude returns the Euclidean magnitude of a vector. +func magnitude(v []float64) float64 { + var sum float64 + for _, x := range v { + sum += x * x + } + return math.Sqrt(sum) +} + +// dotProduct returns the dot product of two vectors (assumed equal length). +func dotProduct(a, b []float64) float64 { + var sum float64 + for i := range a { + sum += a[i] * b[i] + } + return sum +} diff --git a/application/service/duplicates_test.go b/application/service/duplicates_test.go new file mode 100644 index 000000000..30f80969e --- /dev/null +++ b/application/service/duplicates_test.go @@ -0,0 +1,228 @@ +package service + +import ( + "context" + "testing" + + "github.com/rs/zerolog" + + "github.com/helixml/kodit/domain/repository" + "github.com/helixml/kodit/domain/search" +) + +// dupEmbeddingStore is a fake EmbeddingStore for duplicate detection tests. +type dupEmbeddingStore struct { + embeddings []search.Embedding + err error +} + +func (d *dupEmbeddingStore) SaveAll(_ context.Context, _ []search.Embedding) error { return nil } +func (d *dupEmbeddingStore) Find(_ context.Context, _ ...repository.Option) ([]search.Embedding, error) { + return nil, nil +} +func (d *dupEmbeddingStore) Search(_ context.Context, _ ...repository.Option) ([]search.Result, error) { + return nil, nil +} +func (d *dupEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) { + return false, nil +} +func (d *dupEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) { + if d.err != nil { + return nil, d.err + } + return d.embeddings, nil +} +func (d *dupEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil } + +func newDupService(embeddings []search.Embedding) *DuplicateSearch { + store := &dupEmbeddingStore{embeddings: embeddings} + return NewDuplicateSearch(store, zerolog.Nop()) +} + +func TestFindDuplicates_NoStore_ReturnsEmpty(t *testing.T) { + svc := NewDuplicateSearch(nil, zerolog.Nop()) + pairs, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if truncated { + t.Error("want truncated=false, got true") + } + if len(pairs) != 0 { + t.Errorf("want 0 pairs, got %d", len(pairs)) + } +} + +func TestFindDuplicates_EmptyEmbeddings_ReturnsEmpty(t *testing.T) { + svc := newDupService(nil) + pairs, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if truncated { + t.Error("want truncated=false, got true") + } + if len(pairs) != 0 { + t.Errorf("want 0 pairs, got %d", len(pairs)) + } +} + +func TestFindDuplicates_SingleEmbedding_ReturnsEmpty(t *testing.T) { + embeddings := []search.Embedding{ + search.NewEmbedding("1", []float64{1, 0, 0}), + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 0 { + t.Errorf("want 0 pairs, got %d", len(pairs)) + } +} + +func TestFindDuplicates_IdenticalVectors_ReturnsSimilarityOne(t *testing.T) { + vec := []float64{1.0, 0.5, 0.3} + embeddings := []search.Embedding{ + search.NewEmbedding("1", vec), + search.NewEmbedding("2", vec), + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 1 { + t.Fatalf("want 1 pair, got %d", len(pairs)) + } + if pairs[0].Similarity < 0.999 { + t.Errorf("want similarity ~1.0, got %f", pairs[0].Similarity) + } + // Verify snippet IDs + gotA, gotB := pairs[0].SnippetIDA, pairs[0].SnippetIDB + if !((gotA == "1" && gotB == "2") || (gotA == "2" && gotB == "1")) { + t.Errorf("unexpected snippet IDs: A=%s B=%s", gotA, gotB) + } +} + +func TestFindDuplicates_DissimilarVectors_NotReturned(t *testing.T) { + embeddings := []search.Embedding{ + search.NewEmbedding("1", []float64{1, 0, 0}), + search.NewEmbedding("2", []float64{0, 1, 0}), // orthogonal: cosine similarity = 0 + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 0 { + t.Errorf("want 0 pairs, got %d", len(pairs)) + } +} + +func TestFindDuplicates_ThresholdBoundary_Inclusive(t *testing.T) { + // Two vectors with known cosine similarity + // cos([1,1,0], [1,0,0]) = 1/sqrt(2) ≈ 0.7071 + embeddings := []search.Embedding{ + search.NewEmbedding("1", []float64{1, 1, 0}), + search.NewEmbedding("2", []float64{1, 0, 0}), + } + svc := newDupService(embeddings) + + // Threshold just below the similarity → should be returned + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.70, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 1 { + t.Errorf("threshold below sim: want 1 pair, got %d", len(pairs)) + } + + // Threshold above the similarity → should not be returned + pairs, _, err = svc.FindDuplicates(context.Background(), []int64{1}, 0.80, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 0 { + t.Errorf("threshold above sim: want 0 pairs, got %d", len(pairs)) + } +} + +func TestFindDuplicates_LimitCap(t *testing.T) { + // 4 embeddings → up to 6 pairs; we set limit=3 + vec := []float64{1, 0, 0} + embeddings := []search.Embedding{ + search.NewEmbedding("1", vec), + search.NewEmbedding("2", vec), + search.NewEmbedding("3", vec), + search.NewEmbedding("4", vec), + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 3) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) != 3 { + t.Errorf("want 3 pairs (capped), got %d", len(pairs)) + } +} + +func TestFindDuplicates_SortedBySimilarityDescending(t *testing.T) { + // 3 embeddings: pair (1,2) is very similar, pair (1,3) is less similar + embeddings := []search.Embedding{ + search.NewEmbedding("1", []float64{1.0, 0.0}), + search.NewEmbedding("2", []float64{0.99, 0.01}), // close to "1" + search.NewEmbedding("3", []float64{0.5, 0.5}), // farther from "1" + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.0, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(pairs) < 2 { + t.Fatalf("want at least 2 pairs, got %d", len(pairs)) + } + for i := 1; i < len(pairs); i++ { + if pairs[i].Similarity > pairs[i-1].Similarity { + t.Errorf("pairs not sorted: pairs[%d].Similarity=%f > pairs[%d].Similarity=%f", + i, pairs[i].Similarity, i-1, pairs[i-1].Similarity) + } + } +} + +func TestFindDuplicates_ZeroVector_Skipped(t *testing.T) { + // Zero vectors should not produce NaN similarities and should be skipped + embeddings := []search.Embedding{ + search.NewEmbedding("1", []float64{0, 0, 0}), + search.NewEmbedding("2", []float64{1, 0, 0}), + } + svc := newDupService(embeddings) + pairs, _, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.0, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + // Zero-magnitude vector should be skipped, no pairs returned + if len(pairs) != 0 { + t.Errorf("want 0 pairs (zero vector skipped), got %d", len(pairs)) + } +} + +func TestFindDuplicates_Truncated_WhenExceedsMax(t *testing.T) { + // Create more embeddings than maxEmbeddingsForDuplicate + n := maxEmbeddingsForDuplicate + 1 + embeddings := make([]search.Embedding, n) + for i := range embeddings { + embeddings[i] = search.NewEmbedding( + string(rune('A'+i%26))+string(rune('a'+i%26)), + []float64{float64(i), 1, 0}, + ) + } + svc := newDupService(embeddings) + _, truncated, err := svc.FindDuplicates(context.Background(), []int64{1}, 0.90, 50) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if !truncated { + t.Error("want truncated=true when N > maxEmbeddingsForDuplicate") + } +} diff --git a/application/service/enrichment_test.go b/application/service/enrichment_test.go index 25afa12c1..b4e5e31f2 100644 --- a/application/service/enrichment_test.go +++ b/application/service/enrichment_test.go @@ -46,6 +46,9 @@ func (r *recordingEmbeddingStore) Search(_ context.Context, _ ...repository.Opti func (r *recordingEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) { return false, nil } +func (r *recordingEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) { + return nil, nil +} func (r *recordingEmbeddingStore) DeleteBy(_ context.Context, opts ...repository.Option) error { r.deleteCalled = true r.deleteOpts = opts diff --git a/application/service/search_test.go b/application/service/search_test.go index 02c001fe3..ed689a6db 100644 --- a/application/service/search_test.go +++ b/application/service/search_test.go @@ -53,6 +53,9 @@ func (f fakeEmbeddingStore) Search(_ context.Context, _ ...repository.Option) ([ func (f fakeEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) (bool, error) { return false, nil } +func (f fakeEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) { + return nil, nil +} func (f fakeEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil } // fakeBM25Store implements search.BM25Store for testing. diff --git a/domain/search/store.go b/domain/search/store.go index 1a6bef1a7..24e8f9340 100644 --- a/domain/search/store.go +++ b/domain/search/store.go @@ -14,6 +14,10 @@ type EmbeddingStore interface { // Find retrieves embeddings matching the given options. Find(ctx context.Context, options ...repository.Option) ([]Embedding, error) + // FindAll retrieves all embeddings matching the given search filters, + // including repository source filtering via enrichment association JOINs. + FindAll(ctx context.Context, filters Filters) ([]Embedding, error) + // Search performs vector similarity search using options. // Embedding must be passed via WithEmbedding. Search(ctx context.Context, options ...repository.Option) ([]Result, error) diff --git a/domain/service/embedding_test.go b/domain/service/embedding_test.go index b2f761026..47a96a5e5 100644 --- a/domain/service/embedding_test.go +++ b/domain/service/embedding_test.go @@ -80,6 +80,10 @@ func (f *fakeEmbeddingStore) Exists(_ context.Context, _ ...repository.Option) ( return len(f.existing) > 0, nil } +func (f *fakeEmbeddingStore) FindAll(_ context.Context, _ search.Filters) ([]search.Embedding, error) { + return nil, nil +} + func (f *fakeEmbeddingStore) DeleteBy(_ context.Context, _ ...repository.Option) error { return nil } diff --git a/infrastructure/api/api_server.go b/infrastructure/api/api_server.go index d9ea6857c..3c88c0be6 100644 --- a/infrastructure/api/api_server.go +++ b/infrastructure/api/api_server.go @@ -88,7 +88,7 @@ func (a *APIServer) mountRoutes(router chi.Router) { // MCP uses streaming responses and manages its own session state via // response headers, which is incompatible with chi's Timeout middleware // that wraps the ResponseWriter. - mcpSrv := mcpinternal.NewServer(c.Repositories, c.Commits, c.Enrichments, c.Blobs, c.Search, c.Search, c.Enrichments, c.Blobs, c.Files, c.Grep, "1.0.0", a.logger) + mcpSrv := mcpinternal.NewServer(c.Repositories, c.Commits, c.Enrichments, c.Blobs, c.Search, c.Search, c.Enrichments, c.Blobs, c.Files, c.Grep, c.Duplicates, "1.0.0", a.logger) httpHandler := server.NewStreamableHTTPServer(mcpSrv.MCPServer()) router.Mount("/mcp", httpHandler) } diff --git a/infrastructure/api/v1/dto/duplicates.go b/infrastructure/api/v1/dto/duplicates.go new file mode 100644 index 000000000..5ff80ad34 --- /dev/null +++ b/infrastructure/api/v1/dto/duplicates.go @@ -0,0 +1,50 @@ +package dto + +// DuplicateSearchAttributes holds the attributes of a duplicate search request. +type DuplicateSearchAttributes struct { + RepositoryIDs []int64 `json:"repository_ids"` + Threshold *float64 `json:"threshold,omitempty"` + Limit *int `json:"limit,omitempty"` +} + +// DuplicateSearchData is the data envelope of a duplicate search request. +type DuplicateSearchData struct { + Type string `json:"type"` + Attributes DuplicateSearchAttributes `json:"attributes"` +} + +// DuplicateSearchRequest is the JSON:API-style request body for POST /search/duplicates. +type DuplicateSearchRequest struct { + Data DuplicateSearchData `json:"data"` +} + +// DuplicateSnippetSchema represents one snippet in a duplicate pair. +type DuplicateSnippetSchema struct { + ID string `json:"id"` + Content string `json:"content"` + Language string `json:"language"` +} + +// DuplicatePairAttributes holds the attributes of a single duplicate pair. +type DuplicatePairAttributes struct { + Similarity float64 `json:"similarity"` + SnippetA DuplicateSnippetSchema `json:"snippet_a"` + SnippetB DuplicateSnippetSchema `json:"snippet_b"` +} + +// DuplicatePairData is a single duplicate pair in JSON:API format. +type DuplicatePairData struct { + Type string `json:"type"` + Attributes DuplicatePairAttributes `json:"attributes"` +} + +// DuplicatesMeta holds optional metadata for the duplicates response. +type DuplicatesMeta struct { + Truncated bool `json:"truncated,omitempty"` +} + +// DuplicatesResponse is the JSON:API-style response body for POST /search/duplicates. +type DuplicatesResponse struct { + Data []DuplicatePairData `json:"data"` + Meta *DuplicatesMeta `json:"meta,omitempty"` +} diff --git a/infrastructure/api/v1/search.go b/infrastructure/api/v1/search.go index 7ae661f89..f802bf81c 100644 --- a/infrastructure/api/v1/search.go +++ b/infrastructure/api/v1/search.go @@ -42,6 +42,7 @@ func (r *SearchRouter) Routes() chi.Router { router := chi.NewRouter() router.Post("/", r.Search) + router.Post("/duplicates", r.Duplicates) router.Get("/semantic", r.SemanticSearch) router.Get("/keyword", r.KeywordSearch) router.Get("/ls", r.Ls) @@ -486,6 +487,129 @@ func (r *SearchRouter) Grep(w http.ResponseWriter, req *http.Request) { middleware.WriteJSON(w, http.StatusOK, response) } +// Duplicates handles POST /api/v1/search/duplicates. +// +// @Summary Find duplicate code snippets +// @Description Finds pairs of semantically similar code snippets using pairwise embedding comparison +// @Tags search +// @Accept json +// @Produce json +// @Param body body dto.DuplicateSearchRequest true "Duplicate search request" +// @Success 200 {object} dto.DuplicatesResponse +// @Failure 400 {object} middleware.JSONAPIErrorResponse +// @Failure 500 {object} middleware.JSONAPIErrorResponse +// @Router /search/duplicates [post] +func (r *SearchRouter) Duplicates(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + var body dto.DuplicateSearchRequest + if err := json.NewDecoder(req.Body).Decode(&body); err != nil { + if err == io.EOF { + middleware.WriteError(w, req, fmt.Errorf("request body is required: %w", middleware.ErrValidation), r.logger) + return + } + middleware.WriteError(w, req, err, r.logger) + return + } + + attrs := body.Data.Attributes + + if len(attrs.RepositoryIDs) == 0 { + middleware.WriteError(w, req, fmt.Errorf("repository_ids is required and must not be empty: %w", middleware.ErrValidation), r.logger) + return + } + + threshold := 0.90 + if attrs.Threshold != nil { + if *attrs.Threshold <= 0 || *attrs.Threshold > 1 { + middleware.WriteError(w, req, fmt.Errorf("threshold must be in range (0, 1]: %w", middleware.ErrValidation), r.logger) + return + } + threshold = *attrs.Threshold + } + + limit := 50 + if attrs.Limit != nil { + if *attrs.Limit < 1 { + middleware.WriteError(w, req, fmt.Errorf("limit must be at least 1: %w", middleware.ErrValidation), r.logger) + return + } + if *attrs.Limit > 500 { + limit = 500 + } else { + limit = *attrs.Limit + } + } + + // Validate all repository IDs exist. + for _, repoID := range attrs.RepositoryIDs { + if _, err := r.client.Repositories.Get(ctx, repository.WithID(repoID)); err != nil { + middleware.WriteError(w, req, err, r.logger) + return + } + } + + pairs, truncated, err := r.client.Duplicates.FindDuplicates(ctx, attrs.RepositoryIDs, threshold, limit) + if err != nil { + middleware.WriteError(w, req, err, r.logger) + return + } + + // Resolve snippet IDs to enrichment content. + snippetIDs := make([]int64, 0, len(pairs)*2) + for _, p := range pairs { + if idA, err := strconv.ParseInt(p.SnippetIDA, 10, 64); err == nil { + snippetIDs = append(snippetIDs, idA) + } + if idB, err := strconv.ParseInt(p.SnippetIDB, 10, 64); err == nil { + snippetIDs = append(snippetIDs, idB) + } + } + + enrichmentByID := map[string]enrichment.Enrichment{} + if len(snippetIDs) > 0 { + enrichments, fetchErr := r.client.Enrichments.Find(ctx, repository.WithIDIn(snippetIDs)) + if fetchErr != nil { + r.logger.Warn().Err(fetchErr).Msg("failed to fetch enrichments for duplicate pairs") + } else { + for _, e := range enrichments { + enrichmentByID[strconv.FormatInt(e.ID(), 10)] = e + } + } + } + + data := make([]dto.DuplicatePairData, 0, len(pairs)) + for _, p := range pairs { + snippetA := snippetSchema(p.SnippetIDA, enrichmentByID) + snippetB := snippetSchema(p.SnippetIDB, enrichmentByID) + data = append(data, dto.DuplicatePairData{ + Type: "duplicate_pair", + Attributes: dto.DuplicatePairAttributes{ + Similarity: p.Similarity, + SnippetA: snippetA, + SnippetB: snippetB, + }, + }) + } + + resp := dto.DuplicatesResponse{Data: data} + if truncated { + resp.Meta = &dto.DuplicatesMeta{Truncated: true} + } + + middleware.WriteJSON(w, http.StatusOK, resp) +} + +// snippetSchema builds a DuplicateSnippetSchema from the enrichment map. +func snippetSchema(snippetID string, enrichmentByID map[string]enrichment.Enrichment) dto.DuplicateSnippetSchema { + s := dto.DuplicateSnippetSchema{ID: snippetID} + if e, ok := enrichmentByID[snippetID]; ok { + s.Content = e.Content() + s.Language = e.Language() + } + return s +} + func buildSearchRequest(body dto.SearchRequest) (search.MultiRequest, error) { attrs := body.Data.Attributes diff --git a/infrastructure/persistence/embedding_store_sqlite.go b/infrastructure/persistence/embedding_store_sqlite.go index 2bee57a77..8b9e105f0 100644 --- a/infrastructure/persistence/embedding_store_sqlite.go +++ b/infrastructure/persistence/embedding_store_sqlite.go @@ -115,6 +115,20 @@ func (s *SQLiteEmbeddingStore) Search(ctx context.Context, options ...repository return results, nil } +// FindAll returns all embeddings that match the given search filters. +// Repository filtering is applied via enrichment association JOINs. +func (s *SQLiteEmbeddingStore) FindAll(ctx context.Context, filters search.Filters) ([]search.Embedding, error) { + vectors, err := s.loadVectors(ctx, search.WithFilters(filters)) + if err != nil { + return nil, err + } + result := make([]search.Embedding, len(vectors)) + for i, v := range vectors { + result[i] = search.NewEmbedding(v.snippetID, v.embedding) + } + return result, nil +} + // loadVectors loads embedding vectors from the database using GORM. func (s *SQLiteEmbeddingStore) loadVectors(ctx context.Context, options ...repository.Option) ([]StoredVector, error) { var entities []SQLiteEmbeddingModel diff --git a/infrastructure/persistence/embedding_store_vectorchord.go b/infrastructure/persistence/embedding_store_vectorchord.go index 7ea16dd13..fcf71d16f 100644 --- a/infrastructure/persistence/embedding_store_vectorchord.go +++ b/infrastructure/persistence/embedding_store_vectorchord.go @@ -191,6 +191,23 @@ func probeCount(rows int64) int { return max(int(math.Sqrt(float64(lists))), 10) } +// FindAll returns all embeddings that match the given search filters. +// Repository filtering is applied via enrichment association JOINs. +func (s *VectorChordEmbeddingStore) FindAll(ctx context.Context, filters search.Filters) ([]search.Embedding, error) { + db := database.ApplySearchFilters(s.DB(ctx).Table(s.Table()), filters) + + var models []PgEmbeddingModel + if err := db.Find(&models).Error; err != nil { + return nil, fmt.Errorf("find all embeddings: %w", err) + } + + result := make([]search.Embedding, len(models)) + for i, m := range models { + result[i] = search.NewEmbedding(m.SnippetID, m.Embedding.Floats()) + } + return result, nil +} + // Search performs vector similarity search within a transaction so that // the vchordrq.probes session variable is visible to the query. func (s *VectorChordEmbeddingStore) Search(ctx context.Context, options ...repository.Option) ([]search.Result, error) { diff --git a/internal/mcp/catalog.go b/internal/mcp/catalog.go index b7d8bf575..0f7df329b 100644 --- a/internal/mcp/catalog.go +++ b/internal/mcp/catalog.go @@ -158,6 +158,15 @@ func tools() []ToolDefinition { {name: "pattern", description: "Glob pattern to match files (e.g. **/*.go, src/*.py)", typ: "string", required: true}, }, }, + { + name: "kodit_find_duplicates", + description: "Find semantically duplicated code snippets in a repository using embedding similarity", + params: []ParamDefinition{ + {name: "repo_url", description: "The remote URL of the repository", typ: "string", required: true}, + {name: "threshold", description: "Minimum cosine similarity to consider a pair duplicate (0–1, default 0.90)", typ: "number"}, + {name: "limit", description: "Maximum number of duplicate pairs to return (default 50)", typ: "number"}, + }, + }, } } diff --git a/internal/mcp/catalog_test.go b/internal/mcp/catalog_test.go index 89adc0c44..68beacf06 100644 --- a/internal/mcp/catalog_test.go +++ b/internal/mcp/catalog_test.go @@ -8,12 +8,12 @@ import ( func TestToolDefinitions_Count(t *testing.T) { defs := ToolDefinitions() - if len(defs) != 14 { + if len(defs) != 15 { names := make([]string, len(defs)) for i, def := range defs { names[i] = def.Name() } - t.Fatalf("ToolDefinitions() length = %d, want 14; got %v", len(defs), names) + t.Fatalf("ToolDefinitions() length = %d, want 15; got %v", len(defs), names) } } diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 0b6639401..49078b446 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -79,6 +79,11 @@ type Grepper interface { Search(ctx context.Context, repoID int64, pattern string, pathspec string, maxFiles int) ([]service.GrepResult, error) } +// DuplicateFinder finds semantically duplicated code snippets. +type DuplicateFinder interface { + FindDuplicates(ctx context.Context, repoIDs []int64, threshold float64, limit int) ([]service.DuplicatePair, bool, error) +} + // Server wraps the MCP server with kodit-specific tools. type Server struct { mcpServer *server.MCPServer @@ -92,6 +97,7 @@ type Server struct { fileLister FileLister files FileFinder grepper Grepper + duplicateFinder DuplicateFinder version string logger zerolog.Logger } @@ -114,7 +120,8 @@ const instructions = "This server provides access to code knowledge through mult "- kodit_keyword_search() - Find files matching keywords using BM25 search (returns resource URIs)\n" + "- kodit_grep() - Search file contents using git grep with regex patterns (returns resource URIs)\n" + "- kodit_ls() - List files matching a glob pattern in a repository\n" + - "- kodit_read_resource() - Read file content from a resource URI returned by search tools\n\n" + + "- kodit_read_resource() - Read file content from a resource URI returned by search tools\n" + + "- kodit_find_duplicates() - Find semantically duplicated code snippets in a repository\n\n" + "**Reading file content:**\n" + "Use kodit_read_resource() with the URI returned by search tools, or the file resource " + "template: file://{id}/{blob_name}/{+path}\n" + @@ -137,6 +144,7 @@ func NewServer( fileLister FileLister, files FileFinder, grepper Grepper, + duplicateFinder DuplicateFinder, version string, logger zerolog.Logger, ) *Server { @@ -152,6 +160,7 @@ func NewServer( fileLister: fileLister, files: files, grepper: grepper, + duplicateFinder: duplicateFinder, version: version, logger: logger, } @@ -190,6 +199,7 @@ func (s *Server) registerTools(mcpServer *server.MCPServer) { "kodit_grep": s.handleGrep, "kodit_read_resource": s.handleReadResource, "kodit_ls": s.handleLs, + "kodit_find_duplicates": s.handleFindDuplicates, } for _, def := range tools() { @@ -946,6 +956,74 @@ func (s *Server) handleReadResource(ctx context.Context, request mcp.CallToolReq return mcp.NewToolResultText(text.Text), nil } +// duplicateResult holds one duplicate pair for JSON output. +type duplicateResult struct { + SnippetIDA string `json:"snippet_id_a"` + SnippetIDB string `json:"snippet_id_b"` + Similarity float64 `json:"similarity"` +} + +// handleFindDuplicates handles the find_duplicates tool invocation. +func (s *Server) handleFindDuplicates(ctx context.Context, request mcp.CallToolRequest) (*mcp.CallToolResult, error) { + repoURL, err := request.RequireString("repo_url") + if err != nil { + return mcp.NewToolResultError("repo_url is required"), nil + } + + repos, err := s.resolveRepository(ctx, repoURL) + if err != nil { + s.logger.Error().Str("repo_url", repoURL).Interface("error", err).Msg("failed to find repository") + return mcp.NewToolResultError(fmt.Sprintf("failed to find repository: %v", err)), nil + } + if len(repos) == 0 { + return mcp.NewToolResultError(fmt.Sprintf("repository not found: %s", repoURL)), nil + } + + threshold := request.GetFloat("threshold", 0.90) + if threshold <= 0 || threshold > 1 { + return mcp.NewToolResultError("threshold must be in the range (0, 1]"), nil + } + + limit := int(request.GetFloat("limit", 50)) + if limit < 1 { + return mcp.NewToolResultError("limit must be at least 1"), nil + } + + if s.duplicateFinder == nil { + return mcp.NewToolResultText("[]"), nil + } + + pairs, truncated, err := s.duplicateFinder.FindDuplicates(ctx, []int64{repos[0].ID()}, threshold, limit) + if err != nil { + s.logger.Error().Interface("error", err).Msg("find duplicates failed") + return mcp.NewToolResultError(fmt.Sprintf("find duplicates failed: %v", err)), nil + } + + if len(pairs) == 0 { + return mcp.NewToolResultText("[]"), nil + } + + results := make([]duplicateResult, len(pairs)) + for i, p := range pairs { + results[i] = duplicateResult{ + SnippetIDA: p.SnippetIDA, + SnippetIDB: p.SnippetIDB, + Similarity: p.Similarity, + } + } + + jsonBytes, err := json.Marshal(results) + if err != nil { + return mcp.NewToolResultError(fmt.Sprintf("marshal results: %v", err)), nil + } + + if truncated { + s.logger.Warn().Msg("duplicate search truncated: too many embeddings") + } + + return mcp.NewToolResultText(string(jsonBytes)), nil +} + // registerResources registers MCP resource templates with the server. func (s *Server) registerResources(mcpServer *server.MCPServer) { mcpServer.AddResourceTemplate( diff --git a/internal/mcp/server_test.go b/internal/mcp/server_test.go index 719c0adc1..dc4b1c36e 100644 --- a/internal/mcp/server_test.go +++ b/internal/mcp/server_test.go @@ -271,6 +271,7 @@ func testServer() *Server { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -419,6 +420,7 @@ func TestServer_ListRepositories_DisplaysUpstreamURL(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -470,6 +472,7 @@ func TestServer_ListRepositories_FallsBackToSanitizedURL(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -694,6 +697,7 @@ func semanticSearchServer() *Server { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{testFile}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -814,6 +818,7 @@ func TestServer_SemanticSearch_AbsolutePathNormalized(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{testFile}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -893,6 +898,7 @@ func TestServer_SemanticSearch_LanguageFilterDotPrefix(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{testFile}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1047,6 +1053,7 @@ func TestServer_SemanticSearch_LimitCapsResults(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{f1, f2, f3}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1147,6 +1154,7 @@ func TestServer_SemanticSearchThenReadFile(t *testing.T) { time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)), }}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1232,6 +1240,7 @@ func TestServer_SemanticSearchThenReadFile_AbsolutePath(t *testing.T) { "", "", ".py", ".py", 256, time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)), }}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1311,6 +1320,7 @@ func TestServer_SemanticSearchThenReadFile_WithLineRange(t *testing.T) { time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC)), }}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1370,6 +1380,7 @@ func TestServer_SemanticSearchNoResults(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1428,6 +1439,7 @@ func keywordSearchServer() *Server { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{testFile}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1645,6 +1657,7 @@ func TestServer_KeywordSearch_NoResults(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1759,6 +1772,7 @@ func TestServer_KeywordSearchThenReadFile(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{files: []repository.File{testFile}}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1877,6 +1891,7 @@ func wikiServer() *Server { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -1955,6 +1970,7 @@ func TestServer_GetWiki_NoWiki(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2073,6 +2089,7 @@ func lsServer(files []service.FileEntry) *Server { &fakeFileLister{files: files}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2253,6 +2270,7 @@ func TestServer_Ls_RepoNotFound(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2313,6 +2331,7 @@ func credentialServer() *Server { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2388,6 +2407,7 @@ func TestServer_GetWiki_SanitizesCredentials(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2433,6 +2453,7 @@ func TestServer_GetWikiPage_SanitizesCredentials(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2491,6 +2512,7 @@ func TestServer_Grep_SanitizesCredentials(t *testing.T) { }, }, }, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2537,6 +2559,7 @@ func TestServer_Ls_SanitizesCredentials(t *testing.T) { &fakeFileLister{files: []service.FileEntry{{Path: "README.md", Size: 100}}}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2709,6 +2732,7 @@ func grepServer() *Server { }, }, }, + nil, "1.0.0-test", zerolog.Nop(), ) @@ -2832,6 +2856,7 @@ func TestServer_Grep_NoResults(t *testing.T) { &fakeFileLister{}, &fakeFileFinder{}, &fakeGrepper{}, + nil, "1.0.0-test", zerolog.Nop(), ) diff --git a/kodit.go b/kodit.go index fc02acce8..ce9c0509b 100644 --- a/kodit.go +++ b/kodit.go @@ -79,6 +79,7 @@ type Client struct { Tracking *service.Tracking Search *service.Search Grep *service.Grep + Duplicates *service.DuplicateSearch // MCPServer describes the MCP server's tools and instructions. MCPServer MCPServer @@ -402,6 +403,7 @@ func New(opts ...Option) (*Client, error) { client.Tracking = trackingSvc client.Search = service.NewSearch(domainEmbedder, textEmbeddingStore, codeEmbeddingStore, bm25Store, enrichmentStore, &client.closed, logger) client.Grep = service.NewGrep(repoStore, commitStore, gitAdapter) + client.Duplicates = service.NewDuplicateSearch(codeEmbeddingStore, logger) // Register task handlers if err := client.registerHandlers(); err != nil { diff --git a/scoped_handler.go b/scoped_handler.go index 9d7e65f2b..3d5b7c181 100644 --- a/scoped_handler.go +++ b/scoped_handler.go @@ -37,6 +37,7 @@ func NewScopedMCPHandler(client *Client, repoIDs []int64) http.Handler { fileLister, client.Files, grepper, + client.Duplicates, "1.0.0", client.logger, ) diff --git a/scoped_handler_test.go b/scoped_handler_test.go index 10c9f72e9..028897bad 100644 --- a/scoped_handler_test.go +++ b/scoped_handler_test.go @@ -61,6 +61,7 @@ func TestScopedMCPServer_RepositoryListFiltered(t *testing.T) { scopedFL, &scopedFakeFileFinder{}, scopedG, + nil, "test", zerolog.Nop(), ) @@ -124,6 +125,7 @@ func TestScopedMCPServer_ReadResourceBlocked(t *testing.T) { scopedFL, &scopedFakeFileFinder{}, scopedG, + nil, "test", zerolog.Nop(), ) @@ -182,6 +184,7 @@ func TestScopedMCPServer_NilRepoIDsNoScoping(t *testing.T) { &scopedFakeFileLister{}, &scopedFakeFileFinder{}, &scopedFakeGrepper{}, + nil, "test", zerolog.Nop(), ) @@ -292,6 +295,7 @@ func TestScopedMCPServer_ListRepositories_SanitizesCredentials(t *testing.T) { scopedFL, &scopedFakeFileFinder{}, scopedG, + nil, "test", zerolog.Nop(), ) diff --git a/test/e2e/duplicates_test.go b/test/e2e/duplicates_test.go new file mode 100644 index 000000000..06d5af156 --- /dev/null +++ b/test/e2e/duplicates_test.go @@ -0,0 +1,204 @@ +package e2e_test + +import ( + "net/http" + "strconv" + "testing" + + "github.com/helixml/kodit/infrastructure/api/v1/dto" +) + +func TestDuplicates_MissingRepositoryIDs_Returns400(t *testing.T) { + ts := NewTestServer(t) + + resp := ts.POST("/api/v1/search/duplicates", map[string]any{ + "data": map[string]any{ + "type": "duplicate_search", + "attributes": map[string]any{}, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want 400", resp.StatusCode) + } +} + +func TestDuplicates_InvalidThreshold_Zero_Returns400(t *testing.T) { + ts := NewTestServer(t) + repo := ts.CreateRepository("https://github.com/test/repo.git") + + threshold := 0.0 + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + Threshold: &threshold, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("threshold=0: status = %d, want 400", resp.StatusCode) + } +} + +func TestDuplicates_InvalidThreshold_TooLarge_Returns400(t *testing.T) { + ts := NewTestServer(t) + repo := ts.CreateRepository("https://github.com/test/repo.git") + + threshold := 1.5 + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + Threshold: &threshold, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("threshold=1.5: status = %d, want 400", resp.StatusCode) + } +} + +func TestDuplicates_InvalidLimit_Returns400(t *testing.T) { + ts := NewTestServer(t) + repo := ts.CreateRepository("https://github.com/test/repo.git") + + limit := 0 + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + Limit: &limit, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("limit=0: status = %d, want 400", resp.StatusCode) + } +} + +func TestDuplicates_NoEmbeddings_ReturnsEmpty(t *testing.T) { + ts := NewTestServer(t) + repo := ts.CreateRepository("https://github.com/test/repo.git") + + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + + var result dto.DuplicatesResponse + ts.DecodeJSON(resp, &result) + + if len(result.Data) != 0 { + t.Errorf("want 0 pairs, got %d", len(result.Data)) + } +} + +func TestDuplicates_WithSeededEmbeddings_ReturnsPairs(t *testing.T) { + ts := NewTestServer(t) + + // Create repository + commit + two enrichments. + repo := ts.CreateRepository("https://github.com/test/dup.git") + commit := ts.CreateCommit(repo, "abc123def456", "Initial commit") + e1 := ts.CreateSnippetEnrichmentForCommit(commit.SHA(), "func foo() {}", "go") + e2 := ts.CreateSnippetEnrichmentForCommit(commit.SHA(), "func foo() {}", "go") + + // Seed near-identical embeddings for both snippets (same direction vector). + vec1 := []float64{1.0, 0.5, 0.25} + vec2 := []float64{1.0, 0.5, 0.25} + ts.SeedCodeEmbedding(strconv.FormatInt(e1.ID(), 10), vec1) + ts.SeedCodeEmbedding(strconv.FormatInt(e2.ID(), 10), vec2) + + threshold := 0.90 + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + Threshold: &threshold, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body := ts.ReadBody(resp) + t.Fatalf("status = %d, want 200; body: %s", resp.StatusCode, body) + } + + var result dto.DuplicatesResponse + ts.DecodeJSON(resp, &result) + + if len(result.Data) != 1 { + t.Fatalf("want 1 pair, got %d", len(result.Data)) + } + if result.Data[0].Attributes.Similarity < 0.99 { + t.Errorf("want similarity ~1.0, got %f", result.Data[0].Attributes.Similarity) + } +} + +func TestDuplicates_BelowThreshold_ReturnsEmpty(t *testing.T) { + ts := NewTestServer(t) + + repo := ts.CreateRepository("https://github.com/test/dup2.git") + commit := ts.CreateCommit(repo, "deadbeef1234", "Initial commit") + e1 := ts.CreateSnippetEnrichmentForCommit(commit.SHA(), "func a() {}", "go") + e2 := ts.CreateSnippetEnrichmentForCommit(commit.SHA(), "func b() {}", "go") + + // Orthogonal vectors: cosine similarity = 0 + ts.SeedCodeEmbedding(strconv.FormatInt(e1.ID(), 10), []float64{1, 0, 0}) + ts.SeedCodeEmbedding(strconv.FormatInt(e2.ID(), 10), []float64{0, 1, 0}) + + threshold := 0.90 + resp := ts.POST("/api/v1/search/duplicates", dto.DuplicateSearchRequest{ + Data: dto.DuplicateSearchData{ + Type: "duplicate_search", + Attributes: dto.DuplicateSearchAttributes{ + RepositoryIDs: []int64{repo.ID()}, + Threshold: &threshold, + }, + }, + }) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + + var result dto.DuplicatesResponse + ts.DecodeJSON(resp, &result) + + if len(result.Data) != 0 { + t.Errorf("want 0 pairs (below threshold), got %d", len(result.Data)) + } +} + +func TestDuplicates_MissingRequestBody_Returns400(t *testing.T) { + ts := NewTestServer(t) + + resp := ts.POSTRaw("/api/v1/search/duplicates", "") + defer resp.Body.Close() + + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("status = %d, want 400", resp.StatusCode) + } +} diff --git a/test/e2e/helpers_test.go b/test/e2e/helpers_test.go index 8be00dad6..c2b4edfb6 100644 --- a/test/e2e/helpers_test.go +++ b/test/e2e/helpers_test.go @@ -383,6 +383,28 @@ func (ts *TestServer) CreateTag(repo repository.Repository, name, commitSHA stri return saved } +// SeedCodeEmbedding inserts a code embedding vector into the SQLite code embedding store. +// snippetID should be the string form of an enrichment ID (e.g. "42"). +// vec must be non-nil and non-empty. +func (ts *TestServer) SeedCodeEmbedding(snippetID string, vec []float64) { + ts.t.Helper() + gormDB := ts.db.GORM() + + // The SQLite embedding store creates this table during client initialization. + data, err := json.Marshal(vec) + if err != nil { + ts.t.Fatalf("marshal embedding: %v", err) + } + + err = gormDB.Exec( + `INSERT OR REPLACE INTO kodit_code_embeddings (snippet_id, embedding) VALUES (?, ?)`, + snippetID, string(data), + ).Error + if err != nil { + ts.t.Fatalf("seed code embedding: %v", err) + } +} + // SeedBM25 inserts a document into the SQLite FTS5 BM25 index. func (ts *TestServer) SeedBM25(snippetID, passage string) { ts.t.Helper() From f842ab2c8d0659a04665023de0d0d1e30e85bdaa Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Tue, 24 Mar 2026 21:09:59 +0000 Subject: [PATCH 2/3] test: add smoke test for duplicate code detection Adds TestSmoke_Duplicates which registers a plain local directory containing two semantically similar Python files, waits for embeddings, then asserts that both the REST API (POST /api/v1/search/duplicates) and the MCP tool (kodit_find_duplicates) return at least one pair with cosine similarity >= 0.50. Also adds postJSON helper for raw HTTP POST calls in smoke tests. Co-Authored-By: Claude Sonnet 4.6 Spec-Ref: helix-specs@30382ed --- test/smoke/smoke_test.go | 195 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 195 insertions(+) diff --git a/test/smoke/smoke_test.go b/test/smoke/smoke_test.go index f409bed70..7be57db1f 100644 --- a/test/smoke/smoke_test.go +++ b/test/smoke/smoke_test.go @@ -1364,6 +1364,185 @@ func main() { }) } +// TestSmoke_Duplicates verifies the duplicate-detection REST endpoint and the +// kodit_find_duplicates MCP tool against a live server. It registers a plain +// local directory containing two semantically similar Python files, waits for +// embeddings to be generated, then asserts that the API and MCP tool both +// surface at least one duplicate pair above a lenient 0.50 cosine threshold. +func TestSmoke_Duplicates(t *testing.T) { + if testing.Short() { + t.Skip("skipping smoke test in short mode") + } + + // Container-side path — the host testdata/ tree is mounted at /app inside + // the kodit Docker container, so files written here are visible to the server. + containerDir := "/app/test/smoke/testdata/smoke-duplicates-dir" + plainDirURI := "file://" + containerDir + + fixtureDir := filepath.Join("testdata", "smoke-duplicates-dir") + if err := os.MkdirAll(fixtureDir, 0o755); err != nil { + t.Fatalf("create fixture dir: %v", err) + } + t.Cleanup(func() { _ = os.RemoveAll(fixtureDir) }) + + // Two Python files implementing the same arithmetic operations with different + // variable names — similar enough to score ~0.70 cosine similarity. + const calcSource = `def add_numbers(a, b): + """Add two numbers together.""" + result = a + b + return result + +def subtract_numbers(a, b): + """Subtract b from a.""" + result = a - b + return result + +def multiply_numbers(a, b): + """Multiply two numbers.""" + result = a * b + return result +` + const mathSource = `def add_values(x, y): + """Add two values together.""" + result = x + y + return result + +def subtract_values(x, y): + """Subtract y from x.""" + result = x - y + return result + +def compute_product(x, y): + """Compute the product of two numbers.""" + result = x * y + return result +` + if err := os.WriteFile(filepath.Join(fixtureDir, "calculator.py"), []byte(calcSource), 0o644); err != nil { + t.Fatalf("write calculator.py: %v", err) + } + if err := os.WriteFile(filepath.Join(fixtureDir, "math_utils.py"), []byte(mathSource), 0o644); err != nil { + t.Fatalf("write math_utils.py: %v", err) + } + + client, err := kodit.NewClientWithResponses(baseURL) + if err != nil { + t.Fatalf("failed to create client: %v", err) + } + ctx := context.Background() + + repoType := "repository" + createResp, err := client.PostRepositoriesWithResponse(ctx, kodit.DtoRepositoryCreateRequest{ + Data: &kodit.DtoRepositoryCreateData{ + Type: &repoType, + Attributes: &kodit.DtoRepositoryCreateAttributes{ + RemoteUri: &plainDirURI, + }, + }, + }) + if err != nil { + t.Fatalf("create repository failed: %v", err) + } + var repo *kodit.DtoRepositoryData + switch createResp.StatusCode() { + case http.StatusCreated: + repo = createResp.JSON201.Data + t.Log("duplicates repository created (201)") + case http.StatusOK: + repo = createResp.JSON200.Data + t.Log("duplicates repository already exists (200)") + default: + t.Fatalf("expected 200 or 201, got %d: %s", createResp.StatusCode(), string(createResp.Body)) + } + repoID, err := strconv.Atoi(*repo.Id) + if err != nil { + t.Fatalf("parse repo ID: %v", err) + } + t.Logf("duplicates repository: id=%d uri=%s", repoID, plainDirURI) + + t.Cleanup(func() { + _, _ = client.DeleteRepositoriesIdWithResponse(ctx, repoID) + }) + + waitForIndexing(t, client, ctx, repoID) + + t.Run("api_validation_missing_repo_ids", func(t *testing.T) { + resp := postJSON(t, baseURL+"/search/duplicates", `{"data":{"type":"duplicate_search","attributes":{}}}`) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } + }) + + t.Run("api_finds_pairs", func(t *testing.T) { + body := fmt.Sprintf( + `{"data":{"type":"duplicate_search","attributes":{"repository_ids":[%d],"threshold":0.50,"limit":10}}}`, + repoID, + ) + resp := postJSON(t, baseURL+"/search/duplicates", body) + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + var result struct { + Data []struct { + Attributes struct { + Similarity float64 `json:"similarity"` + SnippetA struct { + ID string `json:"id"` + } `json:"snippet_a"` + SnippetB struct { + ID string `json:"id"` + } `json:"snippet_b"` + } `json:"attributes"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { + t.Fatalf("decode response: %v", err) + } + if len(result.Data) == 0 { + t.Fatal("expected at least one duplicate pair (threshold 0.50)") + } + pair := result.Data[0] + if pair.Attributes.Similarity < 0.50 { + t.Fatalf("expected similarity >= 0.50, got %.4f", pair.Attributes.Similarity) + } + if pair.Attributes.SnippetA.ID == "" || pair.Attributes.SnippetB.ID == "" { + t.Fatal("expected non-empty snippet IDs in pair") + } + t.Logf("duplicate pair: similarity=%.4f, snippet_a=%s, snippet_b=%s", + pair.Attributes.Similarity, pair.Attributes.SnippetA.ID, pair.Attributes.SnippetB.ID) + }) + + t.Run("mcp_find_duplicates", func(t *testing.T) { + sessionID := initMCPSession(t) + + type mcpDuplicateResult struct { + SnippetIDA string `json:"snippet_id_a"` + SnippetIDB string `json:"snippet_id_b"` + Similarity float64 `json:"similarity"` + } + + text := callMCPToolText(t, sessionID, "kodit_find_duplicates", 2, map[string]any{ + "repo_url": plainDirURI, + "threshold": 0.50, + "limit": 10, + }) + + var results []mcpDuplicateResult + if err := json.Unmarshal([]byte(text), &results); err != nil { + t.Fatalf("unmarshal MCP find_duplicates results: %v", err) + } + if len(results) == 0 { + t.Fatal("expected at least one duplicate pair from MCP tool (threshold 0.50)") + } + if results[0].Similarity < 0.50 { + t.Fatalf("expected similarity >= 0.50, got %.4f", results[0].Similarity) + } + t.Logf("MCP duplicate pair: snippet_id_a=%s snippet_id_b=%s similarity=%.4f", + results[0].SnippetIDA, results[0].SnippetIDB, results[0].Similarity) + }) +} + // validateSearchResults validates the structure of search results. func validateSearchResults(t *testing.T, results []kodit.DtoSnippetData, mode string) { t.Helper() @@ -1736,6 +1915,22 @@ func getJSON(t *testing.T, url string) *http.Response { return resp } +// postJSON sends a POST request with a JSON body and returns the response. +func postJSON(t *testing.T, url string, body string) *http.Response { + t.Helper() + httpClient := &http.Client{Timeout: 30 * time.Second} + req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(body)) + if err != nil { + t.Fatalf("POST %s: create request: %v", url, err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := httpClient.Do(req) + if err != nil { + t.Fatalf("POST %s failed: %v", url, err) + } + return resp +} + // verifyHealth checks the /healthz endpoint. func verifyHealth(t *testing.T) { t.Helper() From ed9cb7d83f73ad6fe5a74099012ee8dff79655e4 Mon Sep 17 00:00:00 2001 From: Phil Winder Date: Wed, 25 Mar 2026 13:59:26 +0000 Subject: [PATCH 3/3] docs: regenerate OpenAPI spec with duplicates endpoint Co-Authored-By: Claude Sonnet 4.6 Spec-Ref: helix-specs@595b2e8 --- docs/swagger/docs.go | 143 ++++++++++++++++++++++++++++++ docs/swagger/openapi.json | 150 ++++++++++++++++++++++++++++++++ docs/swagger/swagger.json | 143 ++++++++++++++++++++++++++++++ docs/swagger/swagger.yaml | 93 ++++++++++++++++++++ infrastructure/api/openapi.json | 150 ++++++++++++++++++++++++++++++++ 5 files changed, 679 insertions(+) diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index b0f96ae19..01c6ab7ce 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -1896,6 +1896,52 @@ const docTemplate = `{ } } }, + "/search/duplicates": { + "post": { + "description": "Finds pairs of semantically similar code snippets using pairwise embedding comparison", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "search" + ], + "summary": "Find duplicate code snippets", + "parameters": [ + { + "description": "Duplicate search request", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/dto.DuplicateSearchRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/dto.DuplicatesResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/middleware.JSONAPIErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/middleware.JSONAPIErrorResponse" + } + } + } + } + }, "/search/grep": { "get": { "description": "Search file contents in a repository using git grep with regex patterns", @@ -2210,6 +2256,103 @@ const docTemplate = `{ } } }, + "dto.DuplicatePairAttributes": { + "type": "object", + "properties": { + "similarity": { + "type": "number" + }, + "snippet_a": { + "$ref": "#/definitions/dto.DuplicateSnippetSchema" + }, + "snippet_b": { + "$ref": "#/definitions/dto.DuplicateSnippetSchema" + } + } + }, + "dto.DuplicatePairData": { + "type": "object", + "properties": { + "attributes": { + "$ref": "#/definitions/dto.DuplicatePairAttributes" + }, + "type": { + "type": "string" + } + } + }, + "dto.DuplicateSearchAttributes": { + "type": "object", + "properties": { + "limit": { + "type": "integer" + }, + "repository_ids": { + "type": "array", + "items": { + "type": "integer" + } + }, + "threshold": { + "type": "number" + } + } + }, + "dto.DuplicateSearchData": { + "type": "object", + "properties": { + "attributes": { + "$ref": "#/definitions/dto.DuplicateSearchAttributes" + }, + "type": { + "type": "string" + } + } + }, + "dto.DuplicateSearchRequest": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/dto.DuplicateSearchData" + } + } + }, + "dto.DuplicateSnippetSchema": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + } + } + }, + "dto.DuplicatesMeta": { + "type": "object", + "properties": { + "truncated": { + "type": "boolean" + } + } + }, + "dto.DuplicatesResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/dto.DuplicatePairData" + } + }, + "meta": { + "$ref": "#/definitions/dto.DuplicatesMeta" + } + } + }, "dto.EnrichmentAttributes": { "type": "object", "properties": { diff --git a/docs/swagger/openapi.json b/docs/swagger/openapi.json index 9fbb6b557..1287897b9 100644 --- a/docs/swagger/openapi.json +++ b/docs/swagger/openapi.json @@ -60,6 +60,103 @@ }, "type": "object" }, + "dto.DuplicatePairAttributes": { + "properties": { + "similarity": { + "type": "number" + }, + "snippet_a": { + "$ref": "#/components/schemas/dto.DuplicateSnippetSchema" + }, + "snippet_b": { + "$ref": "#/components/schemas/dto.DuplicateSnippetSchema" + } + }, + "type": "object" + }, + "dto.DuplicatePairData": { + "properties": { + "attributes": { + "$ref": "#/components/schemas/dto.DuplicatePairAttributes" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicateSearchAttributes": { + "properties": { + "limit": { + "type": "integer" + }, + "repository_ids": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "threshold": { + "type": "number" + } + }, + "type": "object" + }, + "dto.DuplicateSearchData": { + "properties": { + "attributes": { + "$ref": "#/components/schemas/dto.DuplicateSearchAttributes" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicateSearchRequest": { + "properties": { + "data": { + "$ref": "#/components/schemas/dto.DuplicateSearchData" + } + }, + "type": "object" + }, + "dto.DuplicateSnippetSchema": { + "properties": { + "content": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicatesMeta": { + "properties": { + "truncated": { + "type": "boolean" + } + }, + "type": "object" + }, + "dto.DuplicatesResponse": { + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/dto.DuplicatePairData" + }, + "type": "array" + }, + "meta": { + "$ref": "#/components/schemas/dto.DuplicatesMeta" + } + }, + "type": "object" + }, "dto.EnrichmentAttributes": { "properties": { "content": { @@ -3277,6 +3374,59 @@ ] } }, + "/search/duplicates": { + "post": { + "description": "Finds pairs of semantically similar code snippets using pairwise embedding comparison", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dto.DuplicateSearchRequest" + } + } + }, + "description": "Duplicate search request", + "required": true, + "x-originalParamName": "body" + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dto.DuplicatesResponse" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/middleware.JSONAPIErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/middleware.JSONAPIErrorResponse" + } + } + }, + "description": "Internal Server Error" + } + }, + "summary": "Find duplicate code snippets", + "tags": [ + "search" + ] + } + }, "/search/grep": { "get": { "description": "Search file contents in a repository using git grep with regex patterns", diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index c1e32c16c..7a420996b 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -1890,6 +1890,52 @@ } } }, + "/search/duplicates": { + "post": { + "description": "Finds pairs of semantically similar code snippets using pairwise embedding comparison", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "search" + ], + "summary": "Find duplicate code snippets", + "parameters": [ + { + "description": "Duplicate search request", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/dto.DuplicateSearchRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/dto.DuplicatesResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/middleware.JSONAPIErrorResponse" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/middleware.JSONAPIErrorResponse" + } + } + } + } + }, "/search/grep": { "get": { "description": "Search file contents in a repository using git grep with regex patterns", @@ -2204,6 +2250,103 @@ } } }, + "dto.DuplicatePairAttributes": { + "type": "object", + "properties": { + "similarity": { + "type": "number" + }, + "snippet_a": { + "$ref": "#/definitions/dto.DuplicateSnippetSchema" + }, + "snippet_b": { + "$ref": "#/definitions/dto.DuplicateSnippetSchema" + } + } + }, + "dto.DuplicatePairData": { + "type": "object", + "properties": { + "attributes": { + "$ref": "#/definitions/dto.DuplicatePairAttributes" + }, + "type": { + "type": "string" + } + } + }, + "dto.DuplicateSearchAttributes": { + "type": "object", + "properties": { + "limit": { + "type": "integer" + }, + "repository_ids": { + "type": "array", + "items": { + "type": "integer" + } + }, + "threshold": { + "type": "number" + } + } + }, + "dto.DuplicateSearchData": { + "type": "object", + "properties": { + "attributes": { + "$ref": "#/definitions/dto.DuplicateSearchAttributes" + }, + "type": { + "type": "string" + } + } + }, + "dto.DuplicateSearchRequest": { + "type": "object", + "properties": { + "data": { + "$ref": "#/definitions/dto.DuplicateSearchData" + } + } + }, + "dto.DuplicateSnippetSchema": { + "type": "object", + "properties": { + "content": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + } + } + }, + "dto.DuplicatesMeta": { + "type": "object", + "properties": { + "truncated": { + "type": "boolean" + } + } + }, + "dto.DuplicatesResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/dto.DuplicatePairData" + } + }, + "meta": { + "$ref": "#/definitions/dto.DuplicatesMeta" + } + } + }, "dto.EnrichmentAttributes": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.yaml b/docs/swagger/swagger.yaml index 714da19e4..44df3f2ce 100644 --- a/docs/swagger/swagger.yaml +++ b/docs/swagger/swagger.yaml @@ -38,6 +38,68 @@ definitions: data: $ref: '#/definitions/dto.CommitData' type: object + dto.DuplicatePairAttributes: + properties: + similarity: + type: number + snippet_a: + $ref: '#/definitions/dto.DuplicateSnippetSchema' + snippet_b: + $ref: '#/definitions/dto.DuplicateSnippetSchema' + type: object + dto.DuplicatePairData: + properties: + attributes: + $ref: '#/definitions/dto.DuplicatePairAttributes' + type: + type: string + type: object + dto.DuplicateSearchAttributes: + properties: + limit: + type: integer + repository_ids: + items: + type: integer + type: array + threshold: + type: number + type: object + dto.DuplicateSearchData: + properties: + attributes: + $ref: '#/definitions/dto.DuplicateSearchAttributes' + type: + type: string + type: object + dto.DuplicateSearchRequest: + properties: + data: + $ref: '#/definitions/dto.DuplicateSearchData' + type: object + dto.DuplicateSnippetSchema: + properties: + content: + type: string + id: + type: string + language: + type: string + type: object + dto.DuplicatesMeta: + properties: + truncated: + type: boolean + type: object + dto.DuplicatesResponse: + properties: + data: + items: + $ref: '#/definitions/dto.DuplicatePairData' + type: array + meta: + $ref: '#/definitions/dto.DuplicatesMeta' + type: object dto.EnrichmentAttributes: properties: content: @@ -1892,6 +1954,37 @@ paths: summary: Search code tags: - search + /search/duplicates: + post: + consumes: + - application/json + description: Finds pairs of semantically similar code snippets using pairwise + embedding comparison + parameters: + - description: Duplicate search request + in: body + name: body + required: true + schema: + $ref: '#/definitions/dto.DuplicateSearchRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/dto.DuplicatesResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/middleware.JSONAPIErrorResponse' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/middleware.JSONAPIErrorResponse' + summary: Find duplicate code snippets + tags: + - search /search/grep: get: description: Search file contents in a repository using git grep with regex diff --git a/infrastructure/api/openapi.json b/infrastructure/api/openapi.json index 9fbb6b557..1287897b9 100644 --- a/infrastructure/api/openapi.json +++ b/infrastructure/api/openapi.json @@ -60,6 +60,103 @@ }, "type": "object" }, + "dto.DuplicatePairAttributes": { + "properties": { + "similarity": { + "type": "number" + }, + "snippet_a": { + "$ref": "#/components/schemas/dto.DuplicateSnippetSchema" + }, + "snippet_b": { + "$ref": "#/components/schemas/dto.DuplicateSnippetSchema" + } + }, + "type": "object" + }, + "dto.DuplicatePairData": { + "properties": { + "attributes": { + "$ref": "#/components/schemas/dto.DuplicatePairAttributes" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicateSearchAttributes": { + "properties": { + "limit": { + "type": "integer" + }, + "repository_ids": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "threshold": { + "type": "number" + } + }, + "type": "object" + }, + "dto.DuplicateSearchData": { + "properties": { + "attributes": { + "$ref": "#/components/schemas/dto.DuplicateSearchAttributes" + }, + "type": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicateSearchRequest": { + "properties": { + "data": { + "$ref": "#/components/schemas/dto.DuplicateSearchData" + } + }, + "type": "object" + }, + "dto.DuplicateSnippetSchema": { + "properties": { + "content": { + "type": "string" + }, + "id": { + "type": "string" + }, + "language": { + "type": "string" + } + }, + "type": "object" + }, + "dto.DuplicatesMeta": { + "properties": { + "truncated": { + "type": "boolean" + } + }, + "type": "object" + }, + "dto.DuplicatesResponse": { + "properties": { + "data": { + "items": { + "$ref": "#/components/schemas/dto.DuplicatePairData" + }, + "type": "array" + }, + "meta": { + "$ref": "#/components/schemas/dto.DuplicatesMeta" + } + }, + "type": "object" + }, "dto.EnrichmentAttributes": { "properties": { "content": { @@ -3277,6 +3374,59 @@ ] } }, + "/search/duplicates": { + "post": { + "description": "Finds pairs of semantically similar code snippets using pairwise embedding comparison", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dto.DuplicateSearchRequest" + } + } + }, + "description": "Duplicate search request", + "required": true, + "x-originalParamName": "body" + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/dto.DuplicatesResponse" + } + } + }, + "description": "OK" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/middleware.JSONAPIErrorResponse" + } + } + }, + "description": "Bad Request" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/middleware.JSONAPIErrorResponse" + } + } + }, + "description": "Internal Server Error" + } + }, + "summary": "Find duplicate code snippets", + "tags": [ + "search" + ] + } + }, "/search/grep": { "get": { "description": "Search file contents in a repository using git grep with regex patterns",