Skip to content

Commit 9314fbd

Browse files
randalmurphalclaude
andcommitted
fix: Comprehensive fixes from code review findings
Chunker improvements: - Code chunker: implement MaxChunkSize, MinChunkSize, ChunkGranularity - Code chunker: add granularity validation with constants - Code chunker: fix extractReceiver for generics and complex types - Markdown chunker: use original heading level in continuations - Markdown chunker: add ChunkOverlap config for context preservation Entity extraction improvements: - Add word boundaries to patterns to prevent partial matches - Add missing entity types: Version, Email, Date, Commit, IP - Update Extract to handle capturing groups properly - Add priority system for new entity types Enricher improvements: - Add ContinueOnError option to enricher chain - Make Chain.Add() thread-safe with sync.RWMutex - Add NewXxxValidated() constructors for error handling - Fix DRY violation in relationship enricher (shared enrichInternal) - Add document context to error messages - Add missing relationship types: Calls, Implements, Extends, etc. Summarizer improvements: - UTF-8 safe truncation using runes instead of bytes - Add document ID to error context Test updates: - Update nil constructor tests to expect nil return - Fix URL trailing punctuation test expectation - Update custom pattern test to use non-conflicting pattern 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
1 parent 436c56a commit 9314fbd

13 files changed

Lines changed: 619 additions & 146 deletions

File tree

‎pkg/graphrag/indexing/chunker/code/go.go‎

Lines changed: 226 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ func (c *GoChunker) ChunkCode(ctx context.Context, id, filePath, content string)
7070
default:
7171
}
7272

73+
// Handle file-level granularity early
74+
if c.config.ChunkGranularity == GranularityFile {
75+
return c.chunkAsFile(id, filePath, content), nil
76+
}
77+
7378
// Parse the Go source file
7479
fset := token.NewFileSet()
7580
file, err := parser.ParseFile(fset, filePath, content, parser.ParseComments)
@@ -131,9 +136,206 @@ func (c *GoChunker) ChunkCode(ctx context.Context, id, filePath, content string)
131136
}
132137
}
133138

139+
// Apply granularity grouping if type-level
140+
if c.config.ChunkGranularity == GranularityType {
141+
chunks = c.groupByType(chunks, id)
142+
}
143+
144+
// Apply size limits
145+
chunks = c.applySizeLimits(chunks, id)
146+
134147
return chunks, nil
135148
}
136149

150+
// chunkAsFile returns the entire file as a single chunk.
151+
func (c *GoChunker) chunkAsFile(id, filePath, content string) []CodeChunk {
152+
return []CodeChunk{{
153+
Chunk: chunker.Chunk{
154+
ID: c.generateID(id, "file", 0),
155+
Content: content,
156+
Index: 0,
157+
ParentID: id,
158+
Metadata: map[string]any{
159+
"symbol_type": string(SymbolPackage),
160+
"granularity": GranularityFile,
161+
},
162+
},
163+
Language: "go",
164+
SymbolType: SymbolPackage,
165+
FilePath: filePath,
166+
StartLine: 1,
167+
}}
168+
}
169+
170+
// groupByType groups types with their methods into single chunks.
171+
func (c *GoChunker) groupByType(chunks []CodeChunk, parentID string) []CodeChunk {
172+
// Map receiver types to their methods
173+
typeMethods := make(map[string][]CodeChunk)
174+
var result []CodeChunk
175+
176+
// First pass: collect methods by receiver type
177+
for _, chunk := range chunks {
178+
if chunk.SymbolType == SymbolMethod && chunk.Receiver != "" {
179+
// Strip pointer prefix for grouping
180+
baseReceiver := strings.TrimPrefix(chunk.Receiver, "*")
181+
// Also strip generic suffixes
182+
if idx := strings.Index(baseReceiver, "["); idx > 0 {
183+
baseReceiver = baseReceiver[:idx]
184+
}
185+
typeMethods[baseReceiver] = append(typeMethods[baseReceiver], chunk)
186+
}
187+
}
188+
189+
// Second pass: merge types with their methods
190+
newIdx := 0
191+
usedMethods := make(map[string]bool)
192+
193+
for _, chunk := range chunks {
194+
if chunk.SymbolType == SymbolMethod {
195+
continue // Skip methods, they'll be merged with types
196+
}
197+
198+
if chunk.SymbolType == SymbolTypeDecl {
199+
// Check if this type has methods
200+
if methods, ok := typeMethods[chunk.SymbolName]; ok && len(methods) > 0 {
201+
// Merge type with its methods
202+
var combined strings.Builder
203+
combined.WriteString(chunk.Content)
204+
205+
for _, m := range methods {
206+
combined.WriteString("\n\n")
207+
combined.WriteString(m.Content)
208+
usedMethods[m.ID] = true
209+
}
210+
211+
merged := chunk
212+
merged.Chunk.Content = combined.String()
213+
merged.Chunk.Index = newIdx
214+
merged.Chunk.ID = c.generateID(parentID, chunk.SymbolName+"_with_methods", newIdx)
215+
merged.Chunk.Metadata["method_count"] = len(methods)
216+
result = append(result, merged)
217+
newIdx++
218+
continue
219+
}
220+
}
221+
222+
// Keep non-method, non-merged chunks
223+
chunk.Chunk.Index = newIdx
224+
result = append(result, chunk)
225+
newIdx++
226+
}
227+
228+
// Add any orphan methods (methods without a type in this file)
229+
for _, chunk := range chunks {
230+
if chunk.SymbolType == SymbolMethod && !usedMethods[chunk.ID] {
231+
chunk.Chunk.Index = newIdx
232+
result = append(result, chunk)
233+
newIdx++
234+
}
235+
}
236+
237+
return result
238+
}
239+
240+
// applySizeLimits applies MaxChunkSize splitting and MinChunkSize combining.
241+
func (c *GoChunker) applySizeLimits(chunks []CodeChunk, parentID string) []CodeChunk {
242+
// Skip if no size limits configured
243+
if c.config.MaxChunkSize <= 0 && c.config.MinChunkSize <= 0 {
244+
return chunks
245+
}
246+
247+
var result []CodeChunk
248+
newIdx := 0
249+
250+
for i := 0; i < len(chunks); i++ {
251+
chunk := chunks[i]
252+
253+
// Handle MaxChunkSize: split large chunks
254+
if c.config.MaxChunkSize > 0 && len(chunk.Content) > c.config.MaxChunkSize {
255+
splitChunks := c.splitLargeChunk(chunk, parentID, &newIdx)
256+
result = append(result, splitChunks...)
257+
continue
258+
}
259+
260+
// Handle MinChunkSize: combine small chunks
261+
if c.config.MinChunkSize > 0 && len(chunk.Content) < c.config.MinChunkSize {
262+
// Try to combine with next chunk if it's also small and same type category
263+
if i+1 < len(chunks) {
264+
next := chunks[i+1]
265+
combinedSize := len(chunk.Content) + len(next.Content) + 2 // +2 for newlines
266+
267+
// Only combine if result won't exceed MaxChunkSize (if set)
268+
canCombine := c.config.MaxChunkSize <= 0 || combinedSize <= c.config.MaxChunkSize
269+
// Only combine similar types (both consts, both vars, etc.)
270+
sameCategory := chunk.SymbolType == next.SymbolType
271+
272+
if canCombine && sameCategory && len(next.Content) < c.config.MinChunkSize {
273+
combined := chunk
274+
combined.Content = chunk.Content + "\n\n" + next.Content
275+
combined.SymbolName = chunk.SymbolName + ", " + next.SymbolName
276+
combined.Chunk.Index = newIdx
277+
combined.Chunk.ID = c.generateID(parentID, combined.SymbolName, newIdx)
278+
combined.Chunk.Metadata["combined"] = true
279+
result = append(result, combined)
280+
newIdx++
281+
i++ // Skip next chunk since we combined it
282+
continue
283+
}
284+
}
285+
}
286+
287+
// Keep chunk as-is
288+
chunk.Chunk.Index = newIdx
289+
result = append(result, chunk)
290+
newIdx++
291+
}
292+
293+
return result
294+
}
295+
296+
// splitLargeChunk splits a chunk that exceeds MaxChunkSize.
297+
func (c *GoChunker) splitLargeChunk(chunk CodeChunk, parentID string, idx *int) []CodeChunk {
298+
content := chunk.Content
299+
maxSize := c.config.MaxChunkSize
300+
var result []CodeChunk
301+
302+
partNum := 0
303+
for len(content) > 0 {
304+
// Find split point (prefer line boundaries)
305+
splitAt := maxSize
306+
if splitAt > len(content) {
307+
splitAt = len(content)
308+
} else {
309+
// Look for last newline within limit
310+
lastNewline := strings.LastIndex(content[:splitAt], "\n")
311+
if lastNewline > maxSize/2 { // Only use if not too far back
312+
splitAt = lastNewline + 1
313+
}
314+
}
315+
316+
part := content[:splitAt]
317+
content = content[splitAt:]
318+
319+
partChunk := chunk
320+
partChunk.Content = strings.TrimSpace(part)
321+
partChunk.Chunk.Index = *idx
322+
partChunk.Chunk.ID = c.generateID(parentID, fmt.Sprintf("%s_part%d", chunk.SymbolName, partNum), *idx)
323+
partChunk.Chunk.Metadata["part"] = partNum
324+
partChunk.Chunk.Metadata["split"] = true
325+
326+
if partNum > 0 {
327+
// Add context header for continuation
328+
partChunk.Content = fmt.Sprintf("// %s (continued)\n%s", chunk.SymbolName, partChunk.Content)
329+
}
330+
331+
result = append(result, partChunk)
332+
*idx++
333+
partNum++
334+
}
335+
336+
return result
337+
}
338+
137339
// extractPackage creates a chunk for the package declaration.
138340
func (c *GoChunker) extractPackage(fset *token.FileSet, file *ast.File, id, filePath, content string, idx int) *CodeChunk {
139341
var builder strings.Builder
@@ -393,13 +595,35 @@ func (c *GoChunker) extractFuncDecl(fset *token.FileSet, file *ast.File, fn *ast
393595
}
394596

395597
// extractReceiver extracts the receiver type from a method.
598+
// Handles value receivers, pointer receivers, and generic receivers.
396599
func (c *GoChunker) extractReceiver(field *ast.Field) string {
397-
switch t := field.Type.(type) {
600+
return c.typeToReceiverString(field.Type)
601+
}
602+
603+
// typeToReceiverString converts a type expression to a receiver string.
604+
func (c *GoChunker) typeToReceiverString(expr ast.Expr) string {
605+
switch t := expr.(type) {
398606
case *ast.Ident:
399607
return t.Name
400608
case *ast.StarExpr:
609+
inner := c.typeToReceiverString(t.X)
610+
if inner != "" {
611+
return "*" + inner
612+
}
613+
case *ast.IndexExpr:
614+
// Generic type with single type parameter: Type[T]
615+
if ident, ok := t.X.(*ast.Ident); ok {
616+
return ident.Name + "[T]"
617+
}
618+
case *ast.IndexListExpr:
619+
// Generic type with multiple type parameters: Type[T, U]
401620
if ident, ok := t.X.(*ast.Ident); ok {
402-
return "*" + ident.Name
621+
return ident.Name + "[...]"
622+
}
623+
case *ast.SelectorExpr:
624+
// Qualified type: pkg.Type
625+
if x, ok := t.X.(*ast.Ident); ok {
626+
return x.Name + "." + t.Sel.Name
403627
}
404628
}
405629
return ""

‎pkg/graphrag/indexing/chunker/code/options.go‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,24 @@
11
package code
22

3+
// Granularity constants for chunk granularity.
4+
const (
5+
GranularityFunction = "function" // Each function/method is a separate chunk (default)
6+
GranularityType = "type" // Each type and its methods are grouped together
7+
GranularityFile = "file" // Entire file is one chunk
8+
)
9+
10+
// validGranularities is the set of valid granularity values.
11+
var validGranularities = map[string]bool{
12+
GranularityFunction: true,
13+
GranularityType: true,
14+
GranularityFile: true,
15+
}
16+
17+
// IsValidGranularity checks if a granularity value is valid.
18+
func IsValidGranularity(g string) bool {
19+
return validGranularities[g]
20+
}
21+
322
// Option configures a code chunker.
423
type Option func(*Config)
524

@@ -44,13 +63,14 @@ func WithMinChunkSize(size int) Option {
4463
}
4564

4665
// WithChunkGranularity sets the chunking level.
47-
// Valid values:
48-
// - "function": Each function/method is a separate chunk (default)
49-
// - "type": Each type and its methods are grouped together
50-
// - "file": Entire file is one chunk
66+
// Valid values: GranularityFunction, GranularityType, GranularityFile.
67+
// Invalid values are ignored (use IsValidGranularity to check beforehand).
5168
func WithChunkGranularity(granularity string) Option {
5269
return func(c *Config) {
53-
c.ChunkGranularity = granularity
70+
if IsValidGranularity(granularity) {
71+
c.ChunkGranularity = granularity
72+
}
73+
// Invalid values are silently ignored - use IsValidGranularity to validate
5474
}
5575
}
5676

0 commit comments

Comments
 (0)