⚡ Bolt: [performance improvement] Optimize Lexer Keyword Lookup

.jules/bolt.md

@@ -0,0 +1,3 @@
+## 2024-07-06 - Optimized Lexer Keyword Lookup
+**Learning:** Replaced the hashmap `keywords` and its associated case-insensitive string parsing logic with an auto-generated, length-based `switch` statement in the lexer. This prevents unnecessary string allocations and hashmap lookups during parsing. The `keywords` map definition was moved to `cmd/gen_keywords/main.go` and is no longer part of the compiled application binary.
+**Action:** When working on lexer or parser performance, prefer generated switch statements over hashmaps or regexes. Always remove the original hashmap from the source file to prevent silent failure for future developers who might try to add to the old map.

cmd/gen_keywords/main.go

@@ -0,0 +1,194 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/format"
+	"log"
+	"os"
+	"sort"
+	"text/template"
+)
+
+// The template for the generated code
+var tmpl = template.Must(template.New("keywords").Parse(`// Code generated by go generate; DO NOT EDIT.
+
+package lexer
+
+func lookupKeywordFast(s string) (TokenKind, bool) {
+	switch len(s) {
+{{- range $len, $kws := .ByLength }}
+	case {{$len}}:
+{{- range $kws }}
+		if {{.Condition}} {
+			return {{.Kind}}, true
+		}
+{{- end }}
+{{- end }}
+	}
+	return 0, false
+}
+`))
+
+type Keyword struct {
+	Word      string
+	Kind      string
+	Condition string
+}
+
+func main() {
+	keywords := map[string]string{
+		"GEO_BBOX":     "TokenKindGeoBbox",
+		"GEO_RADIUS":   "TokenKindGeoRadius",
+		"VALUES_COUNT": "TokenKindValuesCount",
+		"HAS_VECTOR":   "TokenKindHasVector",
+		"BOOST":        "TokenKindBoost",
+		"DEFAULTS":     "TokenKindDefaults",
+		"CASE":         "TokenKindCase",
+		"WHEN":         "TokenKindWhen",
+		"THEN":         "TokenKindThen",
+		"ELSE":         "TokenKindElse",
+		"END":          "TokenKindEnd",
+		"INSERT":       "TokenKindInsert",
+		"INTO":         "TokenKindInto",
+		"COLLECTION":   "TokenKindCollection",
+		"VALUES":       "TokenKindValues",
+		"USING":        "TokenKindUsing",
+		"MODEL":        "TokenKindModel",
+		"HYBRID":       "TokenKindHybrid",
+		"DENSE":        "TokenKindDense",
+		"SPARSE":       "TokenKindSparse",
+		"RERANK":       "TokenKindRerank",
+		"EXACT":        "TokenKindExact",
+		"WITH":         "TokenKindWith",
+		"AS":           "TokenKindAs",
+		"ACORN":        "TokenKindAcorn",
+		"QUANTIZE":     "TokenKindQuantize",
+		"SCALAR":       "TokenKindScalar",
+		"BINARY":       "TokenKindBinary",
+		"PRODUCT":      "TokenKindProduct",
+		"TURBO":        "TokenKindTurbo",
+		"BITS":         "TokenKindBits",
+		"QUANTILE":     "TokenKindQuantile",
+		"ALWAYS":       "TokenKindAlways",
+		"RAM":          "TokenKindRam",
+		"HNSW":         "TokenKindHnsw",
+		"VECTORS":      "TokenKindVectors",
+		"OPTIMIZERS":   "TokenKindOptimizers",
+		"PARAMS":       "TokenKindParams",
+		"DISABLED":     "TokenKindDisabled",
+		"CREATE":       "TokenKindCreate",
+		"ALTER":        "TokenKindAlter",
+		"DROP":         "TokenKindDrop",
+		"SHOW":         "TokenKindShow",
+		"COLLECTIONS":  "TokenKindCollections",
+		"SELECT":       "TokenKindSelect",
+		"SCROLL":       "TokenKindScroll",
+		"AFTER":        "TokenKindAfter",
+		"RECOMMEND":    "TokenKindRecommend",
+		"QUERY":        "TokenKindQuery",
+		"NEAREST":      "TokenKindNearest",
+		"CONTEXT":      "TokenKindContext",
+		"DISCOVER":     "TokenKindDiscover",
+		"PAIRS":        "TokenKindPairs",
+		"TARGET":       "TokenKindTarget",
+		"ORDER":        "TokenKindOrder",
+		"ASC":          "TokenKindAsc",
+		"DESC":         "TokenKindDesc",
+		"LIMIT":        "TokenKindLimit",
+		"GROUP":        "TokenKindGroup",
+		"BY":           "TokenKindBy",
+		"GROUP_SIZE":   "TokenKindGroupSize",
+		"STRATEGY":     "TokenKindStrategy",
+		"DELETE":       "TokenKindDelete",
+		"UPDATE":       "TokenKindUpdate",
+		"SET":          "TokenKindSet",
+		"VECTOR":       "TokenKindVector",
+		"PAYLOAD":      "TokenKindPayload",
+		"FROM":         "TokenKindFrom",
+		"WHERE":        "TokenKindWhere",
+		"ID":           "TokenKindId",
+		"INDEX":        "TokenKindIndex",
+		"ON":           "TokenKindOn",
+		"FOR":          "TokenKindFor",
+		"TYPE":         "TokenKindType",
+		"AND":          "TokenKindAnd",
+		"OR":           "TokenKindOr",
+		"NOT":          "TokenKindNot",
+		"IN":           "TokenKindIn",
+		"BETWEEN":      "TokenKindBetween",
+		"IS":           "TokenKindIs",
+		"NULL":         "TokenKindNull",
+		"EMPTY":        "TokenKindEmpty",
+		"MATCH":        "TokenKindMatch",
+		"ANY":          "TokenKindAny",
+		"PHRASE":       "TokenKindPhrase",
+		"OFFSET":       "TokenKindOffset",
+		"SCORE":        "TokenKindScore",
+		"THRESHOLD":    "TokenKindThreshold",
+		"LOOKUP":       "TokenKindLookup",
+		"COSINE":       "TokenKindCosine",
+		"DOT":          "TokenKindDot",
+		"EUCLID":       "TokenKindEuclid",
+		"MANHATTAN":    "TokenKindManhattan",
+		"PREFETCH":     "TokenKindPrefetch",
+		"FUSION":       "TokenKindFusion",
+		"SAMPLE":       "TokenKindSample",
+		"RELEVANCE":    "TokenKindRelevance",
+		"FEEDBACK":     "TokenKindFeedback",
+	}
+
+	byLength := make(map[int][]Keyword)
+
+	for word, kind := range keywords {
+		cond := ""
+		for i := 0; i < len(word); i++ {
+			if i > 0 {
+				cond += " && "
+			}
+			c := word[i]
+			lower := c + 32
+			if c == '_' {
+				cond += fmt.Sprintf("s[%d] == '_'", i)
+			} else {
+				cond += fmt.Sprintf("(s[%d] == '%c' || s[%d] == '%c')", i, c, i, lower)
+			}
+		}
+
+		kw := Keyword{
+			Word:      word,
+			Kind:      kind,
+			Condition: cond,
+		}
+		byLength[len(word)] = append(byLength[len(word)], kw)
+	}
+
+	for _, kws := range byLength {
+		sort.Slice(kws, func(i, j int) bool {
+			return kws[i].Word < kws[j].Word
+		})
+	}
+
+	data := struct {
+		ByLength map[int][]Keyword
+	}{
+		ByLength: byLength,
+	}
+
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, data); err != nil {
+		log.Fatalf("Template execution failed: %v", err)
+	}
+
+	formatted, err := format.Source(buf.Bytes())
+	if err != nil {
+		log.Printf("Formatting failed (writing unformatted code for debugging): %v", err)
+		formatted = buf.Bytes()
+	}
+
+	outPath := "keywords_fast.go"
+	if err := os.WriteFile(outPath, formatted, 0644); err != nil {
+		log.Fatalf("Write failed: %v", err)
+	}
+	fmt.Printf("Successfully wrote %s\n", outPath)
+}