diff --git a/.jules/bolt.md b/.jules/bolt.md new file mode 100644 index 0000000..fb14938 --- /dev/null +++ b/.jules/bolt.md @@ -0,0 +1,3 @@ +## 2024-07-06 - Optimized Lexer Keyword Lookup +**Learning:** Replaced the hashmap `keywords` and its associated case-insensitive string parsing logic with an auto-generated, length-based `switch` statement in the lexer. This prevents unnecessary string allocations and hashmap lookups during parsing. The `keywords` map definition was moved to `cmd/gen_keywords/main.go` and is no longer part of the compiled application binary. +**Action:** When working on lexer or parser performance, prefer generated switch statements over hashmaps or regexes. Always remove the original hashmap from the source file to prevent silent failure for future developers who might try to add to the old map. diff --git a/cmd/gen_keywords/main.go b/cmd/gen_keywords/main.go new file mode 100644 index 0000000..27b35d7 --- /dev/null +++ b/cmd/gen_keywords/main.go @@ -0,0 +1,194 @@ +package main + +import ( + "bytes" + "fmt" + "go/format" + "log" + "os" + "sort" + "text/template" +) + +// The template for the generated code +var tmpl = template.Must(template.New("keywords").Parse(`// Code generated by go generate; DO NOT EDIT. + +package lexer + +func lookupKeywordFast(s string) (TokenKind, bool) { + switch len(s) { +{{- range $len, $kws := .ByLength }} + case {{$len}}: +{{- range $kws }} + if {{.Condition}} { + return {{.Kind}}, true + } +{{- end }} +{{- end }} + } + return 0, false +} +`)) + +type Keyword struct { + Word string + Kind string + Condition string +} + +func main() { + keywords := map[string]string{ + "GEO_BBOX": "TokenKindGeoBbox", + "GEO_RADIUS": "TokenKindGeoRadius", + "VALUES_COUNT": "TokenKindValuesCount", + "HAS_VECTOR": "TokenKindHasVector", + "BOOST": "TokenKindBoost", + "DEFAULTS": "TokenKindDefaults", + "CASE": "TokenKindCase", + "WHEN": "TokenKindWhen", + "THEN": "TokenKindThen", + "ELSE": "TokenKindElse", + "END": "TokenKindEnd", + "INSERT": "TokenKindInsert", + "INTO": "TokenKindInto", + "COLLECTION": "TokenKindCollection", + "VALUES": "TokenKindValues", + "USING": "TokenKindUsing", + "MODEL": "TokenKindModel", + "HYBRID": "TokenKindHybrid", + "DENSE": "TokenKindDense", + "SPARSE": "TokenKindSparse", + "RERANK": "TokenKindRerank", + "EXACT": "TokenKindExact", + "WITH": "TokenKindWith", + "AS": "TokenKindAs", + "ACORN": "TokenKindAcorn", + "QUANTIZE": "TokenKindQuantize", + "SCALAR": "TokenKindScalar", + "BINARY": "TokenKindBinary", + "PRODUCT": "TokenKindProduct", + "TURBO": "TokenKindTurbo", + "BITS": "TokenKindBits", + "QUANTILE": "TokenKindQuantile", + "ALWAYS": "TokenKindAlways", + "RAM": "TokenKindRam", + "HNSW": "TokenKindHnsw", + "VECTORS": "TokenKindVectors", + "OPTIMIZERS": "TokenKindOptimizers", + "PARAMS": "TokenKindParams", + "DISABLED": "TokenKindDisabled", + "CREATE": "TokenKindCreate", + "ALTER": "TokenKindAlter", + "DROP": "TokenKindDrop", + "SHOW": "TokenKindShow", + "COLLECTIONS": "TokenKindCollections", + "SELECT": "TokenKindSelect", + "SCROLL": "TokenKindScroll", + "AFTER": "TokenKindAfter", + "RECOMMEND": "TokenKindRecommend", + "QUERY": "TokenKindQuery", + "NEAREST": "TokenKindNearest", + "CONTEXT": "TokenKindContext", + "DISCOVER": "TokenKindDiscover", + "PAIRS": "TokenKindPairs", + "TARGET": "TokenKindTarget", + "ORDER": "TokenKindOrder", + "ASC": "TokenKindAsc", + "DESC": "TokenKindDesc", + "LIMIT": "TokenKindLimit", + "GROUP": "TokenKindGroup", + "BY": "TokenKindBy", + "GROUP_SIZE": "TokenKindGroupSize", + "STRATEGY": "TokenKindStrategy", + "DELETE": "TokenKindDelete", + "UPDATE": "TokenKindUpdate", + "SET": "TokenKindSet", + "VECTOR": "TokenKindVector", + "PAYLOAD": "TokenKindPayload", + "FROM": "TokenKindFrom", + "WHERE": "TokenKindWhere", + "ID": "TokenKindId", + "INDEX": "TokenKindIndex", + "ON": "TokenKindOn", + "FOR": "TokenKindFor", + "TYPE": "TokenKindType", + "AND": "TokenKindAnd", + "OR": "TokenKindOr", + "NOT": "TokenKindNot", + "IN": "TokenKindIn", + "BETWEEN": "TokenKindBetween", + "IS": "TokenKindIs", + "NULL": "TokenKindNull", + "EMPTY": "TokenKindEmpty", + "MATCH": "TokenKindMatch", + "ANY": "TokenKindAny", + "PHRASE": "TokenKindPhrase", + "OFFSET": "TokenKindOffset", + "SCORE": "TokenKindScore", + "THRESHOLD": "TokenKindThreshold", + "LOOKUP": "TokenKindLookup", + "COSINE": "TokenKindCosine", + "DOT": "TokenKindDot", + "EUCLID": "TokenKindEuclid", + "MANHATTAN": "TokenKindManhattan", + "PREFETCH": "TokenKindPrefetch", + "FUSION": "TokenKindFusion", + "SAMPLE": "TokenKindSample", + "RELEVANCE": "TokenKindRelevance", + "FEEDBACK": "TokenKindFeedback", + } + + byLength := make(map[int][]Keyword) + + for word, kind := range keywords { + cond := "" + for i := 0; i < len(word); i++ { + if i > 0 { + cond += " && " + } + c := word[i] + lower := c + 32 + if c == '_' { + cond += fmt.Sprintf("s[%d] == '_'", i) + } else { + cond += fmt.Sprintf("(s[%d] == '%c' || s[%d] == '%c')", i, c, i, lower) + } + } + + kw := Keyword{ + Word: word, + Kind: kind, + Condition: cond, + } + byLength[len(word)] = append(byLength[len(word)], kw) + } + + for _, kws := range byLength { + sort.Slice(kws, func(i, j int) bool { + return kws[i].Word < kws[j].Word + }) + } + + data := struct { + ByLength map[int][]Keyword + }{ + ByLength: byLength, + } + + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + log.Fatalf("Template execution failed: %v", err) + } + + formatted, err := format.Source(buf.Bytes()) + if err != nil { + log.Printf("Formatting failed (writing unformatted code for debugging): %v", err) + formatted = buf.Bytes() + } + + outPath := "keywords_fast.go" + if err := os.WriteFile(outPath, formatted, 0644); err != nil { + log.Fatalf("Write failed: %v", err) + } + fmt.Printf("Successfully wrote %s\n", outPath) +} diff --git a/internal/lexer/keywords_fast.go b/internal/lexer/keywords_fast.go new file mode 100644 index 0000000..d57bd58 --- /dev/null +++ b/internal/lexer/keywords_fast.go @@ -0,0 +1,314 @@ +// Code generated by go generate; DO NOT EDIT. + +package lexer + +func lookupKeywordFast(s string) (TokenKind, bool) { + switch len(s) { + case 2: + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'S' || s[1] == 's') { + return TokenKindAs, true + } + if (s[0] == 'B' || s[0] == 'b') && (s[1] == 'Y' || s[1] == 'y') { + return TokenKindBy, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'D' || s[1] == 'd') { + return TokenKindId, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') { + return TokenKindIn, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'S' || s[1] == 's') { + return TokenKindIs, true + } + if (s[0] == 'O' || s[0] == 'o') && (s[1] == 'N' || s[1] == 'n') { + return TokenKindOn, true + } + if (s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') { + return TokenKindOr, true + } + case 3: + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') { + return TokenKindAnd, true + } + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'Y' || s[2] == 'y') { + return TokenKindAny, true + } + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'S' || s[1] == 's') && (s[2] == 'C' || s[2] == 'c') { + return TokenKindAsc, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') { + return TokenKindDot, true + } + if (s[0] == 'E' || s[0] == 'e') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') { + return TokenKindEnd, true + } + if (s[0] == 'F' || s[0] == 'f') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'R' || s[2] == 'r') { + return TokenKindFor, true + } + if (s[0] == 'N' || s[0] == 'n') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'T' || s[2] == 't') { + return TokenKindNot, true + } + if (s[0] == 'R' || s[0] == 'r') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'M' || s[2] == 'm') { + return TokenKindRam, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'T' || s[2] == 't') { + return TokenKindSet, true + } + case 4: + if (s[0] == 'B' || s[0] == 'b') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'S' || s[3] == 's') { + return TokenKindBits, true + } + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'E' || s[3] == 'e') { + return TokenKindCase, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'C' || s[3] == 'c') { + return TokenKindDesc, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'P' || s[3] == 'p') { + return TokenKindDrop, true + } + if (s[0] == 'E' || s[0] == 'e') && (s[1] == 'L' || s[1] == 'l') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'E' || s[3] == 'e') { + return TokenKindElse, true + } + if (s[0] == 'F' || s[0] == 'f') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'M' || s[3] == 'm') { + return TokenKindFrom, true + } + if (s[0] == 'H' || s[0] == 'h') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'W' || s[3] == 'w') { + return TokenKindHnsw, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'O' || s[3] == 'o') { + return TokenKindInto, true + } + if (s[0] == 'N' || s[0] == 'n') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'L' || s[3] == 'l') { + return TokenKindNull, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'W' || s[3] == 'w') { + return TokenKindShow, true + } + if (s[0] == 'T' || s[0] == 't') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'N' || s[3] == 'n') { + return TokenKindThen, true + } + if (s[0] == 'T' || s[0] == 't') && (s[1] == 'Y' || s[1] == 'y') && (s[2] == 'P' || s[2] == 'p') && (s[3] == 'E' || s[3] == 'e') { + return TokenKindType, true + } + if (s[0] == 'W' || s[0] == 'w') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'N' || s[3] == 'n') { + return TokenKindWhen, true + } + if (s[0] == 'W' || s[0] == 'w') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'H' || s[3] == 'h') { + return TokenKindWith, true + } + case 5: + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'C' || s[1] == 'c') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'N' || s[4] == 'n') { + return TokenKindAcorn, true + } + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'F' || s[1] == 'f') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'R' || s[4] == 'r') { + return TokenKindAfter, true + } + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'L' || s[1] == 'l') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'R' || s[4] == 'r') { + return TokenKindAlter, true + } + if (s[0] == 'B' || s[0] == 'b') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'S' || s[3] == 's') && (s[4] == 'T' || s[4] == 't') { + return TokenKindBoost, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'N' || s[2] == 'n') && (s[3] == 'S' || s[3] == 's') && (s[4] == 'E' || s[4] == 'e') { + return TokenKindDense, true + } + if (s[0] == 'E' || s[0] == 'e') && (s[1] == 'M' || s[1] == 'm') && (s[2] == 'P' || s[2] == 'p') && (s[3] == 'T' || s[3] == 't') && (s[4] == 'Y' || s[4] == 'y') { + return TokenKindEmpty, true + } + if (s[0] == 'E' || s[0] == 'e') && (s[1] == 'X' || s[1] == 'x') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'C' || s[3] == 'c') && (s[4] == 'T' || s[4] == 't') { + return TokenKindExact, true + } + if (s[0] == 'G' || s[0] == 'g') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'U' || s[3] == 'u') && (s[4] == 'P' || s[4] == 'p') { + return TokenKindGroup, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'D' || s[2] == 'd') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'X' || s[4] == 'x') { + return TokenKindIndex, true + } + if (s[0] == 'L' || s[0] == 'l') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'M' || s[2] == 'm') && (s[3] == 'I' || s[3] == 'i') && (s[4] == 'T' || s[4] == 't') { + return TokenKindLimit, true + } + if (s[0] == 'M' || s[0] == 'm') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'C' || s[3] == 'c') && (s[4] == 'H' || s[4] == 'h') { + return TokenKindMatch, true + } + if (s[0] == 'M' || s[0] == 'm') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'D' || s[2] == 'd') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'L' || s[4] == 'l') { + return TokenKindModel, true + } + if (s[0] == 'O' || s[0] == 'o') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'D' || s[2] == 'd') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'R' || s[4] == 'r') { + return TokenKindOrder, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'I' || s[2] == 'i') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'S' || s[4] == 's') { + return TokenKindPairs, true + } + if (s[0] == 'Q' || s[0] == 'q') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'Y' || s[4] == 'y') { + return TokenKindQuery, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'C' || s[1] == 'c') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'E' || s[4] == 'e') { + return TokenKindScore, true + } + if (s[0] == 'T' || s[0] == 't') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'B' || s[3] == 'b') && (s[4] == 'O' || s[4] == 'o') { + return TokenKindTurbo, true + } + if (s[0] == 'U' || s[0] == 'u') && (s[1] == 'S' || s[1] == 's') && (s[2] == 'I' || s[2] == 'i') && (s[3] == 'N' || s[3] == 'n') && (s[4] == 'G' || s[4] == 'g') { + return TokenKindUsing, true + } + if (s[0] == 'W' || s[0] == 'w') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'E' || s[4] == 'e') { + return TokenKindWhere, true + } + case 6: + if (s[0] == 'A' || s[0] == 'a') && (s[1] == 'L' || s[1] == 'l') && (s[2] == 'W' || s[2] == 'w') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'Y' || s[4] == 'y') && (s[5] == 'S' || s[5] == 's') { + return TokenKindAlways, true + } + if (s[0] == 'B' || s[0] == 'b') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'N' || s[2] == 'n') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'R' || s[4] == 'r') && (s[5] == 'Y' || s[5] == 'y') { + return TokenKindBinary, true + } + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'I' || s[3] == 'i') && (s[4] == 'N' || s[4] == 'n') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindCosine, true + } + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindCreate, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindDelete, true + } + if (s[0] == 'E' || s[0] == 'e') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'C' || s[2] == 'c') && (s[3] == 'L' || s[3] == 'l') && (s[4] == 'I' || s[4] == 'i') && (s[5] == 'D' || s[5] == 'd') { + return TokenKindEuclid, true + } + if (s[0] == 'F' || s[0] == 'f') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'I' || s[3] == 'i') && (s[4] == 'O' || s[4] == 'o') && (s[5] == 'N' || s[5] == 'n') { + return TokenKindFusion, true + } + if (s[0] == 'H' || s[0] == 'h') && (s[1] == 'Y' || s[1] == 'y') && (s[2] == 'B' || s[2] == 'b') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'I' || s[4] == 'i') && (s[5] == 'D' || s[5] == 'd') { + return TokenKindHybrid, true + } + if (s[0] == 'I' || s[0] == 'i') && (s[1] == 'N' || s[1] == 'n') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'R' || s[4] == 'r') && (s[5] == 'T' || s[5] == 't') { + return TokenKindInsert, true + } + if (s[0] == 'L' || s[0] == 'l') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'K' || s[3] == 'k') && (s[4] == 'U' || s[4] == 'u') && (s[5] == 'P' || s[5] == 'p') { + return TokenKindLookup, true + } + if (s[0] == 'O' || s[0] == 'o') && (s[1] == 'F' || s[1] == 'f') && (s[2] == 'F' || s[2] == 'f') && (s[3] == 'S' || s[3] == 's') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'T' || s[5] == 't') { + return TokenKindOffset, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'M' || s[4] == 'm') && (s[5] == 'S' || s[5] == 's') { + return TokenKindParams, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'S' || s[4] == 's') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindPhrase, true + } + if (s[0] == 'R' || s[0] == 'r') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'N' || s[4] == 'n') && (s[5] == 'K' || s[5] == 'k') { + return TokenKindRerank, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'M' || s[2] == 'm') && (s[3] == 'P' || s[3] == 'p') && (s[4] == 'L' || s[4] == 'l') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindSample, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'C' || s[1] == 'c') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'L' || s[3] == 'l') && (s[4] == 'A' || s[4] == 'a') && (s[5] == 'R' || s[5] == 'r') { + return TokenKindScalar, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'C' || s[1] == 'c') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'O' || s[3] == 'o') && (s[4] == 'L' || s[4] == 'l') && (s[5] == 'L' || s[5] == 'l') { + return TokenKindScroll, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'C' || s[4] == 'c') && (s[5] == 'T' || s[5] == 't') { + return TokenKindSelect, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'P' || s[1] == 'p') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'S' || s[4] == 's') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindSparse, true + } + if (s[0] == 'T' || s[0] == 't') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'G' || s[3] == 'g') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'T' || s[5] == 't') { + return TokenKindTarget, true + } + if (s[0] == 'U' || s[0] == 'u') && (s[1] == 'P' || s[1] == 'p') && (s[2] == 'D' || s[2] == 'd') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'E' || s[5] == 'e') { + return TokenKindUpdate, true + } + if (s[0] == 'V' || s[0] == 'v') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'U' || s[3] == 'u') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'S' || s[5] == 's') { + return TokenKindValues, true + } + if (s[0] == 'V' || s[0] == 'v') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'C' || s[2] == 'c') && (s[3] == 'T' || s[3] == 't') && (s[4] == 'O' || s[4] == 'o') && (s[5] == 'R' || s[5] == 'r') { + return TokenKindVector, true + } + case 7: + if (s[0] == 'B' || s[0] == 'b') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'W' || s[3] == 'w') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'E' || s[5] == 'e') && (s[6] == 'N' || s[6] == 'n') { + return TokenKindBetween, true + } + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'N' || s[2] == 'n') && (s[3] == 'T' || s[3] == 't') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'X' || s[5] == 'x') && (s[6] == 'T' || s[6] == 't') { + return TokenKindContext, true + } + if (s[0] == 'N' || s[0] == 'n') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'R' || s[3] == 'r') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'S' || s[5] == 's') && (s[6] == 'T' || s[6] == 't') { + return TokenKindNearest, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'Y' || s[2] == 'y') && (s[3] == 'L' || s[3] == 'l') && (s[4] == 'O' || s[4] == 'o') && (s[5] == 'A' || s[5] == 'a') && (s[6] == 'D' || s[6] == 'd') { + return TokenKindPayload, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'D' || s[3] == 'd') && (s[4] == 'U' || s[4] == 'u') && (s[5] == 'C' || s[5] == 'c') && (s[6] == 'T' || s[6] == 't') { + return TokenKindProduct, true + } + if (s[0] == 'V' || s[0] == 'v') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'C' || s[2] == 'c') && (s[3] == 'T' || s[3] == 't') && (s[4] == 'O' || s[4] == 'o') && (s[5] == 'R' || s[5] == 'r') && (s[6] == 'S' || s[6] == 's') { + return TokenKindVectors, true + } + case 8: + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'F' || s[2] == 'f') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'U' || s[4] == 'u') && (s[5] == 'L' || s[5] == 'l') && (s[6] == 'T' || s[6] == 't') && (s[7] == 'S' || s[7] == 's') { + return TokenKindDefaults, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'B' || s[4] == 'b') && (s[5] == 'L' || s[5] == 'l') && (s[6] == 'E' || s[6] == 'e') && (s[7] == 'D' || s[7] == 'd') { + return TokenKindDisabled, true + } + if (s[0] == 'D' || s[0] == 'd') && (s[1] == 'I' || s[1] == 'i') && (s[2] == 'S' || s[2] == 's') && (s[3] == 'C' || s[3] == 'c') && (s[4] == 'O' || s[4] == 'o') && (s[5] == 'V' || s[5] == 'v') && (s[6] == 'E' || s[6] == 'e') && (s[7] == 'R' || s[7] == 'r') { + return TokenKindDiscover, true + } + if (s[0] == 'F' || s[0] == 'f') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'D' || s[3] == 'd') && (s[4] == 'B' || s[4] == 'b') && (s[5] == 'A' || s[5] == 'a') && (s[6] == 'C' || s[6] == 'c') && (s[7] == 'K' || s[7] == 'k') { + return TokenKindFeedback, true + } + if (s[0] == 'G' || s[0] == 'g') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'O' || s[2] == 'o') && s[3] == '_' && (s[4] == 'B' || s[4] == 'b') && (s[5] == 'B' || s[5] == 'b') && (s[6] == 'O' || s[6] == 'o') && (s[7] == 'X' || s[7] == 'x') { + return TokenKindGeoBbox, true + } + if (s[0] == 'P' || s[0] == 'p') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'E' || s[2] == 'e') && (s[3] == 'F' || s[3] == 'f') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'T' || s[5] == 't') && (s[6] == 'C' || s[6] == 'c') && (s[7] == 'H' || s[7] == 'h') { + return TokenKindPrefetch, true + } + if (s[0] == 'Q' || s[0] == 'q') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'N' || s[3] == 'n') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'I' || s[5] == 'i') && (s[6] == 'L' || s[6] == 'l') && (s[7] == 'E' || s[7] == 'e') { + return TokenKindQuantile, true + } + if (s[0] == 'Q' || s[0] == 'q') && (s[1] == 'U' || s[1] == 'u') && (s[2] == 'A' || s[2] == 'a') && (s[3] == 'N' || s[3] == 'n') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'I' || s[5] == 'i') && (s[6] == 'Z' || s[6] == 'z') && (s[7] == 'E' || s[7] == 'e') { + return TokenKindQuantize, true + } + if (s[0] == 'S' || s[0] == 's') && (s[1] == 'T' || s[1] == 't') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'A' || s[3] == 'a') && (s[4] == 'T' || s[4] == 't') && (s[5] == 'E' || s[5] == 'e') && (s[6] == 'G' || s[6] == 'g') && (s[7] == 'Y' || s[7] == 'y') { + return TokenKindStrategy, true + } + case 9: + if (s[0] == 'M' || s[0] == 'm') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'N' || s[2] == 'n') && (s[3] == 'H' || s[3] == 'h') && (s[4] == 'A' || s[4] == 'a') && (s[5] == 'T' || s[5] == 't') && (s[6] == 'T' || s[6] == 't') && (s[7] == 'A' || s[7] == 'a') && (s[8] == 'N' || s[8] == 'n') { + return TokenKindManhattan, true + } + if (s[0] == 'R' || s[0] == 'r') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'C' || s[2] == 'c') && (s[3] == 'O' || s[3] == 'o') && (s[4] == 'M' || s[4] == 'm') && (s[5] == 'M' || s[5] == 'm') && (s[6] == 'E' || s[6] == 'e') && (s[7] == 'N' || s[7] == 'n') && (s[8] == 'D' || s[8] == 'd') { + return TokenKindRecommend, true + } + if (s[0] == 'R' || s[0] == 'r') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'V' || s[4] == 'v') && (s[5] == 'A' || s[5] == 'a') && (s[6] == 'N' || s[6] == 'n') && (s[7] == 'C' || s[7] == 'c') && (s[8] == 'E' || s[8] == 'e') { + return TokenKindRelevance, true + } + if (s[0] == 'T' || s[0] == 't') && (s[1] == 'H' || s[1] == 'h') && (s[2] == 'R' || s[2] == 'r') && (s[3] == 'E' || s[3] == 'e') && (s[4] == 'S' || s[4] == 's') && (s[5] == 'H' || s[5] == 'h') && (s[6] == 'O' || s[6] == 'o') && (s[7] == 'L' || s[7] == 'l') && (s[8] == 'D' || s[8] == 'd') { + return TokenKindThreshold, true + } + case 10: + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'L' || s[3] == 'l') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'C' || s[5] == 'c') && (s[6] == 'T' || s[6] == 't') && (s[7] == 'I' || s[7] == 'i') && (s[8] == 'O' || s[8] == 'o') && (s[9] == 'N' || s[9] == 'n') { + return TokenKindCollection, true + } + if (s[0] == 'G' || s[0] == 'g') && (s[1] == 'E' || s[1] == 'e') && (s[2] == 'O' || s[2] == 'o') && s[3] == '_' && (s[4] == 'R' || s[4] == 'r') && (s[5] == 'A' || s[5] == 'a') && (s[6] == 'D' || s[6] == 'd') && (s[7] == 'I' || s[7] == 'i') && (s[8] == 'U' || s[8] == 'u') && (s[9] == 'S' || s[9] == 's') { + return TokenKindGeoRadius, true + } + if (s[0] == 'G' || s[0] == 'g') && (s[1] == 'R' || s[1] == 'r') && (s[2] == 'O' || s[2] == 'o') && (s[3] == 'U' || s[3] == 'u') && (s[4] == 'P' || s[4] == 'p') && s[5] == '_' && (s[6] == 'S' || s[6] == 's') && (s[7] == 'I' || s[7] == 'i') && (s[8] == 'Z' || s[8] == 'z') && (s[9] == 'E' || s[9] == 'e') { + return TokenKindGroupSize, true + } + if (s[0] == 'H' || s[0] == 'h') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'S' || s[2] == 's') && s[3] == '_' && (s[4] == 'V' || s[4] == 'v') && (s[5] == 'E' || s[5] == 'e') && (s[6] == 'C' || s[6] == 'c') && (s[7] == 'T' || s[7] == 't') && (s[8] == 'O' || s[8] == 'o') && (s[9] == 'R' || s[9] == 'r') { + return TokenKindHasVector, true + } + if (s[0] == 'O' || s[0] == 'o') && (s[1] == 'P' || s[1] == 'p') && (s[2] == 'T' || s[2] == 't') && (s[3] == 'I' || s[3] == 'i') && (s[4] == 'M' || s[4] == 'm') && (s[5] == 'I' || s[5] == 'i') && (s[6] == 'Z' || s[6] == 'z') && (s[7] == 'E' || s[7] == 'e') && (s[8] == 'R' || s[8] == 'r') && (s[9] == 'S' || s[9] == 's') { + return TokenKindOptimizers, true + } + case 11: + if (s[0] == 'C' || s[0] == 'c') && (s[1] == 'O' || s[1] == 'o') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'L' || s[3] == 'l') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'C' || s[5] == 'c') && (s[6] == 'T' || s[6] == 't') && (s[7] == 'I' || s[7] == 'i') && (s[8] == 'O' || s[8] == 'o') && (s[9] == 'N' || s[9] == 'n') && (s[10] == 'S' || s[10] == 's') { + return TokenKindCollections, true + } + case 12: + if (s[0] == 'V' || s[0] == 'v') && (s[1] == 'A' || s[1] == 'a') && (s[2] == 'L' || s[2] == 'l') && (s[3] == 'U' || s[3] == 'u') && (s[4] == 'E' || s[4] == 'e') && (s[5] == 'S' || s[5] == 's') && s[6] == '_' && (s[7] == 'C' || s[7] == 'c') && (s[8] == 'O' || s[8] == 'o') && (s[9] == 'U' || s[9] == 'u') && (s[10] == 'N' || s[10] == 'n') && (s[11] == 'T' || s[11] == 't') { + return TokenKindValuesCount, true + } + } + return 0, false +} diff --git a/internal/lexer/lexer.go b/internal/lexer/lexer.go index d30c801..bd94fe3 100644 --- a/internal/lexer/lexer.go +++ b/internal/lexer/lexer.go @@ -1,109 +1,10 @@ +//go:generate go run ../../cmd/gen_keywords/main.go package lexer import ( "github.com/srimon12/qql-go/internal/errors" ) -var keywords = map[string]TokenKind{ - "GEO_BBOX": TokenKindGeoBbox, - "GEO_RADIUS": TokenKindGeoRadius, - "VALUES_COUNT": TokenKindValuesCount, - "HAS_VECTOR": TokenKindHasVector, - "BOOST": TokenKindBoost, - "DEFAULTS": TokenKindDefaults, - "CASE": TokenKindCase, - "WHEN": TokenKindWhen, - "THEN": TokenKindThen, - "ELSE": TokenKindElse, - "END": TokenKindEnd, - "INSERT": TokenKindInsert, - "INTO": TokenKindInto, - "COLLECTION": TokenKindCollection, - "VALUES": TokenKindValues, - "USING": TokenKindUsing, - "MODEL": TokenKindModel, - "HYBRID": TokenKindHybrid, - "DENSE": TokenKindDense, - "SPARSE": TokenKindSparse, - "RERANK": TokenKindRerank, - "EXACT": TokenKindExact, - "WITH": TokenKindWith, - "AS": TokenKindAs, - "ACORN": TokenKindAcorn, - "QUANTIZE": TokenKindQuantize, - "SCALAR": TokenKindScalar, - "BINARY": TokenKindBinary, - "PRODUCT": TokenKindProduct, - "TURBO": TokenKindTurbo, - "BITS": TokenKindBits, - "QUANTILE": TokenKindQuantile, - "ALWAYS": TokenKindAlways, - "RAM": TokenKindRam, - "HNSW": TokenKindHnsw, - "VECTORS": TokenKindVectors, - "OPTIMIZERS": TokenKindOptimizers, - "PARAMS": TokenKindParams, - "DISABLED": TokenKindDisabled, - "CREATE": TokenKindCreate, - "ALTER": TokenKindAlter, - "DROP": TokenKindDrop, - "SHOW": TokenKindShow, - "COLLECTIONS": TokenKindCollections, - "SELECT": TokenKindSelect, - "SCROLL": TokenKindScroll, - "AFTER": TokenKindAfter, - "RECOMMEND": TokenKindRecommend, - "QUERY": TokenKindQuery, - "NEAREST": TokenKindNearest, - "CONTEXT": TokenKindContext, - "DISCOVER": TokenKindDiscover, - "PAIRS": TokenKindPairs, - "TARGET": TokenKindTarget, - "ORDER": TokenKindOrder, - "ASC": TokenKindAsc, - "DESC": TokenKindDesc, - "LIMIT": TokenKindLimit, - "GROUP": TokenKindGroup, - "BY": TokenKindBy, - "GROUP_SIZE": TokenKindGroupSize, - "STRATEGY": TokenKindStrategy, - "DELETE": TokenKindDelete, - "UPDATE": TokenKindUpdate, - "SET": TokenKindSet, - "VECTOR": TokenKindVector, - "PAYLOAD": TokenKindPayload, - "FROM": TokenKindFrom, - "WHERE": TokenKindWhere, - "ID": TokenKindId, - "INDEX": TokenKindIndex, - "ON": TokenKindOn, - "FOR": TokenKindFor, - "TYPE": TokenKindType, - "AND": TokenKindAnd, - "OR": TokenKindOr, - "NOT": TokenKindNot, - "IN": TokenKindIn, - "BETWEEN": TokenKindBetween, - "IS": TokenKindIs, - "NULL": TokenKindNull, - "EMPTY": TokenKindEmpty, - "MATCH": TokenKindMatch, - "ANY": TokenKindAny, - "PHRASE": TokenKindPhrase, - "OFFSET": TokenKindOffset, - "SCORE": TokenKindScore, - "THRESHOLD": TokenKindThreshold, - "LOOKUP": TokenKindLookup, - "COSINE": TokenKindCosine, - "DOT": TokenKindDot, - "EUCLID": TokenKindEuclid, - "MANHATTAN": TokenKindManhattan, - "PREFETCH": TokenKindPrefetch, - "FUSION": TokenKindFusion, - "SAMPLE": TokenKindSample, - "RELEVANCE": TokenKindRelevance, - "FEEDBACK": TokenKindFeedback, -} type Lexer struct{} @@ -322,44 +223,7 @@ func (l *Lexer) readIdentifier(query string, start int) Token { } func lookupKeyword(s string) (TokenKind, bool) { - if kind, ok := keywords[s]; ok { - return kind, true - } - - if len(s) <= 16 { - var buf [16]byte - for i := 0; i < len(s); i++ { - c := s[i] - if c >= 'a' && c <= 'z' { - c -= 32 - } - buf[i] = c - } - if kind, ok := keywords[string(buf[:len(s)])]; ok { - return kind, true - } - return 0, false - } - - for kw, kind := range keywords { - if len(kw) == len(s) && hasPrefixCaseInsensitive(s, kw) { - return kind, true - } - } - return 0, false -} - -func hasPrefixCaseInsensitive(s, upper string) bool { - for i := 0; i < len(s); i++ { - c := s[i] - if c >= 'a' && c <= 'z' { - c -= 32 - } - if c != upper[i] { - return false - } - } - return true + return lookupKeywordFast(s) } func findDot(s string) int {