eql-parser/token.go at main · CraftedSignal/eql-parser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
package eql

import "fmt"

// TokenType identifies the lexical class of a token.
type TokenType int

const (
	TokenEOF TokenType = iota
	TokenError

	// Literals and names
	TokenIdent         // process, endsWith, endsWith~ (Tilde=true)
	TokenBacktickIdent // `my-field` (Value holds the unescaped name)
	TokenString        // "...", """...""", '...', ?"..." (Value holds the decoded value)
	TokenNumber        // 42, 3.14, 1e-5, .5

	// Keywords
	TokenAnd
	TokenOr
	TokenNot
	TokenIn         // in
	TokenInTilde    // in~
	TokenLike       // like
	TokenLikeTilde  // like~
	TokenRegex      // regex
	TokenRegexTilde // regex~
	TokenWhere
	TokenSequence
	TokenJoin
	TokenSample
	TokenUntil
	TokenBy
	TokenWith
	TokenMaxspan
	TokenOf
	TokenAny
	TokenTrue
	TokenFalse
	TokenNull

	// Operators and punctuation
	TokenEQ       // ==
	TokenNEQ      // !=
	TokenLT       // <
	TokenLTE      // <=
	TokenGT       // >
	TokenGTE      // >=
	TokenColon    // :
	TokenAssign   // =
	TokenPlus     // +
	TokenMinus    // -
	TokenStar     // *
	TokenSlash    // /
	TokenPercent  // %
	TokenPipe     // |
	TokenComma    // ,
	TokenLParen   // (
	TokenRParen   // )
	TokenLBrack   // [
	TokenRBrack   // ]
	TokenMissing  // ![ (missing-event subquery opener)
	TokenDot      // .
	TokenOptional // ? (optional field marker)
)

var tokenNames = map[TokenType]string{
	TokenEOF:           "EOF",
	TokenError:         "error",
	TokenIdent:         "identifier",
	TokenBacktickIdent: "backtick identifier",
	TokenString:        "string",
	TokenNumber:        "number",
	TokenAnd:           "and",
	TokenOr:            "or",
	TokenNot:           "not",
	TokenIn:            "in",
	TokenInTilde:       "in~",
	TokenLike:          "like",
	TokenLikeTilde:     "like~",
	TokenRegex:         "regex",
	TokenRegexTilde:    "regex~",
	TokenWhere:         "where",
	TokenSequence:      "sequence",
	TokenJoin:          "join",
	TokenSample:        "sample",
	TokenUntil:         "until",
	TokenBy:            "by",
	TokenWith:          "with",
	TokenMaxspan:       "maxspan",
	TokenOf:            "of",
	TokenAny:           "any",
	TokenTrue:          "true",
	TokenFalse:         "false",
	TokenNull:          "null",
	TokenEQ:            "==",
	TokenNEQ:           "!=",
	TokenLT:            "<",
	TokenLTE:           "<=",
	TokenGT:            ">",
	TokenGTE:           ">=",
	TokenColon:         ":",
	TokenAssign:        "=",
	TokenPlus:          "+",
	TokenMinus:         "-",
	TokenStar:          "*",
	TokenSlash:         "/",
	TokenPercent:       "%",
	TokenPipe:          "|",
	TokenComma:         ",",
	TokenLParen:        "(",
	TokenRParen:        ")",
	TokenLBrack:        "[",
	TokenRBrack:        "]",
	TokenMissing:       "![",
	TokenDot:           ".",
	TokenOptional:      "?",
}

func (t TokenType) String() string {
	if s, ok := tokenNames[t]; ok {
		return s
	}
	return fmt.Sprintf("token(%d)", int(t))
}

// keywords maps lowercase keyword text to its token type. EQL keywords are
// lowercase; the lexer matches case-insensitively for robustness against
// hand-edited content and records the original spelling in Token.Text.
var keywords = map[string]TokenType{
	"and":      TokenAnd,
	"or":       TokenOr,
	"not":      TokenNot,
	"in":       TokenIn,
	"like":     TokenLike,
	"regex":    TokenRegex,
	"where":    TokenWhere,
	"sequence": TokenSequence,
	"join":     TokenJoin,
	"sample":   TokenSample,
	"until":    TokenUntil,
	"by":       TokenBy,
	"with":     TokenWith,
	"maxspan":  TokenMaxspan,
	"of":       TokenOf,
	"any":      TokenAny,
	"true":     TokenTrue,
	"false":    TokenFalse,
	"null":     TokenNull,
}

// Token is a single lexical unit with position information.
type Token struct {
	Type  TokenType
	Text  string // raw source text (original spelling)
	Value string // decoded value for strings and backtick identifiers, lowercase-insensitive-normalized for keywords
	Pos   int    // byte offset in input
	Line  int    // 1-based line
	Col   int    // 1-based column (bytes)

	Tilde        bool // identifier carried a ~ suffix (case-insensitive function)
	Raw          bool // string was raw (triple-quoted or ?"..." form)
	TripleQuoted bool // string used """..."""
	SingleQuoted bool // string used '...' (legacy Endgame form)
}

// isKeyword reports whether the token is any reserved keyword.
func (t Token) isKeyword() bool {
	switch t.Type {
	case TokenAnd, TokenOr, TokenNot, TokenIn, TokenInTilde, TokenLike, TokenLikeTilde,
		TokenRegex, TokenRegexTilde, TokenWhere, TokenSequence, TokenJoin, TokenSample,
		TokenUntil, TokenBy, TokenWith, TokenMaxspan, TokenOf, TokenAny, TokenTrue,
		TokenFalse, TokenNull:
		return true
	}
	return false
}