diff --git a/README.md b/README.md index 3129e8e..b8a12db 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ A sanitization-based swear filter for Go. # Installing -`go get github.com/JoshuaDoes/gofuckyourself` +Just `import github.com/capossele/swearfilter` if using go modules. # Example ```Go @@ -16,11 +16,11 @@ package main import ( "fmt" - swearfilter "github.com/JoshuaDoes/gofuckyourself" + "github.com/capossele/swearfilter" ) -var message = "This is a fûçking message with shitty swear words." -var swears = []string{"fuck", "shit"} +var message = "This is a fûçking message with shitty swear words asswipe." +var swears = []string{"fuck", "shit", "^ass"} func main() { filter := swearfilter.New(false, false, false, false, false, swears...) @@ -34,10 +34,17 @@ func main() { ``` > go run main.go Swear found: true -Swears tripped: [fuck shit] +Swears tripped: [fuck shit ^ass] Error: ``` +## Options +By default substring testing is performed, e.g. so `abc` will match any of `1abc`, `1abc2` and `abc2`. + +To help keep word lists concise but performance good, simple (simulated) regex matching is supported. The only control characters supported are `^` and `$`. These will perform prefix/suffix string match tests respectively. E.g. so `^ass` will match `asses` but not `pass`. These simple regexes aren't compiled to regexes internally so may be faster (but they haven't been benchmarked). + +Full regex support can be enabled by passing the relevant parameter when calling `swearfilter.New`. In this case, each swear word will be compiled to a regex and tested with regex matching. + ## License The source code for gofuckyourself is released under the MIT License. See LICENSE for more details. diff --git a/swearfilter.go b/swearfilter.go index 674ca75..a638abe 100644 --- a/swearfilter.go +++ b/swearfilter.go @@ -18,21 +18,24 @@ type SwearFilter struct { DisableMultiWhitespaceStripping bool //Disables stripping down multiple whitespaces (ex: hello[space][space]world -> hello[space]world) DisableZeroWidthStripping bool //Disables stripping zero-width spaces EnableSpacedBypass bool //Disables testing for spaced bypasses (if hell is in filter, look for occurrences of h and detect only alphabetic characters that follow; ex: h[space]e[space]l[space]l[space] -> hell) + DisableSimpleRegex bool //Disables using strings.HasPrefix if the string starts with ^ and strings.HasSuffix if it ends with $. Only strings.Contains will be used + EnableFullRegex bool //Enables treating each word in the wordlist as a regex //A list of words to check against the filters - BadWords map[string]struct{} - mutex sync.RWMutex + BadWords map[string]struct{} + BadWordRegexps map[string]*regexp.Regexp + mutex sync.RWMutex } //NewSwearFilter returns an initialized SwearFilter struct to check messages against -func NewSwearFilter(enableSpacedBypass bool, uhohwords ...string) (filter *SwearFilter) { +func NewSwearFilter(enableSpacedBypass bool, enableFullRegex bool, uhohwords ...string) (filter *SwearFilter) { filter = &SwearFilter{ EnableSpacedBypass: enableSpacedBypass, + EnableFullRegex: enableFullRegex, BadWords: make(map[string]struct{}), } - for _, word := range uhohwords { - filter.BadWords[word] = struct{}{} - } + + filter.Add(uhohwords...) return } @@ -41,8 +44,14 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error) filter.mutex.RLock() defer filter.mutex.RUnlock() - if filter.BadWords == nil || len(filter.BadWords) == 0 { - return nil, nil + if filter.EnableFullRegex { + if filter.BadWordRegexps == nil || len(filter.BadWordRegexps) == 0 { + return nil, nil + } + } else { + if filter.BadWords == nil || len(filter.BadWords) == 0 { + return nil, nil + } } message := strings.ToLower(msg) @@ -80,21 +89,37 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error) trippedWords = make([]string, 0) checkSpace := false - for swear := range filter.BadWords { - if swear == " " { - checkSpace = true - continue - } - if strings.Contains(message, swear) { - trippedWords = append(trippedWords, swear) - continue + if filter.EnableFullRegex { + for swear := range filter.BadWordRegexps { + if swear == " " { + checkSpace = true + continue + } + + if filter.scan(message, swear) { + trippedWords = append(trippedWords, swear) + } else if filter.EnableSpacedBypass { + nospaceMessage := strings.Replace(message, " ", "", -1) + if filter.scan(nospaceMessage, swear) { + trippedWords = append(trippedWords, swear) + } + } } + } else { + for swear := range filter.BadWords { + if swear == " " { + checkSpace = true + continue + } - if filter.EnableSpacedBypass { - nospaceMessage := strings.Replace(message, " ", "", -1) - if strings.Contains(nospaceMessage, swear) { + if filter.scan(message, swear) { trippedWords = append(trippedWords, swear) + } else if filter.EnableSpacedBypass { + nospaceMessage := strings.Replace(message, " ", "", -1) + if filter.scan(nospaceMessage, swear) { + trippedWords = append(trippedWords, swear) + } } } } @@ -106,17 +131,57 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error) return } +func (filter *SwearFilter) scan(message string, swear string) bool { + if filter.EnableFullRegex { + return filter.BadWordRegexps[swear].MatchString(message) + } else { + if filter.DisableSimpleRegex { + return strings.Contains(message, swear) + } else { + hasSimplePrefix := false + if string(swear[0]) == "^" { + hasSimplePrefix = true + return strings.HasPrefix(message, swear[1:]) + } + + hasSimpleSuffix := false + strLen := len(swear) + if string(swear[strLen-1]) == "$" { + hasSimpleSuffix = true + return strings.HasSuffix(message, swear[:strLen-1]) + } + + // fallback to substring matching + if !hasSimplePrefix && !hasSimpleSuffix { + return strings.Contains(message, swear) + } + } + } + + return false +} + //Add appends the given word to the uhohwords list func (filter *SwearFilter) Add(badWords ...string) { filter.mutex.Lock() defer filter.mutex.Unlock() - if filter.BadWords == nil { - filter.BadWords = make(map[string]struct{}) - } + if filter.EnableFullRegex { + if filter.BadWordRegexps == nil { + filter.BadWordRegexps = make(map[string]*regexp.Regexp) + } - for _, word := range badWords { - filter.BadWords[word] = struct{}{} + for _, word := range badWords { + filter.BadWordRegexps[word] = regexp.MustCompile(word) + } + } else { + if filter.BadWords == nil { + filter.BadWords = make(map[string]struct{}) + } + + for _, word := range badWords { + filter.BadWords[word] = struct{}{} + } } } @@ -125,8 +190,14 @@ func (filter *SwearFilter) Delete(badWords ...string) { filter.mutex.Lock() defer filter.mutex.Unlock() - for _, word := range badWords { - delete(filter.BadWords, word) + if filter.EnableFullRegex { + for _, word := range badWords { + delete(filter.BadWordRegexps, word) + } + } else { + for _, word := range badWords { + delete(filter.BadWords, word) + } } } @@ -135,12 +206,22 @@ func (filter *SwearFilter) Load() (activeWords []string) { filter.mutex.RLock() defer filter.mutex.RUnlock() - if filter.BadWords == nil { - return nil - } + if filter.EnableFullRegex { + if filter.BadWordRegexps == nil { + return nil + } - for word := range filter.BadWords { - activeWords = append(activeWords, word) + for word := range filter.BadWordRegexps { + activeWords = append(activeWords, word) + } + } else { + if filter.BadWords == nil { + return nil + } + + for word := range filter.BadWords { + activeWords = append(activeWords, word) + } } return } diff --git a/swearfilter_test.go b/swearfilter_test.go index 73ccfda..08eb0c1 100644 --- a/swearfilter_test.go +++ b/swearfilter_test.go @@ -7,7 +7,7 @@ import ( ) func TestNew(t *testing.T) { - filter := NewSwearFilter(true, "fuck", "hell") + filter := NewSwearFilter(true, false, "fuck", "hell") if filter.DisableNormalize { t.Errorf("Filter option DisableNormalize was incorrect, got: %t, want: %t", filter.DisableNormalize, false) } @@ -26,10 +26,16 @@ func TestNew(t *testing.T) { if len(filter.BadWords) != 2 { t.Errorf("Filter option BadWords was incorrect, got length: %d, want length: %d", len(filter.BadWords), 2) } + if filter.DisableSimpleRegex { + t.Errorf("Filter option DisableSimpleRegex was incorrect, got: %t, want: %t", filter.DisableSimpleRegex, false) + } + if filter.EnableFullRegex { + t.Errorf("Filter option EnableFullRegex was incorrect, got: %t, want: %t", filter.EnableFullRegex, false) + } } func TestCheck(t *testing.T) { - filter := NewSwearFilter(true, "fuck") + filter := NewSwearFilter(true, false, "fuck") messages := []string{"fucking", "fûçk", "asdf", "what the f u c k dude"} for i := 0; i < len(messages); i++ { @@ -56,7 +62,7 @@ func TestCheck(t *testing.T) { } func TestCheck2(t *testing.T) { - filter := NewSwearFilter(true, "fuck") + filter := NewSwearFilter(true, false, "fuck") messages := []string{"FuCking", "fûçk", "asdf", "what the f u c k dude"} for i := 0; i < len(messages); i++ { @@ -65,3 +71,82 @@ func TestCheck2(t *testing.T) { t.Log(trippers) } } + +func TestCheckSimpleRegex(t *testing.T) { + tests := map[string]bool{ + "fuckx": true, + "xfuck": false, + "xthing": true, + "thingx": true, + "xthingx": true, + "xthat": true, + "thatx": false, + "ok": false, + } + + for _, spacedBypass := range []bool{true, false} { + filter := NewSwearFilter(true, false, "^fuck", "thing", "that$") + filter.EnableSpacedBypass = spacedBypass + + for test, expected := range tests { + trippers, err := filter.Check(test) + t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers) + require.NoError(t, err) + require.Equal(t, expected, len(trippers) > 0) + } + } +} + +func TestCheckSimpleRegexDisabled(t *testing.T) { + tests := map[string]bool{ + "fuckx": false, + "xfuck": false, + "xthing": true, + "thingx": true, + "xthingx": true, + "xthat": false, + "thatx": false, + "ok": false, + } + + for _, spacedBypass := range []bool{true, false} { + filter := NewSwearFilter(true, false, "^fuck", "thing", "that$") + filter.DisableSimpleRegex = true + filter.EnableSpacedBypass = spacedBypass + + for test, expected := range tests { + trippers, err := filter.Check(test) + t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers) + require.NoError(t, err) + require.Equal(t, expected, len(trippers) > 0) + } + } +} + +func TestCheckFullRegex(t *testing.T) { + tests := map[string]bool{ + "fuckx": true, + "xfuck": false, + "xthing": true, + "thingx": true, + "xthingx": true, + "xthat": true, + "thatx": false, + "abc": false, + "abcd": true, + "abcde": false, + "ok": false, + } + + for _, spacedBypass := range []bool{true, false} { + filter := NewSwearFilter(true, true, "^fuck", "thing", "that$", "abc.$") + filter.EnableSpacedBypass = spacedBypass + + for test, expected := range tests { + trippers, err := filter.Check(test) + t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers) + require.NoError(t, err) + require.Equal(t, expected, len(trippers) > 0) + } + } +}