Add support for simple and full regex matching

README.md

@@ -7,7 +7,7 @@
 A sanitization-based swear filter for Go.
 
 # Installing
-`go get github.com/JoshuaDoes/gofuckyourself`
+Just `import github.com/capossele/swearfilter` if using go modules.
 
 # Example
 ```Go
@@ -16,11 +16,11 @@ package main
 import (
 	"fmt"
 
-	swearfilter "github.com/JoshuaDoes/gofuckyourself"
+	"github.com/capossele/swearfilter"
 )
 
-var message = "This is a fûçking message with shitty swear words."
-var swears = []string{"fuck", "shit"}
+var message = "This is a fûçking message with shitty swear words asswipe."
+var swears = []string{"fuck", "shit", "^ass"}
 
 func main() {
 	filter := swearfilter.New(false, false, false, false, false, swears...)
@@ -34,10 +34,17 @@ func main() {
 ```
 > go run main.go
 Swear found:  true
-Swears tripped:  [fuck shit]
+Swears tripped:  [fuck shit ^ass]
 Error:  <nil>
 ```
 
+## Options
+By default substring testing is performed, e.g. so `abc` will match any of `1abc`, `1abc2` and `abc2`.
+
+To help keep word lists concise but performance good, simple (simulated) regex matching is supported. The only control characters supported are `^` and `$`. These will perform prefix/suffix string match tests respectively. E.g. so `^ass` will match `asses` but not `pass`. These simple regexes aren't compiled to regexes internally so may be faster (but they haven't been benchmarked).
+
+Full regex support can be enabled by passing the relevant parameter when calling `swearfilter.New`. In this case, each swear word will be compiled to a regex and tested with regex matching.
+
 ## License
 The source code for gofuckyourself is released under the MIT License. See LICENSE for more details.
 

swearfilter.go

@@ -18,21 +18,24 @@ type SwearFilter struct {
 	DisableMultiWhitespaceStripping bool //Disables stripping down multiple whitespaces (ex: hello[space][space]world -> hello[space]world)
 	DisableZeroWidthStripping       bool //Disables stripping zero-width spaces
 	EnableSpacedBypass              bool //Disables testing for spaced bypasses (if hell is in filter, look for occurrences of h and detect only alphabetic characters that follow; ex: h[space]e[space]l[space]l[space] -> hell)
+	DisableSimpleRegex              bool //Disables using strings.HasPrefix if the string starts with ^ and strings.HasSuffix if it ends with $. Only strings.Contains will be used
+	EnableFullRegex                 bool //Enables treating each word in the wordlist as a regex
 
 	//A list of words to check against the filters
-	BadWords map[string]struct{}
-	mutex    sync.RWMutex
+	BadWords       map[string]struct{}
+	BadWordRegexps map[string]*regexp.Regexp
+	mutex          sync.RWMutex
 }
 
 //NewSwearFilter returns an initialized SwearFilter struct to check messages against
-func NewSwearFilter(enableSpacedBypass bool, uhohwords ...string) (filter *SwearFilter) {
+func NewSwearFilter(enableSpacedBypass bool, enableFullRegex bool, uhohwords ...string) (filter *SwearFilter) {
 	filter = &SwearFilter{
 		EnableSpacedBypass: enableSpacedBypass,
+		EnableFullRegex:    enableFullRegex,
 		BadWords:           make(map[string]struct{}),
 	}
-	for _, word := range uhohwords {
-		filter.BadWords[word] = struct{}{}
-	}
+
+	filter.Add(uhohwords...)
 	return
 }
 
@@ -41,8 +44,14 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error)
 	filter.mutex.RLock()
 	defer filter.mutex.RUnlock()
 
-	if filter.BadWords == nil || len(filter.BadWords) == 0 {
-		return nil, nil
+	if filter.EnableFullRegex {
+		if filter.BadWordRegexps == nil || len(filter.BadWordRegexps) == 0 {
+			return nil, nil
+		}
+	} else {
+		if filter.BadWords == nil || len(filter.BadWords) == 0 {
+			return nil, nil
+		}
 	}
 
 	message := strings.ToLower(msg)
@@ -80,21 +89,37 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error)
 
 	trippedWords = make([]string, 0)
 	checkSpace := false
-	for swear := range filter.BadWords {
-		if swear == " " {
-			checkSpace = true
-			continue
-		}
 
-		if strings.Contains(message, swear) {
-			trippedWords = append(trippedWords, swear)
-			continue
+	if filter.EnableFullRegex {
+		for swear := range filter.BadWordRegexps {
+			if swear == " " {
+				checkSpace = true
+				continue
+			}
+
+			if filter.scan(message, swear) {
+				trippedWords = append(trippedWords, swear)
+			} else if filter.EnableSpacedBypass {
+				nospaceMessage := strings.Replace(message, " ", "", -1)
+				if filter.scan(nospaceMessage, swear) {
+					trippedWords = append(trippedWords, swear)
+				}
+			}
 		}
+	} else {
+		for swear := range filter.BadWords {
+			if swear == " " {
+				checkSpace = true
+				continue
+			}
 
-		if filter.EnableSpacedBypass {
-			nospaceMessage := strings.Replace(message, " ", "", -1)
-			if strings.Contains(nospaceMessage, swear) {
+			if filter.scan(message, swear) {
 				trippedWords = append(trippedWords, swear)
+			} else if filter.EnableSpacedBypass {
+				nospaceMessage := strings.Replace(message, " ", "", -1)
+				if filter.scan(nospaceMessage, swear) {
+					trippedWords = append(trippedWords, swear)
+				}
 			}
 		}
 	}
@@ -106,17 +131,57 @@ func (filter *SwearFilter) Check(msg string) (trippedWords []string, err error)
 	return
 }
 
+func (filter *SwearFilter) scan(message string, swear string) bool {
+	if filter.EnableFullRegex {
+		return filter.BadWordRegexps[swear].MatchString(message)
+	} else {
+		if filter.DisableSimpleRegex {
+			return strings.Contains(message, swear)
+		} else {
+			hasSimplePrefix := false
+			if string(swear[0]) == "^" {
+				hasSimplePrefix = true
+				return strings.HasPrefix(message, swear[1:])
+			}
+
+			hasSimpleSuffix := false
+			strLen := len(swear)
+			if string(swear[strLen-1]) == "$" {
+				hasSimpleSuffix = true
+				return strings.HasSuffix(message, swear[:strLen-1])
+			}
+
+			// fallback to substring matching
+			if !hasSimplePrefix && !hasSimpleSuffix {
+				return strings.Contains(message, swear)
+			}
+		}
+	}
+
+	return false
+}
+
 //Add appends the given word to the uhohwords list
 func (filter *SwearFilter) Add(badWords ...string) {
 	filter.mutex.Lock()
 	defer filter.mutex.Unlock()
 
-	if filter.BadWords == nil {
-		filter.BadWords = make(map[string]struct{})
-	}
+	if filter.EnableFullRegex {
+		if filter.BadWordRegexps == nil {
+			filter.BadWordRegexps = make(map[string]*regexp.Regexp)
+		}
 
-	for _, word := range badWords {
-		filter.BadWords[word] = struct{}{}
+		for _, word := range badWords {
+			filter.BadWordRegexps[word] = regexp.MustCompile(word)
+		}
+	} else {
+		if filter.BadWords == nil {
+			filter.BadWords = make(map[string]struct{})
+		}
+
+		for _, word := range badWords {
+			filter.BadWords[word] = struct{}{}
+		}
 	}
 }
 
@@ -125,8 +190,14 @@ func (filter *SwearFilter) Delete(badWords ...string) {
 	filter.mutex.Lock()
 	defer filter.mutex.Unlock()
 
-	for _, word := range badWords {
-		delete(filter.BadWords, word)
+	if filter.EnableFullRegex {
+		for _, word := range badWords {
+			delete(filter.BadWordRegexps, word)
+		}
+	} else {
+		for _, word := range badWords {
+			delete(filter.BadWords, word)
+		}
 	}
 }
 
@@ -135,12 +206,22 @@ func (filter *SwearFilter) Load() (activeWords []string) {
 	filter.mutex.RLock()
 	defer filter.mutex.RUnlock()
 
-	if filter.BadWords == nil {
-		return nil
-	}
+	if filter.EnableFullRegex {
+		if filter.BadWordRegexps == nil {
+			return nil
+		}
 
-	for word := range filter.BadWords {
-		activeWords = append(activeWords, word)
+		for word := range filter.BadWordRegexps {
+			activeWords = append(activeWords, word)
+		}
+	} else {
+		if filter.BadWords == nil {
+			return nil
+		}
+
+		for word := range filter.BadWords {
+			activeWords = append(activeWords, word)
+		}
 	}
 	return
 }

swearfilter_test.go

@@ -7,7 +7,7 @@ import (
 )
 
 func TestNew(t *testing.T) {
-	filter := NewSwearFilter(true, "fuck", "hell")
+	filter := NewSwearFilter(true, false, "fuck", "hell")
 	if filter.DisableNormalize {
 		t.Errorf("Filter option DisableNormalize was incorrect, got: %t, want: %t", filter.DisableNormalize, false)
 	}
@@ -26,10 +26,16 @@ func TestNew(t *testing.T) {
 	if len(filter.BadWords) != 2 {
 		t.Errorf("Filter option BadWords was incorrect, got length: %d, want length: %d", len(filter.BadWords), 2)
 	}
+	if filter.DisableSimpleRegex {
+		t.Errorf("Filter option DisableSimpleRegex was incorrect, got: %t, want: %t", filter.DisableSimpleRegex, false)
+	}
+	if filter.EnableFullRegex {
+		t.Errorf("Filter option EnableFullRegex was incorrect, got: %t, want: %t", filter.EnableFullRegex, false)
+	}
 }
 
 func TestCheck(t *testing.T) {
-	filter := NewSwearFilter(true, "fuck")
+	filter := NewSwearFilter(true, false, "fuck")
 	messages := []string{"fucking", "fûçk", "asdf", "what the f u c k dude"}
 
 	for i := 0; i < len(messages); i++ {
@@ -56,7 +62,7 @@ func TestCheck(t *testing.T) {
 }
 
 func TestCheck2(t *testing.T) {
-	filter := NewSwearFilter(true, "fuck")
+	filter := NewSwearFilter(true, false, "fuck")
 	messages := []string{"FuCking", "fûçk", "asdf", "what the f u c k dude"}
 
 	for i := 0; i < len(messages); i++ {
@@ -65,3 +71,82 @@ func TestCheck2(t *testing.T) {
 		t.Log(trippers)
 	}
 }
+
+func TestCheckSimpleRegex(t *testing.T) {
+	tests := map[string]bool{
+		"fuckx":   true,
+		"xfuck":   false,
+		"xthing":  true,
+		"thingx":  true,
+		"xthingx": true,
+		"xthat":   true,
+		"thatx":   false,
+		"ok":      false,
+	}
+
+	for _, spacedBypass := range []bool{true, false} {
+		filter := NewSwearFilter(true, false, "^fuck", "thing", "that$")
+		filter.EnableSpacedBypass = spacedBypass
+
+		for test, expected := range tests {
+			trippers, err := filter.Check(test)
+			t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers)
+			require.NoError(t, err)
+			require.Equal(t, expected, len(trippers) > 0)
+		}
+	}
+}
+
+func TestCheckSimpleRegexDisabled(t *testing.T) {
+	tests := map[string]bool{
+		"fuckx":   false,
+		"xfuck":   false,
+		"xthing":  true,
+		"thingx":  true,
+		"xthingx": true,
+		"xthat":   false,
+		"thatx":   false,
+		"ok":      false,
+	}
+
+	for _, spacedBypass := range []bool{true, false} {
+		filter := NewSwearFilter(true, false, "^fuck", "thing", "that$")
+		filter.DisableSimpleRegex = true
+		filter.EnableSpacedBypass = spacedBypass
+
+		for test, expected := range tests {
+			trippers, err := filter.Check(test)
+			t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers)
+			require.NoError(t, err)
+			require.Equal(t, expected, len(trippers) > 0)
+		}
+	}
+}
+
+func TestCheckFullRegex(t *testing.T) {
+	tests := map[string]bool{
+		"fuckx":   true,
+		"xfuck":   false,
+		"xthing":  true,
+		"thingx":  true,
+		"xthingx": true,
+		"xthat":   true,
+		"thatx":   false,
+		"abc":     false,
+		"abcd":    true,
+		"abcde":   false,
+		"ok":      false,
+	}
+
+	for _, spacedBypass := range []bool{true, false} {
+		filter := NewSwearFilter(true, true, "^fuck", "thing", "that$", "abc.$")
+		filter.EnableSpacedBypass = spacedBypass
+
+		for test, expected := range tests {
+			trippers, err := filter.Check(test)
+			t.Logf("test=%s, expected=%v, trippers=%v\n", test, expected, trippers)
+			require.NoError(t, err)
+			require.Equal(t, expected, len(trippers) > 0)
+		}
+	}
+}