Add fuzz tests for ParseXXX and token.QuoteXXX functions

.github/workflows/fuzz.yml

@@ -0,0 +1,51 @@
+name: Fuzz
+
+on:
+  schedule:
+    # Weekly, Monday 03:00 UTC
+    - cron: '0 3 * * 1'
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+jobs:
+  fuzz:
+    name: Fuzz
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Set up Go
+      uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
+      with:
+        go-version: "1.26"
+
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+    - name: Fuzz parser entry points
+      run: |
+        status=0
+        for target in FuzzParseStatements FuzzParseQuery FuzzParseExpr FuzzParseType FuzzParseSchemaType FuzzParseGQLGraphPattern FuzzSplitRawStatements; do
+          go test -run '^$' -fuzz "^${target}\$" -fuzztime 2m . || status=1
+        done
+        exit $status
+
+    - name: Fuzz token quoting
+      if: always()
+      run: |
+        status=0
+        for target in FuzzQuoteSQLString FuzzQuoteSQLBytes FuzzQuoteSQLIdent; do
+          go test -run '^$' -fuzz "^${target}\$" -fuzztime 2m ./token || status=1
+        done
+        exit $status
+
+    - name: Upload failing inputs
+      if: failure()
+      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+      with:
+        name: fuzz-failing-inputs
+        path: |
+          testdata/fuzz
+          token/testdata/fuzz
+        if-no-files-found: ignore

fuzz_test.go

@@ -0,0 +1,172 @@
+package memefish_test
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/cloudspannerecosystem/memefish"
+	"github.com/cloudspannerecosystem/memefish/ast"
+)
+
+// addSeedsFromTestdata seeds the fuzzing corpus with the *.sql files under
+// testdata/input/<dir> for each given dir.
+func addSeedsFromTestdata(f *testing.F, dirs ...string) {
+	f.Helper()
+
+	for _, dir := range dirs {
+		paths, err := filepath.Glob(filepath.Join("testdata", "input", dir, "*.sql"))
+		if err != nil {
+			f.Fatal(err)
+		}
+		if len(paths) == 0 {
+			f.Fatalf("no seed files found in testdata/input/%s", dir)
+		}
+		for _, path := range paths {
+			b, err := os.ReadFile(path)
+			if err != nil {
+				f.Fatal(err)
+			}
+			f.Add(string(b))
+		}
+	}
+}
+
+// checkRoundTrip checks that the SQL rendering of a successfully parsed node
+// re-parses without error and renders to the same SQL again (idempotence).
+func checkRoundTrip[T ast.Node](t *testing.T, parse func(s string) (T, error), input string, node T) {
+	t.Helper()
+
+	sql1 := node.SQL()
+	node2, err := parse(sql1)
+	if err != nil {
+		t.Fatalf("failed to re-parse the SQL rendering of a parsed node\ninput: %q\nrendered SQL: %q\nerror: %v", input, sql1, err)
+	}
+	sql2 := node2.SQL()
+	if sql1 != sql2 {
+		t.Fatalf("SQL rendering is not idempotent\ninput: %q\nfirst rendering: %q\nsecond rendering: %q", input, sql1, sql2)
+	}
+}
+
+func FuzzParseStatements(f *testing.F) {
+	addSeedsFromTestdata(f, "statement", "query", "ddl", "dml", "gql")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		stmts, err := memefish.ParseStatements("", s)
+		if err != nil {
+			return
+		}
+		for _, stmt := range stmts {
+			checkRoundTrip(t, func(s string) (ast.Statement, error) { return memefish.ParseStatement("", s) }, s, stmt)
+		}
+	})
+}
+
+func FuzzParseQuery(f *testing.F) {
+	addSeedsFromTestdata(f, "query", "gql")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		stmt, err := memefish.ParseQuery("", s)
+		if err != nil {
+			return
+		}
+		checkRoundTrip(t, func(s string) (*ast.QueryStatement, error) { return memefish.ParseQuery("", s) }, s, stmt)
+	})
+}
+
+func FuzzParseExpr(f *testing.F) {
+	addSeedsFromTestdata(f, "expr")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		expr, err := memefish.ParseExpr("", s)
+		if err != nil {
+			return
+		}
+		checkRoundTrip(t, func(s string) (ast.Expr, error) { return memefish.ParseExpr("", s) }, s, expr)
+	})
+}
+
+func FuzzParseType(f *testing.F) {
+	for _, seed := range []string{
+		"INT64",
+		"FLOAT64",
+		"BOOL",
+		"STRING",
+		"BYTES",
+		"TIMESTAMP",
+		"ARRAY<INT64>",
+		"ARRAY<ARRAY<STRING>>",
+		"STRUCT<>",
+		"STRUCT<x INT64, y FLOAT64>",
+		"STRUCT<arr ARRAY<STRUCT<n INT64>>>",
+	} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, s string) {
+		typ, err := memefish.ParseType("", s)
+		if err != nil {
+			return
+		}
+		checkRoundTrip(t, func(s string) (ast.Type, error) { return memefish.ParseType("", s) }, s, typ)
+	})
+}
+
+func FuzzParseSchemaType(f *testing.F) {
+	for _, seed := range []string{
+		"INT64",
+		"BOOL",
+		"STRING(MAX)",
+		"STRING(42)",
+		"BYTES(1024)",
+		"NUMERIC",
+		"TIMESTAMP",
+		"ARRAY<STRING(MAX)>",
+		"ARRAY<BYTES(MAX)>",
+	} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, s string) {
+		typ, err := memefish.ParseSchemaType("", s)
+		if err != nil {
+			return
+		}
+		checkRoundTrip(t, func(s string) (ast.SchemaType, error) { return memefish.ParseSchemaType("", s) }, s, typ)
+	})
+}
+
+func FuzzParseGQLGraphPattern(f *testing.F) {
+	addSeedsFromTestdata(f, "gql_graph_pattern")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		pattern, err := memefish.ParseGQLGraphPattern("", s)
+		if err != nil {
+			return
+		}
+		checkRoundTrip(t, func(s string) (ast.GQLGraphPatternNode, error) { return memefish.ParseGQLGraphPattern("", s) }, s, pattern)
+	})
+}
+
+func FuzzSplitRawStatements(f *testing.F) {
+	addSeedsFromTestdata(f, "statement", "query", "ddl", "dml", "gql")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		stmts, err := memefish.SplitRawStatements("", s)
+		if err != nil {
+			return
+		}
+		for i, raw := range stmts {
+			// The empty fallback result has zero positions and an empty statement.
+			if raw.Pos < 0 || raw.End < raw.Pos || len(s) < int(raw.End) {
+				t.Fatalf("statement %d has out-of-range positions [%d, %d) for input of length %d\ninput: %q", i, raw.Pos, raw.End, len(s), s)
+			}
+			if len(stmts) == 1 && raw.Statement == "" && raw.Pos == 0 && raw.End == 0 {
+				continue
+			}
+			if raw.Statement != s[raw.Pos:raw.End] {
+				t.Fatalf("statement %d text %q does not match input range [%d, %d) %q\ninput: %q", i, raw.Statement, raw.Pos, raw.End, s[raw.Pos:raw.End], s)
+			}
+		}
+	})
+}

testdata/fuzz/FuzzParseExpr/511261eb981fd65f

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("NEW A00{A0}")

testdata/fuzz/FuzzParseGQLGraphPattern/caae31d51114bd40

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\"\\0")

testdata/fuzz/FuzzParseQuery/caae31d51114bd40

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\"\\0")

testdata/fuzz/FuzzParseSchemaType/e8ccbf2886d3e4c8

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\"000000\n0000\\0")

testdata/fuzz/FuzzParseStatements/5eb45458063dee86

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("SELECT 0 .A0000 #0000000000000")

testdata/fuzz/FuzzParseType/2cf6add4e5042424

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("0\"\\0")

testdata/fuzz/FuzzSplitRawStatements/caae31d51114bd40

@@ -0,0 +1,2 @@
+go test fuzz v1
+string("\"\\0")

token/fuzz_test.go

@@ -0,0 +1,90 @@
+package token_test
+
+import (
+	"testing"
+	"unicode/utf8"
+
+	"github.com/cloudspannerecosystem/memefish"
+	"github.com/cloudspannerecosystem/memefish/token"
+)
+
+// lexSingleToken lexes s and returns its sole token, failing if s does not
+// lex as exactly one token followed by EOF.
+func lexSingleToken(t *testing.T, s string) token.Token {
+	t.Helper()
+
+	l := &memefish.Lexer{File: &token.File{Buffer: s}}
+	if err := l.NextToken(); err != nil {
+		t.Fatalf("failed to lex %q: %v", s, err)
+	}
+	tok := l.Token
+	if err := l.NextToken(); err != nil {
+		t.Fatalf("failed to lex %q after the first token: %v", s, err)
+	}
+	if l.Token.Kind != token.TokenEOF {
+		t.Fatalf("expected a single token in %q, but found trailing %s token", s, l.Token.Kind)
+	}
+	return tok
+}
+
+func FuzzQuoteSQLString(f *testing.F) {
+	for _, seed := range []string{"", "hello", `foo "bar" 'baz'`, "\\n\\", "\n\r\t\a\b\f\v", "日本語", "a\x00b", "\U0001F600"} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, s string) {
+		// A SQL STRING value is Unicode; invalid UTF-8 cannot round-trip.
+		if !utf8.ValidString(s) {
+			t.Skip()
+		}
+
+		q := token.QuoteSQLString(s)
+		tok := lexSingleToken(t, q)
+		if tok.Kind != token.TokenString {
+			t.Fatalf("QuoteSQLString(%q) = %q lexed as %s, not a string literal", s, q, tok.Kind)
+		}
+		if tok.AsString != s {
+			t.Fatalf("QuoteSQLString(%q) = %q lexed back as %q", s, q, tok.AsString)
+		}
+	})
+}
+
+func FuzzQuoteSQLBytes(f *testing.F) {
+	for _, seed := range [][]byte{{}, []byte("hello"), []byte(`"'`), {0x00, 0xff, 0x7f}, []byte("\\x00"), []byte("日本語")} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, bs []byte) {
+		q := token.QuoteSQLBytes(bs)
+		tok := lexSingleToken(t, q)
+		if tok.Kind != token.TokenBytes {
+			t.Fatalf("QuoteSQLBytes(%q) = %q lexed as %s, not a bytes literal", bs, q, tok.Kind)
+		}
+		if tok.AsString != string(bs) {
+			t.Fatalf("QuoteSQLBytes(%q) = %q lexed back as %q", bs, q, []byte(tok.AsString))
+		}
+	})
+}
+
+func FuzzQuoteSQLIdent(f *testing.F) {
+	for _, seed := range []string{"", "foo", "SELECT", "foo bar", "1foo", "_foo", "foo`bar", "日本語", "a\nb"} {
+		f.Add(seed)
+	}
+
+	f.Fuzz(func(t *testing.T, s string) {
+		// An identifier is non-empty Unicode text; QuoteSQLIdent formats without
+		// validating, so invalid identifiers cannot round-trip through the lexer.
+		if s == "" || !utf8.ValidString(s) {
+			t.Skip()
+		}
+
+		q := token.QuoteSQLIdent(s)
+		tok := lexSingleToken(t, q)
+		if tok.Kind != token.TokenIdent {
+			t.Fatalf("QuoteSQLIdent(%q) = %q lexed as %s, not an identifier", s, q, tok.Kind)
+		}
+		if tok.AsString != s {
+			t.Fatalf("QuoteSQLIdent(%q) = %q lexed back as %q", s, q, tok.AsString)
+		}
+	})
+}