Add genception, a Bazel-backed go/packages driver

changelog/kasey_genception-packages-driver.md

@@ -0,0 +1,5 @@
+### Added
+
+- `tools/genception`, a `GOPACKAGESDRIVER` that answers `go/packages` queries
+  from a Bazel-supplied package inventory, so code generators that load types
+  through `go/packages` can run inside the Bazel sandbox.

tools/genception/cmd/BUILD.bazel

@@ -0,0 +1,16 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary")
+load("@prysm//tools/go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "github.com/OffchainLabs/prysm/v7/tools/genception/cmd",
+    visibility = ["//visibility:private"],
+    deps = ["//tools/genception/driver:go_default_library"],
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    visibility = ["//visibility:public"],
+)

tools/genception/cmd/main.go

@@ -0,0 +1,53 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+
+	"github.com/OffchainLabs/prysm/v7/tools/genception/driver"
+)
+
+var log = driver.Logger
+
+func run(_ context.Context, in io.Reader, out io.Writer, args []string) error {
+	// NewJSONDriver builds the (IO-heavy) package registry once per invocation.
+	pd, err := driver.NewJSONDriver()
+	if err != nil {
+		return fmt.Errorf("unable to load JSON files: %w", err)
+	}
+	// Logged after construction so it lands in the configured log file.
+	log.WithField("args", strings.Join(args, " ")).Info("genception lookup")
+	// Note: we are returning all files required to build a specific package.
+	// For file queries (`file=`), this means that the CompiledGoFiles will
+	// include more than the only file being specified.
+	resp, err := pd.Handle(in, args)
+	if err != nil {
+		log.WithError(err).Error("unable to handle driver request")
+	}
+	_, writeErr := out.Write(resp)
+	if writeErr != nil {
+		log.WithError(writeErr).Error("unable to write driver response")
+	}
+	return errors.Join(err, writeErr)
+}
+
+func main() {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
+	defer stop()
+
+	if err := run(ctx, os.Stdin, os.Stdout, os.Args[1:]); err != nil {
+		_, err := fmt.Fprintf(os.Stderr, "error: %v", err)
+		if err != nil {
+			log.WithError(err).Error("unhandled error in package resolution")
+		}
+		// gopls will check the packages driver exit code, and if there is an
+		// error, it will fall back to go list. Obviously we don't want that,
+		// so force a 0 exit code.
+		os.Exit(0)
+	}
+}

tools/genception/driver/BUILD.bazel

@@ -0,0 +1,37 @@
+load("@prysm//tools/go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "env.go",
+        "flatpackage.go",
+        "inventory.go",
+        "jsondriver.go",
+        "logger.go",
+        "recorder.go",
+        "registry.go",
+        "resolver.go",
+        "response.go",
+        "tagfilt.go",
+    ],
+    importpath = "github.com/OffchainLabs/prysm/v7/tools/genception/driver",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_pkg_errors//:go_default_library",
+        "@com_github_sirupsen_logrus//:go_default_library",
+        "@org_golang_x_tools//go/packages:go_default_library",
+    ],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "env_test.go",
+        "inventory_test.go",
+        "registry_test.go",
+        "response_test.go",
+    ],
+    data = glob(["testdata/**"]),
+    embed = [":go_default_library"],
+    deps = ["//testing/require:go_default_library"],
+)

tools/genception/driver/env.go

@@ -0,0 +1,52 @@
+package driver
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	ENV_JSON_INDEX_PATH = "PACKAGE_JSON_INVENTORY"
+	ENV_PACKAGES_BASE   = "PACKAGES_BASE"
+	ENV_PWD             = "PWD"
+	ENV_LOG_PATH        = "GOPACKAGESDRIVER_LOG_PATH"
+	ENV_GO_TAGS         = "GOTAGS"
+	ENV_RECORDER_PATH   = "GOPACKAGESDRIVER_RECORDER_PATH"
+)
+
+var errUnsetEnvVar = errors.New("required env var not set")
+
+type environment struct {
+	inventoryIndexPath string
+	packagesBase       string
+	pwd                string
+	logPath            string
+	goTags             []string
+	recorderPath       string
+}
+
+func loadEnv() (*environment, error) {
+	e := &environment{}
+	e.goTags = strings.Split(os.Getenv(ENV_GO_TAGS), ",")
+	e.inventoryIndexPath = os.Getenv(ENV_JSON_INDEX_PATH)
+	if e.inventoryIndexPath == "" {
+		return nil, errors.Wrap(errUnsetEnvVar, ENV_JSON_INDEX_PATH)
+	}
+	e.packagesBase = os.Getenv(ENV_PACKAGES_BASE)
+	if e.packagesBase == "" {
+		return nil, errors.Wrap(errUnsetEnvVar, ENV_PACKAGES_BASE)
+	}
+	e.pwd = os.Getenv(ENV_PWD)
+	if e.pwd == "" {
+		return nil, errors.Wrap(errUnsetEnvVar, ENV_PWD)
+	}
+	e.logPath = os.Getenv(ENV_LOG_PATH)
+	if e.logPath == "" {
+		e.logPath = filepath.Join(e.pwd, "genception.log")
+	}
+	e.recorderPath = os.Getenv(ENV_RECORDER_PATH)
+	return e, nil
+}

tools/genception/driver/env_test.go

@@ -0,0 +1,93 @@
+package driver
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/OffchainLabs/prysm/v7/testing/require"
+)
+
+func TestJsonIndexPathFromEnv(t *testing.T) {
+	getIdxFile := func(env *environment) string { return env.inventoryIndexPath }
+	pkgBase := func(env *environment) string { return env.packagesBase }
+	pwd := func(env *environment) string { return env.pwd }
+	setAll := map[string]string{
+		ENV_JSON_INDEX_PATH: "/path/to/file",
+		ENV_PACKAGES_BASE:   "/path/to/base",
+		ENV_PWD:             "derp",
+	}
+	cases := []struct {
+		val     string
+		err     error
+		envname string
+		set     map[string]string
+		getter  func(*environment) string
+	}{
+		{
+			getter: getIdxFile,
+			set: map[string]string{
+				ENV_PACKAGES_BASE: "/path/to/base",
+				ENV_PWD:           "derp",
+			},
+			err: errUnsetEnvVar,
+		},
+		{
+			getter: getIdxFile,
+			set:    setAll,
+			val:    "/path/to/file",
+		},
+		{
+			getter: pkgBase,
+			set: map[string]string{
+				ENV_JSON_INDEX_PATH: "/path/to/file",
+				ENV_PWD:             "derp",
+			},
+			err: errUnsetEnvVar,
+		},
+		{
+			getter: pkgBase,
+			val:    "/path/to/base",
+			set:    setAll,
+		},
+		{
+			getter: pwd,
+			set: map[string]string{
+				ENV_JSON_INDEX_PATH: "/path/to/file",
+				ENV_PACKAGES_BASE:   "/path/to/base",
+			},
+			val: os.Getenv("PWD"), // PWD is a special case because it's ya know THE pwd
+		},
+		{
+			getter: pwd,
+			val:    "derp",
+			set:    setAll,
+		},
+	}
+
+	for i, c := range cases {
+		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
+			reset := make(map[string]string)
+			defer func() {
+				for k, v := range reset {
+					if v == "" {
+						require.NoError(t, os.Unsetenv(k))
+					} else {
+						t.Setenv(k, v)
+					}
+				}
+			}()
+			for k, v := range c.set {
+				reset[k] = os.Getenv(k)
+				t.Setenv(k, v)
+			}
+			v, err := loadEnv()
+			if c.err != nil {
+				require.ErrorIs(t, err, c.err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, c.val, c.getter(v))
+		})
+	}
+}