diff --git a/changelog/kasey_genception-packages-driver.md b/changelog/kasey_genception-packages-driver.md new file mode 100644 index 000000000000..f3db12953084 --- /dev/null +++ b/changelog/kasey_genception-packages-driver.md @@ -0,0 +1,5 @@ +### Added + +- `tools/genception`, a `GOPACKAGESDRIVER` that answers `go/packages` queries + from a Bazel-supplied package inventory, so code generators that load types + through `go/packages` can run inside the Bazel sandbox. diff --git a/tools/genception/cmd/BUILD.bazel b/tools/genception/cmd/BUILD.bazel new file mode 100644 index 000000000000..f860328e160e --- /dev/null +++ b/tools/genception/cmd/BUILD.bazel @@ -0,0 +1,16 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary") +load("@prysm//tools/go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["main.go"], + importpath = "github.com/OffchainLabs/prysm/v7/tools/genception/cmd", + visibility = ["//visibility:private"], + deps = ["//tools/genception/driver:go_default_library"], +) + +go_binary( + name = "cmd", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/tools/genception/cmd/main.go b/tools/genception/cmd/main.go new file mode 100644 index 000000000000..080cd0c2414f --- /dev/null +++ b/tools/genception/cmd/main.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "os/signal" + "strings" + + "github.com/OffchainLabs/prysm/v7/tools/genception/driver" +) + +var log = driver.Logger + +func run(_ context.Context, in io.Reader, out io.Writer, args []string) error { + // NewJSONDriver builds the (IO-heavy) package registry once per invocation. + pd, err := driver.NewJSONDriver() + if err != nil { + return fmt.Errorf("unable to load JSON files: %w", err) + } + // Logged after construction so it lands in the configured log file. + log.WithField("args", strings.Join(args, " ")).Info("genception lookup") + // Note: we are returning all files required to build a specific package. + // For file queries (`file=`), this means that the CompiledGoFiles will + // include more than the only file being specified. + resp, err := pd.Handle(in, args) + if err != nil { + log.WithError(err).Error("unable to handle driver request") + } + _, writeErr := out.Write(resp) + if writeErr != nil { + log.WithError(writeErr).Error("unable to write driver response") + } + return errors.Join(err, writeErr) +} + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + if err := run(ctx, os.Stdin, os.Stdout, os.Args[1:]); err != nil { + _, err := fmt.Fprintf(os.Stderr, "error: %v", err) + if err != nil { + log.WithError(err).Error("unhandled error in package resolution") + } + // gopls will check the packages driver exit code, and if there is an + // error, it will fall back to go list. Obviously we don't want that, + // so force a 0 exit code. + os.Exit(0) + } +} diff --git a/tools/genception/driver/BUILD.bazel b/tools/genception/driver/BUILD.bazel new file mode 100644 index 000000000000..51191faf339a --- /dev/null +++ b/tools/genception/driver/BUILD.bazel @@ -0,0 +1,37 @@ +load("@prysm//tools/go:def.bzl", "go_library", "go_test") + +go_library( + name = "go_default_library", + srcs = [ + "env.go", + "flatpackage.go", + "inventory.go", + "jsondriver.go", + "logger.go", + "recorder.go", + "registry.go", + "resolver.go", + "response.go", + "tagfilt.go", + ], + importpath = "github.com/OffchainLabs/prysm/v7/tools/genception/driver", + visibility = ["//visibility:public"], + deps = [ + "@com_github_pkg_errors//:go_default_library", + "@com_github_sirupsen_logrus//:go_default_library", + "@org_golang_x_tools//go/packages:go_default_library", + ], +) + +go_test( + name = "go_default_test", + srcs = [ + "env_test.go", + "inventory_test.go", + "registry_test.go", + "response_test.go", + ], + data = glob(["testdata/**"]), + embed = [":go_default_library"], + deps = ["//testing/require:go_default_library"], +) diff --git a/tools/genception/driver/env.go b/tools/genception/driver/env.go new file mode 100644 index 000000000000..9495ebfb8d8f --- /dev/null +++ b/tools/genception/driver/env.go @@ -0,0 +1,52 @@ +package driver + +import ( + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +const ( + ENV_JSON_INDEX_PATH = "PACKAGE_JSON_INVENTORY" + ENV_PACKAGES_BASE = "PACKAGES_BASE" + ENV_PWD = "PWD" + ENV_LOG_PATH = "GOPACKAGESDRIVER_LOG_PATH" + ENV_GO_TAGS = "GOTAGS" + ENV_RECORDER_PATH = "GOPACKAGESDRIVER_RECORDER_PATH" +) + +var errUnsetEnvVar = errors.New("required env var not set") + +type environment struct { + inventoryIndexPath string + packagesBase string + pwd string + logPath string + goTags []string + recorderPath string +} + +func loadEnv() (*environment, error) { + e := &environment{} + e.goTags = strings.Split(os.Getenv(ENV_GO_TAGS), ",") + e.inventoryIndexPath = os.Getenv(ENV_JSON_INDEX_PATH) + if e.inventoryIndexPath == "" { + return nil, errors.Wrap(errUnsetEnvVar, ENV_JSON_INDEX_PATH) + } + e.packagesBase = os.Getenv(ENV_PACKAGES_BASE) + if e.packagesBase == "" { + return nil, errors.Wrap(errUnsetEnvVar, ENV_PACKAGES_BASE) + } + e.pwd = os.Getenv(ENV_PWD) + if e.pwd == "" { + return nil, errors.Wrap(errUnsetEnvVar, ENV_PWD) + } + e.logPath = os.Getenv(ENV_LOG_PATH) + if e.logPath == "" { + e.logPath = filepath.Join(e.pwd, "genception.log") + } + e.recorderPath = os.Getenv(ENV_RECORDER_PATH) + return e, nil +} diff --git a/tools/genception/driver/env_test.go b/tools/genception/driver/env_test.go new file mode 100644 index 000000000000..aae265efd088 --- /dev/null +++ b/tools/genception/driver/env_test.go @@ -0,0 +1,93 @@ +package driver + +import ( + "fmt" + "os" + "testing" + + "github.com/OffchainLabs/prysm/v7/testing/require" +) + +func TestJsonIndexPathFromEnv(t *testing.T) { + getIdxFile := func(env *environment) string { return env.inventoryIndexPath } + pkgBase := func(env *environment) string { return env.packagesBase } + pwd := func(env *environment) string { return env.pwd } + setAll := map[string]string{ + ENV_JSON_INDEX_PATH: "/path/to/file", + ENV_PACKAGES_BASE: "/path/to/base", + ENV_PWD: "derp", + } + cases := []struct { + val string + err error + envname string + set map[string]string + getter func(*environment) string + }{ + { + getter: getIdxFile, + set: map[string]string{ + ENV_PACKAGES_BASE: "/path/to/base", + ENV_PWD: "derp", + }, + err: errUnsetEnvVar, + }, + { + getter: getIdxFile, + set: setAll, + val: "/path/to/file", + }, + { + getter: pkgBase, + set: map[string]string{ + ENV_JSON_INDEX_PATH: "/path/to/file", + ENV_PWD: "derp", + }, + err: errUnsetEnvVar, + }, + { + getter: pkgBase, + val: "/path/to/base", + set: setAll, + }, + { + getter: pwd, + set: map[string]string{ + ENV_JSON_INDEX_PATH: "/path/to/file", + ENV_PACKAGES_BASE: "/path/to/base", + }, + val: os.Getenv("PWD"), // PWD is a special case because it's ya know THE pwd + }, + { + getter: pwd, + val: "derp", + set: setAll, + }, + } + + for i, c := range cases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + reset := make(map[string]string) + defer func() { + for k, v := range reset { + if v == "" { + require.NoError(t, os.Unsetenv(k)) + } else { + t.Setenv(k, v) + } + } + }() + for k, v := range c.set { + reset[k] = os.Getenv(k) + t.Setenv(k, v) + } + v, err := loadEnv() + if c.err != nil { + require.ErrorIs(t, err, c.err) + return + } + require.NoError(t, err) + require.Equal(t, c.val, c.getter(v)) + }) + } +} diff --git a/tools/genception/driver/flatpackage.go b/tools/genception/driver/flatpackage.go new file mode 100644 index 000000000000..c1f44a561f3b --- /dev/null +++ b/tools/genception/driver/flatpackage.go @@ -0,0 +1,151 @@ +package driver + +import ( + "go/parser" + "go/token" + "maps" + "slices" + "strconv" + "strings" + + "golang.org/x/tools/go/packages" +) + +// flatPackage is the JSON form of a packages.Package, maintaining only the essential +// fields that are necessary to respond to the driver request and encapsulating the +// data transformations for translating between the bazel json package metadata information +// and what the driver expects outside the bazel build context. +type flatPackage struct { + ID string + Name string `json:",omitempty"` + PkgPath string `json:",omitempty"` + Errors []packages.Error `json:",omitempty"` + GoFiles []string `json:",omitempty"` + CompiledGoFiles []string `json:",omitempty"` + OtherFiles []string `json:",omitempty"` + ExportFile string `json:",omitempty"` + Imports map[string]string `json:",omitempty"` + Standard bool `json:",omitempty"` +} + +func newFlatPackage() *flatPackage { + return &flatPackage{ + Imports: make(map[string]string), + } +} + +func resolvePathsInPlace(prf pathResolverFunc, paths []string) { + for i, path := range paths { + paths[i] = prf(path) + } +} + +func (fp *flatPackage) resolvePaths(prf pathResolverFunc, tagFilter tagFilterFunc) { + resolvePathsInPlace(prf, fp.CompiledGoFiles) + resolvePathsInPlace(prf, fp.GoFiles) + resolvePathsInPlace(prf, fp.OtherFiles) + fp.ExportFile = prf(fp.ExportFile) + + // filter down to files that would be included based on go build tags + fp.GoFiles = tagFilter(fp.GoFiles) + fp.CompiledGoFiles = tagFilter(fp.CompiledGoFiles) +} + +const fileParseWarning = "unable to parse source file package name; file dropped from inventory" + +func (fp *flatPackage) groupTestFiles(files []string) (testFiles, xTestFiles, nonTestFiles []string) { + for _, filename := range files { + if strings.HasSuffix(filename, "_test.go") { + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, filename, nil, parser.PackageClauseOnly) + if err != nil { + log.WithError(err).WithField("package", fp.PkgPath).WithField("filename", filename).Warn(fileParseWarning) + continue + } + if f.Name.Name == fp.Name { + testFiles = append(testFiles, filename) + } else { + xTestFiles = append(xTestFiles, filename) + } + } else { + nonTestFiles = append(nonTestFiles, filename) + } + } + return +} + +func (fp *flatPackage) deriveTestPackage() *flatPackage { + internalTests, externalTests, nonTests := fp.groupTestFiles(fp.GoFiles) + // Internal test files compile into the package itself; recombine them once and + // share the slice between GoFiles and CompiledGoFiles (they are identical here). + compiled := slices.Concat(nonTests, internalTests) + fp.GoFiles = compiled + fp.CompiledGoFiles = compiled + + if len(externalTests) == 0 { + return nil + } + + newImports := make(map[string]string, len(fp.Imports)) + maps.Copy(newImports, fp.Imports) + + newImports[fp.PkgPath] = fp.ID + + // Clone package, only xtgf files + return &flatPackage{ + ID: fp.ID + "_xtest", + Name: fp.Name + "_test", + PkgPath: fp.PkgPath + "_test", + Imports: newImports, + Errors: fp.Errors, + GoFiles: slices.Clone(externalTests), + CompiledGoFiles: slices.Clone(externalTests), + OtherFiles: fp.OtherFiles, + ExportFile: fp.ExportFile, + Standard: fp.Standard, + } +} + +func (fp *flatPackage) isStdlib() bool { + return fp.Standard +} + +// resolveStdlib reconstructs stdlib imports, which bazel omits from the package JSON, so they can be resolved at query time. +func (fp *flatPackage) resolveStdlib(stdlibId func(string) string) error { + // Stdlib packages are already complete import wise + if fp.isStdlib() { + return nil + } + + fset := token.NewFileSet() + for _, file := range fp.CompiledGoFiles { + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + return err + } + // If the name is not provided, fetch it from the sources + if fp.Name == "" { + fp.Name = f.Name.Name + } + + for _, rawImport := range f.Imports { + imp, err := strconv.Unquote(rawImport.Path.Value) + if err != nil { + continue + } + // ignore CGo packages + if imp == "C" { + continue + } + if _, ok := fp.Imports[imp]; ok { + continue + } + + if key := stdlibId(imp); key != "" { + fp.Imports[imp] = key + } + } + } + + return nil +} diff --git a/tools/genception/driver/inventory.go b/tools/genception/driver/inventory.go new file mode 100644 index 000000000000..68a22decd011 --- /dev/null +++ b/tools/genception/driver/inventory.go @@ -0,0 +1,25 @@ +package driver + +import ( + "encoding/json" + "os" +) + +// loadJsonListing reads the list of json package index files created by the bazel gopackagesdriver aspect: +// https://github.com/bazelbuild/rules_go/blob/master/go/tools/gopackagesdriver/aspect.bzl +// This list is serialized as a []string paths, relative to the bazel exec root. +func loadJsonListing(env environment) ([]string, error) { + path := env.inventoryIndexPath + b, err := os.ReadFile(path) // #nosec G304 -- path comes from the GOPACKAGESDRIVER env, set by our own Bazel rule + if err != nil { + return nil, err + } + log.WithField("path", path).Info("Read json index file") + + var um []string + if err := json.Unmarshal(b, &um); err != nil { + return nil, err + } + + return um, nil +} diff --git a/tools/genception/driver/inventory_test.go b/tools/genception/driver/inventory_test.go new file mode 100644 index 000000000000..e7123bff2e1b --- /dev/null +++ b/tools/genception/driver/inventory_test.go @@ -0,0 +1,14 @@ +package driver + +import ( + "testing" + + "github.com/OffchainLabs/prysm/v7/testing/require" +) + +func TestJsonList(t *testing.T) { + path := "testdata/json-list.json" + files, err := loadJsonListing(environment{inventoryIndexPath: path}) + require.NoError(t, err) + require.Equal(t, 4, len(files)) +} diff --git a/tools/genception/driver/jsondriver.go b/tools/genception/driver/jsondriver.go new file mode 100644 index 000000000000..b6047969233e --- /dev/null +++ b/tools/genception/driver/jsondriver.go @@ -0,0 +1,104 @@ +package driver + +import ( + "encoding/json" + "fmt" + "io" + "os" + "runtime" + + "github.com/pkg/errors" + "golang.org/x/tools/go/packages" +) + +type JSONDriver struct { + registry *registry + recorder *recorder +} + +func NewJSONDriver() (*JSONDriver, error) { + envPtr, err := loadEnv() + if err != nil { + return nil, fmt.Errorf("unable to load environment: %w", err) + } + env := *envPtr + configureLog(env) + + jsonFiles, err := loadJsonListing(env) + if err != nil { + return nil, errors.Wrap(err, "unable to lookup package") + } + rec, err := newRecorder(env) + if err != nil { + return nil, fmt.Errorf("unable to initialize recorder: %w", err) + } + d := &JSONDriver{ + registry: newRegistry(env), + recorder: rec, + } + + for _, f := range jsonFiles { + if err := d.load(f); err != nil { + return nil, fmt.Errorf("unable to walk json: %w", err) + } + } + + if err := d.registry.resolvePackages(); err != nil { + return nil, fmt.Errorf("unable to resolve paths: %w", err) + } + + return d, nil +} + +func (d *JSONDriver) load(path string) error { + f, err := os.Open(path) // #nosec G304 -- trusted input at build time from our own Bazel machinery + if err != nil { + return fmt.Errorf("unable to open package JSON file: %w", err) + } + defer func() { + if err := f.Close(); err != nil { + log.WithError(err).WithField("file", f.Name()).Error("unable to close file") + } + }() + + decoder := json.NewDecoder(f) + for decoder.More() { + pkg := newFlatPackage() + if err := decoder.Decode(&pkg); err != nil { + return fmt.Errorf("unable to decode package in %s: %w", f.Name(), err) + } + d.registry.add(pkg) + } + return nil +} + +func (d *JSONDriver) Handle(in io.Reader, queries []string) ([]byte, error) { + req := &packages.DriverRequest{} + if err := json.NewDecoder(in).Decode(&req); err != nil { + return nil, fmt.Errorf("unable to decode driver request: %w", err) + } + if err := d.recorder.recordRequest(queries, req); err != nil { + return nil, fmt.Errorf("unable to record request: %w", err) + } + + r, p := d.registry.query(queries) + resp := &driverResponse{ + NotHandled: false, + Compiler: "gc", + Arch: runtime.GOARCH, + GoVersion: goVersion(), + Roots: r, + Packages: p, + } + + data, err := json.Marshal(resp) + if err != nil { + return nil, fmt.Errorf("unable to marshal response: %w", err) + } + + if err := d.recorder.recordResponse(data); err != nil { + return nil, fmt.Errorf("unable to record response: %w", err) + } + + return data, nil +} diff --git a/tools/genception/driver/logger.go b/tools/genception/driver/logger.go new file mode 100644 index 000000000000..2a8bc6a36986 --- /dev/null +++ b/tools/genception/driver/logger.go @@ -0,0 +1,26 @@ +package driver + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +// log is the package-internal logger; Logger is the same instance exported for the +// cmd entrypoint. Both are non-nil from package load and default to stderr until +// configureLog redirects them to the configured log file. +var log = logrus.New() +var Logger = log + +// configureLog redirects logging to the file named by the environment, falling back to +// the default stderr output if the file cannot be opened. It is called once during +// driver construction rather than from init() so that importing the package has no +// filesystem side effects. +func configureLog(env environment) { + file, err := os.OpenFile(env.logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + log.Info("Failed to log to file, using default stderr") + return + } + log.Out = file +} diff --git a/tools/genception/driver/recorder.go b/tools/genception/driver/recorder.go new file mode 100644 index 000000000000..9a9431a9d760 --- /dev/null +++ b/tools/genception/driver/recorder.go @@ -0,0 +1,60 @@ +package driver + +import ( + "encoding/json" + "os" + "path" + "strconv" + "time" + + "golang.org/x/tools/go/packages" +) + +type recorder struct { + env environment + t time.Time +} + +func newRecorder(env environment) (*recorder, error) { + r := &recorder{env: env, t: time.Now()} + if env.recorderPath == "" { + log.Info(ENV_RECORDER_PATH + " not set; set this to a writable directory path if you want to record the driver response for debugging") + } else { + if err := r.mkdir(); err != nil { + return nil, err + } + } + return r, nil +} + +func (r *recorder) dir() string { + return path.Join(r.env.recorderPath, strconv.FormatInt(r.t.UTC().UnixNano(), 10)) +} + +func (r *recorder) mkdir() error { + return os.MkdirAll(r.dir(), 0750) +} + +func (r *recorder) recordRequest(args []string, req *packages.DriverRequest) error { + if r.env.recorderPath == "" { + return nil + } + b, err := json.Marshal(struct { + Args []string + Request *packages.DriverRequest + }{ + Args: args, + Request: req, + }) + if err != nil { + return err + } + return os.WriteFile(path.Join(r.dir(), "request.json"), b, 0600) +} + +func (r *recorder) recordResponse(resp []byte) error { + if r.env.recorderPath == "" { + return nil + } + return os.WriteFile(path.Join(r.dir(), "response.json"), resp, 0600) +} diff --git a/tools/genception/driver/registry.go b/tools/genception/driver/registry.go new file mode 100644 index 000000000000..fb6c13b40b71 --- /dev/null +++ b/tools/genception/driver/registry.go @@ -0,0 +1,210 @@ +package driver + +import ( + "strings" +) + +const stdlibPrefix = "@@io_bazel_rules_go//stdlib:" + +// Query prefixes defined by the go/packages driver protocol. A `file=` query asks for +// the package enclosing a source file; a `pattern=` query escapes its argument so it is +// passed through to the build tool verbatim. Anything else is a bare package pattern. +const ( + queryFilePrefix = "file=" + queryPatternPrefix = "pattern=" +) + +type registry struct { + env environment + packages map[string]*flatPackage + stdlib map[string]string + files map[string]string + resolve pathResolverFunc + tagFilter tagFilterFunc +} + +func newRegistry(env environment) *registry { + return ®istry{ + env: env, + packages: map[string]*flatPackage{}, + stdlib: map[string]string{}, + files: map[string]string{}, + resolve: newPathResolver(env), + tagFilter: goTagFilter(env), + } +} + +// --- ingest: take raw flatPackages from the JSON inventory into the registry --- + +func (r *registry) add(pkgs ...*flatPackage) *registry { + for _, pkg := range pkgs { + rewritePackage(pkg) + r.update(pkg) + + if pkg.isStdlib() { + r.stdlib[pkg.PkgPath] = pkg.ID + } + } + return r +} + +// update merges the contents of 2 packages together in the instance where they have the same package path. +// This can happen when the gopackages aspect traverses to a child label and generates separate json files transitive targets. +// For example, in //proto/prysm/v1alpha1 we see both `:go_default_library` and `:go_proto` from `//proto/engine/v1`. +// Without the merge, `:go_proto` can overwrite `:go_default_library`, leaving sources files out of the final graph. +// When the incoming package's source set is a superset of the existing one, it is the more complete view of the +// package, so we keep it wholesale (GoFiles/CompiledGoFiles/Imports all come from the same target and must stay +// consistent). Otherwise the existing entry is at least as complete, so we keep it. +func (r *registry) update(pkg *flatPackage) { + existing, ok := r.packages[pkg.PkgPath] + if !ok || isSuperset(pkg.GoFiles, existing.GoFiles) { + r.packages[pkg.PkgPath] = pkg + } +} + +func rewritePackage(pkg *flatPackage) { + pkg.ID = pkg.PkgPath + for k, v := range pkg.Imports { + // rewrite package ID mapping to be the same as the path + pkg.Imports[k] = canonicalizeID(k, v, pkg) + } +} + +func canonicalizeID(path, id string, pkg *flatPackage) string { + if strings.HasPrefix(id, stdlibPrefix) { + return id[len(stdlibPrefix):] + } + if pkg.isStdlib() { + return id + } + return path +} + +// isSuperset reports whether b is contained in a as an ordered subsequence (i.e. a +// is a superset of b). It relies on both slices being in the same sorted order, which +// holds for the GoFiles lists Bazel emits in the package JSON. +func isSuperset(a, b []string) bool { + if len(b) == 0 { + return true + } + if len(a) < len(b) { + return false + } + bi := 0 + for i := range a { + if a[i] == b[bi] { + bi++ + if bi == len(b) { + return true + } + } + } + return false +} + +// --- resolve: post-process the ingested packages into a complete, queryable graph --- + +// resolvePackages performs post-processing on the json package data to add context that will +// be needed to give complete information to the package data requester outside the bazel environment. +// - adds stdlib imports to packages. This is required because stdlib packages are not part of the +// JSON file exports as bazel is unaware of them. +// - We need to process all the bazel file paths to replace symbolic path names with +// fully qualified paths. This is done by the path resolver func. +func (r *registry) resolvePackages() error { + testPackages := make([]*flatPackage, 0) + for _, pkg := range r.packages { + pkg.resolvePaths(r.resolve, r.tagFilter) + if err := pkg.resolveStdlib(r.stdlibPkgID); err != nil { + return err + } + // extract test sources and imports into their own flatPackage + testFp := pkg.deriveTestPackage() + if testFp != nil { + testPackages = append(testPackages, testFp) + } + } + for _, pkg := range testPackages { + r.packages[pkg.ID] = pkg + } + + // Build the file->package index once paths are absolute and test packages exist, + // so file= queries can be resolved to the package that owns the file. + r.indexFiles() + + return nil +} + +// indexFiles maps each resolved source file path to the ID of the package that owns it. +// Each file belongs to exactly one package (internal test files stay with the base +// package, external test files move to the derived _xtest package), so there is no +// ambiguity to resolve here. +func (r *registry) indexFiles() { + for _, pkg := range r.packages { + for _, f := range pkg.GoFiles { + r.files[f] = pkg.ID + } + } +} + +func (r *registry) stdlibPkgID(importPath string) string { + return r.stdlib[importPath] +} + +// --- query: resolve driver queries to a package and its transitive dependencies --- + +// resolveQueryID maps a single driver query to a known package ID. It understands the +// file= and pattern= query forms from the go/packages driver protocol; any other query +// is treated as a bare package pattern, which after rewritePackage is just the import +// path. The boolean is false when the query matches no known package. +func (r *registry) resolveQueryID(query string) (string, bool) { + switch { + case strings.HasPrefix(query, queryFilePrefix): + id, ok := r.files[query[len(queryFilePrefix):]] + return id, ok + case strings.HasPrefix(query, queryPatternPrefix): + query = query[len(queryPatternPrefix):] + } + _, ok := r.packages[query] + return query, ok +} + +func (r *registry) query(queries []string) ([]string, []*flatPackage) { + walkedPackages := map[string]*flatPackage{} + retRoots := make([]string, 0, len(queries)) + seenRoot := make(map[string]bool, len(queries)) + for _, q := range queries { + id, ok := r.resolveQueryID(q) + if !ok { + log.WithField("query", q).Warn("driver query did not match any known package") + continue + } + if !seenRoot[id] { + seenRoot[id] = true + retRoots = append(retRoots, id) + } + r.walk(walkedPackages, id) + } + + retPkgs := make([]*flatPackage, 0, len(walkedPackages)) + for _, pkg := range walkedPackages { + retPkgs = append(retPkgs, pkg) + } + + return retRoots, retPkgs +} + +func (r *registry) walk(acc map[string]*flatPackage, root string) { + pkg := r.packages[root] + + if pkg == nil { + log.WithField("root", root).Error("package ID not found") + return + } + + acc[pkg.ID] = pkg + for _, pkgID := range pkg.Imports { + if _, ok := acc[pkgID]; !ok { + r.walk(acc, pkgID) + } + } +} diff --git a/tools/genception/driver/registry_test.go b/tools/genception/driver/registry_test.go new file mode 100644 index 000000000000..3b7d7176d9b2 --- /dev/null +++ b/tools/genception/driver/registry_test.go @@ -0,0 +1,111 @@ +package driver + +import ( + "slices" + "strings" + "testing" + + "github.com/OffchainLabs/prysm/v7/testing/require" +) + +func TestIsSuperset(t *testing.T) { + cases := []struct { + a []string + b []string + expected bool + }{ + {[]string{"a", "b", "c", "d"}, []string{"a", "b"}, true}, + {[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d"}, true}, + {[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d", "e"}, false}, + {[]string{"a", "b", "c", "d"}, []string{"a", "b", "c"}, true}, + {[]string{}, []string{"a"}, false}, + } + for _, c := range cases { + t.Run(strings.Join(c.a, "_")+"__"+strings.Join(c.b, "_"), func(t *testing.T) { + if isSuperset(c.a, c.b) != c.expected { + t.Errorf("isSuperset(%v, %v) != %v", c.a, c.b, c.expected) + } + }) + } +} + +// TestRegistryMergeKeepsSuperset guards against the regression where two bazel targets +// mapping to the same Go package path (e.g. :go_proto and :go_default_library for +// //proto/prysm/v1alpha1) would clobber each other in the registry, dropping source files. +// Whichever target carries the more complete source set must win, regardless of add order. +func TestRegistryMergeKeepsSuperset(t *testing.T) { + const pkgPath = "github.com/example/p" + full := []string{"a.go", "b.go", "c.go"} + subset := []string{"a.go", "c.go"} + + mkPkg := func(files []string) *flatPackage { + return &flatPackage{ + PkgPath: pkgPath, + Name: "p", + GoFiles: slices.Clone(files), + CompiledGoFiles: slices.Clone(files), + } + } + + cases := []struct { + name string + first []string + second []string + }{ + {"superset added first", full, subset}, + {"subset added first", subset, full}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + r := newRegistry(environment{}) + r.add(mkPkg(c.first)) + r.add(mkPkg(c.second)) + + got, ok := r.packages[pkgPath] + require.Equal(t, true, ok) + // The complete source set wins, and CompiledGoFiles stays consistent with it. + require.DeepEqual(t, full, got.GoFiles) + require.DeepEqual(t, full, got.CompiledGoFiles) + }) + } +} + +func testRegistry() *registry { + r := newRegistry(environment{}) + r.packages["example.com/p"] = &flatPackage{ID: "example.com/p", PkgPath: "example.com/p"} + r.files["/abs/p/a.go"] = "example.com/p" + return r +} + +func TestResolveQueryID(t *testing.T) { + cases := []struct { + name string + query string + want string + ok bool + }{ + {"bare hit", "example.com/p", "example.com/p", true}, + {"bare miss", "example.com/q", "example.com/q", false}, + {"pattern hit", "pattern=example.com/p", "example.com/p", true}, + {"pattern miss", "pattern=example.com/q", "example.com/q", false}, + {"file hit", "file=/abs/p/a.go", "example.com/p", true}, + {"file miss", "file=/abs/p/missing.go", "", false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + id, ok := testRegistry().resolveQueryID(c.query) + require.Equal(t, c.ok, ok) + require.Equal(t, c.want, id) + }) + } +} + +// TestQueryRootsResolved checks that Roots holds resolved package IDs (a file= query +// reports the owning package, not the raw "file=" string) and that queries matching no +// package are dropped from Roots rather than emitted as dangling roots. +func TestQueryRootsResolved(t *testing.T) { + roots, pkgs := testRegistry().query([]string{"file=/abs/p/a.go", "example.com/missing"}) + require.DeepEqual(t, []string{"example.com/p"}, roots) + require.Equal(t, 1, len(pkgs)) + require.Equal(t, "example.com/p", pkgs[0].ID) +} diff --git a/tools/genception/driver/resolver.go b/tools/genception/driver/resolver.go new file mode 100644 index 000000000000..0cc9239305ac --- /dev/null +++ b/tools/genception/driver/resolver.go @@ -0,0 +1,48 @@ +package driver + +import ( + "os" + "strings" +) + +// pathResolverFunc rewrites a single bazel-symbolic source path into a real, +// absolute filesystem path. newPathResolver returns an implementation. +type pathResolverFunc func(path string) string + +func newPathResolver(env environment) pathResolverFunc { + res := &pathResolver{ + execRoot: env.pwd, + outputBase: env.packagesBase, + } + return res.resolve +} + +type pathResolver struct { + outputBase string + execRoot string +} + +const ( + prefixExecRoot = "__BAZEL_EXECROOT__" + prefixOutputBase = "__BAZEL_OUTPUT_BASE__" + prefixWorkspace = "__BAZEL_WORKSPACE__" +) + +var prefixes = []string{prefixExecRoot, prefixOutputBase, prefixWorkspace} + +func (r pathResolver) resolve(path string) string { + for _, prefix := range prefixes { + if strings.HasPrefix(path, prefix) { + for _, rpl := range []string{r.execRoot, r.outputBase} { + rp := strings.Replace(path, prefix, rpl, 1) + _, err := os.Stat(rp) + if err == nil { + return rp + } + } + return path + } + } + log.WithField("path", path).Warn("unrecognized path prefix when resolving source paths in json import metadata") + return path +} diff --git a/tools/genception/driver/response.go b/tools/genception/driver/response.go new file mode 100644 index 000000000000..e4306f351497 --- /dev/null +++ b/tools/genception/driver/response.go @@ -0,0 +1,46 @@ +package driver + +import ( + "runtime" + "strconv" + "strings" +) + +// driverResponse is a copy of packages.DriverResponse, but with the +// `Packages` field using flatPackage instead of packages.Package. +type driverResponse struct { + NotHandled bool + Compiler string + Arch string + // GoVersion is the minor version of the Go toolchain (e.g. 21 for go1.21.x). + // The driver protocol defines this field; zero means unknown. + GoVersion int + Roots []string `json:",omitempty"` + Packages []*flatPackage +} + +// goVersion reports the Go minor version of the toolchain that built genception, +// which inside bazel is the rules_go toolchain used for codegen. +func goVersion() int { + return parseGoMinorVersion(runtime.Version()) +} + +// parseGoMinorVersion extracts the minor version from a runtime.Version() string +// such as "go1.21.5", "go1.22rc1", or "go1.21". It returns 0 (unknown) for any +// value it can't parse, e.g. "devel ..." development builds. +func parseGoMinorVersion(v string) int { + const prefix = "go1." + if !strings.HasPrefix(v, prefix) { + return 0 + } + v = v[len(prefix):] + end := 0 + for end < len(v) && v[end] >= '0' && v[end] <= '9' { + end++ + } + n, err := strconv.Atoi(v[:end]) + if err != nil { + return 0 + } + return n +} diff --git a/tools/genception/driver/response_test.go b/tools/genception/driver/response_test.go new file mode 100644 index 000000000000..ff7f65808b3d --- /dev/null +++ b/tools/genception/driver/response_test.go @@ -0,0 +1,29 @@ +package driver + +import ( + "testing" + + "github.com/OffchainLabs/prysm/v7/testing/require" +) + +func TestParseGoMinorVersion(t *testing.T) { + cases := []struct { + in string + want int + }{ + {"go1.21.5", 21}, + {"go1.21", 21}, + {"go1.22rc1", 22}, + {"go1.20.0", 20}, + {"go1.100.1", 100}, + {"devel +abc123", 0}, + {"go1.", 0}, + {"go1", 0}, + {"", 0}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + require.Equal(t, c.want, parseGoMinorVersion(c.in)) + }) + } +} diff --git a/tools/genception/driver/tagfilt.go b/tools/genception/driver/tagfilt.go new file mode 100644 index 000000000000..27829b6a067d --- /dev/null +++ b/tools/genception/driver/tagfilt.go @@ -0,0 +1,37 @@ +package driver + +import ( + "go/build" + "path/filepath" +) + +// tagFilterFunc keeps only the files that the active build tags would include. +// goTagFilter returns an implementation. +type tagFilterFunc func([]string) []string + +// goTagFilter returns a function that filters files based on the build tags +// specified in the environment. It uses the go/build package to determine +// whether a file would be included in the build/compilation context. +func goTagFilter(env environment) tagFilterFunc { + bctx := build.Default + bctx.BuildTags = env.goTags + return func(files []string) []string { + ret := make([]string, 0, len(files)) + for _, f := range files { + dir, filename := filepath.Split(f) + ext := filepath.Ext(f) + if ext == "" { + ret = append(ret, f) + continue + } + match, err := bctx.MatchFile(dir, filename) + if err != nil { + log.WithError(err).WithField("file", f).Warn("error matching file") + } + if match { + ret = append(ret, f) + } + } + return ret + } +} diff --git a/tools/genception/driver/testdata/json-list.json b/tools/genception/driver/testdata/json-list.json new file mode 100644 index 000000000000..0c138858d4e3 --- /dev/null +++ b/tools/genception/driver/testdata/json-list.json @@ -0,0 +1,6 @@ +[ + "bazel-out/darwin_arm64-fastbuild/bin/external/io_bazel_rules_go/stdlib_/stdlib.pkg.json", + "bazel-out/darwin_arm64-fastbuild/bin/external/com_github_thomaso_mirodin_intmath/constants/c64/c64.pkg.json", + "bazel-out/darwin_arm64-fastbuild/bin/external/com_github_thomaso_mirodin_intmath/u64/u64.pkg.json", + "bazel-out/darwin_arm64-fastbuild/bin/proto/prysm/v1alpha1/go_proto.pkg.json" +]