diff --git a/.github/workflows/java.yaml b/.github/workflows/java.yaml index 628b950cf7e..fa0af8469ad 100644 --- a/.github/workflows/java.yaml +++ b/.github/workflows/java.yaml @@ -85,7 +85,7 @@ jobs: with: path: | ~/.cache/librarian - key: librarian-java-tools-v2-${{ runner.os }}-${{ hashFiles('internal/config/**/*.go') }}-${{ hashFiles('google-cloud-java/librarian.yaml', 'google-cloud-java/sdk-platform-java/gapic-generator-java/**') }} + key: librarian-java-tools-v3-${{ runner.os }}-${{ hashFiles('internal/config/**/*.go', 'internal/tool/maven/**/*.go', 'internal/librarian/java/**/*.go') }}-${{ hashFiles('google-cloud-java/librarian.yaml', 'google-cloud-java/sdk-platform-java/gapic-generator-java/**') }} - name: Verify Java and Maven installation if: steps.cache-tools.outputs.cache-hit != 'true' run: | @@ -116,7 +116,7 @@ jobs: working-directory: google-cloud-java run: librarian generate secretmanager - name: Run librarian generate all (integration test) - if: matrix.task == 'integration' && github.event_name == 'push' && github.ref == 'refs/heads/main' + if: matrix.task == 'integration' working-directory: google-cloud-java run: librarian generate --all create-issue-on-failure: diff --git a/.github/workflows/sidekick.yaml b/.github/workflows/sidekick.yaml index 2e853c1e26e..1049d668197 100644 --- a/.github/workflows/sidekick.yaml +++ b/.github/workflows/sidekick.yaml @@ -28,7 +28,7 @@ jobs: - name: Display Go version run: go version - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - name: Display Cargo version run: cargo version - name: Display rustc version @@ -49,14 +49,16 @@ jobs: persist-credentials: false - uses: ./.github/actions/setup-librarian - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable - - uses: ./.github/actions/install-taplo + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable - name: Checkout google-cloud-rust uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: repository: googleapis/google-cloud-rust path: google-cloud-rust persist-credentials: false + - name: Run librarian install + working-directory: google-cloud-rust + run: librarian install -v - name: Run librarian generate working-directory: google-cloud-rust run: librarian generate --all diff --git a/doc/config-schema.md b/doc/config-schema.md index df1a27a04cd..dd189a0b97f 100644 --- a/doc/config-schema.md +++ b/doc/config-schema.md @@ -642,7 +642,6 @@ This document describes the schema for the librarian.yaml. | `api_path` | string | Is the proto path to generate from (e.g., "google/storage/v2"). | | `module_type` | string | Is the type of module to generate (e.g., "swift-protobuf", "convert-swift", or empty/"default" for standard GAPIC). | | `include_list` | list of string | Is a subset of proto files under the target API path to include. This is typically reserved for special cases to avoid generating unused/dead code. For example, in Storage we need Protobuf gencode for a subset of the protos in the google/type directory. This code is private to the package (google-cloud-storage in Rust, GoogleCloudStorage in Swift). All other files in google/type would be dead code. | -| `included_ids` | list of string | Is a list of proto IDs to include in generation for this module. If set, only these IDs and their dependencies are generated. | | `skipped_ids` | list of string | Is a list of proto IDs to skip in generation for this module. | | `module_path` | string | Is the module import path or target containing stubs (used by convert-swift and gRPC transports). | @@ -653,7 +652,6 @@ This document describes the schema for the librarian.yaml. | (embedded) | [SwiftDefault](#swiftdefault-configuration) | | | `library_name_override` | string | Overrides the default library name.

In Swift, each GAPIC package consists of a single product (the library), which contains a single target and module name. For example, the package for the google/cloud/secretmanager/v1 API is called google-cloud-secretmanager-v1, and contains a single product: `GoogleCloudSecretManagerV1`, which in turn contains a single target and module of the same name.

To use the library applications use this import:

``` import GoogleCloudSecretManagerV1 ```

Normally the name is derived from:
- If the Protobuf namespace overrides for PHP, Ruby, and C# are consistent, sidekick uses this name.
- Otherwise, the name implied by the Protobuf package
- Or the package set in the service config yaml file | | `include_list` | list of string | Is a subset of proto files under the target API path to include (e.g., ["date.proto", "expr.proto"]). | -| `included_ids` | list of string | Is a list of proto IDs to include in generation for the package. If set, only these IDs and their dependencies are generated. | | `skipped_ids` | list of string | Is a list of proto IDs to skip in generation for the package. | | `modules` | list of [SwiftModule](#swiftmodule-configuration) (optional) | Specifies generation targets for veneers and test packages.

Each module defines a source proto path, and output location. | | `package_name_override` | string | Overrides the package name.

This may be useful if the protobuf package lacks the necessary prefixes, e.g. `grafeas.v1` may be published as `google-grafeas-v1` to match the other packages. | diff --git a/internal/config/swift.go b/internal/config/swift.go index 2f60725eae9..0697c61f7fc 100644 --- a/internal/config/swift.go +++ b/internal/config/swift.go @@ -52,10 +52,6 @@ type SwiftPackage struct { // include (e.g., ["date.proto", "expr.proto"]). IncludeList []string `yaml:"include_list,omitempty"` - // IncludedIDs is a list of proto IDs to include in generation for the package. - // If set, only these IDs and their dependencies are generated. - IncludedIDs []string `yaml:"included_ids,omitempty"` - // SkippedIds is a list of proto IDs to skip in generation for the package. SkippedIds []string `yaml:"skipped_ids,omitempty"` @@ -162,10 +158,6 @@ type SwiftModule struct { // in Swift). All other files in google/type would be dead code. IncludeList []string `yaml:"include_list,omitempty"` - // IncludedIDs is a list of proto IDs to include in generation for this module. - // If set, only these IDs and their dependencies are generated. - IncludedIDs []string `yaml:"included_ids,omitempty"` - // SkippedIds is a list of proto IDs to skip in generation for this module. SkippedIds []string `yaml:"skipped_ids,omitempty"` diff --git a/internal/librarian/generate.go b/internal/librarian/generate.go index 0f64deaf006..cecce21992a 100644 --- a/internal/librarian/generate.go +++ b/internal/librarian/generate.go @@ -20,6 +20,7 @@ import ( "fmt" "runtime" "strings" + "time" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/librarian/dart" @@ -241,15 +242,41 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con } return g.Wait() case config.LanguageJava: + daemon, err := java.StartDaemonIfConfigured(ctx, cfg) + if err == nil && daemon != nil { + defer daemon.Stop() + } + genStart := time.Now() + g, gctx := errgroup.WithContext(ctx) + g.SetLimit(max(runtime.NumCPU()*2, 4)) for _, library := range libraries { - if err := java.Generate(ctx, cfg, library, src); err != nil { - return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err) - } + g.Go(func() error { + if err := java.Generate(gctx, cfg, library, src); err != nil { + return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err) + } + return nil + }) + } + if err := g.Wait(); err != nil { + return err } + durGen := time.Since(genStart) + fmt.Printf("[BENCHMARK-CI] Phase 1: Java Code Generation Step Completed: %v\n", durGen) + + fmtStart := time.Now() if err := java.Format(ctx, libraries...); err != nil { return fmt.Errorf("format java libraries (%s): %w", cfg.Language, err) } - return java.PostGenerate(ctx, ".", cfg) + durFmt := time.Since(fmtStart) + fmt.Printf("[BENCHMARK-CI] Phase 2: Java Code Formatting Step Completed: %v\n", durFmt) + + postStart := time.Now() + if err := java.PostGenerate(ctx, ".", cfg); err != nil { + return err + } + durPost := time.Since(postStart) + fmt.Printf("[BENCHMARK-CI] Phase 3: Java Post-Generate Step Completed: %v\n", durPost) + return nil case config.LanguageNodejs: g, gctx := errgroup.WithContext(ctx) g.SetLimit(runtime.NumCPU()) diff --git a/internal/librarian/java/daemon.go b/internal/librarian/java/daemon.go new file mode 100644 index 00000000000..0c9947cdc59 --- /dev/null +++ b/internal/librarian/java/daemon.go @@ -0,0 +1,168 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "context" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/googleapis/librarian/internal/config" +) + +var ( + activeDaemonPorts []int + activeDaemonMu sync.RWMutex +) + +// GetDaemonPorts returns the slice of active Nailgun daemon ports. +func GetDaemonPorts() []int { + activeDaemonMu.RLock() + defer activeDaemonMu.RUnlock() + if len(activeDaemonPorts) > 0 { + ports := make([]int, len(activeDaemonPorts)) + copy(ports, activeDaemonPorts) + return ports + } + if portStr := os.Getenv("NAILGUN_PORT"); portStr != "" { + var p int + if _, err := fmt.Sscanf(portStr, "%d", &p); err == nil && p > 0 { + return []int{p} + } + } + return nil +} + +// GAPICDaemon manages long-running background JVM daemon processes using Nailgun. +type GAPICDaemon struct { + cmds []*exec.Cmd + Port int +} + +// StartGAPICDaemon starts a background JVM daemon process running com.martiansoftware.nailgun.NGServer. +func StartGAPICDaemon(ctx context.Context, toolsEnv map[string]string, classpath string, port int) (*GAPICDaemon, error) { + cmd := exec.CommandContext(ctx, "java", + "-Xms384m", + "-Xmx1792m", + "-XX:+UseG1GC", + "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "--add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "--add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED", + "-cp", classpath, + "com.martiansoftware.nailgun.NGServer", + "127.0.0.1", + fmt.Sprintf("%d", port), + ) + cmd.Env = os.Environ() + for k, v := range toolsEnv { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v)) + } + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start nailgun daemon: %w", err) + } + + // Wait for socket port to become ready + address := fmt.Sprintf("127.0.0.1:%d", port) + for i := 0; i < 50; i++ { + conn, err := net.DialTimeout("tcp", address, 100*time.Millisecond) + if err == nil { + conn.Close() + return &GAPICDaemon{cmds: []*exec.Cmd{cmd}, Port: port}, nil + } + time.Sleep(100 * time.Millisecond) + } + + _ = cmd.Process.Kill() + return nil, fmt.Errorf("nailgun daemon failed to respond on port %d within 5 seconds", port) +} + +// Stop terminates the background JVM daemon processes. +func (d *GAPICDaemon) Stop() error { + os.Unsetenv("NAILGUN_PORT") + activeDaemonMu.Lock() + activeDaemonPorts = nil + activeDaemonMu.Unlock() + if d != nil { + for _, cmd := range d.cmds { + if cmd != nil && cmd.Process != nil { + _ = cmd.Process.Kill() + } + } + } + return nil +} + +// StartDaemonIfConfigured attempts to start GAPIC JVM daemons if configured tools are present. +func StartDaemonIfConfigured(ctx context.Context, cfg *config.Config) (*GAPICDaemon, error) { + if cfg == nil || cfg.Tools == nil { + return nil, nil + } + env, err := getToolsEnv() + if err != nil { + return nil, err + } + libDir, err := getLibDir() + if err != nil { + return nil, err + } + nailgunMatches, _ := filepath.Glob(filepath.Join(libDir, "nailgun-server-*.jar")) + if len(nailgunMatches) == 0 { + return nil, nil // Safe fallback if nailgun server jar is not installed + } + + allJars, _ := filepath.Glob(filepath.Join(libDir, "*.jar")) + if len(allJars) == 0 { + return nil, nil + } + classpath := strings.Join(allJars, ":") + + numDaemons := min(runtime.NumCPU(), 2) + basePort := 2113 + var cmds []*exec.Cmd + var ports []int + + for i := 0; i < numDaemons; i++ { + port := basePort + i + daemon, err := StartGAPICDaemon(ctx, env, classpath, port) + if err != nil { + for _, c := range cmds { + if c != nil && c.Process != nil { + _ = c.Process.Kill() + } + } + return nil, fmt.Errorf("failed to start daemon on port %d: %w", port, err) + } + cmds = append(cmds, daemon.cmds...) + ports = append(ports, port) + } + + activeDaemonMu.Lock() + activeDaemonPorts = ports + activeDaemonMu.Unlock() + + os.Setenv("NAILGUN_PORT", fmt.Sprintf("%d", ports[0])) + return &GAPICDaemon{cmds: cmds, Port: ports[0]}, nil +} diff --git a/internal/librarian/java/daemon_test.go b/internal/librarian/java/daemon_test.go new file mode 100644 index 00000000000..58dc2422535 --- /dev/null +++ b/internal/librarian/java/daemon_test.go @@ -0,0 +1,38 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package java + +import ( + "context" + "testing" +) + +func TestGAPICDaemon_StopNil(t *testing.T) { + var daemon *GAPICDaemon + if err := daemon.Stop(); err != nil { + t.Errorf("expected no error when stopping nil daemon, got %v", err) + } +} + +func TestStartDaemonIfConfigured_NilConfig(t *testing.T) { + ctx := context.Background() + daemon, err := StartDaemonIfConfigured(ctx, nil) + if err != nil { + t.Fatalf("expected no error for nil config, got %v", err) + } + if daemon != nil { + t.Errorf("expected nil daemon for nil config, got %v", daemon) + } +} diff --git a/internal/librarian/java/format.go b/internal/librarian/java/format.go index 70894f1bb4b..841614f4447 100644 --- a/internal/librarian/java/format.go +++ b/internal/librarian/java/format.go @@ -19,13 +19,16 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" + "time" "github.com/googleapis/librarian/internal/command" "github.com/googleapis/librarian/internal/config" + "golang.org/x/sync/errgroup" ) -const maxFilesPerFormatBatch = 2000 +const maxFilesPerFormatBatch = 1000 // Format formats Java client libraries using google-java-format in batches. func Format(ctx context.Context, libraries ...*config.Library) error { @@ -41,27 +44,59 @@ func Format(ctx context.Context, libraries ...*config.Library) error { if err != nil { return err } - // Batch file paths in chunks of maxFilesPerFormatBatch (2,000 files). - // Passing 2,000 files per CLI invocation avoids exceeding OS command-line length limits (ARG_MAX) - // while preventing JVM heap exhaustion on RAM-constrained CI runners. + daemons := GetDaemonPorts() + g, gctx := errgroup.WithContext(ctx) + concurrency := runtime.NumCPU() + if len(daemons) > 0 { + concurrency = len(daemons) + } + g.SetLimit(concurrency) + batchIdx := 0 for i := 0; i < len(allFiles); i += maxFilesPerFormatBatch { end := min(i+maxFilesPerFormatBatch, len(allFiles)) chunk := allFiles[i:end] - args := append([]string{"--replace"}, chunk...) - if err := command.RunWithEnv(ctx, env, "google-java-format", args...); err != nil { - return fmt.Errorf("failed to format batch [%d:%d]: %w", i, end, err) - } + idx := batchIdx + batchIdx++ + g.Go(func() error { + batchStart := time.Now() + workerEnv := make(map[string]string) + for k, v := range env { + workerEnv[k] = v + } + if len(daemons) > 0 { + port := daemons[idx%len(daemons)] + workerEnv["NAILGUN_PORT"] = fmt.Sprintf("%d", port) + } + args := append([]string{"--replace", "--skip-javadoc-formatting"}, chunk...) + if err := command.RunWithEnv(gctx, workerEnv, "google-java-format", args...); err != nil { + return fmt.Errorf("failed to format batch [%d:%d]: %w", i, end, err) + } + fmt.Printf("[BENCHMARK-CI] Format Batch %d files: %v\n", len(chunk), time.Since(batchStart)) + return nil + }) } - return nil + return g.Wait() } func collectJavaFiles(root string) ([]string, error) { + // Attempt to collect modified/untracked Java files using git status to avoid re-formatting unchanged repository files. + if gitFiles, err := collectGitModifiedJavaFiles(root); err == nil && len(gitFiles) > 0 { + return gitFiles, nil + } + var files []string err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err } - if d.IsDir() || filepath.Ext(path) != ".java" { + if d.IsDir() { + name := d.Name() + if name == "target" || (strings.HasPrefix(name, ".") && name != ".") || strings.HasPrefix(name, "proto-") || strings.HasPrefix(name, "grpc-") { + return filepath.SkipDir + } + return nil + } + if filepath.Ext(path) != ".java" { return nil } // Exclude generated samples and Spanner-specific sample source directory. @@ -77,3 +112,59 @@ func collectJavaFiles(root string) ([]string, error) { }) return files, err } + +func collectGitModifiedJavaFiles(root string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + topLevel, err := command.Output(ctx, "git", "-C", root, "rev-parse", "--show-toplevel") + if err != nil { + return nil, err + } + topLevel = strings.TrimSpace(topLevel) + + absRoot, err := filepath.Abs(root) + if err != nil { + absRoot = root + } + + out, err := command.Output(ctx, "git", "-C", topLevel, "status", "--porcelain", "-u") + if err != nil { + return nil, err + } + var files []string + lines := strings.Split(out, "\n") + for _, line := range lines { + if len(line) < 4 { + continue + } + // git status --porcelain output format: XY filename or XY -> filename + relPath := strings.TrimSpace(line[3:]) + if idx := strings.Index(relPath, " -> "); idx != -1 { + relPath = relPath[idx+4:] + } + relPath = strings.Trim(relPath, "\"") + if filepath.Ext(relPath) != ".java" { + continue + } + if strings.Contains(relPath, "target/") || + strings.Contains(relPath, "/proto-") || + strings.Contains(relPath, "/grpc-") || + strings.HasPrefix(relPath, "proto-") || + strings.HasPrefix(relPath, "grpc-") || + strings.Contains(relPath, filepath.Join("samples", "snippets", "generated")) || + strings.Contains(relPath, filepath.Join("samples", "snippets", "src")) { + continue + } + absPath := filepath.Join(topLevel, relPath) + if strings.HasPrefix(absPath, absRoot) { + if _, err := os.Stat(absPath); err == nil { + files = append(files, absPath) + } + } + } + if len(files) > 0 { + fmt.Printf("[BENCHMARK-CI] Formatting %d modified/untracked Java files via git status in %s\n", len(files), root) + } + return files, nil +} diff --git a/internal/librarian/java/generate.go b/internal/librarian/java/generate.go index faf598795ff..6785755674c 100644 --- a/internal/librarian/java/generate.go +++ b/internal/librarian/java/generate.go @@ -24,6 +24,7 @@ import ( "path" "path/filepath" "strings" + "time" "github.com/googleapis/librarian/internal/config" "github.com/googleapis/librarian/internal/proto" @@ -162,37 +163,62 @@ func generateAPI(ctx context.Context, params generateAPIParams) error { if params.cfg.Tools != nil && params.cfg.Tools.Protoc != nil { pc = params.cfg.Tools.Protoc } - // 1. Generate standard Protocol Buffer Java classes. - if shouldGenerateProto(javaAPI) { - protoProtos := filterProtos(apiProtos, javaAPI.SkipProtoClassGeneration, primaryDir) - protoProtos = append(protoProtos, additionalProtosToGenerateAbs...) - args := protoProtocArgs(protoProtos, params.srcCfg, protoDir) - if err := runProtoc(ctx, pc, args); err != nil { - return fmt.Errorf("failed to generate proto: %w", err) - } - } - // 2. Generate gRPC service stubs (skipped if transport is rest). - transport := params.apiCfg.Transport(config.LanguageJava) - if shouldGenerateGRPC(javaAPI) && transport != "rest" { - if err := runProtoc(ctx, pc, gRPCProtocArgs(apiProtos, params.srcCfg, gRPCDir)); err != nil { - return fmt.Errorf("failed to generate gRPC module: %w", err) - } - } - // 3. Generate GAPIC library. - if shouldGenerateGAPIC(javaAPI) || shouldGenerateResourceNames(javaAPI) { + genProto := shouldGenerateProto(javaAPI) + genGRPC := shouldGenerateGRPC(javaAPI) && params.apiCfg.Transport(config.LanguageJava) != "rest" + genGAPIC := shouldGenerateGAPIC(javaAPI) || shouldGenerateResourceNames(javaAPI) + + protoProtos := filterProtos(apiProtos, javaAPI.SkipProtoClassGeneration, primaryDir) + protoProtos = append(protoProtos, additionalProtosToGenerateAbs...) + sameProtos := len(protoProtos) == len(apiProtos) + + if genProto && genGRPC && genGAPIC && sameProtos { gapicOpts, err := resolveGAPICOptions(params.cfg, params.library, params.api, primaryDir, params.apiCfg) if err != nil { return fmt.Errorf("failed to resolve gapic options: %w", err) } - args := gapicProtocArgs(apiProtos, allAdditionalProtosAbs, params.srcCfg, gapicDir, gapicOpts) + args := baseProtocArgs(params.srcCfg) + args = append(args, fmt.Sprintf("--java_out=%s", protoDir)) + args = append(args, fmt.Sprintf("--java_grpc_out=%s", gRPCDir)) + args = append(args, fmt.Sprintf("--java_gapic_out=metadata:%s", gapicDir)) + args = append(args, "--java_gapic_opt="+strings.Join(gapicOpts, ",")) + args = append(args, apiProtos...) + args = append(args, allAdditionalProtosAbs...) if err := runProtoc(ctx, pc, args); err != nil { - return fmt.Errorf("failed to generate gapic: %w", err) + return fmt.Errorf("failed to generate combined proto/grpc/gapic: %w", err) + } + } else { + // 1. Generate standard Protocol Buffer Java classes. + if genProto { + args := protoProtocArgs(protoProtos, params.srcCfg, protoDir) + if err := runProtoc(ctx, pc, args); err != nil { + return fmt.Errorf("failed to generate proto: %w", err) + } + } + // 2. Generate gRPC service stubs. + if genGRPC { + if err := runProtoc(ctx, pc, gRPCProtocArgs(apiProtos, params.srcCfg, gRPCDir)); err != nil { + return fmt.Errorf("failed to generate gRPC module: %w", err) + } + } + // 3. Generate GAPIC library. + if genGAPIC { + gapicOpts, err := resolveGAPICOptions(params.cfg, params.library, params.api, primaryDir, params.apiCfg) + if err != nil { + return fmt.Errorf("failed to resolve gapic options: %w", err) + } + args := gapicProtocArgs(apiProtos, allAdditionalProtosAbs, params.srcCfg, gapicDir, gapicOpts) + if err := runProtoc(ctx, pc, args); err != nil { + return fmt.Errorf("failed to generate gapic: %w", err) + } } } + postStart := time.Now() if err := postProcessAPI(ctx, postParams); err != nil { return fmt.Errorf("failed to post process: %w", err) } + durPost := time.Since(postStart) + fmt.Printf("[BENCHMARK-CI] API %s postProcessAPI total: %v\n", params.api.Path, durPost) return nil } diff --git a/internal/librarian/java/generate_test.go b/internal/librarian/java/generate_test.go index a911802e5e7..0dd41443f1a 100644 --- a/internal/librarian/java/generate_test.go +++ b/internal/librarian/java/generate_test.go @@ -446,12 +446,12 @@ func TestGenerateAPI_NoTools(t *testing.T) { t.Fatal(err) } - // Verify that runProtoc was called 3 times: proto, grpc, and gapic. - if len(calls) != 3 { - t.Errorf("expected 3 calls to runProtoc, got %d", len(calls)) + // Verify that runProtoc was called 1 time for combined proto, grpc, and gapic generation. + if len(calls) != 1 { + t.Errorf("expected 1 call to runProtoc, got %d", len(calls)) } - // Basic validation of GAPIC generation arguments (the 3rd call). - gapicArgs := calls[2] + // Basic validation of GAPIC generation arguments. + gapicArgs := calls[0] foundGAPICOut := false for _, arg := range gapicArgs { if strings.HasPrefix(arg, "--java_gapic_out=") { diff --git a/internal/librarian/java/install.go b/internal/librarian/java/install.go index 0b2ef04e9fa..d8e3f498214 100644 --- a/internal/librarian/java/install.go +++ b/internal/librarian/java/install.go @@ -99,5 +99,9 @@ func getToolsEnv() (map[string]string, error) { if err != nil { return nil, err } - return map[string]string{envPath: binDir}, nil + env := map[string]string{envPath: binDir} + if port := os.Getenv("NAILGUN_PORT"); port != "" { + env["NAILGUN_PORT"] = port + } + return env, nil } diff --git a/internal/librarian/java/postprocess.go b/internal/librarian/java/postprocess.go index 02d6aab7f7b..b11208d6677 100644 --- a/internal/librarian/java/postprocess.go +++ b/internal/librarian/java/postprocess.go @@ -114,13 +114,18 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { // Unzip the temp-codegen.srcjar into temporary {gapicDir} directory. srcjarPath := filepath.Join(gapicDir, "temp-codegen.srcjar") if _, err := os.Stat(srcjarPath); err == nil { + unzipStart := time.Now() if err := filesystem.Unzip(ctx, srcjarPath, gapicDir); err != nil { return fmt.Errorf("failed to unzip %s: %w", srcjarPath, err) } + fmt.Printf("[BENCHMARK-CI] API %s PostProcess Unzip Srcjar: %v\n", params.apiBase, time.Since(unzipStart)) } + headerStart := time.Now() if err := addHeaders(params, []string{gRPCDir, protoDir}); err != nil { return err } + fmt.Printf("[BENCHMARK-CI] API %s PostProcess Add Headers: %v\n", params.apiBase, time.Since(headerStart)) + if err := copyFiles(params); err != nil { return fmt.Errorf("failed to copy files: %w", err) } @@ -129,9 +134,11 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error { if params.library != nil { keepSet = toKeepSet(params.library.Keep) } + moveStart := time.Now() if err := restructureToLibrary(params, params.outDir, keepSet); err != nil { return fmt.Errorf("failed to restructure to library root: %w", err) } + fmt.Printf("[BENCHMARK-CI] API %s PostProcess Restructure Layout: %v\n", params.apiBase, time.Since(moveStart)) coords := params.coords() // Generate clirr-ignored-differences.xml for the proto module. diff --git a/internal/librarian/swift/generate.go b/internal/librarian/swift/generate.go index 96889611a40..36069466bdd 100644 --- a/internal/librarian/swift/generate.go +++ b/internal/librarian/swift/generate.go @@ -115,11 +115,6 @@ func libraryToModelConfig(library *config.Library, apiCfg *config.API, src *sour specFormat = library.SpecificationFormat } - var includedIDs []string - if library.Swift != nil && len(library.Swift.IncludedIDs) > 0 { - includedIDs = library.Swift.IncludedIDs - } - var skippedIDs []string if library.Swift != nil && len(library.Swift.SkippedIds) > 0 { skippedIDs = library.Swift.SkippedIds @@ -133,8 +128,7 @@ func libraryToModelConfig(library *config.Library, apiCfg *config.API, src *sour Source: sourceConfig, Protoc: pc, Override: api.ModelOverride{ - IncludedIDs: includedIDs, - SkippedIDs: skippedIDs, + SkippedIDs: skippedIDs, }, } if library.Swift != nil && library.Swift.Discovery != nil { diff --git a/internal/librarian/swift/generate_module.go b/internal/librarian/swift/generate_module.go index accf7a2b566..4be336439ed 100644 --- a/internal/librarian/swift/generate_module.go +++ b/internal/librarian/swift/generate_module.go @@ -76,13 +76,6 @@ func moduleToModelConfig(library *config.Library, module *config.SwiftModule, sr sourceConfig.IncludeList = library.Swift.IncludeList } - var includedIDs []string - if len(module.IncludedIDs) > 0 { - includedIDs = module.IncludedIDs - } else if library.Swift != nil && len(library.Swift.IncludedIDs) > 0 { - includedIDs = library.Swift.IncludedIDs - } - var skippedIDs []string if len(module.SkippedIds) > 0 { skippedIDs = module.SkippedIds @@ -102,8 +95,7 @@ func moduleToModelConfig(library *config.Library, module *config.SwiftModule, sr Source: sourceConfig, Protoc: pc, Override: api.ModelOverride{ - IncludedIDs: includedIDs, - SkippedIDs: skippedIDs, + SkippedIDs: skippedIDs, }, } } diff --git a/internal/librarian/swift/generate_module_test.go b/internal/librarian/swift/generate_module_test.go index dede7cbb269..4ec292d7048 100644 --- a/internal/librarian/swift/generate_module_test.go +++ b/internal/librarian/swift/generate_module_test.go @@ -363,40 +363,3 @@ func TestModuleToModelConfig_SkippedIds(t *testing.T) { } }) } - -func TestModuleToModelConfig_IncludedIds(t *testing.T) { - src := &sources.Sources{} - - t.Run("module level included_ids", func(t *testing.T) { - library := &config.Library{ - Swift: &config.SwiftPackage{ - IncludedIDs: []string{".google.type.Color"}, - }, - } - module := &config.SwiftModule{ - APIPath: "google/type", - IncludedIDs: []string{".google.type.Money"}, - } - modelCfg := moduleToModelConfig(library, module, src) - expected := []string{".google.type.Money"} - if diff := cmp.Diff(expected, modelCfg.Override.IncludedIDs); diff != "" { - t.Errorf("moduleToModelConfig() mismatch (-want +got):\n%s", diff) - } - }) - - t.Run("library level fallback included_ids", func(t *testing.T) { - library := &config.Library{ - Swift: &config.SwiftPackage{ - IncludedIDs: []string{".google.type.Color"}, - }, - } - module := &config.SwiftModule{ - APIPath: "google/type", - } - modelCfg := moduleToModelConfig(library, module, src) - expected := []string{".google.type.Color"} - if diff := cmp.Diff(expected, modelCfg.Override.IncludedIDs); diff != "" { - t.Errorf("moduleToModelConfig() mismatch (-want +got):\n%s", diff) - } - }) -} diff --git a/internal/librarian/swift/generate_test.go b/internal/librarian/swift/generate_test.go index b1339676dc9..fdb3fea5e8e 100644 --- a/internal/librarian/swift/generate_test.go +++ b/internal/librarian/swift/generate_test.go @@ -293,32 +293,6 @@ func TestLibraryToModelConfig(t *testing.T) { }, }, }, - { - name: "library level included_ids", - library: &config.Library{ - Name: "google-cloud-secretmanager", - Version: "1.2.3", - SpecificationFormat: config.SpecProtobuf, - Swift: &config.SwiftPackage{ - IncludedIDs: []string{".google.cloud.secretmanager.v1.SecretManagerService.GetSecret"}, - }, - }, - api: &config.API{ - Path: "google/cloud/secretmanager/v1", - }, - want: &parser.ModelConfig{ - Language: config.LanguageSwift, - SpecificationFormat: config.SpecProtobuf, - SpecificationSource: "google/cloud/secretmanager/v1", - ServiceConfig: "google/cloud/secretmanager/v1/secretmanager_v1.yaml", - Source: &sources.SourceConfig{ - ActiveRoots: []string{"googleapis"}, - }, - Override: api.ModelOverride{ - IncludedIDs: []string{".google.cloud.secretmanager.v1.SecretManagerService.GetSecret"}, - }, - }, - }, { name: "discovery config", library: &config.Library{ diff --git a/internal/sidekick/api/field.go b/internal/sidekick/api/field.go index 1b64c6f6f23..4a540cac36d 100644 --- a/internal/sidekick/api/field.go +++ b/internal/sidekick/api/field.go @@ -186,6 +186,14 @@ func (f *Field) IsObject() bool { return f.Typez == TypezMessage } +// IsWktAny returns true if the field is of type ".google.protobuf.Any" +// +// This is a well-known type that requires special treatment in some +// sidekick gencode <-> Protobuf gencode conversions. +func (f *Field) IsWktAny() bool { + return f.TypezID == WktAnyID +} + // IsResourceReference returns true if the field is annotated with google.api.resource_reference. func (f *Field) IsResourceReference() bool { return f.ResourceReference != nil diff --git a/internal/sidekick/api/well_known_types.go b/internal/sidekick/api/well_known_types.go index 3631b2a2658..a3f971fc584 100644 --- a/internal/sidekick/api/well_known_types.go +++ b/internal/sidekick/api/well_known_types.go @@ -14,6 +14,43 @@ package api +const ( + // WktAnyID is the well-known type ID for google.protobuf.Any. + WktAnyID = ".google.protobuf.Any" + // WktStructID is the well-known type ID for google.protobuf.Struct. + WktStructID = ".google.protobuf.Struct" + // WktValueID is the well-known type ID for google.protobuf.Value. + WktValueID = ".google.protobuf.Value" + // WktListValueID is the well-known type ID for google.protobuf.ListValue. + WktListValueID = ".google.protobuf.ListValue" + // WktEmptyID is the well-known type ID for google.protobuf.Empty. + WktEmptyID = ".google.protobuf.Empty" + // WktFieldMaskID is the well-known type ID for google.protobuf.FieldMask. + WktFieldMaskID = ".google.protobuf.FieldMask" + // WktDurationID is the well-known type ID for google.protobuf.Duration. + WktDurationID = ".google.protobuf.Duration" + // WktTimestampID is the well-known type ID for google.protobuf.Timestamp. + WktTimestampID = ".google.protobuf.Timestamp" + // WktNullValueID is the well-known type ID for google.protobuf.NullValue. + WktNullValueID = ".google.protobuf.NullValue" + // WktBytesValueID is the well-known type ID for google.protobuf.BytesValue. + WktBytesValueID = ".google.protobuf.BytesValue" + // WktUInt64ValueID is the well-known type ID for google.protobuf.UInt64Value. + WktUInt64ValueID = ".google.protobuf.UInt64Value" + // WktInt64ValueID is the well-known type ID for google.protobuf.Int64Value. + WktInt64ValueID = ".google.protobuf.Int64Value" + // WktUInt32ValueID is the well-known type ID for google.protobuf.UInt32Value. + WktUInt32ValueID = ".google.protobuf.UInt32Value" + // WktInt32ValueID is the well-known type ID for google.protobuf.Int32Value. + WktInt32ValueID = ".google.protobuf.Int32Value" + // WktFloatValueID is the well-known type ID for google.protobuf.FloatValue. + WktFloatValueID = ".google.protobuf.FloatValue" + // WktDoubleValueID is the well-known type ID for google.protobuf.DoubleValue. + WktDoubleValueID = ".google.protobuf.DoubleValue" + // WktBoolValueID is the well-known type ID for google.protobuf.BoolValue. + WktBoolValueID = ".google.protobuf.BoolValue" +) + // LoadWellKnownTypes adds well-known types to `state`. // // Some source specification formats (Discovery, OpenAPI) must manually add the @@ -24,40 +61,40 @@ func (model *API) LoadWellKnownTypes() { model.AddMessage(message) } model.AddEnum(&Enum{ + ID: WktNullValueID, Name: "NullValue", Package: "google.protobuf", - ID: ".google.protobuf.NullValue", }) } var wellKnownMessages = []*Message{ { - ID: ".google.protobuf.Any", + ID: WktAnyID, Name: "Any", Package: "google.protobuf", }, { - ID: ".google.protobuf.Struct", + ID: WktStructID, Name: "Struct", Package: "google.protobuf", }, { - ID: ".google.protobuf.Value", + ID: WktValueID, Name: "Value", Package: "google.protobuf", }, { - ID: ".google.protobuf.ListValue", + ID: WktListValueID, Name: "ListValue", Package: "google.protobuf", }, { - ID: ".google.protobuf.Empty", + ID: WktEmptyID, Name: "Empty", Package: "google.protobuf", }, { - ID: ".google.protobuf.FieldMask", + ID: WktFieldMaskID, Name: "FieldMask", Package: "google.protobuf", Fields: []*Field{ @@ -70,21 +107,21 @@ var wellKnownMessages = []*Message{ }, }, { - ID: ".google.protobuf.Duration", + ID: WktDurationID, Name: "Duration", Package: "google.protobuf", }, { - ID: ".google.protobuf.Timestamp", + ID: WktTimestampID, Name: "Timestamp", Package: "google.protobuf", }, - {ID: ".google.protobuf.BytesValue", Name: "BytesValue", Package: "google.protobuf"}, - {ID: ".google.protobuf.UInt64Value", Name: "UInt64Value", Package: "google.protobuf"}, - {ID: ".google.protobuf.Int64Value", Name: "Int64Value", Package: "google.protobuf"}, - {ID: ".google.protobuf.UInt32Value", Name: "UInt32Value", Package: "google.protobuf"}, - {ID: ".google.protobuf.Int32Value", Name: "Int32Value", Package: "google.protobuf"}, - {ID: ".google.protobuf.FloatValue", Name: "FloatValue", Package: "google.protobuf"}, - {ID: ".google.protobuf.DoubleValue", Name: "DoubleValue", Package: "google.protobuf"}, - {ID: ".google.protobuf.BoolValue", Name: "BoolValue", Package: "google.protobuf"}, + {ID: WktBytesValueID, Name: "BytesValue", Package: "google.protobuf"}, + {ID: WktUInt64ValueID, Name: "UInt64Value", Package: "google.protobuf"}, + {ID: WktInt64ValueID, Name: "Int64Value", Package: "google.protobuf"}, + {ID: WktUInt32ValueID, Name: "UInt32Value", Package: "google.protobuf"}, + {ID: WktInt32ValueID, Name: "Int32Value", Package: "google.protobuf"}, + {ID: WktFloatValueID, Name: "FloatValue", Package: "google.protobuf"}, + {ID: WktDoubleValueID, Name: "DoubleValue", Package: "google.protobuf"}, + {ID: WktBoolValueID, Name: "BoolValue", Package: "google.protobuf"}, } diff --git a/internal/sidekick/parser/protobuf.go b/internal/sidekick/parser/protobuf.go index adf3827e633..939b5549fdc 100644 --- a/internal/sidekick/parser/protobuf.go +++ b/internal/sidekick/parser/protobuf.go @@ -262,7 +262,12 @@ const ( // From https://pkg.go.dev/google.golang.org/protobuf/types/descriptorpb#EnumDescriptorProto - enumDescriptorValue = 2 + enumDescriptorName = 1 + enumDescriptorValue = 2 + enumDescriptorOptions = 3 + enumDescriptorReservedRange = 4 + enumDescriptorReservedName = 5 + enumDescriptorVisibility = 6 ) func makeAPIForProtobuf(serviceConfig *serviceconfig.Service, req *pluginpb.CodeGeneratorRequest) (*api.API, error) { @@ -873,12 +878,17 @@ func addMessageDocumentation(model *api.API, m *descriptorpb.DescriptorProto, p // addEnumDocumentation adds documentation to an enum. func addEnumDocumentation(model *api.API, p []int32, doc string, eFQN string) { - if len(p) == 0 { + switch { + case len(p) == 0: // This is a comment for an enum model.Enum(eFQN).Documentation = trimLeadingSpacesInDocumentation(doc) - } else if len(p) == 2 && p[0] == enumDescriptorValue { + case p[0] == enumDescriptorValue && len(p) == 2: model.Enum(eFQN).Values[p[1]].Documentation = trimLeadingSpacesInDocumentation(doc) - } else { + case p[0] == enumDescriptorValue: + slog.Warn("enumValue comments with missing index", "loc", p, "docs", doc) + case p[0] == enumDescriptorReservedRange: + // A comment for a reserved range, ignore, it does not emit any generated code. + default: slog.Warn("enum dropped documentation", "loc", p, "docs", doc) } } diff --git a/internal/sidekick/rust/annotate_field.go b/internal/sidekick/rust/annotate_field.go index ef7f8ae1d1e..217409124dd 100644 --- a/internal/sidekick/rust/annotate_field.go +++ b/internal/sidekick/rust/annotate_field.go @@ -22,6 +22,17 @@ import ( "github.com/googleapis/librarian/internal/sidekick/language" ) +var ( + // The annotations print a warning when generating code that converts between sidekick-gencode + // and Prost-gencode **and** has a field of type Any. The warning is suppressed for some + // well-known cases where we decided it was fine. + suppressProstConvertAndAnyWarnings = map[string]struct{}{ + ".google.rpc.Status": {}, + ".google.longrunning.Operation": {}, + ".google.storage.control.v2.ObjectFullContext": {}, + } +) + type fieldAnnotations struct { // In Rust, message fields are fields inside a struct. These must be // `snake_case`. Possibly mangled with `r#` if the name is a Rust reserved @@ -195,6 +206,11 @@ func (c *codec) annotateField(field *api.Field, message *api.Message, model *api if err != nil { return nil, err } + if field.TypezID == api.WktAnyID && c.templateOverride == templateConvertProst { + if _, ok := suppressProstConvertAndAnyWarnings[message.ID]; !ok { + fmt.Printf("WARNING: unknown field of type wkt::Any, conversion skipped, consider ad-hoc code, message: %s\n", message.ID) + } + } ann := &fieldAnnotations{ FieldName: toSnake(field.Name), SetterName: toSnakeNoMangling(field.Name), diff --git a/internal/sidekick/rust/generate.go b/internal/sidekick/rust/generate.go index 65bafd8ef51..f2ab1193863 100644 --- a/internal/sidekick/rust/generate.go +++ b/internal/sidekick/rust/generate.go @@ -27,6 +27,10 @@ import ( //go:embed all:templates var templates embed.FS +const ( + templateConvertProst = "templates/convert-prost" +) + // Generate generates Rust code from the model. func Generate(ctx context.Context, model *api.API, outdir string, cfg *parser.ModelConfig) error { c, err := newCodec(cfg.SpecificationFormat, cfg.Codec) diff --git a/internal/sidekick/rust/generate_bidi_streaming_test.go b/internal/sidekick/rust/generate_bidi_streaming_test.go index af1438167e6..85b338bb12c 100644 --- a/internal/sidekick/rust/generate_bidi_streaming_test.go +++ b/internal/sidekick/rust/generate_bidi_streaming_test.go @@ -17,6 +17,7 @@ package rust import ( "os" "path/filepath" + "strconv" "strings" "testing" @@ -89,7 +90,7 @@ func TestGenerateBidiStreaming(t *testing.T) { return files[relPath] } - for _, tc := range []struct { + for _, test := range []struct { name string file string startStr string @@ -235,6 +236,19 @@ func TestGenerateBidiStreaming(t *testing.T) { google_cloud_gax::streaming::RequestSender, google_cloud_gax::streaming::ResponseReceiver, )> {`, + }, + { + name: "transport: request params without routing", + file: "src/transport.rs", + startStr: " let req = req.ok_or_else(|| {\n", + endStr: " let x_goog_request_params = \"\";", + want: ` let req = req.ok_or_else(|| { + google_cloud_gax::error::Error::binding( + "a request is required" + ) + })?; + + let x_goog_request_params = "";`, }, { name: "transport: eager bidi_stream call", @@ -251,7 +265,7 @@ func TestGenerateBidiStreaming(t *testing.T) { req_stream, options.into(), &crate::info::X_GOOG_API_CLIENT_HEADER, - x_goog_request_params, + &x_goog_request_params, ) .await?;`, }, @@ -271,7 +285,10 @@ func TestGenerateBidiStreaming(t *testing.T) { .send(prost_item) .await .map_err(|_| { - google_cloud_gax::error::Error::io("cannot send request: stream is closed") + google_cloud_gax::error::Error::io(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "cannot send request: stream is closed", + )) }) } }, @@ -287,12 +304,115 @@ func TestGenerateBidiStreaming(t *testing.T) { }`, }, } { - t.Run(tc.name, func(t *testing.T) { - content := readFile(tc.file) - got := extractBlock(t, content, tc.startStr, tc.endStr) - if diff := cmp.Diff(tc.want, got); diff != "" { + t.Run(test.name, func(t *testing.T) { + content := readFile(test.file) + got := extractBlock(t, content, test.startStr, test.endStr) + if diff := cmp.Diff(test.want, got); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } }) } } + +func TestGenerateBidiStreamingWithRouting(t *testing.T) { + for _, test := range []struct { + name string + routingRequired bool + wantSubstrings []string + wantAbsent []string + }{ + { + name: "without routing required", + routingRequired: false, + wantSubstrings: []string{ + "let x_goog_request_params = {", + "gaxi::routing_parameter::format(&[", + `.map(|v| ("table_name", v))`, + }, + wantAbsent: []string{ + "BindingError", + "PathMismatchBuilder", + }, + }, + { + name: "with routing required", + routingRequired: true, + wantSubstrings: []string{ + "let x_goog_request_params = {", + "if x_goog_request_params.is_empty() {", + "use google_cloud_gax::error::binding::BindingError;", + "use gaxi::path_parameter::PathMismatchBuilder;", + "let builder = PathMismatchBuilder::default();", + `"projects/*/datasets/*/tables/*"`, + "return Err(google_cloud_gax::error::Error::binding(BindingError { paths }))", + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + outDir := t.TempDir() + + request := api.NewTestMessage("Request").WithPackage("test.v1") + request.Fields = []*api.Field{ + { + Name: "table_name", + JSONName: "tableName", + ID: ".test.v1.Request.table_name", + Typez: api.TypezString, + }, + } + response := api.NewTestMessage("Response").WithPackage("test.v1") + + bidiMethod := api.NewTestMethod("AppendRows").WithInput(request).WithOutput(response).WithBidiStreaming() + bidiMethod.PathInfo = &api.PathInfo{ + Bindings: []*api.PathBinding{{Verb: "GET", PathTemplate: &api.PathTemplate{}}}, + } + bidiMethod.Routing = []*api.RoutingInfo{ + { + Name: "table_name", + Variants: []*api.RoutingInfoVariant{ + { + FieldPath: []string{"table_name"}, + Matching: api.RoutingPathSpec{Segments: []string{"projects", "*", "datasets", "*", "tables", "*"}}, + }, + }, + }, + } + service := api.NewTestService("WriteStream").WithPackage("test.v1").WithMethods(bidiMethod) + + model := api.NewTestAPI([]*api.Message{request, response}, []*api.Enum{}, []*api.Service{service}) + model.PackageName = "test.v1" + if err := api.CrossReference(model); err != nil { + t.Fatal(err) + } + + cfg := &parser.ModelConfig{ + SpecificationFormat: libconfig.SpecProtobuf, + Codec: map[string]string{ + "package:wkt": "source=google.protobuf,package=google-cloud-wkt", + "include-bidi-streaming-methods": "true", + "routing-required": strconv.FormatBool(test.routingRequired), + }, + } + if err := Generate(t.Context(), model, outDir, cfg); err != nil { + t.Fatal(err) + } + + transportContent, err := os.ReadFile(filepath.Join(outDir, "src/transport.rs")) + if err != nil { + t.Fatal(err) + } + content := string(transportContent) + + for _, sub := range test.wantSubstrings { + if !strings.Contains(content, sub) { + t.Errorf("missing expected substring %q in generated transport.rs", sub) + } + } + for _, absent := range test.wantAbsent { + if strings.Contains(content, absent) { + t.Errorf("unexpected substring %q found in generated transport.rs", absent) + } + } + }) + } +} diff --git a/internal/sidekick/rust/templates/convert-prost/message.mustache b/internal/sidekick/rust/templates/convert-prost/message.mustache index f44a444545c..b1f30778105 100644 --- a/internal/sidekick/rust/templates/convert-prost/message.mustache +++ b/internal/sidekick/rust/templates/convert-prost/message.mustache @@ -31,6 +31,10 @@ impl gaxi::prost::ToProto<{{Codec.RelativeName}}> for {{Codec.QualifiedName}} { Ok(Self::Output { {{#Codec.BasicFields}} {{#Singular}} + {{#IsWktAny}} + {{Codec.FieldName}}: None, + {{/IsWktAny}} + {{^IsWktAny}} {{^Optional}} {{Codec.FieldName}}: self.{{Codec.FieldName}}.to_proto()?, {{/Optional}} @@ -42,7 +46,9 @@ impl gaxi::prost::ToProto<{{Codec.RelativeName}}> for {{Codec.QualifiedName}} { {{Codec.FieldName}}: self.{{Codec.FieldName}}.map(|v| v.to_proto().map(std::boxed::Box::new)).transpose()?, {{/Codec.MapToBoxed}} {{/Optional}} + {{/IsWktAny}} {{/Singular}} + {{! A sequence or map with `google.protobuf.Any` is unlikely. If it ever happens, we see a break and fix it with more understanding }} {{#Repeated}} {{Codec.FieldName}}: self.{{Codec.FieldName}} .into_iter() @@ -71,13 +77,16 @@ impl gaxi::prost::FromProto<{{Codec.QualifiedName}}> for {{Codec.RelativeName}} {{Codec.QualifiedName}}::new() {{#Codec.BasicFields}} {{#Singular}} + {{^IsWktAny}} {{^Optional}} .set_{{Codec.SetterName}}(self.{{Codec.FieldName}}) {{/Optional}} {{#Optional}} .set_or_clear_{{Codec.SetterName}}(self.{{Codec.FieldName}}.map(|v| v.cnv()).transpose()?) {{/Optional}} + {{/IsWktAny}} {{/Singular}} + {{! A sequence or map with `google.protobuf.Any` is unlikely. If it ever happens, we see a break and fix it with more understanding }} {{#Repeated}} {{^IsEnum}} .set_{{Codec.SetterName}}(self.{{Codec.FieldName}}.into_iter().map(|v| v.cnv()) diff --git a/internal/sidekick/rust/templates/crate/src/transport.rs.mustache b/internal/sidekick/rust/templates/crate/src/transport.rs.mustache index 9076004c8e3..c0ae8fc2311 100644 --- a/internal/sidekick/rust/templates/crate/src/transport.rs.mustache +++ b/internal/sidekick/rust/templates/crate/src/transport.rs.mustache @@ -147,6 +147,17 @@ impl super::stub::{{Codec.Name}} for {{Codec.Name}} { ) })?; + {{! + Per AIP-4222, fallback to HTTP path templates only applies to unary RPCs. + Streaming RPCs only extract routing parameters if explicitly annotated. + }} + {{#HasRouting}} + {{> /templates/grpc-client/routinginfo}} + {{/HasRouting}} + {{^HasRouting}} + let x_goog_request_params = ""; + {{/HasRouting}} + let first_req = req .to_proto() .map_err(google_cloud_gax::error::Error::ser)?; @@ -167,8 +178,6 @@ impl super::stub::{{Codec.Name}} for {{Codec.Name}} { let path = http::uri::PathAndQuery::from_static( "/{{SourceService.Package}}.{{SourceService.Name}}/{{Name}}" ); - {{! TODO(#6835) - derive request params from initial request }} - let x_goog_request_params = ""; let result = self.grpc_inner .bidi_stream::< @@ -180,7 +189,7 @@ impl super::stub::{{Codec.Name}} for {{Codec.Name}} { req_stream, options.into(), &crate::info::X_GOOG_API_CLIENT_HEADER, - x_goog_request_params, + &x_goog_request_params, ) .await?; @@ -195,8 +204,10 @@ impl super::stub::{{Codec.Name}} for {{Codec.Name}} { .send(prost_item) .await .map_err(|_| { - {{! TODO(#6835) - use a custom error kind for stream closed errors }} - google_cloud_gax::error::Error::io("cannot send request: stream is closed") + google_cloud_gax::error::Error::io(std::io::Error::new( + std::io::ErrorKind::BrokenPipe, + "cannot send request: stream is closed", + )) }) } }, diff --git a/internal/tool/maven/maven.go b/internal/tool/maven/maven.go index ccc85a7ef04..bd0ee0b17a9 100644 --- a/internal/tool/maven/maven.go +++ b/internal/tool/maven/maven.go @@ -47,7 +47,11 @@ type pomProject struct { // Install installs Maven tool dependencies. func Install(ctx context.Context, tools []*config.MavenTool, binDir, libDir string) error { + hasGAPIC := false for _, mvnTool := range tools { + if mvnTool.Name == "protoc-gen-java_gapic" { + hasGAPIC = true + } var err error if mvnTool.LocalPath != "" { err = installLocalMavenTool(ctx, mvnTool, binDir, libDir) @@ -58,6 +62,18 @@ func Install(ctx context.Context, tools []*config.MavenTool, binDir, libDir stri return fmt.Errorf("failed to install maven tool %s: %w", mvnTool.Name, err) } } + if hasGAPIC { + nailgunTool := &config.MavenTool{ + Name: "nailgun-server", + GroupID: "com.martiansoftware", + ArtifactID: "nailgun-server", + Version: "1.0.0", + Packaging: "jar", + } + if err := installExternalMavenTool(ctx, nailgunTool, binDir, libDir); err != nil { + // Log warning or ignore failure to ensure safe fallback + } + } return nil } @@ -196,12 +212,61 @@ func copyArtifactToLib(srcPath, libDir string, makeExecutable bool) (string, err // createBinWrapper creates a shell wrapper script in the bin directory that forwards executions to the library file. func createBinWrapper(wrapperName, destPath, binDir string, isExecutable bool, mainClass string) error { wrapperPath := filepath.Join(binDir, wrapperName) + if mainClass == "" && wrapperName == "google-java-format" { + mainClass = "com.google.googlejavaformat.java.Main" + } var content string switch { case isExecutable: content = fmt.Sprintf("#!/bin/sh\nexec %q \"$@\"\n", destPath) case mainClass != "": - content = fmt.Sprintf("#!/bin/sh\nexec java -cp %q %q \"$@\"\n", destPath, mainClass) + content = fmt.Sprintf(`#!/bin/sh +if [ -n "$NAILGUN_PORT" ]; then + python3 -c " +import os, socket, sys +port = int(sys.argv[1]) +main_class = sys.argv[2] +cli_args = sys.argv[3:] +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect(('127.0.0.1', port)) +for arg in cli_args: + b_arg = arg.encode('utf-8') + s.sendall(b'A' + len(b_arg).to_bytes(4, 'big') + b_arg) +cwd = os.getcwd().encode('utf-8') +s.sendall(b'W' + len(cwd).to_bytes(4, 'big') + cwd) +b_class = main_class.encode('utf-8') +s.sendall(b'C' + len(b_class).to_bytes(4, 'big') + b_class) +if not sys.stdin.isatty(): + stdin_data = sys.stdin.buffer.read() + if stdin_data: + s.sendall(b'0' + len(stdin_data).to_bytes(4, 'big') + stdin_data) +s.sendall(b'S\x00\x00\x00\x00') +exit_code = 0 +while True: + chunk_header = s.recv(5) + if not chunk_header or len(chunk_header) < 5: + break + c_type = chunk_header[0:1] + c_len = int.from_bytes(chunk_header[1:5], 'big') + payload = b'' + while len(payload) < c_len: + payload += s.recv(c_len - len(payload)) + if c_type == b'1': + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + elif c_type == b'2': + sys.stderr.buffer.write(payload) + sys.stderr.buffer.flush() + elif c_type == b'X': + if len(payload) > 0: + exit_code = int(payload.decode('utf-8').strip()) + break +s.close() +sys.exit(exit_code) +" "$NAILGUN_PORT" %q "$@" && exit 0 +fi +exec java --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -cp %q %q "$@" +`, mainClass, destPath, mainClass) default: content = fmt.Sprintf("#!/bin/sh\nexec java -jar %q \"$@\"\n", destPath) } diff --git a/internal/tool/maven/maven_test.go b/internal/tool/maven/maven_test.go index c21b45338fc..19dd5bff51c 100644 --- a/internal/tool/maven/maven_test.go +++ b/internal/tool/maven/maven_test.go @@ -123,7 +123,8 @@ func TestInstall(t *testing.T) { wantMvn := "mvn dependency:get -Dartifact=com.google.googlejavaformat:google-java-format:1.25.2:jar:all-deps\n" + "mvn dependency:get -Dartifact=io.grpc:protoc-gen-grpc-java:1.81.0:exe:linux-x86_64\n" + "mvn package -B -ntp -T 1.5C -DskipTests -Dcheckstyle.skip -Dclirr.skip -Denforcer.skip -Dfmt.skip " + - "-pl sdk-platform-java/gapic-generator-java --also-make" + "-pl sdk-platform-java/gapic-generator-java --also-make\n" + + "mvn dependency:get -Dartifact=com.martiansoftware:nailgun-server:1.0.0:jar" if diff := cmp.Diff(wantMvn, gotMvn); diff != "" { t.Errorf("mismatch (-want +got):\n%s", diff) } @@ -139,7 +140,53 @@ func TestInstall(t *testing.T) { filename: "google-java-format-1.25.2-all-deps.jar", wantContent: "gjf jar content", wrapperName: "google-java-format", - wantFormat: "#!/bin/sh\nexec java -jar %q \"$@\"\n", + wantFormat: `#!/bin/sh +if [ -n "$NAILGUN_PORT" ]; then + python3 -c " +import os, socket, sys +port = int(sys.argv[1]) +main_class = sys.argv[2] +cli_args = sys.argv[3:] +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect(('127.0.0.1', port)) +for arg in cli_args: + b_arg = arg.encode('utf-8') + s.sendall(b'A' + len(b_arg).to_bytes(4, 'big') + b_arg) +cwd = os.getcwd().encode('utf-8') +s.sendall(b'W' + len(cwd).to_bytes(4, 'big') + cwd) +b_class = main_class.encode('utf-8') +s.sendall(b'C' + len(b_class).to_bytes(4, 'big') + b_class) +if not sys.stdin.isatty(): + stdin_data = sys.stdin.buffer.read() + if stdin_data: + s.sendall(b'0' + len(stdin_data).to_bytes(4, 'big') + stdin_data) +s.sendall(b'S\x00\x00\x00\x00') +exit_code = 0 +while True: + chunk_header = s.recv(5) + if not chunk_header or len(chunk_header) < 5: + break + c_type = chunk_header[0:1] + c_len = int.from_bytes(chunk_header[1:5], 'big') + payload = b'' + while len(payload) < c_len: + payload += s.recv(c_len - len(payload)) + if c_type == b'1': + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + elif c_type == b'2': + sys.stderr.buffer.write(payload) + sys.stderr.buffer.flush() + elif c_type == b'X': + if len(payload) > 0: + exit_code = int(payload.decode('utf-8').strip()) + break +s.close() +sys.exit(exit_code) +" "$NAILGUN_PORT" "com.google.googlejavaformat.java.Main" "$@" && exit 0 +fi +exec java --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -cp %q "com.google.googlejavaformat.java.Main" "$@" +`, }, { name: "protoc-gen-java_grpc", @@ -153,7 +200,53 @@ func TestInstall(t *testing.T) { filename: "gapic-generator-java-2.28.0-SNAPSHOT.jar", wantContent: "local gapic jar content", wrapperName: "protoc-gen-java_gapic", - wantFormat: "#!/bin/sh\nexec java -cp %q \"com.google.api.generator.Main\" \"$@\"\n", + wantFormat: `#!/bin/sh +if [ -n "$NAILGUN_PORT" ]; then + python3 -c " +import os, socket, sys +port = int(sys.argv[1]) +main_class = sys.argv[2] +cli_args = sys.argv[3:] +s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +s.connect(('127.0.0.1', port)) +for arg in cli_args: + b_arg = arg.encode('utf-8') + s.sendall(b'A' + len(b_arg).to_bytes(4, 'big') + b_arg) +cwd = os.getcwd().encode('utf-8') +s.sendall(b'W' + len(cwd).to_bytes(4, 'big') + cwd) +b_class = main_class.encode('utf-8') +s.sendall(b'C' + len(b_class).to_bytes(4, 'big') + b_class) +if not sys.stdin.isatty(): + stdin_data = sys.stdin.buffer.read() + if stdin_data: + s.sendall(b'0' + len(stdin_data).to_bytes(4, 'big') + stdin_data) +s.sendall(b'S\x00\x00\x00\x00') +exit_code = 0 +while True: + chunk_header = s.recv(5) + if not chunk_header or len(chunk_header) < 5: + break + c_type = chunk_header[0:1] + c_len = int.from_bytes(chunk_header[1:5], 'big') + payload = b'' + while len(payload) < c_len: + payload += s.recv(c_len - len(payload)) + if c_type == b'1': + sys.stdout.buffer.write(payload) + sys.stdout.buffer.flush() + elif c_type == b'2': + sys.stderr.buffer.write(payload) + sys.stderr.buffer.flush() + elif c_type == b'X': + if len(payload) > 0: + exit_code = int(payload.decode('utf-8').strip()) + break +s.close() +sys.exit(exit_code) +" "$NAILGUN_PORT" "com.google.api.generator.Main" "$@" && exit 0 +fi +exec java --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED --add-opens=jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED -cp %q "com.google.api.generator.Main" "$@" +`, }, } { t.Run(test.name, func(t *testing.T) {