diff --git a/README.md b/README.md index c50a788..90b3239 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,8 @@ httpClient, err := client.NewHTTPFromFiles("https://"+target, certFile, keyFile, ``` See [Client SDK](docs/clients.md) for complete examples, request-ID -propagation, error handling, and regeneration instructions. +propagation, and regeneration instructions. Error reasons and retryability are +documented in [Client Errors and Retries](docs/client-errors.md). ## Install the agent diff --git a/docs/client-errors.md b/docs/client-errors.md new file mode 100644 index 0000000..e4014b0 --- /dev/null +++ b/docs/client-errors.md @@ -0,0 +1,162 @@ +# Client Errors and Retries + +The Go client exposes the agent's gRPC status codes and `google.rpc.ErrorInfo` +details through both transports. + +- gRPC returns a standard `*status.Status`. +- HTTP returns `*client.Error`, which implements `GRPCStatus()`. +- `client.ErrorInfo(err)` extracts the `kouja.v1` `ErrorInfo` from either transport. +- `client.IsRetryable(err)` applies the server's `retryable` metadata, with a + code-based fallback when metadata is absent. + +Use `IsRetryable` instead of deciding from the gRPC code alone. The same code can +be retryable in one phase and not another. For example, `Unavailable` during +execution may be retryable, while `Unavailable` from a request guard may not. + +## Error metadata + +The agent sets `ErrorInfo.domain` to `kouja.v1`. The metadata contains: + +| Field | Meaning | +|---|---| +| `retryable` | `"true"` or `"false"` according to the server | +| `operation_id` | Journaled operation ID, when the error came from an operation | + +HTTP error bodies are `google.rpc.Status` JSON objects with the same details. + +## Retry decision table + +`Retryable` below is the value the server puts in `ErrorInfo.metadata`. When the +metadata is absent, `IsRetryable` falls back to the gRPC code. + +| Reason | gRPC code | HTTP | Server `retryable` | Guidance | +|---|---|---|---|---| +| `INVALID_ARGUMENT` | `InvalidArgument` | 400 | no | Fix the request. | +| `NOT_FOUND` | `NotFound` | 404 | no | Check the volume, VG, or owner. | +| `FORBIDDEN` | `PermissionDenied` | 403 | no | Fix caller identity or ownership. | +| `UNAUTHENTICATED` | `Unauthenticated` | 401 | no | Fix or renew the client certificate. | +| `VOLUME_IN_DRIFT` | `FailedPrecondition` | 400 | no | Resolve drift through the admin API. | +| `VOLUME_ALREADY_EXPORTED` | `FailedPrecondition` | 400 | no | Use the existing initiator or unexport first. | +| `VOLUME_REQUEST_MISMATCH` | `FailedPrecondition` | 400 | no | Reconcile with the existing volume; resize separately. | +| `SHRINK_NOT_SUPPORTED` | `FailedPrecondition` | 400 | no | Request a grow operation instead. | +| `SESSION_ACTIVE` | `FailedPrecondition` | 400 | no | Log out or disconnect the initiator, then retry. | +| `SESSION_UNKNOWN` | `FailedPrecondition` | 400 | no | Inspect target sessions; admin intervention may be needed. | +| `OPERATION_NEEDS_ATTENTION` | `FailedPrecondition` | 400 | no | Resolve the operation through the admin API. | +| `OPERATION_IN_PROGRESS` | `FailedPrecondition` | 400 | yes | Retry with the same `request_id` after a backoff. | +| `CONFLICT` | `Aborted` | 409 | no | Resolve the conflicting state. | +| `RESOURCE_EXHAUSTED` | `ResourceExhausted` | 429 | no | Wait for capacity or headroom; do not hot-loop. | +| `ADMISSION_REJECTED` | `ResourceExhausted` | 429 | no | Back off and reduce concurrent operations. | +| `UNAVAILABLE` | `Unavailable` | 503 | sometimes | Retry with backoff only when `IsRetryable` is true. | +| `DEADLINE_EXCEEDED` | `DeadlineExceeded` | 504 | sometimes | Retry idempotent requests with backoff when `IsRetryable` is true. | +| `CANCELED` | `Canceled` | 499 | no | The caller canceled; retry only intentionally. | +| `INTERNAL` | `Internal` | 500 | no | Report the failure; retry only for transient infrastructure issues. | +| `UNIMPLEMENTED` | `Unimplemented` | 501 | no | Do not retry; check the agent version and API support. | +| `RETRY_UNAVAILABLE` | `Internal` | 500 | no | Do not retry blindly; inspect the operation. | +| `OPERATION_SUPERSEDED` | `Internal` | 500 | no | Do not retry; inspect the latest operation. | + +HTTP status mapping follows the grpc-gateway defaults. In particular, +`FailedPrecondition` is HTTP 400, not 412 or 409. + +## Retryable operations + +Read-only calls are safe to retry: + +- `GetInfo` +- `GetCapacity` +- `ListStorageClasses` +- `GetVolume` +- `ListVolumes` +- `GetCredential` +- `ListVGs` +- `ListOrphans` + +For mutations, reuse the same `request_id` when retrying: + +- `CreateVolume` +- `ExportVolume` +- `UnexportVolume` +- `ResizeVolume` +- `DeleteVolume` +- `RotateCredential` +- `RegisterVG` +- `DeregisterVG` +- `ImportVolume` +- `ResolveDrift` +- `WipeVolume` +- `TransferOwnership` +- `AdminDeleteVolume` +- `GrantCredential` +- `RevokeGrant` + +Reusing the `request_id` lets the agent replay the journaled operation and +return the original result instead of starting a second operation. Do not retry +a mutation with a new ID unless you have checked the volume's current state. + +## Special cases + +### `SESSION_ACTIVE` + +`SESSION_ACTIVE` is a precondition failure, not an immediately retryable +error. Retrying the same request without disconnecting the initiator will fail +again. Log the session out, then retry the unexport, delete, or rotate request. + +### `RESOURCE_EXHAUSTED` and `ADMISSION_REJECTED` + +These are not hot-loop retries. Wait for capacity to be freed or concurrency to +drop, then try again with backoff. The agent marks these guards +`retryable=false`. + +### `UNAVAILABLE` and `DEADLINE_EXCEEDED` + +These are sometimes retryable. The server marks execution failures retryable +when the operation can be replayed, but guard or observation failures may be +marked false. Always use `IsRetryable`. + +### Transport errors + +If no agent response was received, `IsRetryable` cannot classify the error. A +mutating request may still have been applied. Retry with the same `request_id` +and use `GetVolume` or `ListVolumes` to reconcile when possible. + +## HTTP example + +```go +volume, err := httpClient.CreateVolume(ctx, request) +if err != nil { + var clientErr *client.Error + if errors.As(err, &clientErr) { + log.Printf( + "code=%s reason=%s retryable=%t operation=%s message=%s", + clientErr.Code, + clientErr.Reason, + client.IsRetryable(err), + clientErr.OperationID, + clientErr.Message, + ) + } + if client.IsRetryable(err) { + // Back off, then retry with the same request.RequestId. + } +} +``` + +`clientErr.Retryable` is the raw server metadata. `clientErr.IsRetryable()` and +`client.IsRetryable(err)` apply the code-based fallback when metadata is absent. + +## gRPC example + +```go +_, err := grpcClient.CreateVolume(ctx, request) +if err != nil { + info, ok := client.ErrorInfo(err) + if ok { + log.Printf("reason=%s retryable=%t operation=%s", info.GetReason(), client.IsRetryable(err), info.GetMetadata()["operation_id"]) + } + if client.IsRetryable(err) { + // Back off, then retry with the same request.RequestId. + } +} +``` + +`client.IsRetryable` works with the raw gRPC status returned by the generated +stubs and with `*client.Error` from the HTTP client. diff --git a/docs/clients.md b/docs/clients.md index ac83e4d..413ded7 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -110,9 +110,9 @@ sends it as `x-request-id` metadata and the HTTP client sends it as the `X-Request-Id` header, matching the agent's request-ID propagation. HTTP errors are returned as `*client.Error`. It exposes the HTTP status, the -gRPC code, the message, and any `google.rpc.Status` details. It also implements -`GRPCStatus()`, so `status.Code` and `status.Convert` work across both -transports: +gRPC code, the kouja reason, the operation ID, and any `google.rpc.Status` +details. It also implements `GRPCStatus()`, so `status.Code` and +`status.Convert` work across both transports: ```go _, err := httpClient.GetVolume(ctx, &koujav1.GetVolumeRequest{VolumeId: "missing"}) @@ -121,6 +121,10 @@ if status.Code(err) == codes.NotFound { } ``` +Use `client.IsRetryable(err)` to decide whether a request is safe to retry. +The complete reason table, retry rules, and mutation idempotency guidance are +in [Client Errors and Retries](client-errors.md). + ## Regenerating clients and the OpenAPI document Install the pinned tools once, then regenerate: diff --git a/gen/client/client.gen.go b/gen/client/client.gen.go index fc4c33c..d7e91c7 100644 --- a/gen/client/client.gen.go +++ b/gen/client/client.gen.go @@ -15,6 +15,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -24,6 +25,7 @@ import ( "strings" pb "github.com/voidstarit/kouja/gen/kouja/v1" + errdetails "google.golang.org/genproto/googleapis/rpc/errdetails" statuspb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -259,13 +261,27 @@ func (client *HTTP) CloseIdleConnections() { } // Error is returned when the agent responds with a non-2xx HTTP status. The -// code and message come from the grpc-gateway google.rpc.Status error body when -// one is present. +// code, message, and details come from the grpc-gateway google.rpc.Status +// error body when one is present. type Error struct { HTTPStatus int Code codes.Code - Message string - Details []*anypb.Any + // Reason is the stable kouja reason from google.rpc.ErrorInfo, such as + // "SESSION_ACTIVE" or "UNAVAILABLE". + Reason string + // Message is the human-readable error message. + Message string + // OperationID identifies the journaled operation, when the error came from + // an operation rather than a request guard. + OperationID string + // Retryable is the server's retryable flag from google.rpc.ErrorInfo. It is + // false when ErrorInfo was not present; use [IsRetryable] to apply the + // code-based fallback. + Retryable bool + // Details contains the raw google.rpc.Status details. + Details []*anypb.Any + + hasErrorInfo bool } // Error implements error. @@ -289,7 +305,22 @@ func (err *Error) Error() string { return fmt.Sprintf("client: %s: %s", err.Code, err.Message) } -// GRPCStatus lets grpc/status translate an HTTP error into a gRPC status. +// IsRetryable reports whether the server marked this HTTP error as retryable. +// If the response did not contain google.rpc.ErrorInfo, it falls back to the +// gRPC code: Unavailable, DeadlineExceeded, and ResourceExhausted are treated +// as retryable, and every other code is not. +func (err *Error) IsRetryable() bool { + if err == nil { + return false + } + if err.hasErrorInfo { + return err.Retryable + } + return retryableCode(err.Code) +} + +// GRPCStatus lets grpc/status translate an HTTP error into a gRPC status. The +// original google.rpc.Status details, including kouja ErrorInfo, are preserved. func (err *Error) GRPCStatus() *status.Status { if err == nil { return status.New(codes.Unknown, "") @@ -298,7 +329,84 @@ func (err *Error) GRPCStatus() *status.Status { if code == codes.OK { code = codes.Unknown } - return status.New(code, err.Message) + return status.FromProto(&statuspb.Status{ + Code: int32(code), //nolint:gosec // gRPC codes are non-negative + Message: err.Message, + Details: err.Details, + }) +} + +// IsRetryable reports whether err is safe to retry according to the agent's +// google.rpc.ErrorInfo metadata. If the error has no ErrorInfo, it falls back +// to the gRPC code: Unavailable, DeadlineExceeded, and ResourceExhausted are +// retryable, and every other code is not. +// +// Transport errors that never produced an agent response are not classified; +// retry those with backoff only when the request is idempotent. +func IsRetryable(err error) bool { + if err == nil { + return false + } + var httpErr *Error + if errors.As(err, &httpErr) { + return httpErr.IsRetryable() + } + if info, ok := ErrorInfo(err); ok { + return strings.EqualFold(info.GetMetadata()["retryable"], "true") + } + return retryableCode(status.Code(err)) +} + +// ErrorInfo returns the kouja google.rpc.ErrorInfo detail from an agent error. +// It works for errors returned by both the gRPC and HTTP clients. +func ErrorInfo(err error) (*errdetails.ErrorInfo, bool) { + if err == nil { + return nil, false + } + var httpErr *Error + if errors.As(err, &httpErr) { + return errorInfoFromDetails(httpErr.Details) + } + for _, detail := range status.Convert(err).Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok { + return info, true + } + } + return nil, false +} + +func errorInfoFromDetails(details []*anypb.Any) (*errdetails.ErrorInfo, bool) { + for _, detail := range details { + if detail == nil || !detail.MessageIs(&errdetails.ErrorInfo{}) { + continue + } + info := &errdetails.ErrorInfo{} + if err := detail.UnmarshalTo(info); err != nil { + continue + } + return info, true + } + return nil, false +} + +func populateErrorInfo(result *Error) { + info, ok := errorInfoFromDetails(result.Details) + if !ok { + return + } + result.Reason = info.GetReason() + result.OperationID = info.GetMetadata()["operation_id"] + result.Retryable = strings.EqualFold(info.GetMetadata()["retryable"], "true") + result.hasErrorInfo = true +} + +func retryableCode(code codes.Code) bool { + switch code { + case codes.Unavailable, codes.DeadlineExceeded, codes.ResourceExhausted: + return true + default: + return false + } } func (client *HTTP) do(ctx context.Context, method, path string, query url.Values, request, response proto.Message) error { @@ -363,6 +471,7 @@ func decodeHTTPError(response *http.Response) error { result.Code = codes.Code(protoStatus.GetCode()) //nolint:gosec // google.rpc.Code values are non-negative result.Message = protoStatus.GetMessage() result.Details = protoStatus.GetDetails() + populateErrorInfo(result) } else { result.Message = strings.TrimSpace(string(payload)) } diff --git a/gen/client/errors_test.go b/gen/client/errors_test.go new file mode 100644 index 0000000..49137f0 --- /dev/null +++ b/gen/client/errors_test.go @@ -0,0 +1,155 @@ +package client + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "testing" + + koujav1 "github.com/voidstarit/kouja/gen/kouja/v1" + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func bddTestErrorRetryability(t testing.TB) { + server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusServiceUnavailable) + _, _ = response.Write([]byte(`{ + "code": 14, + "message": "backend unavailable", + "details": [{ + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "UNAVAILABLE", + "domain": "kouja.v1", + "metadata": {"retryable": "true", "operation_id": "op-1"} + }] + }`)) + })) + defer server.Close() + + client, err := NewHTTP(server.URL) + if err != nil { + t.Fatalf("NewHTTP: %v", err) + } + _, err = client.GetInfo(context.Background(), &koujav1.GetInfoRequest{}) + var httpErr *Error + if !errors.As(err, &httpErr) { + t.Fatalf("error = %v, want *Error", err) + } + if httpErr.HTTPStatus != http.StatusServiceUnavailable || httpErr.Code != codes.Unavailable { + t.Fatalf("error = %#v", httpErr) + } + if httpErr.Reason != "UNAVAILABLE" || httpErr.OperationID != "op-1" || !httpErr.Retryable { + t.Fatalf("error metadata = %#v", httpErr) + } + if !httpErr.IsRetryable() || !IsRetryable(err) { + t.Fatal("retryable error was not reported as retryable") + } + info, ok := ErrorInfo(err) + if !ok || info.GetReason() != "UNAVAILABLE" || info.GetMetadata()["operation_id"] != "op-1" { + t.Fatalf("ErrorInfo = %#v, %t", info, ok) + } + if status.Code(err) != codes.Unavailable { + t.Fatalf("status.Code = %v", status.Code(err)) + } + found := false + for _, detail := range status.Convert(err).Details() { + if _, ok := detail.(*errdetails.ErrorInfo); ok { + found = true + } + } + if !found { + t.Fatal("status details did not preserve ErrorInfo") + } + + nonRetryableServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusServiceUnavailable) + _, _ = response.Write([]byte(`{ + "code": 14, + "message": "guarded unavailable", + "details": [{ + "@type": "type.googleapis.com/google.rpc.ErrorInfo", + "reason": "UNAVAILABLE", + "domain": "kouja.v1", + "metadata": {"retryable": "false"} + }] + }`)) + })) + defer nonRetryableServer.Close() + nonRetryableClient, err := NewHTTP(nonRetryableServer.URL) + if err != nil { + t.Fatalf("NewHTTP: %v", err) + } + _, err = nonRetryableClient.GetInfo(context.Background(), &koujav1.GetInfoRequest{}) + if !errors.As(err, &httpErr) { + t.Fatalf("error = %v, want *Error", err) + } + if httpErr.Retryable || httpErr.IsRetryable() || IsRetryable(err) { + t.Fatal("server retryable=false was ignored") + } + + fallbackServer := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, _ *http.Request) { + response.Header().Set("Content-Type", "application/json") + response.WriteHeader(http.StatusTooManyRequests) + _, _ = response.Write([]byte(`{"code":8,"message":"capacity"}`)) + })) + defer fallbackServer.Close() + fallbackClient, err := NewHTTP(fallbackServer.URL) + if err != nil { + t.Fatalf("NewHTTP: %v", err) + } + _, err = fallbackClient.GetInfo(context.Background(), &koujav1.GetInfoRequest{}) + if !errors.As(err, &httpErr) { + t.Fatalf("error = %v, want *Error", err) + } + if !httpErr.IsRetryable() || !IsRetryable(err) { + t.Fatal("ResourceExhausted fallback was not retryable") + } + + retryableStatus, err := status.New(codes.Unavailable, "backend").WithDetails(&errdetails.ErrorInfo{ + Reason: "UNAVAILABLE", + Domain: "kouja.v1", + Metadata: map[string]string{"retryable": "true"}, + }) + if err != nil { + t.Fatalf("status.WithDetails: %v", err) + } + if !IsRetryable(retryableStatus.Err()) { + t.Fatal("gRPC retryable ErrorInfo was not retryable") + } + if info, ok := ErrorInfo(retryableStatus.Err()); !ok || info.GetReason() != "UNAVAILABLE" { + t.Fatalf("gRPC ErrorInfo = %#v, %t", info, ok) + } + + guardedStatus, err := status.New(codes.Unavailable, "guarded").WithDetails(&errdetails.ErrorInfo{ + Reason: "UNAVAILABLE", + Domain: "kouja.v1", + Metadata: map[string]string{"retryable": "false"}, + }) + if err != nil { + t.Fatalf("status.WithDetails: %v", err) + } + if IsRetryable(guardedStatus.Err()) { + t.Fatal("gRPC retryable=false was ignored") + } + if !IsRetryable(status.Error(codes.Unavailable, "transport")) { + t.Fatal("Unavailable fallback was not retryable") + } + if IsRetryable(status.Error(codes.FailedPrecondition, "session")) { + t.Fatal("FailedPrecondition was retryable") + } + if IsRetryable(errors.New("network")) { + t.Fatal("unclassified transport error was retryable") + } + if IsRetryable(nil) { + t.Fatal("nil error was retryable") + } + var nilErr *Error + if nilErr.IsRetryable() { + t.Fatal("nil *Error was retryable") + } +} diff --git a/gen/client/ginkgo_suite_test.go b/gen/client/ginkgo_suite_test.go index a8fc042..c782e33 100644 --- a/gen/client/ginkgo_suite_test.go +++ b/gen/client/ginkgo_suite_test.go @@ -37,6 +37,9 @@ var _ = Describe("unit tests", func() { It("TestHTTPClientErrors", func() { bddTestHTTPClientErrors(GinkgoTB()) }) + It("TestErrorRetryability", func() { + bddTestErrorRetryability(GinkgoTB()) + }) It("TestHTTPClientValidation", func() { bddTestHTTPClientValidation(GinkgoTB()) }) diff --git a/hack/protoc-gen-kouja-client/client.go.tmpl b/hack/protoc-gen-kouja-client/client.go.tmpl index 1c8e12a..6c2ceb9 100644 --- a/hack/protoc-gen-kouja-client/client.go.tmpl +++ b/hack/protoc-gen-kouja-client/client.go.tmpl @@ -15,6 +15,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -24,6 +25,7 @@ import ( "strings" pb "{{.ProtoImport}}" + errdetails "google.golang.org/genproto/googleapis/rpc/errdetails" statuspb "google.golang.org/genproto/googleapis/rpc/status" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -259,13 +261,27 @@ func (client *HTTP) CloseIdleConnections() { } // Error is returned when the agent responds with a non-2xx HTTP status. The -// code and message come from the grpc-gateway google.rpc.Status error body when -// one is present. +// code, message, and details come from the grpc-gateway google.rpc.Status +// error body when one is present. type Error struct { HTTPStatus int Code codes.Code - Message string - Details []*anypb.Any + // Reason is the stable kouja reason from google.rpc.ErrorInfo, such as + // "SESSION_ACTIVE" or "UNAVAILABLE". + Reason string + // Message is the human-readable error message. + Message string + // OperationID identifies the journaled operation, when the error came from + // an operation rather than a request guard. + OperationID string + // Retryable is the server's retryable flag from google.rpc.ErrorInfo. It is + // false when ErrorInfo was not present; use [IsRetryable] to apply the + // code-based fallback. + Retryable bool + // Details contains the raw google.rpc.Status details. + Details []*anypb.Any + + hasErrorInfo bool } // Error implements error. @@ -289,7 +305,22 @@ func (err *Error) Error() string { return fmt.Sprintf("client: %s: %s", err.Code, err.Message) } -// GRPCStatus lets grpc/status translate an HTTP error into a gRPC status. +// IsRetryable reports whether the server marked this HTTP error as retryable. +// If the response did not contain google.rpc.ErrorInfo, it falls back to the +// gRPC code: Unavailable, DeadlineExceeded, and ResourceExhausted are treated +// as retryable, and every other code is not. +func (err *Error) IsRetryable() bool { + if err == nil { + return false + } + if err.hasErrorInfo { + return err.Retryable + } + return retryableCode(err.Code) +} + +// GRPCStatus lets grpc/status translate an HTTP error into a gRPC status. The +// original google.rpc.Status details, including kouja ErrorInfo, are preserved. func (err *Error) GRPCStatus() *status.Status { if err == nil { return status.New(codes.Unknown, "") @@ -298,7 +329,84 @@ func (err *Error) GRPCStatus() *status.Status { if code == codes.OK { code = codes.Unknown } - return status.New(code, err.Message) + return status.FromProto(&statuspb.Status{ + Code: int32(code), //nolint:gosec // gRPC codes are non-negative + Message: err.Message, + Details: err.Details, + }) +} + +// IsRetryable reports whether err is safe to retry according to the agent's +// google.rpc.ErrorInfo metadata. If the error has no ErrorInfo, it falls back +// to the gRPC code: Unavailable, DeadlineExceeded, and ResourceExhausted are +// retryable, and every other code is not. +// +// Transport errors that never produced an agent response are not classified; +// retry those with backoff only when the request is idempotent. +func IsRetryable(err error) bool { + if err == nil { + return false + } + var httpErr *Error + if errors.As(err, &httpErr) { + return httpErr.IsRetryable() + } + if info, ok := ErrorInfo(err); ok { + return strings.EqualFold(info.GetMetadata()["retryable"], "true") + } + return retryableCode(status.Code(err)) +} + +// ErrorInfo returns the kouja google.rpc.ErrorInfo detail from an agent error. +// It works for errors returned by both the gRPC and HTTP clients. +func ErrorInfo(err error) (*errdetails.ErrorInfo, bool) { + if err == nil { + return nil, false + } + var httpErr *Error + if errors.As(err, &httpErr) { + return errorInfoFromDetails(httpErr.Details) + } + for _, detail := range status.Convert(err).Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok { + return info, true + } + } + return nil, false +} + +func errorInfoFromDetails(details []*anypb.Any) (*errdetails.ErrorInfo, bool) { + for _, detail := range details { + if detail == nil || !detail.MessageIs(&errdetails.ErrorInfo{}) { + continue + } + info := &errdetails.ErrorInfo{} + if err := detail.UnmarshalTo(info); err != nil { + continue + } + return info, true + } + return nil, false +} + +func populateErrorInfo(result *Error) { + info, ok := errorInfoFromDetails(result.Details) + if !ok { + return + } + result.Reason = info.GetReason() + result.OperationID = info.GetMetadata()["operation_id"] + result.Retryable = strings.EqualFold(info.GetMetadata()["retryable"], "true") + result.hasErrorInfo = true +} + +func retryableCode(code codes.Code) bool { + switch code { + case codes.Unavailable, codes.DeadlineExceeded, codes.ResourceExhausted: + return true + default: + return false + } } func (client *HTTP) do(ctx context.Context, method, path string, query url.Values, request, response proto.Message) error { @@ -363,6 +471,7 @@ func decodeHTTPError(response *http.Response) error { result.Code = codes.Code(protoStatus.GetCode()) //nolint:gosec // google.rpc.Code values are non-negative result.Message = protoStatus.GetMessage() result.Details = protoStatus.GetDetails() + populateErrorInfo(result) } else { result.Message = strings.TrimSpace(string(payload)) }