From 11d3bcb65adb664b96940106324fb8a3fdfc2e87 Mon Sep 17 00:00:00 2001 From: Antoine Toussaint <312536060+antoinetoussaint-byte@users.noreply.github.com> Date: Tue, 3 Nov 2026 23:54:04 -0500 Subject: [PATCH] fix(render): promote the secret keys the environment generates for a service A hosted render of a composition pinning obin-ai/module-model-gateway stopped, from the postgres agent, with postgres deployment requires a typed Kubernetes Secret reference for CODEFLY__SERVICE_SECRET_CONFIGURATION__MODEL__STORE__POSTGRES__POSTGRES_USER The render promotes into a typed Secret reference every secret key a service's own configuration declares (promotableConfiguration), and that configuration is read from the service's own directory, `configurations//`. A composition resolves a pinned module into a module cache it never writes to, and module-model-gateway, by decision, ships no `*.secret.env` (the run that composes the store generates it). A local run does generate it (cmd/run/generated_secrets.go, before loading); a hosted render has no analogue: `codefly deploy secrets` mints, in the cell's store, only the keys a render already emitted. So nothing declared the four primitives, the render promoted nothing, and the agent refused. The environment's service-secrets.generate IS the declaration that those keys exist for that service and that the cell's store holds them. The restricted render now adds, to a service's configuration, every key a generator naming the service declares that the configuration does not, as a secret with no value, and promotes it with the rest: the key name reaches the manifest and its ExternalSecret, `deploy secrets` mints the value, and no module tree is written or widened. A key the service declares itself keeps its declaration; a generator naming other services, or the workspace, adds nothing. Evidence: the new test fails to compile on main (the seam does not exist) and passes here. On obin-ai/platform-obin (blue), v0.1.234 and main refuse model/store; the same four key names supplied through the module tree instead clear the postgres deployment, which isolates the missing declaration as the cause. A full render with this build is still owed: every run since the workspace moved robin to dc892ada stops earlier, in the build phase, on robin/chat:0.1.36 "already published from other code", which is a module-robin release matter and not this seam. Co-Authored-By: Claude Fable 5.1 --- docs/commands.md | 23 +++++++ pkg/environments/environment.go | 6 +- pkg/orchestration/builder_deploy.go | 88 ++++++++++++++++++++++++ pkg/orchestration/builder_deploy_test.go | 73 ++++++++++++++++++++ 4 files changed, 189 insertions(+), 1 deletion(-) diff --git a/docs/commands.md b/docs/commands.md index ce31d68ec..e607d7b33 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -679,6 +679,29 @@ rule, the mechanisms and the tests that pin them are in [the orchestration engine's workspace configuration groups](orchestration.md#workspace-configuration-groups). +#### A service whose module ships no secret file + +A restricted render promotes every secret key a service's own configuration +declares into a `secretKeyRef` on `secret-`, and reads that +configuration from the service's own `configurations//`. A composition +resolves a pinned module into a module cache it never writes to, and a module +that commits no `*.secret.env` (its store's credentials are generated by the run +that composes it) therefore declares no key to the render — where the postgres +agent refuses the deployment: `postgres deployment requires a typed Kubernetes +Secret reference for CODEFLY__SERVICE_SECRET_CONFIGURATION______POSTGRES__POSTGRES_USER`. +A local run has no such gap because it generates the file before loading +([generated secrets](#generated-secrets)); a hosted render's store is seeded +from the render itself. + +So the render also reads the environment's `service-secrets.generate`: a +`service` generator naming the service declares that its keys exist and that +the store holds them, and the render promotes each key the service's +configuration does not declare itself as if it had, key name only. The +projected ExternalSecret reads it, `deploy secrets` plans it as `generate`, and +no module tree is written or widened. A key the service does declare keeps its +own declaration, and a generator naming other services, or the workspace, adds +nothing to this one. The render says which keys it promoted this way. + #### A root group's credential, and what `deploy secrets` can and cannot seed diff --git a/pkg/environments/environment.go b/pkg/environments/environment.go index 2a6e7c166..8e3910f92 100644 --- a/pkg/environments/environment.go +++ b/pkg/environments/environment.go @@ -840,7 +840,11 @@ type EnvironmentServiceSecrets struct { // holds as values minted at random rather than supplied from outside — the // only thing `codefly deploy secrets` generates. A key no generator names // and no stored secret already holds is reported as one the operator must - // supply; it is never guessed. + // supply; it is never guessed. A restricted render also reads it as a + // declaration: a service generator's keys that the named service's own + // configuration does not declare (a module shipping no *.secret.env) are + // promoted to typed Secret references all the same + // (orchestration.withEnvironmentDeclaredSecrets). Generate []EnvironmentSecretGenerator `yaml:"generate,omitempty"` // MayBeEmpty names secret configuration keys whose correct value in this // environment is the empty string. A stored property is otherwise refused diff --git a/pkg/orchestration/builder_deploy.go b/pkg/orchestration/builder_deploy.go index 59da50c13..980df400e 100644 --- a/pkg/orchestration/builder_deploy.go +++ b/pkg/orchestration/builder_deploy.go @@ -5,6 +5,7 @@ import ( "fmt" "os/exec" "path/filepath" + "slices" "sort" "strings" @@ -61,6 +62,7 @@ func (b *Builder) Deploy(ctx context.Context) (*OutputProperty, error) { var secretReferences map[string]*builderv0.KubernetesSecretKeyReference if profile == builderv0.KubernetesOutputProfile_KUBERNETES_OUTPUT_PROFILE_RESTRICTED_PORTABLE_V1 { secretName := "secret-" + b.instance.Service.Name + conf = withEnvironmentDeclaredSecrets(ctx, conf, b.instance.Identity, b.world.Env) conf, dependenciesConfigurations, secretReferences, err = promotableDeploymentConfigurations( conf, dependenciesConfigurations, @@ -426,6 +428,92 @@ func promotableDeploymentConfigurations( return own, safeDependencies, references, nil } +// withEnvironmentDeclaredSecrets adds to a service's own configuration the +// secret keys the environment's service-secrets.generate declares for it and the +// configuration does not declare itself. The environment has declared that its +// store holds those keys, minted by `codefly deploy secrets`, so the render +// promotes them into typed Secret references exactly as it promotes a key read +// from the service's *.secret.env: the key name reaches the manifest and the +// value never exists here. A module that ships no local secret file then +// renders — a composition reads its module cache and never writes into it, and +// a cell's store is seeded from the render, so nothing else could have declared +// the keys — where before the postgres agent refused the deployment for the +// first primitive it could not find. +// +// Only a generator naming the service adds anything, the rule a local run's +// provisionGeneratedSecrets applies; a key the configuration already declares +// stays as declared, with the value its file carries; and the result is a copy, +// so the manager's configuration is never widened for anyone else. +func withEnvironmentDeclaredSecrets( + ctx context.Context, + configuration *basev0.Configuration, + identity *resources.ServiceIdentity, + env *environments.Environment, +) *basev0.Configuration { + if env == nil || env.ServiceSecrets == nil || identity == nil { + return configuration + } + w := wool.Get(ctx).In("Builder.withEnvironmentDeclaredSecrets") + unique := identity.Unique() + var declared *basev0.Configuration + for index := range env.ServiceSecrets.Generate { + generator := &env.ServiceSecrets.Generate[index] + if generator.Scope != environments.SecretGeneratorScopeService || !slices.Contains(generator.Services, unique) { + continue + } + for _, key := range generator.Keys { + key = strings.TrimSpace(key) + if declaresConfigurationKey(configuration, generator.Configuration, key) || + declaresConfigurationKey(declared, generator.Configuration, key) { + continue + } + if declared == nil { + declared = &basev0.Configuration{Origin: unique} + if configuration != nil { + declared = proto.CloneOf(configuration) + } + } + info := configurationInformationNamed(declared, generator.Configuration) + info.ConfigurationValues = append(info.ConfigurationValues, &basev0.ConfigurationValue{Key: key, Secret: true}) + w.Info("secret key declared by the environment's service-secrets.generate, not by the service's own configuration; rendering its reference", + wool.Field("service", unique), wool.Field("configuration", generator.Configuration), wool.Field("key", key)) + } + } + if declared == nil { + return configuration + } + return declared +} + +// declaresConfigurationKey reports whether the configuration carries key in the +// named group, matched the way core matches configuration names and keys. +func declaresConfigurationKey(configuration *basev0.Configuration, group, key string) bool { + for _, info := range configuration.GetInfos() { + if !resources.Match(info.GetName(), group) { + continue + } + for _, value := range info.GetConfigurationValues() { + if resources.Match(value.GetKey(), key) { + return true + } + } + } + return false +} + +// configurationInformationNamed returns the configuration's group of that name, +// appending an empty one when it has none. +func configurationInformationNamed(configuration *basev0.Configuration, name string) *basev0.ConfigurationInformation { + for _, info := range configuration.GetInfos() { + if resources.Match(info.GetName(), name) { + return info + } + } + info := &basev0.ConfigurationInformation{Name: name} + configuration.Infos = append(configuration.Infos, info) + return info +} + // restrictedRenderRejects reports whether restricted render will refuse a value // that reaches it inline. It mirrors the per-value half of core's // validateRestrictedDeploymentRequest guard (core agents/services/base_builder.go): diff --git a/pkg/orchestration/builder_deploy_test.go b/pkg/orchestration/builder_deploy_test.go index ef3fce29a..372ab1ad3 100644 --- a/pkg/orchestration/builder_deploy_test.go +++ b/pkg/orchestration/builder_deploy_test.go @@ -5,6 +5,7 @@ import ( "errors" "os" "path/filepath" + "sort" "testing" "github.com/codefly-dev/cli/pkg/deployments" @@ -65,6 +66,78 @@ func TestPromotableDeploymentInputsReplaceSecretValuesWithReferences(t *testing. } } +// A composition that pins a module reads the module's own tree and never writes +// into it, so a service whose module ships no *.secret.env declares no secret +// key to the render — and the postgres agent refuses the deployment for the +// first primitive without a typed reference. The environment's +// service-secrets.generate is the declaration that the keys exist and that the +// cell's store holds them, so the render promotes them from it. +func TestEnvironmentDeclaredSecretsPromoteToReferencesWhenTheServiceShipsNoFile(t *testing.T) { + ctx := context.Background() + identity := &resources.ServiceIdentity{Module: "model", Name: "store"} + env := &environments.Environment{ServiceSecrets: &environments.EnvironmentServiceSecrets{ + SecretStore: environments.EnvironmentSecretStoreReference{Kind: "ClusterSecretStore", Name: "cell-secrets"}, + Generate: []environments.EnvironmentSecretGenerator{ + {Scope: environments.SecretGeneratorScopeService, Configuration: "postgres", Format: "identifier", + Services: []string{"model/store", "robin/store"}, Keys: []string{"POSTGRES_USER"}}, + {Scope: environments.SecretGeneratorScopeService, Configuration: "postgres", + Services: []string{"model/store", "robin/store"}, Keys: []string{"POSTGRES_USER", "POSTGRES_PASSWORD"}}, + // Another service's keys and a workspace group are not this service's. + {Scope: environments.SecretGeneratorScopeService, Configuration: "postgres", + Services: []string{"robin/store"}, Keys: []string{"POSTGRES_READER_PASSWORD"}}, + {Scope: environments.SecretGeneratorScopeWorkspace, Configuration: "internal-auth", Keys: []string{"CODEFLY_INTERNAL_TOKEN"}}, + }, + }} + + // No configuration at all: the module ships no file for the service. + declared := withEnvironmentDeclaredSecrets(ctx, nil, identity, env) + _, _, references, err := promotableDeploymentConfigurations(declared, nil, "secret-store") + require.NoError(t, err) + keys := make([]string, 0, len(references)) + for key := range references { + keys = append(keys, key) + } + sort.Strings(keys) + require.Equal(t, []string{ + "CODEFLY__SERVICE_SECRET_CONFIGURATION__MODEL__STORE__POSTGRES__POSTGRES_PASSWORD", + "CODEFLY__SERVICE_SECRET_CONFIGURATION__MODEL__STORE__POSTGRES__POSTGRES_USER", + }, keys) + for _, key := range keys { + require.Equal(t, "secret-store", references[key].GetName()) + require.Equal(t, key, references[key].GetKey()) + } + + // A key the service declares itself stays its own; only the missing one is added, + // and the manager's configuration is left as it was. + own := &basev0.Configuration{ + Origin: "model/store", + Infos: []*basev0.ConfigurationInformation{{ + Name: "postgres", + ConfigurationValues: []*basev0.ConfigurationValue{ + {Key: "POSTGRES_USER", Value: "user", Secret: true}, + {Key: "POSTGRES_HOST", Value: "store.model.svc"}, + }, + }}, + } + original := proto.CloneOf(own) + declared = withEnvironmentDeclaredSecrets(ctx, own, identity, env) + require.True(t, proto.Equal(original, own)) + require.Len(t, declared.GetInfos(), 1) + values := declared.GetInfos()[0].GetConfigurationValues() + require.Len(t, values, 3) + require.Equal(t, "POSTGRES_USER", values[0].GetKey()) + require.Equal(t, "user", values[0].GetValue()) + require.Equal(t, "POSTGRES_PASSWORD", values[2].GetKey()) + require.True(t, values[2].GetSecret()) + require.Empty(t, values[2].GetValue()) + + // A service no generator names, and an environment with no secrets block, + // get back exactly what they had. + other := &resources.ServiceIdentity{Module: "model", Name: "model"} + require.Same(t, own, withEnvironmentDeclaredSecrets(ctx, own, other, env)) + require.Nil(t, withEnvironmentDeclaredSecrets(ctx, nil, identity, &environments.Environment{})) +} + func TestPromotableDeploymentInputsRejectMisplacedSecretEndpointKeys(t *testing.T) { configuration := &basev0.Configuration{ Origin: "users/accounts",