From 9ab78c875bfb3843c0a50beff0945ee6bb564f90 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 14 Jul 2026 18:55:42 +0530 Subject: [PATCH 01/38] feat(infrastructure): add scaffolding for the infrastructure kind --- api/v1alpha1/casting.schema.json | 3 + api/v1alpha1/casting_kind.go | 3 +- .../collectionagent/casting.schema.json | 11 +- api/v1alpha1/infrastructure/casting.go | 76 ++++ .../infrastructure/casting.schema.json | 343 ++++++++++++++++++ api/v1alpha1/infrastructure/schema.go | 18 + api/v1alpha1/infrastructure/schema_test.go | 26 ++ api/v1alpha1/installation/casting.schema.json | 11 +- api/v1alpha1/molding_kind.go | 3 +- api/v1alpha1/resource.go | 31 ++ cmd/foundryctl/gen.go | 26 +- internal/casting/infrastructure/casting.go | 15 + internal/casting/infrastructure/planner.go | 99 +++++ internal/casting/infrastructure/registry.go | 62 ++++ .../terraformcasting/casting.go | 36 ++ .../terraformcasting/enricher.go | 20 + internal/config/yamlconfig/config.go | 35 ++ internal/foundry/foundry.go | 5 + internal/molding/infrastructure/molding.go | 17 + .../resourcemolding/resource.go | 31 ++ 20 files changed, 850 insertions(+), 21 deletions(-) create mode 100644 api/v1alpha1/infrastructure/casting.go create mode 100644 api/v1alpha1/infrastructure/casting.schema.json create mode 100644 api/v1alpha1/infrastructure/schema.go create mode 100644 api/v1alpha1/infrastructure/schema_test.go create mode 100644 api/v1alpha1/resource.go create mode 100644 internal/casting/infrastructure/casting.go create mode 100644 internal/casting/infrastructure/planner.go create mode 100644 internal/casting/infrastructure/registry.go create mode 100644 internal/casting/infrastructure/terraformcasting/casting.go create mode 100644 internal/casting/infrastructure/terraformcasting/enricher.go create mode 100644 internal/molding/infrastructure/molding.go create mode 100644 internal/molding/infrastructure/resourcemolding/resource.go diff --git a/api/v1alpha1/casting.schema.json b/api/v1alpha1/casting.schema.json index 56078887..6e9ecddf 100644 --- a/api/v1alpha1/casting.schema.json +++ b/api/v1alpha1/casting.schema.json @@ -7,6 +7,9 @@ }, { "$ref": "collectionagent/casting.schema.json" + }, + { + "$ref": "infrastructure/casting.schema.json" } ] } \ No newline at end of file diff --git a/api/v1alpha1/casting_kind.go b/api/v1alpha1/casting_kind.go index 3d710907..f0002878 100644 --- a/api/v1alpha1/casting_kind.go +++ b/api/v1alpha1/casting_kind.go @@ -19,6 +19,7 @@ var _ jsonschema.Enum = (*Kind)(nil) var ( KindInstallation Kind = Kind{s: "Installation"} KindCollectionAgent Kind = Kind{s: "CollectionAgent"} + KindInfrastructure Kind = Kind{s: "Infrastructure"} ) // Kind discriminates between top-level casting resource types. @@ -33,7 +34,7 @@ func (kind Kind) String() string { } func Kinds() []Kind { - return []Kind{KindInstallation, KindCollectionAgent} + return []Kind{KindInstallation, KindCollectionAgent, KindInfrastructure} } func (kind Kind) MarshalJSON() ([]byte, error) { diff --git a/api/v1alpha1/collectionagent/casting.schema.json b/api/v1alpha1/collectionagent/casting.schema.json index 8998b545..9ab23179 100644 --- a/api/v1alpha1/collectionagent/casting.schema.json +++ b/api/v1alpha1/collectionagent/casting.schema.json @@ -98,7 +98,9 @@ }, "V1Alpha1Kind": { "enum": [ - "CollectionAgent" + "Installation", + "CollectionAgent", + "Infrastructure" ], "type": "string" }, @@ -364,8 +366,11 @@ "type": "string" }, "kind": { - "$ref": "#/definitions/V1Alpha1Kind", - "description": "Kind of the casting resource." + "description": "Kind of the casting resource.", + "enum": [ + "CollectionAgent" + ], + "type": "string" }, "metadata": { "$ref": "#/definitions/V1Alpha1TypeMetadata", diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go new file mode 100644 index 00000000..b8e471a0 --- /dev/null +++ b/api/v1alpha1/infrastructure/casting.go @@ -0,0 +1,76 @@ +package infrastructure + +import ( + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/domain" +) + +// Casting is the Infrastructure kind. +type Casting struct { + v1alpha1.CastingMeta `json:",inline" yaml:",inline"` + Spec Spec `json:"spec" yaml:"spec" required:"true" description:"Infrastructure specification"` + _ struct{} `additionalProperties:"false"` +} + +// Spec is the Infrastructure-specific configuration. +type Spec struct { + Deployment v1alpha1.TypeDeployment `json:"deployment" yaml:"deployment" required:"true" description:"Deployment configuration for the platform"` + Resource v1alpha1.TypeResourceRef `json:"resource" yaml:"resource" required:"true" description:"The resource this infrastructure serves"` + Patches []v1alpha1.PatchEntry `json:"patches,omitempty" yaml:"patches,omitempty" description:"Patch operations to apply to generated materials"` + _ struct{} `additionalProperties:"false"` +} + +var _ v1alpha1.Machinery = (*Casting)(nil) + +// Default returns an Infrastructure casting with defaults initialised. +func Default() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindInfrastructure, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + Spec: Spec{ + Deployment: v1alpha1.TypeDeployment{Flavor: v1alpha1.FlavorTerraform}, + Resource: v1alpha1.TypeResourceRef{ + APIVersion: "v1alpha1", + Kind: v1alpha1.KindInstallation, + Name: "signoz", + }, + }, + } +} + +// Example returns a minimal Infrastructure; the forge pipeline fills in +// defaults. +func Example() *Casting { + return &Casting{ + CastingMeta: v1alpha1.CastingMeta{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + Kind: v1alpha1.KindInfrastructure, + Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, + }, + } +} + +// Kind reports the casting kind. Shadows the embedded CastingMeta.Kind field; +// the field stays reachable as c.CastingMeta.Kind. +func (c *Casting) Kind() v1alpha1.Kind { + return v1alpha1.KindInfrastructure +} + +// MergeStatusIntoSpec folds molding-written status into spec. The resource +// reference's status is its own home; nothing shadows spec fields. +func (c *Casting) MergeStatusIntoSpec() error { + return nil +} + +// TrackableProperties returns analytics tags for the casting. +func (c *Casting) TrackableProperties() domain.Properties { + return domain.NewProperties(). + Set("kind", v1alpha1.KindInfrastructure.String()). + Set("platform", c.Spec.Deployment.Platform.String()). + Set("flavor", c.Spec.Deployment.Flavor.String()). + Set("resource_kind", c.Spec.Resource.Kind.String()). + Set("patches_count", len(c.Spec.Patches)) +} diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json new file mode 100644 index 00000000..1367af9c --- /dev/null +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -0,0 +1,343 @@ +{ + "required": [ + "apiVersion", + "kind", + "metadata", + "spec" + ], + "additionalProperties": false, + "definitions": { + "InfrastructureSpec": { + "required": [ + "deployment", + "resource" + ], + "additionalProperties": false, + "properties": { + "deployment": { + "$ref": "#/definitions/V1Alpha1TypeDeployment", + "description": "Deployment configuration for the platform" + }, + "patches": { + "description": "Patch operations to apply to generated materials", + "items": { + "$ref": "#/definitions/V1Alpha1PatchEntry" + }, + "type": "array" + }, + "resource": { + "$ref": "#/definitions/V1Alpha1TypeResourceRef", + "description": "The resource this infrastructure serves" + } + }, + "type": "object" + }, + "V1Alpha1Flavor": { + "enum": [ + "compose", + "swarm", + "binary", + "kustomize", + "helm", + "blueprint", + "stack", + "template", + "terraform" + ], + "type": "string" + }, + "V1Alpha1Kind": { + "enum": [ + "Installation", + "CollectionAgent", + "Infrastructure" + ], + "type": "string" + }, + "V1Alpha1Mode": { + "enum": [ + "docker", + "systemd", + "kubernetes", + "ec2" + ], + "type": "string" + }, + "V1Alpha1PatchEntry": { + "required": [ + "target", + "operations" + ], + "additionalProperties": false, + "properties": { + "operations": { + "description": "JSON Patch (RFC 6902) operations to apply. Used by the jsonpatch driver.", + "items": { + "$ref": "#/definitions/V1Alpha1PatchOperation" + }, + "minItems": 1, + "type": [ + "array", + "null" + ] + }, + "target": { + "description": "Target output file to patch", + "examples": [ + "compose.yaml", + "signoz/deployment.yaml", + "values.yaml", + "telemetrystore/telemtrystore-clickhouse-0-*.yaml" + ], + "minLength": 1, + "type": "string" + }, + "type": { + "description": "Patch driver type. Defaults to jsonpatch.", + "default": "jsonpatch", + "examples": [ + "jsonpatch" + ], + "enum": [ + "", + "jsonpatch" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1PatchOperation": { + "required": [ + "op", + "path" + ], + "additionalProperties": false, + "properties": { + "from": { + "description": "Source JSON Pointer for move and copy operations", + "examples": [ + "/services/clickhouse/old_field" + ], + "pattern": "^/", + "type": "string" + }, + "op": { + "description": "JSON Patch (RFC 6902) operation type", + "enum": [ + "add", + "remove", + "replace", + "move", + "copy", + "test" + ], + "type": "string" + }, + "path": { + "description": "JSON Pointer (RFC 6901) to the target location", + "examples": [ + "/services/clickhouse/mem_limit" + ], + "pattern": "^/", + "type": "string" + }, + "value": { + "description": "Value for add, replace, or test operations" + } + }, + "type": "object" + }, + "V1Alpha1Platform": { + "enum": [ + "render", + "coolify", + "railway", + "ecs", + "aws", + "gcp", + "azure" + ], + "type": "string" + }, + "V1Alpha1Status": { + "additionalProperties": false, + "properties": { + "checksum": { + "description": "Checksum of the casting file", + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1TypeCondition": { + "required": [ + "type", + "status" + ], + "additionalProperties": false, + "properties": { + "message": { + "description": "Human-readable message for the condition.", + "type": "string" + }, + "reason": { + "description": "Machine-readable reason for the condition.", + "type": "string" + }, + "status": { + "description": "Status of the condition.", + "examples": [ + "True", + "False" + ], + "type": "string" + }, + "type": { + "description": "Type of the condition.", + "examples": [ + "ResolvedRefs", + "Accepted", + "Programmed" + ], + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1TypeDeployment": { + "additionalProperties": false, + "properties": { + "flavor": { + "$ref": "#/definitions/V1Alpha1Flavor", + "description": "Flavor of mode for the deployment" + }, + "mode": { + "$ref": "#/definitions/V1Alpha1Mode", + "description": "Type of installation method" + }, + "platform": { + "$ref": "#/definitions/V1Alpha1Platform", + "description": "Provider where an installation runs on" + } + }, + "type": "object" + }, + "V1Alpha1TypeMetadata": { + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "annotations": { + "description": "Annotations is an unstructured key-value map for arbitrary metadata. Can be used to specify deployment-specific settings.", + "additionalProperties": { + "type": "string" + }, + "type": [ + "object", + "null" + ] + }, + "name": { + "description": "The name of this installation. This name is used to identify the installation.", + "default": "signoz", + "examples": [ + "signoz" + ], + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + } + }, + "type": "object" + }, + "V1Alpha1TypeResourceRef": { + "required": [ + "kind", + "name" + ], + "additionalProperties": false, + "properties": { + "apiVersion": { + "description": "API version of the referenced casting.", + "examples": [ + "v1alpha1" + ], + "type": "string" + }, + "kind": { + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the referenced casting." + }, + "name": { + "description": "Name of the referenced casting.", + "type": "string" + }, + "path": { + "description": "Relative path to the casting file containing the referenced casting, resolved against the declaring file's directory. Empty means the declaring file itself.", + "type": "string" + }, + "status": { + "$ref": "#/definitions/V1Alpha1TypeResourceRefStatus", + "description": "Status of the reference. This is populated by foundry and written to the lock file." + } + }, + "type": "object" + }, + "V1Alpha1TypeResourceRefStatus": { + "additionalProperties": false, + "properties": { + "checksum": { + "description": "Checksum of the referenced casting source, as seen at forge.", + "type": "string" + }, + "conditions": { + "description": "Conditions observed while resolving the reference.", + "items": { + "$ref": "#/definitions/V1Alpha1TypeCondition" + }, + "type": "array" + }, + "path": { + "description": "Resolved source of the referenced casting.", + "type": "string" + } + }, + "type": "object" + } + }, + "properties": { + "apiVersion": { + "description": "API Version of the configuration schema.", + "default": "v1alpha1", + "examples": [ + "v1alpha1" + ], + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "kind": { + "description": "Kind of the casting resource.", + "enum": [ + "Infrastructure" + ], + "type": "string" + }, + "metadata": { + "$ref": "#/definitions/V1Alpha1TypeMetadata", + "description": "Metadata of the casting configuration" + }, + "spec": { + "$ref": "#/definitions/InfrastructureSpec", + "description": "Infrastructure specification" + }, + "status": { + "$ref": "#/definitions/V1Alpha1Status", + "description": "Status of the casting" + } + }, + "type": "object" +} \ No newline at end of file diff --git a/api/v1alpha1/infrastructure/schema.go b/api/v1alpha1/infrastructure/schema.go new file mode 100644 index 00000000..44dfafe3 --- /dev/null +++ b/api/v1alpha1/infrastructure/schema.go @@ -0,0 +1,18 @@ +package infrastructure + +import ( + _ "embed" + + "github.com/google/jsonschema-go/jsonschema" + "github.com/signoz/foundry/api/v1alpha1" +) + +//go:embed casting.schema.json +var schemaJSON []byte + +var schema = v1alpha1.MustResolveSchema(schemaJSON) + +// Schema returns the resolved JSON schema for an Infrastructure casting. +func Schema() *jsonschema.Resolved { + return schema +} diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go new file mode 100644 index 00000000..f9aaf2b6 --- /dev/null +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -0,0 +1,26 @@ +package infrastructure + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSchema(t *testing.T) { + t.Parallel() + assert.NotNil(t, Schema()) +} + +func TestSchemaValidatesDefault(t *testing.T) { + t.Parallel() + + contents, err := json.Marshal(Default()) + require.NoError(t, err) + + var payload map[string]any + require.NoError(t, json.Unmarshal(contents, &payload)) + + assert.NoError(t, Schema().Validate(payload)) +} diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json index 01a00b04..4f4202b7 100644 --- a/api/v1alpha1/installation/casting.schema.json +++ b/api/v1alpha1/installation/casting.schema.json @@ -488,7 +488,9 @@ }, "V1Alpha1Kind": { "enum": [ - "Installation" + "Installation", + "CollectionAgent", + "Infrastructure" ], "type": "string" }, @@ -754,8 +756,11 @@ "type": "string" }, "kind": { - "$ref": "#/definitions/V1Alpha1Kind", - "description": "Kind of the casting resource." + "description": "Kind of the casting resource.", + "enum": [ + "Installation" + ], + "type": "string" }, "metadata": { "$ref": "#/definitions/V1Alpha1TypeMetadata", diff --git a/api/v1alpha1/molding_kind.go b/api/v1alpha1/molding_kind.go index 488848a8..8ac5ad74 100644 --- a/api/v1alpha1/molding_kind.go +++ b/api/v1alpha1/molding_kind.go @@ -19,6 +19,7 @@ var ( MoldingKindSignoz MoldingKind = MoldingKind{s: "signoz"} MoldingKindCollector MoldingKind = MoldingKind{s: "collector"} MoldingKindMCP MoldingKind = MoldingKind{s: "mcp"} + MoldingKindResource MoldingKind = MoldingKind{s: "resource"} ) type MoldingKind struct { @@ -30,7 +31,7 @@ func (kind MoldingKind) String() string { } func MoldingKinds() []MoldingKind { - return []MoldingKind{MoldingKindIngester, MoldingKindTelemetryStore, MoldingKindTelemetryKeeper, MoldingKindMetaStore, MoldingKindSignoz, MoldingKindCollector} + return []MoldingKind{MoldingKindIngester, MoldingKindTelemetryStore, MoldingKindTelemetryKeeper, MoldingKindMetaStore, MoldingKindSignoz, MoldingKindCollector, MoldingKindResource} } func (kind *MoldingKind) UnmarshalText(text []byte) error { diff --git a/api/v1alpha1/resource.go b/api/v1alpha1/resource.go new file mode 100644 index 00000000..6d36673a --- /dev/null +++ b/api/v1alpha1/resource.go @@ -0,0 +1,31 @@ +package v1alpha1 + +// TypeResourceRef references a casting by identity. Path qualifies the file +// containing the referenced casting when it lives outside the declaring file; +// when empty, the reference resolves among the declaring file's own documents. +type TypeResourceRef struct { + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" description:"API version of the referenced casting." example:"v1alpha1"` + Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the referenced casting."` + Name string `json:"name" yaml:"name" required:"true" description:"Name of the referenced casting."` + Path string `json:"path,omitempty" yaml:"path,omitempty" description:"Relative path to the casting file containing the referenced casting, resolved against the declaring file's directory. Empty means the declaring file itself."` + Status *TypeResourceRefStatus `json:"status,omitempty" yaml:"status,omitempty" description:"Status of the reference. This is populated by foundry and written to the lock file."` + _ struct{} `additionalProperties:"false"` +} + +// TypeResourceRefStatus records how a resource reference resolved at forge time. +type TypeResourceRefStatus struct { + Conditions []TypeCondition `json:"conditions,omitempty" yaml:"conditions,omitempty" description:"Conditions observed while resolving the reference."` + Path string `json:"path,omitempty" yaml:"path,omitempty" description:"Resolved source of the referenced casting."` + Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty" description:"Checksum of the referenced casting source, as seen at forge."` + _ struct{} `additionalProperties:"false"` +} + +// TypeCondition is a single observed condition, mirroring the kubernetes +// condition vocabulary. +type TypeCondition struct { + Type string `json:"type" yaml:"type" required:"true" description:"Type of the condition." examples:"[\"ResolvedRefs\",\"Accepted\",\"Programmed\"]"` + Status string `json:"status" yaml:"status" required:"true" description:"Status of the condition." examples:"[\"True\",\"False\"]"` + Reason string `json:"reason,omitempty" yaml:"reason,omitempty" description:"Machine-readable reason for the condition."` + Message string `json:"message,omitempty" yaml:"message,omitempty" description:"Human-readable message for the condition."` + _ struct{} `additionalProperties:"false"` +} diff --git a/cmd/foundryctl/gen.go b/cmd/foundryctl/gen.go index 839bfe3e..0bcbb989 100644 --- a/cmd/foundryctl/gen.go +++ b/cmd/foundryctl/gen.go @@ -11,6 +11,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/collectionagent" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/api/v1alpha1/installation" installationcasting "github.com/signoz/foundry/internal/casting/installation" "github.com/signoz/foundry/internal/domain" @@ -30,6 +31,7 @@ type schemaTarget struct { var schemaTargets = []schemaTarget{ {v1alpha1.KindInstallation, installation.Casting{}}, {v1alpha1.KindCollectionAgent, collectionagent.Casting{}}, + {v1alpha1.KindInfrastructure, infrastructure.Casting{}}, } func registerGenCmd(rootCmd *cobra.Command) { @@ -100,29 +102,27 @@ func runGenExamples(ctx context.Context, logger *slog.Logger) error { func runGenSchemas(_ context.Context) error { var oneOf []jsonschema.SchemaOrBool - kindType := reflect.TypeFor[v1alpha1.Kind]() for _, t := range schemaTargets { target := t reflector := jsonschema.Reflector{} - // v1alpha1.Kind's Enum() returns all Kinds (the type permits any). - // For this per-Kind schema, the kind field is always this Casting's - // Kind, so we narrow the enum at reflection. - reflector.DefaultOptions = append(reflector.DefaultOptions, - jsonschema.InterceptSchema(func(params jsonschema.InterceptSchemaParams) (bool, error) { - if !params.Processed || params.Value.Type() != kindType { - return false, nil - } - params.Schema.Enum = []any{target.kind.String()} - return false, nil - }), - ) schema, err := reflector.Reflect(target.val) if err != nil { return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "reflect %T", target.val) } + // v1alpha1.Kind's Enum() returns all Kinds (the type permits any, and + // nested Kind-typed fields such as resource refs must keep the full + // enum). Only the TOP-LEVEL kind field is always this Casting's Kind, + // so it is narrowed here with an inline schema instead of the shared + // Kind definition. + narrowedKind := (&jsonschema.Schema{}). + WithType(jsonschema.String.Type()). + WithEnum(target.kind.String()). + WithDescription("Kind of the casting resource.") + schema.WithPropertiesItem("kind", narrowedKind.ToSchemaOrBool()) + contents, err := json.MarshalIndent(schema, "", " ") if err != nil { return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "marshal %T", target.val) diff --git a/internal/casting/infrastructure/casting.go b/internal/casting/infrastructure/casting.go new file mode 100644 index 00000000..52a2d3a5 --- /dev/null +++ b/internal/casting/infrastructure/casting.go @@ -0,0 +1,15 @@ +package infrastructure + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +type Casting interface { + Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) + Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) + Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error +} diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go new file mode 100644 index 00000000..215a5b4c --- /dev/null +++ b/internal/casting/infrastructure/planner.go @@ -0,0 +1,99 @@ +package infrastructure + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/signoz/foundry/internal/planner" + "github.com/signoz/foundry/internal/tooler" +) + +var _ planner.Planner = (*Planner)(nil) + +// Planner is the Infrastructure Kind's per-Kind orchestrator. It satisfies +// the foundry planner contract by exposing this Kind's moldings, enricher, +// and casting strategy as verbs on a single value. +type Planner struct { + config *infrastructure.Casting + logger *slog.Logger + casting Casting + toolers []tooler.Tooler + enricher infrastructuremolding.MoldingEnricher + moldings []infrastructuremolding.Molding +} + +func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Logger) (planner.Planner, error) { + registry := NewRegistry(logger) + + castingStrategy, err := registry.Casting(c.Spec.Deployment) + if err != nil { + return nil, err + } + + toolers, err := registry.Toolers(c.Spec.Deployment) + if err != nil { + return nil, err + } + + enricher, err := castingStrategy.Enricher(ctx, c) + if err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") + } + + moldings := []infrastructuremolding.Molding{ + resourcemolding.New(logger), + } + + return &Planner{ + config: c, + logger: logger, + casting: castingStrategy, + toolers: toolers, + enricher: enricher, + moldings: moldings, + }, nil +} + +func (p *Planner) Machinery() v1alpha1.Machinery { return p.config } +func (p *Planner) Patches() []v1alpha1.PatchEntry { return p.config.Spec.Patches } + +func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { + kinds := make([]v1alpha1.MoldingKind, len(p.moldings)) + for i, m := range p.moldings { + kinds[i] = m.Kind() + } + return kinds +} + +func (p *Planner) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error { + return p.enricher.EnrichStatus(ctx, kind, p.config) +} + +func (p *Planner) Mold(ctx context.Context, kind v1alpha1.MoldingKind) error { + for _, m := range p.moldings { + if m.Kind() == kind { + return m.MoldV1Alpha1(ctx, p.config) + } + } + return foundryerrors.Newf(foundryerrors.TypeInternal, "molding %q not registered for infrastructure planner", kind) +} + +func (p *Planner) MergeStatusIntoSpec() error { + return p.config.MergeStatusIntoSpec() +} + +func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, error) { + return p.casting.Forge(ctx, *p.config, target) +} + +func (p *Planner) Cast(ctx context.Context, poursPath string) error { + return p.casting.Cast(ctx, *p.config, poursPath) +} + +func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go new file mode 100644 index 00000000..e453799a --- /dev/null +++ b/internal/casting/infrastructure/registry.go @@ -0,0 +1,62 @@ +package infrastructure + +import ( + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting/infrastructure/terraformcasting" + foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" +) + +type CastingItem struct { + Casting Casting + Toolers []tooler.Tooler +} + +type Registry struct { + castings map[v1alpha1.TypeDeployment]CastingItem +} + +func NewRegistry(logger *slog.Logger) *Registry { + return &Registry{ + castings: map[v1alpha1.TypeDeployment]CastingItem{ + { + Flavor: v1alpha1.FlavorTerraform, + }: { + Casting: terraformcasting.New(logger), + Toolers: []tooler.Tooler{terraformtooler.New()}, + }, + }, + } +} + +func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingItem, bool) { + if item, ok := registry.castings[deployment]; ok { + return item, true + } + // Fall back to matching without platform: the platform selects the provider + // inside the casting, not the casting itself. + if deployment.Platform != (v1alpha1.Platform{}) { + item, ok := registry.castings[v1alpha1.TypeDeployment{Mode: deployment.Mode, Flavor: deployment.Flavor}] + return item, ok + } + return CastingItem{}, false +} + +func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (Casting, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "infrastructure deployment '%+v' is not supported", deployment) + } + return item.Casting, nil +} + +func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler.Tooler, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "infrastructure deployment '%+v' is not supported", deployment) + } + return item.Toolers, nil +} diff --git a/internal/casting/infrastructure/terraformcasting/casting.go b/internal/casting/infrastructure/terraformcasting/casting.go new file mode 100644 index 00000000..4d24445b --- /dev/null +++ b/internal/casting/infrastructure/terraformcasting/casting.go @@ -0,0 +1,36 @@ +package terraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +type terraformCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *terraformCasting { + return &terraformCasting{logger: logger} +} + +func (c *terraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return &enricher{logger: c.logger}, nil +} + +// Forge renders the terraform materials for the provisioned infrastructure. +// Scaffolding: rendering lands with the provision templates. +func (c *terraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { + c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, no materials generated yet") + return nil, nil +} + +// Cast applies the forged terraform and captures its outputs. +// Scaffolding: applying lands with the forge implementation. +func (c *terraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { + c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, nothing to cast yet") + return nil +} diff --git a/internal/casting/infrastructure/terraformcasting/enricher.go b/internal/casting/infrastructure/terraformcasting/enricher.go new file mode 100644 index 00000000..9f9d9038 --- /dev/null +++ b/internal/casting/infrastructure/terraformcasting/enricher.go @@ -0,0 +1,20 @@ +package terraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) + +type enricher struct { + logger *slog.Logger +} + +func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { + return nil +} diff --git a/internal/config/yamlconfig/config.go b/internal/config/yamlconfig/config.go index 520dabca..ca05ed46 100644 --- a/internal/config/yamlconfig/config.go +++ b/internal/config/yamlconfig/config.go @@ -9,6 +9,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/collectionagent" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/api/v1alpha1/installation" installationcompat "github.com/signoz/foundry/internal/compat/installation" "github.com/signoz/foundry/internal/config" @@ -28,6 +29,7 @@ func New(logger *slog.Logger) config.Config { c.loaders = map[v1alpha1.Kind]loaderFn{ v1alpha1.KindInstallation: c.loadInstallation, v1alpha1.KindCollectionAgent: c.loadCollectionAgent, + v1alpha1.KindInfrastructure: c.loadInfrastructure, } return c } @@ -126,6 +128,33 @@ func (*yamlConfig) loadCollectionAgent(bytes []byte, path string) (v1alpha1.Mach return base, nil } +func (*yamlConfig) loadInfrastructure(bytes []byte, path string) (v1alpha1.Machinery, error) { + var loaded infrastructure.Casting + if err := domain.UnmarshalYAML(bytes, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal infrastructure casting") + } + + base := infrastructure.Default() + if err := v1alpha1.Merge(base, &loaded); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge default infrastructure casting") + } + + contents, err := json.Marshal(base) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal infrastructure casting") + } + toValidate := map[string]any{} + if err := json.Unmarshal(contents, &toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal infrastructure casting for validation") + } + + if err := infrastructure.Schema().Validate(toValidate); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid casting file %s", path) + } + + return base, nil +} + // CreateV1Alpha1Lock writes the resolved casting to the lock file. func (*yamlConfig) CreateV1Alpha1Lock(ctx context.Context, machinery v1alpha1.Machinery, path string) error { contents, err := domain.MarshalYAML(machinery) @@ -165,6 +194,12 @@ func (*yamlConfig) GetV1Alpha1Lock(ctx context.Context, path string) (v1alpha1.M return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal collectionagent casting") } return &c, nil + case v1alpha1.KindInfrastructure: + var c infrastructure.Casting + if err := domain.UnmarshalYAML(bytes, &c); err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal infrastructure casting") + } + return &c, nil } return nil, errors.Newf(errors.TypeUnsupported, "unknown casting kind %q", kind) } diff --git a/internal/foundry/foundry.go b/internal/foundry/foundry.go index d544a132..b4f4ec7b 100644 --- a/internal/foundry/foundry.go +++ b/internal/foundry/foundry.go @@ -6,8 +6,10 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/collectionagent" + infrastructurev1alpha1 "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/api/v1alpha1/installation" collectionagentcasting "github.com/signoz/foundry/internal/casting/collectionagent" + infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure" installationcasting "github.com/signoz/foundry/internal/casting/installation" "github.com/signoz/foundry/internal/config" "github.com/signoz/foundry/internal/config/yamlconfig" @@ -52,6 +54,9 @@ func New(logger *slog.Logger) (*Foundry, error) { v1alpha1.KindCollectionAgent: func(ctx context.Context, m v1alpha1.Machinery, logger *slog.Logger) (planner.Planner, error) { return collectionagentcasting.NewPlanner(ctx, m.(*collectionagent.Casting), logger) }, + v1alpha1.KindInfrastructure: func(ctx context.Context, m v1alpha1.Machinery, logger *slog.Logger) (planner.Planner, error) { + return infrastructurecasting.NewPlanner(ctx, m.(*infrastructurev1alpha1.Casting), logger) + }, }, InfrastructureGenerator: terraformgenerator.New(logger), }, nil diff --git a/internal/molding/infrastructure/molding.go b/internal/molding/infrastructure/molding.go new file mode 100644 index 00000000..f0c4d629 --- /dev/null +++ b/internal/molding/infrastructure/molding.go @@ -0,0 +1,17 @@ +package infrastructure + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" +) + +type MoldingEnricher interface { + EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error +} + +type Molding interface { + Kind() v1alpha1.MoldingKind + MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error +} diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go new file mode 100644 index 00000000..3f9d4536 --- /dev/null +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -0,0 +1,31 @@ +package resourcemolding + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +var _ infrastructuremolding.Molding = (*resourceMolding)(nil) + +type resourceMolding struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *resourceMolding { + return &resourceMolding{logger: logger} +} + +func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { + return v1alpha1.MoldingKindResource +} + +// MoldV1Alpha1 derives the resource's infrastructure record from the resolved +// reference. Scaffolding: reference resolution and derivation land with the +// forge implementation. +func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { + return nil +} From e5ebf041dfeb2b1c0db4f3b4519bc26defd01bac Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 15:01:27 +0530 Subject: [PATCH 02/38] refactor(infrastructure): slim resource ref to identity and add status outputs --- api/v1alpha1/casting_meta.go | 8 ++- .../collectionagent/casting.schema.json | 5 ++ .../infrastructure/casting.schema.json | 67 ++----------------- api/v1alpha1/installation/casting.schema.json | 5 ++ api/v1alpha1/resource.go | 33 ++------- 5 files changed, 25 insertions(+), 93 deletions(-) diff --git a/api/v1alpha1/casting_meta.go b/api/v1alpha1/casting_meta.go index 483d2d75..9a33b6e4 100644 --- a/api/v1alpha1/casting_meta.go +++ b/api/v1alpha1/casting_meta.go @@ -10,8 +10,10 @@ type CastingMeta struct { _ struct{} `additionalProperties:"false"` } -// Status carries the casting file's checksum. +// Status carries the casting file's checksum and, for kinds whose cast +// captures provider outputs, those outputs verbatim. type Status struct { - Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` - _ struct{} `additionalProperties:"false"` + Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` + Outputs map[string]any `json:"outputs,omitempty" yaml:"outputs,omitempty" description:"Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge."` + _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/collectionagent/casting.schema.json b/api/v1alpha1/collectionagent/casting.schema.json index 9ab23179..35b62792 100644 --- a/api/v1alpha1/collectionagent/casting.schema.json +++ b/api/v1alpha1/collectionagent/casting.schema.json @@ -260,6 +260,11 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" + }, + "outputs": { + "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", + "additionalProperties": {}, + "type": "object" } }, "type": "object" diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 1367af9c..d566fa6e 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -166,41 +166,11 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" - } - }, - "type": "object" - }, - "V1Alpha1TypeCondition": { - "required": [ - "type", - "status" - ], - "additionalProperties": false, - "properties": { - "message": { - "description": "Human-readable message for the condition.", - "type": "string" }, - "reason": { - "description": "Machine-readable reason for the condition.", - "type": "string" - }, - "status": { - "description": "Status of the condition.", - "examples": [ - "True", - "False" - ], - "type": "string" - }, - "type": { - "description": "Type of the condition.", - "examples": [ - "ResolvedRefs", - "Accepted", - "Programmed" - ], - "type": "string" + "outputs": { + "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", + "additionalProperties": {}, + "type": "object" } }, "type": "object" @@ -273,35 +243,6 @@ "name": { "description": "Name of the referenced casting.", "type": "string" - }, - "path": { - "description": "Relative path to the casting file containing the referenced casting, resolved against the declaring file's directory. Empty means the declaring file itself.", - "type": "string" - }, - "status": { - "$ref": "#/definitions/V1Alpha1TypeResourceRefStatus", - "description": "Status of the reference. This is populated by foundry and written to the lock file." - } - }, - "type": "object" - }, - "V1Alpha1TypeResourceRefStatus": { - "additionalProperties": false, - "properties": { - "checksum": { - "description": "Checksum of the referenced casting source, as seen at forge.", - "type": "string" - }, - "conditions": { - "description": "Conditions observed while resolving the reference.", - "items": { - "$ref": "#/definitions/V1Alpha1TypeCondition" - }, - "type": "array" - }, - "path": { - "description": "Resolved source of the referenced casting.", - "type": "string" } }, "type": "object" diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json index 4f4202b7..6955282b 100644 --- a/api/v1alpha1/installation/casting.schema.json +++ b/api/v1alpha1/installation/casting.schema.json @@ -650,6 +650,11 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" + }, + "outputs": { + "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", + "additionalProperties": {}, + "type": "object" } }, "type": "object" diff --git a/api/v1alpha1/resource.go b/api/v1alpha1/resource.go index 6d36673a..9c7d6788 100644 --- a/api/v1alpha1/resource.go +++ b/api/v1alpha1/resource.go @@ -1,31 +1,10 @@ package v1alpha1 -// TypeResourceRef references a casting by identity. Path qualifies the file -// containing the referenced casting when it lives outside the declaring file; -// when empty, the reference resolves among the declaring file's own documents. +// TypeResourceRef references a casting by identity. References resolve among +// the declaring file's own documents. type TypeResourceRef struct { - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" description:"API version of the referenced casting." example:"v1alpha1"` - Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the referenced casting."` - Name string `json:"name" yaml:"name" required:"true" description:"Name of the referenced casting."` - Path string `json:"path,omitempty" yaml:"path,omitempty" description:"Relative path to the casting file containing the referenced casting, resolved against the declaring file's directory. Empty means the declaring file itself."` - Status *TypeResourceRefStatus `json:"status,omitempty" yaml:"status,omitempty" description:"Status of the reference. This is populated by foundry and written to the lock file."` - _ struct{} `additionalProperties:"false"` -} - -// TypeResourceRefStatus records how a resource reference resolved at forge time. -type TypeResourceRefStatus struct { - Conditions []TypeCondition `json:"conditions,omitempty" yaml:"conditions,omitempty" description:"Conditions observed while resolving the reference."` - Path string `json:"path,omitempty" yaml:"path,omitempty" description:"Resolved source of the referenced casting."` - Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty" description:"Checksum of the referenced casting source, as seen at forge."` - _ struct{} `additionalProperties:"false"` -} - -// TypeCondition is a single observed condition, mirroring the kubernetes -// condition vocabulary. -type TypeCondition struct { - Type string `json:"type" yaml:"type" required:"true" description:"Type of the condition." examples:"[\"ResolvedRefs\",\"Accepted\",\"Programmed\"]"` - Status string `json:"status" yaml:"status" required:"true" description:"Status of the condition." examples:"[\"True\",\"False\"]"` - Reason string `json:"reason,omitempty" yaml:"reason,omitempty" description:"Machine-readable reason for the condition."` - Message string `json:"message,omitempty" yaml:"message,omitempty" description:"Human-readable message for the condition."` - _ struct{} `additionalProperties:"false"` + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" description:"API version of the referenced casting." example:"v1alpha1"` + Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the referenced casting."` + Name string `json:"name" yaml:"name" required:"true" description:"Name of the referenced casting."` + _ struct{} `additionalProperties:"false"` } From d06176c21a6a1c21f5599ab4983471cf60061f19 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 15:32:16 +0530 Subject: [PATCH 03/38] refactor(v1alpha1): drop outputs from the shared casting status --- api/v1alpha1/casting_meta.go | 8 +++----- api/v1alpha1/collectionagent/casting.schema.json | 5 ----- api/v1alpha1/infrastructure/casting.schema.json | 5 ----- api/v1alpha1/installation/casting.schema.json | 5 ----- 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/api/v1alpha1/casting_meta.go b/api/v1alpha1/casting_meta.go index 9a33b6e4..483d2d75 100644 --- a/api/v1alpha1/casting_meta.go +++ b/api/v1alpha1/casting_meta.go @@ -10,10 +10,8 @@ type CastingMeta struct { _ struct{} `additionalProperties:"false"` } -// Status carries the casting file's checksum and, for kinds whose cast -// captures provider outputs, those outputs verbatim. +// Status carries the casting file's checksum. type Status struct { - Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` - Outputs map[string]any `json:"outputs,omitempty" yaml:"outputs,omitempty" description:"Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge."` - _ struct{} `additionalProperties:"false"` + Checksum string `json:"checksum" yaml:"checksum" description:"Checksum of the casting file"` + _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/collectionagent/casting.schema.json b/api/v1alpha1/collectionagent/casting.schema.json index 35b62792..9ab23179 100644 --- a/api/v1alpha1/collectionagent/casting.schema.json +++ b/api/v1alpha1/collectionagent/casting.schema.json @@ -260,11 +260,6 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" - }, - "outputs": { - "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", - "additionalProperties": {}, - "type": "object" } }, "type": "object" diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index d566fa6e..14358082 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -166,11 +166,6 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" - }, - "outputs": { - "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", - "additionalProperties": {}, - "type": "object" } }, "type": "object" diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json index 6955282b..4f4202b7 100644 --- a/api/v1alpha1/installation/casting.schema.json +++ b/api/v1alpha1/installation/casting.schema.json @@ -650,11 +650,6 @@ "checksum": { "description": "Checksum of the casting file", "type": "string" - }, - "outputs": { - "description": "Provider outputs captured by cast, verbatim. Written only by cast and preserved by forge.", - "additionalProperties": {}, - "type": "object" } }, "type": "object" From 9e819c96fbfca461a5786dfa37a922a527231de8 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 15:42:28 +0530 Subject: [PATCH 04/38] refactor(infrastructure): apply schema-first fixes to the scaffolding - resource ref: no fabricated default, apiVersion enum, name pattern - reject absent resource kind before the merge can default it to Installation - reject unsupported consumer kinds (self-reference) at load - drop the no-op resource molding: the kind has zero moldings - rework tests to the Subject_Outcome table idiom; add loader coverage --- api/v1alpha1/infrastructure/casting.go | 5 - .../infrastructure/casting.schema.json | 5 + api/v1alpha1/infrastructure/schema_test.go | 71 ++++++++++++-- api/v1alpha1/molding_kind.go | 3 +- api/v1alpha1/resource.go | 4 +- internal/casting/infrastructure/casting.go | 2 - internal/casting/infrastructure/planner.go | 54 +++-------- .../terraformcasting/casting.go | 9 -- .../terraformcasting/enricher.go | 20 ---- internal/config/yamlconfig/config.go | 11 +++ internal/config/yamlconfig/config_test.go | 96 +++++++++++++++++++ internal/molding/infrastructure/molding.go | 17 ---- .../resourcemolding/resource.go | 31 ------ .../config.clickhouse.v25125.yaml.gotmpl | 2 +- 14 files changed, 192 insertions(+), 138 deletions(-) delete mode 100644 internal/casting/infrastructure/terraformcasting/enricher.go delete mode 100644 internal/molding/infrastructure/molding.go delete mode 100644 internal/molding/infrastructure/resourcemolding/resource.go diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index b8e471a0..8926a3d8 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -32,11 +32,6 @@ func Default() *Casting { }, Spec: Spec{ Deployment: v1alpha1.TypeDeployment{Flavor: v1alpha1.FlavorTerraform}, - Resource: v1alpha1.TypeResourceRef{ - APIVersion: "v1alpha1", - Kind: v1alpha1.KindInstallation, - Name: "signoz", - }, }, } } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 14358082..1202b4e7 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -229,6 +229,9 @@ "examples": [ "v1alpha1" ], + "enum": [ + "v1alpha1" + ], "type": "string" }, "kind": { @@ -237,6 +240,8 @@ }, "name": { "description": "Name of the referenced casting.", + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", "type": "string" } }, diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index f9aaf2b6..cdecacde 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -4,23 +4,76 @@ import ( "encoding/json" "testing" + "github.com/signoz/foundry/api/v1alpha1" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestSchema(t *testing.T) { - t.Parallel() assert.NotNil(t, Schema()) } -func TestSchemaValidatesDefault(t *testing.T) { - t.Parallel() +func TestSchemaValidate(t *testing.T) { + tests := []struct { + name string + mutate func(casting *Casting) + pass bool + }{ + { + name: "ResourceProvided_Valid", + mutate: func(casting *Casting) { + casting.Spec.Resource = v1alpha1.TypeResourceRef{ + APIVersion: "v1alpha1", + Kind: v1alpha1.KindInstallation, + Name: "signoz", + } + }, + pass: true, + }, + { + name: "ResourceAPIVersionOmitted_Valid", + mutate: func(casting *Casting) { + casting.Spec.Resource = v1alpha1.TypeResourceRef{ + Kind: v1alpha1.KindCollectionAgent, + Name: "signoz-gateway", + } + }, + pass: true, + }, + { + name: "ResourceMissing_Invalid", + mutate: func(casting *Casting) {}, + pass: false, + }, + { + name: "ResourceAPIVersionUnknown_Invalid", + mutate: func(casting *Casting) { + casting.Spec.Resource = v1alpha1.TypeResourceRef{ + APIVersion: "v2", + Kind: v1alpha1.KindInstallation, + Name: "signoz", + } + }, + pass: false, + }, + } - contents, err := json.Marshal(Default()) - require.NoError(t, err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + casting := Default() + tt.mutate(casting) - var payload map[string]any - require.NoError(t, json.Unmarshal(contents, &payload)) + contents, err := json.Marshal(casting) + assert.NoError(t, err) - assert.NoError(t, Schema().Validate(payload)) + payload := map[string]any{} + assert.NoError(t, json.Unmarshal(contents, &payload)) + + err = Schema().Validate(payload) + if !tt.pass { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } } diff --git a/api/v1alpha1/molding_kind.go b/api/v1alpha1/molding_kind.go index 8ac5ad74..488848a8 100644 --- a/api/v1alpha1/molding_kind.go +++ b/api/v1alpha1/molding_kind.go @@ -19,7 +19,6 @@ var ( MoldingKindSignoz MoldingKind = MoldingKind{s: "signoz"} MoldingKindCollector MoldingKind = MoldingKind{s: "collector"} MoldingKindMCP MoldingKind = MoldingKind{s: "mcp"} - MoldingKindResource MoldingKind = MoldingKind{s: "resource"} ) type MoldingKind struct { @@ -31,7 +30,7 @@ func (kind MoldingKind) String() string { } func MoldingKinds() []MoldingKind { - return []MoldingKind{MoldingKindIngester, MoldingKindTelemetryStore, MoldingKindTelemetryKeeper, MoldingKindMetaStore, MoldingKindSignoz, MoldingKindCollector, MoldingKindResource} + return []MoldingKind{MoldingKindIngester, MoldingKindTelemetryStore, MoldingKindTelemetryKeeper, MoldingKindMetaStore, MoldingKindSignoz, MoldingKindCollector} } func (kind *MoldingKind) UnmarshalText(text []byte) error { diff --git a/api/v1alpha1/resource.go b/api/v1alpha1/resource.go index 9c7d6788..71ab90de 100644 --- a/api/v1alpha1/resource.go +++ b/api/v1alpha1/resource.go @@ -3,8 +3,8 @@ package v1alpha1 // TypeResourceRef references a casting by identity. References resolve among // the declaring file's own documents. type TypeResourceRef struct { - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" description:"API version of the referenced casting." example:"v1alpha1"` + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" enum:"v1alpha1" description:"API version of the referenced casting." example:"v1alpha1"` Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the referenced casting."` - Name string `json:"name" yaml:"name" required:"true" description:"Name of the referenced casting."` + Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` _ struct{} `additionalProperties:"false"` } diff --git a/internal/casting/infrastructure/casting.go b/internal/casting/infrastructure/casting.go index 52a2d3a5..c63c4f43 100644 --- a/internal/casting/infrastructure/casting.go +++ b/internal/casting/infrastructure/casting.go @@ -5,11 +5,9 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) type Casting interface { - Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error } diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 215a5b4c..59ae10a1 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -8,24 +8,20 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" - "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" "github.com/signoz/foundry/internal/planner" "github.com/signoz/foundry/internal/tooler" ) var _ planner.Planner = (*Planner)(nil) -// Planner is the Infrastructure Kind's per-Kind orchestrator. It satisfies -// the foundry planner contract by exposing this Kind's moldings, enricher, -// and casting strategy as verbs on a single value. +// Planner is the Infrastructure Kind's per-Kind orchestrator. The Kind has no +// moldings: reference validation happens at planner construction and all +// derivation happens inside the casting at forge. type Planner struct { - config *infrastructure.Casting - logger *slog.Logger - casting Casting - toolers []tooler.Tooler - enricher infrastructuremolding.MoldingEnricher - moldings []infrastructuremolding.Molding + config *infrastructure.Casting + logger *slog.Logger + casting Casting + toolers []tooler.Tooler } func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Logger) (planner.Planner, error) { @@ -41,47 +37,25 @@ func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Log return nil, err } - enricher, err := castingStrategy.Enricher(ctx, c) - if err != nil { - return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") - } - - moldings := []infrastructuremolding.Molding{ - resourcemolding.New(logger), - } - return &Planner{ - config: c, - logger: logger, - casting: castingStrategy, - toolers: toolers, - enricher: enricher, - moldings: moldings, + config: c, + logger: logger, + casting: castingStrategy, + toolers: toolers, }, nil } func (p *Planner) Machinery() v1alpha1.Machinery { return p.config } func (p *Planner) Patches() []v1alpha1.PatchEntry { return p.config.Spec.Patches } -func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { - kinds := make([]v1alpha1.MoldingKind, len(p.moldings)) - for i, m := range p.moldings { - kinds[i] = m.Kind() - } - return kinds -} +func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { return nil } func (p *Planner) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error { - return p.enricher.EnrichStatus(ctx, kind, p.config) + return foundryerrors.Newf(foundryerrors.TypeInternal, "infrastructure has no moldings") } func (p *Planner) Mold(ctx context.Context, kind v1alpha1.MoldingKind) error { - for _, m := range p.moldings { - if m.Kind() == kind { - return m.MoldV1Alpha1(ctx, p.config) - } - } - return foundryerrors.Newf(foundryerrors.TypeInternal, "molding %q not registered for infrastructure planner", kind) + return foundryerrors.Newf(foundryerrors.TypeInternal, "infrastructure has no moldings") } func (p *Planner) MergeStatusIntoSpec() error { diff --git a/internal/casting/infrastructure/terraformcasting/casting.go b/internal/casting/infrastructure/terraformcasting/casting.go index 4d24445b..6fd6f367 100644 --- a/internal/casting/infrastructure/terraformcasting/casting.go +++ b/internal/casting/infrastructure/terraformcasting/casting.go @@ -6,7 +6,6 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) type terraformCasting struct { @@ -17,19 +16,11 @@ func New(logger *slog.Logger) *terraformCasting { return &terraformCasting{logger: logger} } -func (c *terraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { - return &enricher{logger: c.logger}, nil -} - -// Forge renders the terraform materials for the provisioned infrastructure. -// Scaffolding: rendering lands with the provision templates. func (c *terraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, no materials generated yet") return nil, nil } -// Cast applies the forged terraform and captures its outputs. -// Scaffolding: applying lands with the forge implementation. func (c *terraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, nothing to cast yet") return nil diff --git a/internal/casting/infrastructure/terraformcasting/enricher.go b/internal/casting/infrastructure/terraformcasting/enricher.go deleted file mode 100644 index 9f9d9038..00000000 --- a/internal/casting/infrastructure/terraformcasting/enricher.go +++ /dev/null @@ -1,20 +0,0 @@ -package terraformcasting - -import ( - "context" - "log/slog" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) - -type enricher struct { - logger *slog.Logger -} - -func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { - return nil -} diff --git a/internal/config/yamlconfig/config.go b/internal/config/yamlconfig/config.go index ca05ed46..8a49f8ce 100644 --- a/internal/config/yamlconfig/config.go +++ b/internal/config/yamlconfig/config.go @@ -134,6 +134,13 @@ func (*yamlConfig) loadInfrastructure(bytes []byte, path string) (v1alpha1.Machi return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal infrastructure casting") } + // Checked before the merge: merging round-trips through JSON, where an + // absent kind unmarshals to the backward-compatibility default and would + // silently pass validation as an Installation reference. + if loaded.Spec.Resource.Kind == (v1alpha1.Kind{}) { + return nil, errors.Newf(errors.TypeInvalidInput, "invalid casting file %s: spec.resource.kind is required", path) + } + base := infrastructure.Default() if err := v1alpha1.Merge(base, &loaded); err != nil { return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge default infrastructure casting") @@ -152,6 +159,10 @@ func (*yamlConfig) loadInfrastructure(bytes []byte, path string) (v1alpha1.Machi return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid casting file %s", path) } + if kind := base.Spec.Resource.Kind; kind != v1alpha1.KindInstallation && kind != v1alpha1.KindCollectionAgent { + return nil, errors.Newf(errors.TypeInvalidInput, "invalid casting file %s: infrastructure cannot serve resource kind %q, supported kinds are %q and %q", path, kind, v1alpha1.KindInstallation, v1alpha1.KindCollectionAgent) + } + return base, nil } diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go index e0bb0717..8d0aea52 100644 --- a/internal/config/yamlconfig/config_test.go +++ b/internal/config/yamlconfig/config_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/api/v1alpha1/installation" "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" @@ -370,3 +371,98 @@ func TestGetV1Alpha1Merge(t *testing.T) { }) } } + +func TestGetV1Alpha1Infrastructure(t *testing.T) { + tests := []struct { + name string + input string + expectedResource v1alpha1.Kind + pass bool + }{ + { + name: "InstallationResource_Valid", + input: ` +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: ecs + flavor: terraform + resource: + kind: Installation + name: signoz +`, + expectedResource: v1alpha1.KindInstallation, + pass: true, + }, + { + name: "ResourceMissing_Invalid", + input: ` +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: ecs + flavor: terraform +`, + pass: false, + }, + { + name: "SelfReference_Invalid", + input: ` +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: ecs + flavor: terraform + resource: + kind: Infrastructure + name: signoz +`, + pass: false, + }, + { + name: "ResourceKindMissing_Invalid", + input: ` +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: ecs + flavor: terraform + resource: + name: signoz +`, + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + castingPath := filepath.Join(t.TempDir(), "casting.yaml") + assert.NoError(t, os.WriteFile(castingPath, []byte(tt.input), 0644)) + + cfg := New(slog.New(slog.DiscardHandler)) + machinery, err := cfg.GetV1Alpha1(context.Background(), castingPath) + if !tt.pass { + assert.Error(t, err) + return + } + assert.NoError(t, err) + + casting, ok := machinery.(*infrastructure.Casting) + assert.True(t, ok) + assert.Equal(t, v1alpha1.KindInfrastructure, casting.Kind()) + assert.Equal(t, tt.expectedResource, casting.Spec.Resource.Kind) + }) + } +} diff --git a/internal/molding/infrastructure/molding.go b/internal/molding/infrastructure/molding.go deleted file mode 100644 index f0c4d629..00000000 --- a/internal/molding/infrastructure/molding.go +++ /dev/null @@ -1,17 +0,0 @@ -package infrastructure - -import ( - "context" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" -) - -type MoldingEnricher interface { - EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error -} - -type Molding interface { - Kind() v1alpha1.MoldingKind - MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error -} diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go deleted file mode 100644 index 3f9d4536..00000000 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ /dev/null @@ -1,31 +0,0 @@ -package resourcemolding - -import ( - "context" - "log/slog" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -var _ infrastructuremolding.Molding = (*resourceMolding)(nil) - -type resourceMolding struct { - logger *slog.Logger -} - -func New(logger *slog.Logger) *resourceMolding { - return &resourceMolding{logger: logger} -} - -func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { - return v1alpha1.MoldingKindResource -} - -// MoldV1Alpha1 derives the resource's infrastructure record from the resolved -// reference. Scaffolding: reference resolution and derivation land with the -// forge implementation. -func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { - return nil -} diff --git a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl index 9b485f81..8671e987 100644 --- a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl +++ b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl @@ -35,7 +35,7 @@ quotas: result_rows: 0 user_directories: users_xml: - path: users.xml + path: config.yaml remote_servers: cluster: shard: From d73e7959420b4913025d03283262b8edc56c6dfd Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 15:48:10 +0530 Subject: [PATCH 05/38] chore(telemetrystore): revert unintended template change --- .../templates/config.clickhouse.v25125.yaml.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl index 8671e987..9b485f81 100644 --- a/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl +++ b/internal/molding/telemetrystoremolding/templates/config.clickhouse.v25125.yaml.gotmpl @@ -35,7 +35,7 @@ quotas: result_rows: 0 user_directories: users_xml: - path: config.yaml + path: users.xml remote_servers: cluster: shard: From 654bbc7f96a2eda19ee8615c9ad1293102151bff Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 18:22:01 +0530 Subject: [PATCH 06/38] refactor(infrastructure): reshape resource as the kind's molding --- api/v1alpha1/casting_ref.go | 9 ++ api/v1alpha1/infrastructure/casting.go | 11 +- .../infrastructure/casting.schema.json | 153 ++++++++++++++---- api/v1alpha1/infrastructure/resource.go | 41 +++++ api/v1alpha1/infrastructure/resource_kind.go | 82 ++++++++++ api/v1alpha1/infrastructure/schema_test.go | 34 ++-- api/v1alpha1/molding_kind.go | 1 + api/v1alpha1/resource.go | 10 -- internal/casting/infrastructure/casting.go | 2 + internal/casting/infrastructure/planner.go | 54 +++++-- internal/casting/infrastructure/registry.go | 18 +-- .../terraformcasting/casting.go | 5 + .../terraformcasting/enricher.go | 20 +++ internal/config/yamlconfig/config.go | 11 -- internal/config/yamlconfig/config_test.go | 18 ++- internal/molding/infrastructure/molding.go | 17 ++ .../resourcemolding/resource.go | 47 ++++++ .../resourcemolding/resource_test.go | 55 +++++++ 18 files changed, 488 insertions(+), 100 deletions(-) create mode 100644 api/v1alpha1/casting_ref.go create mode 100644 api/v1alpha1/infrastructure/resource.go create mode 100644 api/v1alpha1/infrastructure/resource_kind.go delete mode 100644 api/v1alpha1/resource.go create mode 100644 internal/casting/infrastructure/terraformcasting/enricher.go create mode 100644 internal/molding/infrastructure/molding.go create mode 100644 internal/molding/infrastructure/resourcemolding/resource.go create mode 100644 internal/molding/infrastructure/resourcemolding/resource_test.go diff --git a/api/v1alpha1/casting_ref.go b/api/v1alpha1/casting_ref.go new file mode 100644 index 00000000..4b7cbbde --- /dev/null +++ b/api/v1alpha1/casting_ref.go @@ -0,0 +1,9 @@ +package v1alpha1 + +// TypeCastingRef is the identity of a casting: which schema version and which +// name. +type TypeCastingRef struct { + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" enum:"v1alpha1" default:"v1alpha1" description:"API version of the referenced casting." example:"v1alpha1"` + Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` + _ struct{} `additionalProperties:"false"` +} diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index 8926a3d8..3994c8d3 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -14,10 +14,10 @@ type Casting struct { // Spec is the Infrastructure-specific configuration. type Spec struct { - Deployment v1alpha1.TypeDeployment `json:"deployment" yaml:"deployment" required:"true" description:"Deployment configuration for the platform"` - Resource v1alpha1.TypeResourceRef `json:"resource" yaml:"resource" required:"true" description:"The resource this infrastructure serves"` - Patches []v1alpha1.PatchEntry `json:"patches,omitempty" yaml:"patches,omitempty" description:"Patch operations to apply to generated materials"` - _ struct{} `additionalProperties:"false"` + Deployment v1alpha1.TypeDeployment `json:"deployment" yaml:"deployment" required:"true" description:"Deployment configuration for the platform"` + Resource Resource `json:"resource" yaml:"resource" required:"true" description:"The configuration for the resource molding"` + Patches []v1alpha1.PatchEntry `json:"patches,omitempty" yaml:"patches,omitempty" description:"Patch operations to apply to generated materials"` + _ struct{} `additionalProperties:"false"` } var _ v1alpha1.Machinery = (*Casting)(nil) @@ -55,7 +55,8 @@ func (c *Casting) Kind() v1alpha1.Kind { } // MergeStatusIntoSpec folds molding-written status into spec. The resource -// reference's status is its own home; nothing shadows spec fields. +// molding's unit stays in status (addresses are read from status, as on other +// kinds); its spec carries identity only, so nothing merges. func (c *Casting) MergeStatusIntoSpec() error { return nil } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 1202b4e7..99744cdf 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -7,6 +7,112 @@ ], "additionalProperties": false, "definitions": { + "InfrastructureResource": { + "required": [ + "kind", + "spec" + ], + "additionalProperties": false, + "properties": { + "kind": { + "$ref": "#/definitions/InfrastructureResourceKind", + "description": "Kind of the resource this infrastructure serves", + "examples": [ + "Installation" + ] + }, + "spec": { + "$ref": "#/definitions/InfrastructureResourceSpec", + "description": "Specification for the resource" + }, + "status": { + "$ref": "#/definitions/InfrastructureResourceStatus", + "description": "Status of the resource" + } + }, + "type": "object" + }, + "InfrastructureResourceKind": { + "enum": [ + "Installation", + "CollectionAgent" + ], + "type": "string" + }, + "InfrastructureResourceSpec": { + "required": [ + "name" + ], + "additionalProperties": false, + "properties": { + "apiVersion": { + "description": "API version of the referenced casting.", + "default": "v1alpha1", + "examples": [ + "v1alpha1" + ], + "enum": [ + "v1alpha1" + ], + "type": "string" + }, + "name": { + "description": "Name of the referenced casting.", + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" + } + }, + "type": "object" + }, + "InfrastructureResourceStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InfrastructureResourceStatusAddresses", + "description": "Addresses the resource exposes" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InfrastructureResourceStatusAddresses": { + "additionalProperties": false, + "properties": { + "otlp": { + "description": "OTLP addresses", + "items": { + "type": "string" + }, + "type": "array" + }, + "ui": { + "description": "UI addresses", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, "InfrastructureSpec": { "required": [ "deployment", @@ -26,8 +132,8 @@ "type": "array" }, "resource": { - "$ref": "#/definitions/V1Alpha1TypeResourceRef", - "description": "The resource this infrastructure serves" + "$ref": "#/definitions/InfrastructureResource", + "description": "The configuration for the resource molding" } }, "type": "object" @@ -170,6 +276,19 @@ }, "type": "object" }, + "V1Alpha1TypeConfig": { + "additionalProperties": false, + "properties": { + "data": { + "description": "Configuration data as key-value pairs.", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, "V1Alpha1TypeDeployment": { "additionalProperties": false, "properties": { @@ -216,36 +335,6 @@ } }, "type": "object" - }, - "V1Alpha1TypeResourceRef": { - "required": [ - "kind", - "name" - ], - "additionalProperties": false, - "properties": { - "apiVersion": { - "description": "API version of the referenced casting.", - "examples": [ - "v1alpha1" - ], - "enum": [ - "v1alpha1" - ], - "type": "string" - }, - "kind": { - "$ref": "#/definitions/V1Alpha1Kind", - "description": "Kind of the referenced casting." - }, - "name": { - "description": "Name of the referenced casting.", - "maxLength": 63, - "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", - "type": "string" - } - }, - "type": "object" } }, "properties": { diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go new file mode 100644 index 00000000..dbb5123f --- /dev/null +++ b/api/v1alpha1/infrastructure/resource.go @@ -0,0 +1,41 @@ +package infrastructure + +import "github.com/signoz/foundry/api/v1alpha1" + +// Resource is the infrastructure kind's molding: the unit to be hosted. Its +// kinds are the consumer casting kinds. +type Resource struct { + // Kind of the resource this infrastructure serves. + Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` + + // Specification for the resource. + Spec ResourceSpec `json:"spec" yaml:"spec" required:"true" description:"Specification for the resource"` + + // Status of the resource. + Status ResourceStatus `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the resource"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceSpec carries the identity of the casting embodying the resource. +type ResourceSpec struct { + v1alpha1.TypeCastingRef `json:",inline" yaml:",inline"` + + _ struct{} `additionalProperties:"false"` +} + +type ResourceStatus struct { + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` + + Addresses ResourceStatusAddresses `json:"addresses,omitzero" yaml:"addresses,omitempty" description:"Addresses the resource exposes"` + + _ struct{} `additionalProperties:"false"` +} + +type ResourceStatusAddresses struct { + OTLP []string `json:"otlp,omitempty" yaml:"otlp,omitempty" description:"OTLP addresses"` + + UI []string `json:"ui,omitempty" yaml:"ui,omitempty" description:"UI addresses"` + + _ struct{} `additionalProperties:"false"` +} diff --git a/api/v1alpha1/infrastructure/resource_kind.go b/api/v1alpha1/infrastructure/resource_kind.go new file mode 100644 index 00000000..53aa0eb8 --- /dev/null +++ b/api/v1alpha1/infrastructure/resource_kind.go @@ -0,0 +1,82 @@ +package infrastructure + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*ResourceKind)(nil) +var _ yaml.Unmarshaler = (*ResourceKind)(nil) +var _ json.Marshaler = (*ResourceKind)(nil) +var _ json.Unmarshaler = (*ResourceKind)(nil) +var _ fmt.Stringer = (*ResourceKind)(nil) +var _ jsonschema.Enum = (*ResourceKind)(nil) + +var ( + ResourceKindInstallation ResourceKind = ResourceKind{s: "Installation"} + ResourceKindCollectionAgent ResourceKind = ResourceKind{s: "CollectionAgent"} +) + +type ResourceKind struct { + s string +} + +func (kind ResourceKind) String() string { + return kind.s +} + +func ResourceKinds() []ResourceKind { + return []ResourceKind{ResourceKindInstallation, ResourceKindCollectionAgent} +} + +func (kind ResourceKind) MarshalJSON() ([]byte, error) { + return json.Marshal(kind.String()) +} + +func (kind *ResourceKind) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return kind.UnmarshalText([]byte(str)) +} + +func (kind *ResourceKind) UnmarshalText(text []byte) error { + for _, availableKind := range ResourceKinds() { + if availableKind.String() == string(text) { + *kind = availableKind + return nil + } + } + if text == nil { + *kind = ResourceKind{s: ""} + return nil + } + return errors.New("invalid resource kind: " + string(text)) +} + +func (kind ResourceKind) MarshalText() ([]byte, error) { + return []byte(kind.String()), nil +} + +func (kind *ResourceKind) UnmarshalYAML(node *yaml.Node) error { + return kind.UnmarshalText([]byte(node.Value)) +} + +func (kind ResourceKind) MarshalYAML() (any, error) { + return kind.String(), nil +} + +func (kind ResourceKind) Enum() []any { + kinds := []any{} + for _, kind := range ResourceKinds() { + kinds = append(kinds, kind.String()) + } + + return kinds +} diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index cdecacde..750bf9f1 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -21,10 +21,11 @@ func TestSchemaValidate(t *testing.T) { { name: "ResourceProvided_Valid", mutate: func(casting *Casting) { - casting.Spec.Resource = v1alpha1.TypeResourceRef{ - APIVersion: "v1alpha1", - Kind: v1alpha1.KindInstallation, - Name: "signoz", + casting.Spec.Resource = Resource{ + Kind: ResourceKindInstallation, + Spec: ResourceSpec{ + TypeCastingRef: v1alpha1.TypeCastingRef{APIVersion: "v1alpha1", Name: "signoz"}, + }, } }, pass: true, @@ -32,9 +33,11 @@ func TestSchemaValidate(t *testing.T) { { name: "ResourceAPIVersionOmitted_Valid", mutate: func(casting *Casting) { - casting.Spec.Resource = v1alpha1.TypeResourceRef{ - Kind: v1alpha1.KindCollectionAgent, - Name: "signoz-gateway", + casting.Spec.Resource = Resource{ + Kind: ResourceKindCollectionAgent, + Spec: ResourceSpec{ + TypeCastingRef: v1alpha1.TypeCastingRef{Name: "signoz-gateway"}, + }, } }, pass: true, @@ -45,16 +48,23 @@ func TestSchemaValidate(t *testing.T) { pass: false, }, { - name: "ResourceAPIVersionUnknown_Invalid", + name: "ResourceKindMissing_Invalid", mutate: func(casting *Casting) { - casting.Spec.Resource = v1alpha1.TypeResourceRef{ - APIVersion: "v2", - Kind: v1alpha1.KindInstallation, - Name: "signoz", + casting.Spec.Resource = Resource{ + Spec: ResourceSpec{ + TypeCastingRef: v1alpha1.TypeCastingRef{Name: "signoz"}, + }, } }, pass: false, }, + { + name: "ResourceNameMissing_Invalid", + mutate: func(casting *Casting) { + casting.Spec.Resource = Resource{Kind: ResourceKindInstallation} + }, + pass: false, + }, } for _, tt := range tests { diff --git a/api/v1alpha1/molding_kind.go b/api/v1alpha1/molding_kind.go index 488848a8..a6de92af 100644 --- a/api/v1alpha1/molding_kind.go +++ b/api/v1alpha1/molding_kind.go @@ -19,6 +19,7 @@ var ( MoldingKindSignoz MoldingKind = MoldingKind{s: "signoz"} MoldingKindCollector MoldingKind = MoldingKind{s: "collector"} MoldingKindMCP MoldingKind = MoldingKind{s: "mcp"} + MoldingKindResource MoldingKind = MoldingKind{s: "resource"} ) type MoldingKind struct { diff --git a/api/v1alpha1/resource.go b/api/v1alpha1/resource.go deleted file mode 100644 index 71ab90de..00000000 --- a/api/v1alpha1/resource.go +++ /dev/null @@ -1,10 +0,0 @@ -package v1alpha1 - -// TypeResourceRef references a casting by identity. References resolve among -// the declaring file's own documents. -type TypeResourceRef struct { - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" enum:"v1alpha1" description:"API version of the referenced casting." example:"v1alpha1"` - Kind Kind `json:"kind" yaml:"kind" required:"true" description:"Kind of the referenced casting."` - Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` - _ struct{} `additionalProperties:"false"` -} diff --git a/internal/casting/infrastructure/casting.go b/internal/casting/infrastructure/casting.go index c63c4f43..52a2d3a5 100644 --- a/internal/casting/infrastructure/casting.go +++ b/internal/casting/infrastructure/casting.go @@ -5,9 +5,11 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) type Casting interface { + Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error } diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 59ae10a1..215a5b4c 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -8,20 +8,24 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" "github.com/signoz/foundry/internal/planner" "github.com/signoz/foundry/internal/tooler" ) var _ planner.Planner = (*Planner)(nil) -// Planner is the Infrastructure Kind's per-Kind orchestrator. The Kind has no -// moldings: reference validation happens at planner construction and all -// derivation happens inside the casting at forge. +// Planner is the Infrastructure Kind's per-Kind orchestrator. It satisfies +// the foundry planner contract by exposing this Kind's moldings, enricher, +// and casting strategy as verbs on a single value. type Planner struct { - config *infrastructure.Casting - logger *slog.Logger - casting Casting - toolers []tooler.Tooler + config *infrastructure.Casting + logger *slog.Logger + casting Casting + toolers []tooler.Tooler + enricher infrastructuremolding.MoldingEnricher + moldings []infrastructuremolding.Molding } func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Logger) (planner.Planner, error) { @@ -37,25 +41,47 @@ func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Log return nil, err } + enricher, err := castingStrategy.Enricher(ctx, c) + if err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") + } + + moldings := []infrastructuremolding.Molding{ + resourcemolding.New(logger), + } + return &Planner{ - config: c, - logger: logger, - casting: castingStrategy, - toolers: toolers, + config: c, + logger: logger, + casting: castingStrategy, + toolers: toolers, + enricher: enricher, + moldings: moldings, }, nil } func (p *Planner) Machinery() v1alpha1.Machinery { return p.config } func (p *Planner) Patches() []v1alpha1.PatchEntry { return p.config.Spec.Patches } -func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { return nil } +func (p *Planner) MoldingKinds() []v1alpha1.MoldingKind { + kinds := make([]v1alpha1.MoldingKind, len(p.moldings)) + for i, m := range p.moldings { + kinds[i] = m.Kind() + } + return kinds +} func (p *Planner) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error { - return foundryerrors.Newf(foundryerrors.TypeInternal, "infrastructure has no moldings") + return p.enricher.EnrichStatus(ctx, kind, p.config) } func (p *Planner) Mold(ctx context.Context, kind v1alpha1.MoldingKind) error { - return foundryerrors.Newf(foundryerrors.TypeInternal, "infrastructure has no moldings") + for _, m := range p.moldings { + if m.Kind() == kind { + return m.MoldV1Alpha1(ctx, p.config) + } + } + return foundryerrors.Newf(foundryerrors.TypeInternal, "molding %q not registered for infrastructure planner", kind) } func (p *Planner) MergeStatusIntoSpec() error { diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index e453799a..a3af1b9d 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -23,7 +23,9 @@ func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ castings: map[v1alpha1.TypeDeployment]CastingItem{ { - Flavor: v1alpha1.FlavorTerraform, + Platform: v1alpha1.PlatformECS, + Mode: v1alpha1.ModeEC2, + Flavor: v1alpha1.FlavorTerraform, }: { Casting: terraformcasting.New(logger), Toolers: []tooler.Tooler{terraformtooler.New()}, @@ -32,17 +34,11 @@ func NewRegistry(logger *slog.Logger) *Registry { } } +// lookup is an exact match: every platform x mode x flavor combination gets +// its own casting. func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingItem, bool) { - if item, ok := registry.castings[deployment]; ok { - return item, true - } - // Fall back to matching without platform: the platform selects the provider - // inside the casting, not the casting itself. - if deployment.Platform != (v1alpha1.Platform{}) { - item, ok := registry.castings[v1alpha1.TypeDeployment{Mode: deployment.Mode, Flavor: deployment.Flavor}] - return item, ok - } - return CastingItem{}, false + item, ok := registry.castings[deployment] + return item, ok } func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (Casting, error) { diff --git a/internal/casting/infrastructure/terraformcasting/casting.go b/internal/casting/infrastructure/terraformcasting/casting.go index 6fd6f367..e1c91365 100644 --- a/internal/casting/infrastructure/terraformcasting/casting.go +++ b/internal/casting/infrastructure/terraformcasting/casting.go @@ -6,6 +6,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) type terraformCasting struct { @@ -16,6 +17,10 @@ func New(logger *slog.Logger) *terraformCasting { return &terraformCasting{logger: logger} } +func (c *terraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return &enricher{logger: c.logger}, nil +} + func (c *terraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, no materials generated yet") return nil, nil diff --git a/internal/casting/infrastructure/terraformcasting/enricher.go b/internal/casting/infrastructure/terraformcasting/enricher.go new file mode 100644 index 00000000..9f9d9038 --- /dev/null +++ b/internal/casting/infrastructure/terraformcasting/enricher.go @@ -0,0 +1,20 @@ +package terraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) + +type enricher struct { + logger *slog.Logger +} + +func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { + return nil +} diff --git a/internal/config/yamlconfig/config.go b/internal/config/yamlconfig/config.go index 8a49f8ce..ca05ed46 100644 --- a/internal/config/yamlconfig/config.go +++ b/internal/config/yamlconfig/config.go @@ -134,13 +134,6 @@ func (*yamlConfig) loadInfrastructure(bytes []byte, path string) (v1alpha1.Machi return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to unmarshal infrastructure casting") } - // Checked before the merge: merging round-trips through JSON, where an - // absent kind unmarshals to the backward-compatibility default and would - // silently pass validation as an Installation reference. - if loaded.Spec.Resource.Kind == (v1alpha1.Kind{}) { - return nil, errors.Newf(errors.TypeInvalidInput, "invalid casting file %s: spec.resource.kind is required", path) - } - base := infrastructure.Default() if err := v1alpha1.Merge(base, &loaded); err != nil { return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge default infrastructure casting") @@ -159,10 +152,6 @@ func (*yamlConfig) loadInfrastructure(bytes []byte, path string) (v1alpha1.Machi return nil, errors.Wrapf(err, errors.TypeInvalidInput, "invalid casting file %s", path) } - if kind := base.Spec.Resource.Kind; kind != v1alpha1.KindInstallation && kind != v1alpha1.KindCollectionAgent { - return nil, errors.Newf(errors.TypeInvalidInput, "invalid casting file %s: infrastructure cannot serve resource kind %q, supported kinds are %q and %q", path, kind, v1alpha1.KindInstallation, v1alpha1.KindCollectionAgent) - } - return base, nil } diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go index 8d0aea52..e8eada41 100644 --- a/internal/config/yamlconfig/config_test.go +++ b/internal/config/yamlconfig/config_test.go @@ -376,7 +376,7 @@ func TestGetV1Alpha1Infrastructure(t *testing.T) { tests := []struct { name string input string - expectedResource v1alpha1.Kind + expectedResource infrastructure.ResourceKind pass bool }{ { @@ -389,12 +389,14 @@ metadata: spec: deployment: platform: ecs + mode: ec2 flavor: terraform resource: kind: Installation - name: signoz + spec: + name: signoz `, - expectedResource: v1alpha1.KindInstallation, + expectedResource: infrastructure.ResourceKindInstallation, pass: true, }, { @@ -407,6 +409,7 @@ metadata: spec: deployment: platform: ecs + mode: ec2 flavor: terraform `, pass: false, @@ -421,10 +424,12 @@ metadata: spec: deployment: platform: ecs + mode: ec2 flavor: terraform resource: kind: Infrastructure - name: signoz + spec: + name: signoz `, pass: false, }, @@ -438,9 +443,11 @@ metadata: spec: deployment: platform: ecs + mode: ec2 flavor: terraform resource: - name: signoz + spec: + name: signoz `, pass: false, }, @@ -463,6 +470,7 @@ spec: assert.True(t, ok) assert.Equal(t, v1alpha1.KindInfrastructure, casting.Kind()) assert.Equal(t, tt.expectedResource, casting.Spec.Resource.Kind) + assert.Equal(t, "signoz", casting.Spec.Resource.Spec.Name) }) } } diff --git a/internal/molding/infrastructure/molding.go b/internal/molding/infrastructure/molding.go new file mode 100644 index 00000000..f0c4d629 --- /dev/null +++ b/internal/molding/infrastructure/molding.go @@ -0,0 +1,17 @@ +package infrastructure + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" +) + +type MoldingEnricher interface { + EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error +} + +type Molding interface { + Kind() v1alpha1.MoldingKind + MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error +} diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go new file mode 100644 index 00000000..c4b6147e --- /dev/null +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -0,0 +1,47 @@ +package resourcemolding + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +var _ infrastructuremolding.Molding = (*resourceMolding)(nil) + +type resourceMolding struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *resourceMolding { + return &resourceMolding{logger: logger} +} + +func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { + return v1alpha1.MoldingKindResource +} + +// MoldV1Alpha1 molds the unit to be hosted: the exposure and facts every +// casting consumes, dispatched on the resource kind. +func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { + status := &config.Spec.Resource.Status + + switch config.Spec.Resource.Kind { + case infrastructure.ResourceKindInstallation: + status.Addresses = infrastructure.ResourceStatusAddresses{ + OTLP: []string{":4317", ":4318"}, + UI: []string{":8080"}, + } + case infrastructure.ResourceKindCollectionAgent: + status.Addresses = infrastructure.ResourceStatusAddresses{ + OTLP: []string{":4317", ":4318"}, + } + default: + return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported resource kind %q", config.Spec.Resource.Kind) + } + + return nil +} diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go new file mode 100644 index 00000000..bd2ccfd4 --- /dev/null +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -0,0 +1,55 @@ +package resourcemolding + +import ( + "context" + "log/slog" + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/stretchr/testify/assert" +) + +func TestMoldV1Alpha1(t *testing.T) { + tests := []struct { + name string + kind infrastructure.ResourceKind + expectedOTLP []string + expectedUI []string + pass bool + }{ + { + name: "InstallationResource_ExposesOtlpAndUi", + kind: infrastructure.ResourceKindInstallation, + expectedOTLP: []string{":4317", ":4318"}, + expectedUI: []string{":8080"}, + pass: true, + }, + { + name: "CollectionAgentResource_ExposesOtlpOnly", + kind: infrastructure.ResourceKindCollectionAgent, + expectedOTLP: []string{":4317", ":4318"}, + pass: true, + }, + { + name: "UnknownResourceKind_Unsupported", + kind: infrastructure.ResourceKind{}, + pass: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = tt.kind + + err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + if !tt.pass { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expectedOTLP, config.Spec.Resource.Status.Addresses.OTLP) + assert.Equal(t, tt.expectedUI, config.Spec.Resource.Status.Addresses.UI) + }) + } +} From 6d32ccdc669c2ff1c20e507f1a888f3692f5ba75 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 18:31:24 +0530 Subject: [PATCH 07/38] refactor(infrastructure): strip invented exposure content from resource status --- .../infrastructure/casting.schema.json | 24 ------------------ api/v1alpha1/infrastructure/resource.go | 10 -------- .../resourcemolding/resource.go | 20 ++++----------- .../resourcemolding/resource_test.go | 25 +++++++------------ 4 files changed, 14 insertions(+), 65 deletions(-) diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 99744cdf..ae684430 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -68,10 +68,6 @@ "InfrastructureResourceStatus": { "additionalProperties": false, "properties": { - "addresses": { - "$ref": "#/definitions/InfrastructureResourceStatusAddresses", - "description": "Addresses the resource exposes" - }, "config": { "$ref": "#/definitions/V1Alpha1TypeConfig", "description": "Configuration for the molding" @@ -93,26 +89,6 @@ }, "type": "object" }, - "InfrastructureResourceStatusAddresses": { - "additionalProperties": false, - "properties": { - "otlp": { - "description": "OTLP addresses", - "items": { - "type": "string" - }, - "type": "array" - }, - "ui": { - "description": "UI addresses", - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, "InfrastructureSpec": { "required": [ "deployment", diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index dbb5123f..5970b2b1 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -27,15 +27,5 @@ type ResourceSpec struct { type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - Addresses ResourceStatusAddresses `json:"addresses,omitzero" yaml:"addresses,omitempty" description:"Addresses the resource exposes"` - - _ struct{} `additionalProperties:"false"` -} - -type ResourceStatusAddresses struct { - OTLP []string `json:"otlp,omitempty" yaml:"otlp,omitempty" description:"OTLP addresses"` - - UI []string `json:"ui,omitempty" yaml:"ui,omitempty" description:"UI addresses"` - _ struct{} `additionalProperties:"false"` } diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index c4b6147e..a1070b97 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -24,24 +24,14 @@ func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindResource } -// MoldV1Alpha1 molds the unit to be hosted: the exposure and facts every -// casting consumes, dispatched on the resource kind. +// MoldV1Alpha1 molds the unit to be hosted, dispatched on the resource kind. +// Status content enters only with the casting that consumes it, sourced from +// the consumer casting's own molded facts. func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { - status := &config.Spec.Resource.Status - switch config.Spec.Resource.Kind { - case infrastructure.ResourceKindInstallation: - status.Addresses = infrastructure.ResourceStatusAddresses{ - OTLP: []string{":4317", ":4318"}, - UI: []string{":8080"}, - } - case infrastructure.ResourceKindCollectionAgent: - status.Addresses = infrastructure.ResourceStatusAddresses{ - OTLP: []string{":4317", ":4318"}, - } + case infrastructure.ResourceKindInstallation, infrastructure.ResourceKindCollectionAgent: + return nil default: return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported resource kind %q", config.Spec.Resource.Kind) } - - return nil } diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index bd2ccfd4..2a123eee 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -11,24 +11,19 @@ import ( func TestMoldV1Alpha1(t *testing.T) { tests := []struct { - name string - kind infrastructure.ResourceKind - expectedOTLP []string - expectedUI []string - pass bool + name string + kind infrastructure.ResourceKind + pass bool }{ { - name: "InstallationResource_ExposesOtlpAndUi", - kind: infrastructure.ResourceKindInstallation, - expectedOTLP: []string{":4317", ":4318"}, - expectedUI: []string{":8080"}, - pass: true, + name: "InstallationResource_Supported", + kind: infrastructure.ResourceKindInstallation, + pass: true, }, { - name: "CollectionAgentResource_ExposesOtlpOnly", - kind: infrastructure.ResourceKindCollectionAgent, - expectedOTLP: []string{":4317", ":4318"}, - pass: true, + name: "CollectionAgentResource_Supported", + kind: infrastructure.ResourceKindCollectionAgent, + pass: true, }, { name: "UnknownResourceKind_Unsupported", @@ -48,8 +43,6 @@ func TestMoldV1Alpha1(t *testing.T) { return } assert.NoError(t, err) - assert.Equal(t, tt.expectedOTLP, config.Spec.Resource.Status.Addresses.OTLP) - assert.Equal(t, tt.expectedUI, config.Spec.Resource.Status.Addresses.UI) }) } } From 12722a4c2696728fe8dc1ce20ff7c1a7e55d7744 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 18:51:20 +0530 Subject: [PATCH 08/38] feat(infrastructure): record the resolved casting in the resource status --- .../infrastructure/casting.schema.json | 22 +++++++++++++++++++ api/v1alpha1/infrastructure/resource.go | 12 ++++++++++ 2 files changed, 34 insertions(+) diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index ae684430..3550a9c2 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -85,6 +85,28 @@ "type": "string" }, "type": "object" + }, + "resolution": { + "$ref": "#/definitions/InfrastructureResourceStatusResolution", + "description": "Record of the resolution: the casting the infrastructure used for the resource" + } + }, + "type": "object" + }, + "InfrastructureResourceStatusResolution": { + "additionalProperties": false, + "properties": { + "casting": { + "description": "The resolved casting used for the resource, verbatim", + "type": "string" + }, + "checksum": { + "description": "Checksum of the resolved casting", + "type": "string" + }, + "source": { + "description": "The document the resource resolved to", + "type": "string" } }, "type": "object" diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index 5970b2b1..eeb27b27 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -27,5 +27,17 @@ type ResourceSpec struct { type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` + Resolution ResourceStatusResolution `json:"resolution,omitzero" yaml:"resolution,omitempty" description:"Record of the resolution: the casting the infrastructure used for the resource"` + _ struct{} `additionalProperties:"false"` } + +// ResourceStatusResolution records what the resource resolved to. The casting +// field is a record, not an API: castings consume typed facts from the +// status, never parse the embedded document. +type ResourceStatusResolution struct { + Source string `json:"source,omitempty" yaml:"source,omitempty" description:"The document the resource resolved to"` + Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty" description:"Checksum of the resolved casting"` + Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting used for the resource, verbatim"` + _ struct{} `additionalProperties:"false"` +} From ada6fda132c5de9f2fd98ea07ef80de42c6dfc47 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 18:54:06 +0530 Subject: [PATCH 09/38] refactor(infrastructure): trim resolution record and restore schema generator --- .../collectionagent/casting.schema.json | 11 +++------ api/v1alpha1/infrastructure/casting.go | 3 +-- .../infrastructure/casting.schema.json | 17 ++++--------- api/v1alpha1/infrastructure/resource.go | 12 ++++------ api/v1alpha1/installation/casting.schema.json | 11 +++------ cmd/foundryctl/gen.go | 24 ++++++++++--------- .../resourcemolding/resource.go | 3 --- 7 files changed, 28 insertions(+), 53 deletions(-) diff --git a/api/v1alpha1/collectionagent/casting.schema.json b/api/v1alpha1/collectionagent/casting.schema.json index 9ab23179..8998b545 100644 --- a/api/v1alpha1/collectionagent/casting.schema.json +++ b/api/v1alpha1/collectionagent/casting.schema.json @@ -98,9 +98,7 @@ }, "V1Alpha1Kind": { "enum": [ - "Installation", - "CollectionAgent", - "Infrastructure" + "CollectionAgent" ], "type": "string" }, @@ -366,11 +364,8 @@ "type": "string" }, "kind": { - "description": "Kind of the casting resource.", - "enum": [ - "CollectionAgent" - ], - "type": "string" + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the casting resource." }, "metadata": { "$ref": "#/definitions/V1Alpha1TypeMetadata", diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index 3994c8d3..74682a5b 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -55,8 +55,7 @@ func (c *Casting) Kind() v1alpha1.Kind { } // MergeStatusIntoSpec folds molding-written status into spec. The resource -// molding's unit stays in status (addresses are read from status, as on other -// kinds); its spec carries identity only, so nothing merges. +// spec carries identity only; nothing merges. func (c *Casting) MergeStatusIntoSpec() error { return nil } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 3550a9c2..7470a39d 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -88,7 +88,7 @@ }, "resolution": { "$ref": "#/definitions/InfrastructureResourceStatusResolution", - "description": "Record of the resolution: the casting the infrastructure used for the resource" + "description": "The casting the resource resolved to" } }, "type": "object" @@ -97,16 +97,12 @@ "additionalProperties": false, "properties": { "casting": { - "description": "The resolved casting used for the resource, verbatim", + "description": "The resolved casting, verbatim", "type": "string" }, "checksum": { "description": "Checksum of the resolved casting", "type": "string" - }, - "source": { - "description": "The document the resource resolved to", - "type": "string" } }, "type": "object" @@ -152,8 +148,6 @@ }, "V1Alpha1Kind": { "enum": [ - "Installation", - "CollectionAgent", "Infrastructure" ], "type": "string" @@ -348,11 +342,8 @@ "type": "string" }, "kind": { - "description": "Kind of the casting resource.", - "enum": [ - "Infrastructure" - ], - "type": "string" + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the casting resource." }, "metadata": { "$ref": "#/definitions/V1Alpha1TypeMetadata", diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index eeb27b27..5b57b820 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -2,8 +2,7 @@ package infrastructure import "github.com/signoz/foundry/api/v1alpha1" -// Resource is the infrastructure kind's molding: the unit to be hosted. Its -// kinds are the consumer casting kinds. +// Resource is the resource this infrastructure serves. type Resource struct { // Kind of the resource this infrastructure serves. Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` @@ -27,17 +26,14 @@ type ResourceSpec struct { type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - Resolution ResourceStatusResolution `json:"resolution,omitzero" yaml:"resolution,omitempty" description:"Record of the resolution: the casting the infrastructure used for the resource"` + Resolution ResourceStatusResolution `json:"resolution,omitzero" yaml:"resolution,omitempty" description:"The casting the resource resolved to"` _ struct{} `additionalProperties:"false"` } -// ResourceStatusResolution records what the resource resolved to. The casting -// field is a record, not an API: castings consume typed facts from the -// status, never parse the embedded document. +// ResourceStatusResolution records the casting the resource resolved to. type ResourceStatusResolution struct { - Source string `json:"source,omitempty" yaml:"source,omitempty" description:"The document the resource resolved to"` Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty" description:"Checksum of the resolved casting"` - Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting used for the resource, verbatim"` + Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting, verbatim"` _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json index 4f4202b7..01a00b04 100644 --- a/api/v1alpha1/installation/casting.schema.json +++ b/api/v1alpha1/installation/casting.schema.json @@ -488,9 +488,7 @@ }, "V1Alpha1Kind": { "enum": [ - "Installation", - "CollectionAgent", - "Infrastructure" + "Installation" ], "type": "string" }, @@ -756,11 +754,8 @@ "type": "string" }, "kind": { - "description": "Kind of the casting resource.", - "enum": [ - "Installation" - ], - "type": "string" + "$ref": "#/definitions/V1Alpha1Kind", + "description": "Kind of the casting resource." }, "metadata": { "$ref": "#/definitions/V1Alpha1TypeMetadata", diff --git a/cmd/foundryctl/gen.go b/cmd/foundryctl/gen.go index 0bcbb989..3aca084c 100644 --- a/cmd/foundryctl/gen.go +++ b/cmd/foundryctl/gen.go @@ -102,27 +102,29 @@ func runGenExamples(ctx context.Context, logger *slog.Logger) error { func runGenSchemas(_ context.Context) error { var oneOf []jsonschema.SchemaOrBool + kindType := reflect.TypeFor[v1alpha1.Kind]() for _, t := range schemaTargets { target := t reflector := jsonschema.Reflector{} + // v1alpha1.Kind's Enum() returns all Kinds (the type permits any). + // For this per-Kind schema, the kind field is always this Casting's + // Kind, so we narrow the enum at reflection. + reflector.DefaultOptions = append(reflector.DefaultOptions, + jsonschema.InterceptSchema(func(params jsonschema.InterceptSchemaParams) (bool, error) { + if !params.Processed || params.Value.Type() != kindType { + return false, nil + } + params.Schema.Enum = []any{target.kind.String()} + return false, nil + }), + ) schema, err := reflector.Reflect(target.val) if err != nil { return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "reflect %T", target.val) } - // v1alpha1.Kind's Enum() returns all Kinds (the type permits any, and - // nested Kind-typed fields such as resource refs must keep the full - // enum). Only the TOP-LEVEL kind field is always this Casting's Kind, - // so it is narrowed here with an inline schema instead of the shared - // Kind definition. - narrowedKind := (&jsonschema.Schema{}). - WithType(jsonschema.String.Type()). - WithEnum(target.kind.String()). - WithDescription("Kind of the casting resource.") - schema.WithPropertiesItem("kind", narrowedKind.ToSchemaOrBool()) - contents, err := json.MarshalIndent(schema, "", " ") if err != nil { return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "marshal %T", target.val) diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index a1070b97..b53401e1 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -24,9 +24,6 @@ func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindResource } -// MoldV1Alpha1 molds the unit to be hosted, dispatched on the resource kind. -// Status content enters only with the casting that consumes it, sourced from -// the consumer casting's own molded facts. func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { switch config.Spec.Resource.Kind { case infrastructure.ResourceKindInstallation, infrastructure.ResourceKindCollectionAgent: From 4c3766932635452872614c6ab0883f70d5a7babc Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 18:59:54 +0530 Subject: [PATCH 10/38] refactor(v1alpha1): model the casting ref as a spec and status pair --- api/v1alpha1/casting_ref.go | 40 ++++++++++++++++--- api/v1alpha1/infrastructure/casting.go | 7 ++++ .../infrastructure/casting.schema.json | 29 +++++--------- api/v1alpha1/infrastructure/resource.go | 11 +---- api/v1alpha1/infrastructure/schema_test.go | 38 +++++++++--------- internal/config/yamlconfig/config_test.go | 3 ++ 6 files changed, 76 insertions(+), 52 deletions(-) diff --git a/api/v1alpha1/casting_ref.go b/api/v1alpha1/casting_ref.go index 4b7cbbde..c0635ee9 100644 --- a/api/v1alpha1/casting_ref.go +++ b/api/v1alpha1/casting_ref.go @@ -1,9 +1,39 @@ package v1alpha1 -// TypeCastingRef is the identity of a casting: which schema version and which -// name. +import "encoding/json" + +// TypeCastingRef references a casting: the identity it is declared with and +// the casting it resolved to. type TypeCastingRef struct { - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion,omitempty" enum:"v1alpha1" default:"v1alpha1" description:"API version of the referenced casting." example:"v1alpha1"` - Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` - _ struct{} `additionalProperties:"false"` + Spec TypeCastingRefSpec `json:"spec" yaml:"spec" required:"true" description:"Identity of the referenced casting"` + Status TypeCastingRefStatus `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the reference"` + _ struct{} `additionalProperties:"false"` +} + +type TypeCastingRefSpec struct { + TypeVersion `json:",inline" yaml:",inline"` + + Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` + _ struct{} `additionalProperties:"false"` +} + +// MarshalJSON implements json.Marshaler. It manually omits empty fields +// so that the strategic merge patch doesn't overwrite defaults with empty +// values. +func (spec TypeCastingRefSpec) MarshalJSON() ([]byte, error) { + m := map[string]any{} + if spec.APIVersion != "" { + m["apiVersion"] = spec.APIVersion + } + if spec.Name != "" { + m["name"] = spec.Name + } + return json.Marshal(m) +} + +type TypeCastingRefStatus struct { + Status `json:",inline" yaml:",inline"` + + Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting, verbatim"` + _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index 74682a5b..5ae307f4 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -32,6 +32,13 @@ func Default() *Casting { }, Spec: Spec{ Deployment: v1alpha1.TypeDeployment{Flavor: v1alpha1.FlavorTerraform}, + Resource: Resource{ + Spec: ResourceSpec{ + TypeCastingRefSpec: v1alpha1.TypeCastingRefSpec{ + TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, + }, + }, + }, }, } } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 7470a39d..d28533c7 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -41,12 +41,13 @@ }, "InfrastructureResourceSpec": { "required": [ + "apiVersion", "name" ], "additionalProperties": false, "properties": { "apiVersion": { - "description": "API version of the referenced casting.", + "description": "API Version of the configuration schema.", "default": "v1alpha1", "examples": [ "v1alpha1" @@ -68,6 +69,14 @@ "InfrastructureResourceStatus": { "additionalProperties": false, "properties": { + "casting": { + "description": "The resolved casting, verbatim", + "type": "string" + }, + "checksum": { + "description": "Checksum of the casting file", + "type": "string" + }, "config": { "$ref": "#/definitions/V1Alpha1TypeConfig", "description": "Configuration for the molding" @@ -85,24 +94,6 @@ "type": "string" }, "type": "object" - }, - "resolution": { - "$ref": "#/definitions/InfrastructureResourceStatusResolution", - "description": "The casting the resource resolved to" - } - }, - "type": "object" - }, - "InfrastructureResourceStatusResolution": { - "additionalProperties": false, - "properties": { - "casting": { - "description": "The resolved casting, verbatim", - "type": "string" - }, - "checksum": { - "description": "Checksum of the resolved casting", - "type": "string" } }, "type": "object" diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index 5b57b820..f36898b9 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -18,7 +18,7 @@ type Resource struct { // ResourceSpec carries the identity of the casting embodying the resource. type ResourceSpec struct { - v1alpha1.TypeCastingRef `json:",inline" yaml:",inline"` + v1alpha1.TypeCastingRefSpec `json:",inline" yaml:",inline"` _ struct{} `additionalProperties:"false"` } @@ -26,14 +26,7 @@ type ResourceSpec struct { type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - Resolution ResourceStatusResolution `json:"resolution,omitzero" yaml:"resolution,omitempty" description:"The casting the resource resolved to"` + v1alpha1.TypeCastingRefStatus `json:",inline" yaml:",inline"` _ struct{} `additionalProperties:"false"` } - -// ResourceStatusResolution records the casting the resource resolved to. -type ResourceStatusResolution struct { - Checksum string `json:"checksum,omitempty" yaml:"checksum,omitempty" description:"Checksum of the resolved casting"` - Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting, verbatim"` - _ struct{} `additionalProperties:"false"` -} diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index 750bf9f1..d2097856 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -21,24 +21,16 @@ func TestSchemaValidate(t *testing.T) { { name: "ResourceProvided_Valid", mutate: func(casting *Casting) { - casting.Spec.Resource = Resource{ - Kind: ResourceKindInstallation, - Spec: ResourceSpec{ - TypeCastingRef: v1alpha1.TypeCastingRef{APIVersion: "v1alpha1", Name: "signoz"}, - }, - } + casting.Spec.Resource.Kind = ResourceKindInstallation + casting.Spec.Resource.Spec.Name = "signoz" }, pass: true, }, { - name: "ResourceAPIVersionOmitted_Valid", + name: "CollectionAgentResource_Valid", mutate: func(casting *Casting) { - casting.Spec.Resource = Resource{ - Kind: ResourceKindCollectionAgent, - Spec: ResourceSpec{ - TypeCastingRef: v1alpha1.TypeCastingRef{Name: "signoz-gateway"}, - }, - } + casting.Spec.Resource.Kind = ResourceKindCollectionAgent + casting.Spec.Resource.Spec.Name = "signoz-gateway" }, pass: true, }, @@ -50,18 +42,26 @@ func TestSchemaValidate(t *testing.T) { { name: "ResourceKindMissing_Invalid", mutate: func(casting *Casting) { - casting.Spec.Resource = Resource{ - Spec: ResourceSpec{ - TypeCastingRef: v1alpha1.TypeCastingRef{Name: "signoz"}, - }, - } + casting.Spec.Resource.Spec.Name = "signoz" }, pass: false, }, { name: "ResourceNameMissing_Invalid", mutate: func(casting *Casting) { - casting.Spec.Resource = Resource{Kind: ResourceKindInstallation} + casting.Spec.Resource.Kind = ResourceKindInstallation + }, + pass: false, + }, + { + name: "ResourceAPIVersionMissing_Invalid", + mutate: func(casting *Casting) { + casting.Spec.Resource = Resource{ + Kind: ResourceKindInstallation, + Spec: ResourceSpec{ + TypeCastingRefSpec: v1alpha1.TypeCastingRefSpec{Name: "signoz"}, + }, + } }, pass: false, }, diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go index e8eada41..973311b2 100644 --- a/internal/config/yamlconfig/config_test.go +++ b/internal/config/yamlconfig/config_test.go @@ -468,6 +468,9 @@ spec: casting, ok := machinery.(*infrastructure.Casting) assert.True(t, ok) + if !ok { + return + } assert.Equal(t, v1alpha1.KindInfrastructure, casting.Kind()) assert.Equal(t, tt.expectedResource, casting.Spec.Resource.Kind) assert.Equal(t, "signoz", casting.Spec.Resource.Spec.Name) From d1be9757cea048b60743b61ed933da2299d1feb8 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 20:12:17 +0530 Subject: [PATCH 11/38] refactor(v1alpha1): reduce the casting ref to its identity spec --- api/v1alpha1/casting_ref.go | 16 +--------------- api/v1alpha1/infrastructure/casting.go | 1 - api/v1alpha1/infrastructure/casting.schema.json | 4 ---- api/v1alpha1/infrastructure/resource.go | 2 +- api/v1alpha1/infrastructure/resource_kind.go | 5 +++-- 5 files changed, 5 insertions(+), 23 deletions(-) diff --git a/api/v1alpha1/casting_ref.go b/api/v1alpha1/casting_ref.go index c0635ee9..78da586a 100644 --- a/api/v1alpha1/casting_ref.go +++ b/api/v1alpha1/casting_ref.go @@ -2,14 +2,7 @@ package v1alpha1 import "encoding/json" -// TypeCastingRef references a casting: the identity it is declared with and -// the casting it resolved to. -type TypeCastingRef struct { - Spec TypeCastingRefSpec `json:"spec" yaml:"spec" required:"true" description:"Identity of the referenced casting"` - Status TypeCastingRefStatus `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the reference"` - _ struct{} `additionalProperties:"false"` -} - +// TypeCastingRefSpec is the identity a casting is referenced by. type TypeCastingRefSpec struct { TypeVersion `json:",inline" yaml:",inline"` @@ -30,10 +23,3 @@ func (spec TypeCastingRefSpec) MarshalJSON() ([]byte, error) { } return json.Marshal(m) } - -type TypeCastingRefStatus struct { - Status `json:",inline" yaml:",inline"` - - Casting string `json:"casting,omitempty" yaml:"casting,omitempty" description:"The resolved casting, verbatim"` - _ struct{} `additionalProperties:"false"` -} diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index 5ae307f4..96135fe9 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -31,7 +31,6 @@ func Default() *Casting { Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, }, Spec: Spec{ - Deployment: v1alpha1.TypeDeployment{Flavor: v1alpha1.FlavorTerraform}, Resource: Resource{ Spec: ResourceSpec{ TypeCastingRefSpec: v1alpha1.TypeCastingRefSpec{ diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index d28533c7..6ac4baa7 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -69,10 +69,6 @@ "InfrastructureResourceStatus": { "additionalProperties": false, "properties": { - "casting": { - "description": "The resolved casting, verbatim", - "type": "string" - }, "checksum": { "description": "Checksum of the casting file", "type": "string" diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index f36898b9..01688ffa 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -26,7 +26,7 @@ type ResourceSpec struct { type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - v1alpha1.TypeCastingRefStatus `json:",inline" yaml:",inline"` + v1alpha1.Status `json:",inline" yaml:",inline"` _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/infrastructure/resource_kind.go b/api/v1alpha1/infrastructure/resource_kind.go index 53aa0eb8..a08b7cf8 100644 --- a/api/v1alpha1/infrastructure/resource_kind.go +++ b/api/v1alpha1/infrastructure/resource_kind.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" + "github.com/signoz/foundry/api/v1alpha1" "github.com/swaggest/jsonschema-go" "go.yaml.in/yaml/v3" ) @@ -17,8 +18,8 @@ var _ fmt.Stringer = (*ResourceKind)(nil) var _ jsonschema.Enum = (*ResourceKind)(nil) var ( - ResourceKindInstallation ResourceKind = ResourceKind{s: "Installation"} - ResourceKindCollectionAgent ResourceKind = ResourceKind{s: "CollectionAgent"} + ResourceKindInstallation ResourceKind = ResourceKind{s: v1alpha1.KindInstallation.String()} + ResourceKindCollectionAgent ResourceKind = ResourceKind{s: v1alpha1.KindCollectionAgent.String()} ) type ResourceKind struct { From 29993d2a1217ae9d3691aafe6a813dccf2271e6b Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 20:37:31 +0530 Subject: [PATCH 12/38] refactor(infrastructure): name the stub casting for its combination --- .../ecsec2terraformcasting/casting.go | 32 +++++++++++++++++++ .../enricher.go | 2 +- internal/casting/infrastructure/registry.go | 8 ++--- .../terraformcasting/casting.go | 32 ------------------- 4 files changed, 37 insertions(+), 37 deletions(-) create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/casting.go rename internal/casting/infrastructure/{terraformcasting => ecsec2terraformcasting}/enricher.go (93%) delete mode 100644 internal/casting/infrastructure/terraformcasting/casting.go diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/casting.go b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go new file mode 100644 index 00000000..d9725908 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go @@ -0,0 +1,32 @@ +package ecsec2terraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +type ecsEC2TerraformCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *ecsEC2TerraformCasting { + return &ecsEC2TerraformCasting{logger: logger} +} + +func (c *ecsEC2TerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return &enricher{logger: c.logger}, nil +} + +func (c *ecsEC2TerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { + c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, no materials generated") + return nil, nil +} + +func (c *ecsEC2TerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { + c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, nothing to cast") + return nil +} diff --git a/internal/casting/infrastructure/terraformcasting/enricher.go b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go similarity index 93% rename from internal/casting/infrastructure/terraformcasting/enricher.go rename to internal/casting/infrastructure/ecsec2terraformcasting/enricher.go index 9f9d9038..a3355888 100644 --- a/internal/casting/infrastructure/terraformcasting/enricher.go +++ b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go @@ -1,4 +1,4 @@ -package terraformcasting +package ecsec2terraformcasting import ( "context" diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index a3af1b9d..4151ec15 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,7 +4,7 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/internal/casting/infrastructure/terraformcasting" + "github.com/signoz/foundry/internal/casting/infrastructure/ecsec2terraformcasting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" "github.com/signoz/foundry/internal/tooler/terraformtooler" @@ -27,15 +27,15 @@ func NewRegistry(logger *slog.Logger) *Registry { Mode: v1alpha1.ModeEC2, Flavor: v1alpha1.FlavorTerraform, }: { - Casting: terraformcasting.New(logger), + Casting: ecsec2terraformcasting.New(logger), Toolers: []tooler.Tooler{terraformtooler.New()}, }, }, } } -// lookup is an exact match: every platform x mode x flavor combination gets -// its own casting. +// lookup matches the exact deployment; each platform, mode, and flavor +// combination registers its own casting. func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingItem, bool) { item, ok := registry.castings[deployment] return item, ok diff --git a/internal/casting/infrastructure/terraformcasting/casting.go b/internal/casting/infrastructure/terraformcasting/casting.go deleted file mode 100644 index e1c91365..00000000 --- a/internal/casting/infrastructure/terraformcasting/casting.go +++ /dev/null @@ -1,32 +0,0 @@ -package terraformcasting - -import ( - "context" - "log/slog" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - "github.com/signoz/foundry/internal/domain" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -type terraformCasting struct { - logger *slog.Logger -} - -func New(logger *slog.Logger) *terraformCasting { - return &terraformCasting{logger: logger} -} - -func (c *terraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { - return &enricher{logger: c.logger}, nil -} - -func (c *terraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { - c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, no materials generated yet") - return nil, nil -} - -func (c *terraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { - c.logger.InfoContext(ctx, "infrastructure terraform casting is scaffolding, nothing to cast yet") - return nil -} From c0efbf74910112efafb57f01b3f6ec6c579e8663 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 23:52:50 +0530 Subject: [PATCH 13/38] refactor(infrastructure): reduce the resource to its declared identity --- .../infrastructure/casting.schema.json | 45 ------------------- api/v1alpha1/infrastructure/resource.go | 13 +----- api/v1alpha1/infrastructure/schema_test.go | 2 +- 3 files changed, 2 insertions(+), 58 deletions(-) diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 6ac4baa7..019e1d1f 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -24,10 +24,6 @@ "spec": { "$ref": "#/definitions/InfrastructureResourceSpec", "description": "Specification for the resource" - }, - "status": { - "$ref": "#/definitions/InfrastructureResourceStatus", - "description": "Status of the resource" } }, "type": "object" @@ -66,34 +62,6 @@ }, "type": "object" }, - "InfrastructureResourceStatus": { - "additionalProperties": false, - "properties": { - "checksum": { - "description": "Checksum of the casting file", - "type": "string" - }, - "config": { - "$ref": "#/definitions/V1Alpha1TypeConfig", - "description": "Configuration for the molding" - }, - "env": { - "description": "Environment variables for the molding", - "additionalProperties": { - "type": "string" - }, - "type": "object" - }, - "extras": { - "description": "Extra information about the molding", - "additionalProperties": { - "type": "string" - }, - "type": "object" - } - }, - "type": "object" - }, "InfrastructureSpec": { "required": [ "deployment", @@ -255,19 +223,6 @@ }, "type": "object" }, - "V1Alpha1TypeConfig": { - "additionalProperties": false, - "properties": { - "data": { - "description": "Configuration data as key-value pairs.", - "additionalProperties": { - "type": "string" - }, - "type": "object" - } - }, - "type": "object" - }, "V1Alpha1TypeDeployment": { "additionalProperties": false, "properties": { diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index 01688ffa..c3b5ff86 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -2,7 +2,7 @@ package infrastructure import "github.com/signoz/foundry/api/v1alpha1" -// Resource is the resource this infrastructure serves. +// Resource declares the resource this infrastructure is shaped for. type Resource struct { // Kind of the resource this infrastructure serves. Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` @@ -10,9 +10,6 @@ type Resource struct { // Specification for the resource. Spec ResourceSpec `json:"spec" yaml:"spec" required:"true" description:"Specification for the resource"` - // Status of the resource. - Status ResourceStatus `json:"status,omitzero" yaml:"status,omitempty" description:"Status of the resource"` - _ struct{} `additionalProperties:"false"` } @@ -22,11 +19,3 @@ type ResourceSpec struct { _ struct{} `additionalProperties:"false"` } - -type ResourceStatus struct { - v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - - v1alpha1.Status `json:",inline" yaml:",inline"` - - _ struct{} `additionalProperties:"false"` -} diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index d2097856..4edd19ab 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -19,7 +19,7 @@ func TestSchemaValidate(t *testing.T) { pass bool }{ { - name: "ResourceProvided_Valid", + name: "InstallationResource_Valid", mutate: func(casting *Casting) { casting.Spec.Resource.Kind = ResourceKindInstallation casting.Spec.Resource.Spec.Name = "signoz" From a06e5afb539bb99909da2de6b8cb124b7e509336 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 15 Jul 2026 23:52:50 +0530 Subject: [PATCH 14/38] feat(infrastructure): add the aws kubernetes terraform casting --- .../awskubernetesterraformcasting/casting.go | 32 +++++++++++++++++++ .../awskubernetesterraformcasting/enricher.go | 20 ++++++++++++ internal/casting/infrastructure/registry.go | 9 ++++++ 3 files changed, 61 insertions(+) create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/casting.go create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go new file mode 100644 index 00000000..6b4c7b41 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -0,0 +1,32 @@ +package awskubernetesterraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +type awsKubernetesTerraformCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *awsKubernetesTerraformCasting { + return &awsKubernetesTerraformCasting{logger: logger} +} + +func (c *awsKubernetesTerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return &enricher{logger: c.logger}, nil +} + +func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { + c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, no materials generated") + return nil, nil +} + +func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { + c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, nothing to cast") + return nil +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go b/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go new file mode 100644 index 00000000..2587707a --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go @@ -0,0 +1,20 @@ +package awskubernetesterraformcasting + +import ( + "context" + "log/slog" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" +) + +var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) + +type enricher struct { + logger *slog.Logger +} + +func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { + return nil +} diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 4151ec15..d3486bb5 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,6 +4,7 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting/infrastructure/awskubernetesterraformcasting" "github.com/signoz/foundry/internal/casting/infrastructure/ecsec2terraformcasting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" @@ -30,6 +31,14 @@ func NewRegistry(logger *slog.Logger) *Registry { Casting: ecsec2terraformcasting.New(logger), Toolers: []tooler.Tooler{terraformtooler.New()}, }, + { + Platform: v1alpha1.PlatformAWS, + Mode: v1alpha1.ModeKubernetes, + Flavor: v1alpha1.FlavorTerraform, + }: { + Casting: awskubernetesterraformcasting.New(logger), + Toolers: []tooler.Tooler{terraformtooler.New()}, + }, }, } } From a7a53cd0698b0756c2c5e0df248567f3128751a7 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 16 Jul 2026 12:14:07 +0530 Subject: [PATCH 15/38] feat(infrastructure): migrate the eks templates to the aws kubernetes terraform casting --- .../awskubernetesterraformcasting/casting.go | 26 +- .../awskubernetesterraformcasting/embed.go | 17 ++ .../embed_test.go | 33 +++ .../templates/main.tf.json.gotmpl | 223 ++++++++++++++++++ .../templates/outputs.tf.json.gotmpl | 41 ++++ .../templates/providers.tf.json.gotmpl | 14 ++ .../templates/variables.tf.json.gotmpl | 54 +++++ 7 files changed, 405 insertions(+), 3 deletions(-) create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/embed.go create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index 6b4c7b41..53e0cce6 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -3,8 +3,10 @@ package awskubernetesterraformcasting import ( "context" "log/slog" + "path/filepath" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + rootcasting "github.com/signoz/foundry/internal/casting" "github.com/signoz/foundry/internal/domain" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) @@ -22,11 +24,29 @@ func (c *awsKubernetesTerraformCasting) Enricher(ctx context.Context, config *in } func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { - c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, no materials generated") - return nil, nil + items := []struct { + template *domain.Template + path string + }{ + {providersTFTemplate, "providers.tf.json"}, + {mainTFTemplate, "main.tf.json"}, + {variablesTFTemplate, "variables.tf.json"}, + {outputsTFTemplate, "outputs.tf.json"}, + } + + materials := make([]domain.Material, 0, len(items)) + for _, item := range items { + material, err := item.template.Render(config, filepath.Join(rootcasting.DeploymentDir, item.path)) + if err != nil { + return nil, err + } + materials = append(materials, material) + } + + return materials, nil } func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { - c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, nothing to cast") + c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(poursPath, rootcasting.DeploymentDir))) return nil } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go new file mode 100644 index 00000000..f43cf6e4 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go @@ -0,0 +1,17 @@ +package awskubernetesterraformcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) + mainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) + variablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) + outputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/outputs.tf.json.gotmpl", domain.FormatJSON) +) diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go new file mode 100644 index 00000000..d3840de3 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go @@ -0,0 +1,33 @@ +package awskubernetesterraformcasting + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" +) + +func TestTemplates_RenderValidJSON(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation + config.Spec.Resource.Spec.Name = "signoz" + + testCases := []struct { + name string + template *domain.Template + }{ + {name: "ProvidersTemplate_RendersValidJSON", template: providersTFTemplate}, + {name: "MainTemplate_RendersValidJSON", template: mainTFTemplate}, + {name: "VariablesTemplate_RendersValidJSON", template: variablesTFTemplate}, + {name: "OutputsTemplate_RendersValidJSON", template: outputsTFTemplate}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + material, err := tc.template.Render(*config, "out.tf.json") + assert.NoError(t, err) + assert.NotEmpty(t, material.FmtContents()) + }) + } +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl new file mode 100644 index 00000000..e3473613 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl @@ -0,0 +1,223 @@ +{ + "locals": { + "name": "{{ .Metadata.Name }}" + }, + "data": { + "aws_availability_zones": { + "available": { + "state": "available" + } + } + }, + "resource": { + "aws_vpc": { + "main": { + "cidr_block": "${var.vpc_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-vpc", + "kubernetes.io/cluster/${local.name}": "shared" + } + } + }, + "aws_subnet": { + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/internal-elb": "1" + } + }, + "public": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "map_public_ip_on_launch": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/elb": "1" + } + } + }, + "aws_internet_gateway": { + "main": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-igw" + } + } + }, + "aws_eip": { + "nat": { + "count": "${var.az_count}", + "domain": "vpc", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-eip-${count.index}" + } + } + }, + "aws_nat_gateway": { + "main": { + "count": "${var.az_count}", + "allocation_id": "${aws_eip.nat[count.index].id}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-nat-${count.index}" + }, + "depends_on": ["aws_internet_gateway.main"] + } + }, + "aws_route_table": { + "public": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-public-rt" + } + }, + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}-private-rt-${count.index}" + } + } + }, + "aws_route": { + "public_internet": { + "route_table_id": "${aws_route_table.public.id}", + "destination_cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + }, + "private_nat": { + "count": "${var.az_count}", + "route_table_id": "${aws_route_table.private[count.index].id}", + "destination_cidr_block": "0.0.0.0/0", + "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" + } + }, + "aws_route_table_association": { + "public": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "route_table_id": "${aws_route_table.public.id}" + }, + "private": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.private[count.index].id}", + "route_table_id": "${aws_route_table.private[count.index].id}" + } + }, + "aws_iam_role": { + "eks_cluster": { + "name": "${local.name}-eks-cluster-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + }, + "eks_node_group": { + "name": "${local.name}-eks-node-group-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + } + } + }, + "aws_iam_role_policy_attachment": { + "eks_cluster_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "role": "${aws_iam_role.eks_cluster.name}" + }, + "eks_worker_node_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_cni_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_container_registry": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "ebs_csi_driver": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", + "role": "${aws_iam_role.eks_node_group.name}" + } + }, + "aws_eks_cluster": { + "main": { + "name": "${local.name}", + "role_arn": "${aws_iam_role.eks_cluster.arn}", + "version": "${var.kubernetes_version}", + "vpc_config": [{ + "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", + "endpoint_private_access": true, + "endpoint_public_access": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}" + }, + "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] + } + }, + "aws_eks_node_group": { + "main": { + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${var.node_instance_type}"], + "scaling_config": [{ + "desired_size": "${var.node_desired_size}", + "min_size": "${var.node_min_size}", + "max_size": "${var.node_max_size}" + }], + "disk_size": "${var.node_disk_size}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "Name": "${local.name}" + }, + "depends_on": [ + "aws_iam_role_policy_attachment.eks_worker_node_policy", + "aws_iam_role_policy_attachment.eks_cni_policy", + "aws_iam_role_policy_attachment.eks_container_registry" + ] + } + }, + "aws_eks_addon": { + "ebs_csi_driver": { + "cluster_name": "${aws_eks_cluster.main.name}", + "addon_name": "aws-ebs-csi-driver", + "depends_on": ["aws_eks_node_group.main"] + } + } + } +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl new file mode 100644 index 00000000..f9be3cec --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl @@ -0,0 +1,41 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the EKS cluster", + "value": "${aws_eks_cluster.main.name}" + }, + "cluster_endpoint": { + "description": "Endpoint for the EKS cluster API server", + "value": "${aws_eks_cluster.main.endpoint}" + }, + "cluster_ca_certificate": { + "description": "Base64-encoded certificate authority data for the EKS cluster", + "value": "${aws_eks_cluster.main.certificate_authority[0].data}", + "sensitive": true + }, + "cluster_version": { + "description": "Kubernetes version of the EKS cluster", + "value": "${aws_eks_cluster.main.version}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${aws_vpc.main.id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets", + "value": "${aws_subnet.private[*].id}" + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": "${aws_subnet.public[*].id}" + }, + "node_group_arn": { + "description": "ARN of the node group", + "value": "${aws_eks_node_group.main.arn}" + }, + "node_group_status": { + "description": "Status of the node group", + "value": "${aws_eks_node_group.main.status}" + } + } +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl new file mode 100644 index 00000000..b367d253 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl @@ -0,0 +1,14 @@ +{ + "terraform": { + "required_version": ">= 1.0.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + }, + "provider": { + "aws": [{}] + } +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl new file mode 100644 index 00000000..fbb31322 --- /dev/null +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl @@ -0,0 +1,54 @@ +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources", + "type": "string", + "default": "us-east-1" + }, + "vpc_cidr": { + "description": "CIDR block for the VPC", + "type": "string", + "default": "10.0.0.0/16" + }, + "az_count": { + "description": "Number of availability zones to use", + "type": "number", + "default": 2 + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "{{ .Metadata.Name }}" + }, + "kubernetes_version": { + "description": "Kubernetes version for the EKS cluster", + "type": "string", + "default": "1.30" + }, + "node_instance_type": { + "description": "EC2 instance type for the node group", + "type": "string", + "default": "t3.large" + }, + "node_desired_size": { + "description": "Desired number of nodes in the node group", + "type": "number", + "default": 2 + }, + "node_min_size": { + "description": "Minimum number of nodes in the node group", + "type": "number", + "default": 1 + }, + "node_max_size": { + "description": "Maximum number of nodes in the node group", + "type": "number", + "default": 4 + }, + "node_disk_size": { + "description": "Root volume size (GB) for the nodes", + "type": "number", + "default": 50 + } + } +} From 03d467880dbf8ad0115d2a0202300d2cf67310d1 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 16 Jul 2026 14:08:56 +0530 Subject: [PATCH 16/38] feat(infrastructure): adopt foundry discovery tags and honest node group bounds --- .../templates/main.tf.json.gotmpl | 36 ++++++++++++------- .../templates/variables.tf.json.gotmpl | 8 ++--- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl index e3473613..85757305 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl @@ -17,7 +17,8 @@ "enable_dns_support": true, "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-vpc", "kubernetes.io/cluster/${local.name}": "shared" } @@ -31,7 +32,8 @@ "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-private-${count.index}", "kubernetes.io/cluster/${local.name}": "shared", "kubernetes.io/role/internal-elb": "1" @@ -45,7 +47,8 @@ "map_public_ip_on_launch": true, "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-public-${count.index}", "kubernetes.io/cluster/${local.name}": "shared", "kubernetes.io/role/elb": "1" @@ -57,7 +60,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-igw" } } @@ -68,7 +72,8 @@ "domain": "vpc", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-nat-eip-${count.index}" } } @@ -80,7 +85,8 @@ "subnet_id": "${aws_subnet.public[count.index].id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-nat-${count.index}" }, "depends_on": ["aws_internet_gateway.main"] @@ -91,7 +97,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-public-rt" } }, @@ -100,7 +107,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}-private-rt-${count.index}" } } @@ -136,7 +144,8 @@ "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" } }, "eks_node_group": { @@ -144,7 +153,8 @@ "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" } } }, @@ -182,7 +192,8 @@ }], "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" }, "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] } @@ -202,7 +213,8 @@ "disk_size": "${var.node_disk_size}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/name": "{{ .Metadata.Name }}", + "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", "Name": "${local.name}" }, "depends_on": [ diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl index fbb31322..18ae1b16 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl @@ -23,7 +23,7 @@ "kubernetes_version": { "description": "Kubernetes version for the EKS cluster", "type": "string", - "default": "1.30" + "default": "1.33" }, "node_instance_type": { "description": "EC2 instance type for the node group", @@ -38,12 +38,12 @@ "node_min_size": { "description": "Minimum number of nodes in the node group", "type": "number", - "default": 1 + "default": 2 }, "node_max_size": { - "description": "Maximum number of nodes in the node group", + "description": "Maximum number of nodes in the node group; raise together with a cluster autoscaler, bounds alone add no elasticity", "type": "number", - "default": 4 + "default": 2 }, "node_disk_size": { "description": "Root volume size (GB) for the nodes", From 7df32dd326ac181234b03dcfb9c77b2e4d715a71 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 16 Jul 2026 15:38:22 +0530 Subject: [PATCH 17/38] feat(infrastructure): materialize the resource requirement set in status --- .../infrastructure/casting.schema.json | 71 ++++++++++++++ api/v1alpha1/infrastructure/resource.go | 24 +++++ .../infrastructure/resource_config.go | 38 ++++++++ .../awskubernetesterraformcasting/casting.go | 61 +++++++++++- .../embed_test.go | 14 ++- .../templates/main.tf.json.gotmpl | 95 +++++++++++-------- .../templates/outputs.tf.json.gotmpl | 18 ++-- .../templates/variables.tf.json.gotmpl | 34 +++---- .../resourcemolding/resource.go | 59 +++++++++++- .../resourcemolding/resource_test.go | 72 +++++++++++++- 10 files changed, 406 insertions(+), 80 deletions(-) create mode 100644 api/v1alpha1/infrastructure/resource_config.go diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 019e1d1f..7dd8e77d 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -24,6 +24,10 @@ "spec": { "$ref": "#/definitions/InfrastructureResourceSpec", "description": "Specification for the resource" + }, + "status": { + "$ref": "#/definitions/InfrastructureResourceStatus", + "description": "Status of the resource" } }, "type": "object" @@ -62,6 +66,60 @@ }, "type": "object" }, + "InfrastructureResourceStatus": { + "additionalProperties": false, + "properties": { + "addresses": { + "$ref": "#/definitions/InfrastructureResourceStatusAddresses", + "description": "Addresses the resource admits at the substrate's edge" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "extras": { + "description": "Extra information about the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, + "InfrastructureResourceStatusAddresses": { + "additionalProperties": false, + "properties": { + "apiserver": { + "description": "API server addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + }, + "otlp": { + "description": "OTLP addresses", + "items": { + "type": "string" + }, + "type": [ + "array", + "null" + ] + } + }, + "type": "object" + }, "InfrastructureSpec": { "required": [ "deployment", @@ -223,6 +281,19 @@ }, "type": "object" }, + "V1Alpha1TypeConfig": { + "additionalProperties": false, + "properties": { + "data": { + "description": "Configuration data as key-value pairs.", + "additionalProperties": { + "type": "string" + }, + "type": "object" + } + }, + "type": "object" + }, "V1Alpha1TypeDeployment": { "additionalProperties": false, "properties": { diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index c3b5ff86..8ef7edd3 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -10,6 +10,9 @@ type Resource struct { // Specification for the resource. Spec ResourceSpec `json:"spec" yaml:"spec" required:"true" description:"Specification for the resource"` + // Status of the resource. + Status ResourceStatus `json:"status" yaml:"status,omitempty" description:"Status of the resource"` + _ struct{} `additionalProperties:"false"` } @@ -19,3 +22,24 @@ type ResourceSpec struct { _ struct{} `additionalProperties:"false"` } + +// ResourceStatus carries the requirement set a substrate shaped for the +// resource kind must satisfy. +type ResourceStatus struct { + v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` + + // Addresses the resource admits at the substrate's edge. + Addresses ResourceStatusAddresses `json:"addresses" yaml:"addresses,omitempty" description:"Addresses the resource admits at the substrate's edge"` + + _ struct{} `additionalProperties:"false"` +} + +type ResourceStatusAddresses struct { + // OTLP addresses. + OTLP []string `json:"otlp" yaml:"otlp,omitempty" description:"OTLP addresses"` + + // API server addresses. + APIServer []string `json:"apiserver" yaml:"apiserver,omitempty" description:"API server addresses"` + + _ struct{} `additionalProperties:"false"` +} diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go new file mode 100644 index 00000000..89472400 --- /dev/null +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -0,0 +1,38 @@ +package infrastructure + +// ResourceConfig is the resource requirement document (resource.yaml): the +// canonical internal representation of what a substrate shaped for the +// resource kind must provide. It speaks criteria only; platform vocabulary +// never enters it (machines are resolved by castings). +type ResourceConfig struct { + // Storage the resource requires from the substrate. + Storage ResourceConfigStorage `json:"storage"` + + // Node groups the resource requires from the substrate. + NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name"` +} + +// ResourceConfigStorage describes the storage requirement. +type ResourceConfigStorage struct { + // Whether the resource persists data. + Persistent *bool `json:"persistent,omitempty"` +} + +// ResourceConfigNodeGroup sizes a pool of nodes as criteria, never as +// machine types. +type ResourceConfigNodeGroup struct { + // Name of the node group. + Name string `json:"name"` + + // Count of nodes. + Count *int `json:"count,omitempty"` + + // VCPUs per node. + VCPUs *int `json:"vcpus,omitempty"` + + // Memory per node in GiB. + Memory *int `json:"memory,omitempty"` + + // Disk per node in GiB. + Disk *int `json:"disk,omitempty"` +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index 53e0cce6..a3515e84 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -8,9 +8,27 @@ import ( "github.com/signoz/foundry/api/v1alpha1/infrastructure" rootcasting "github.com/signoz/foundry/internal/casting" "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" ) +// Data carries the resolved values the templates render. +type Data struct { + Name string + ResourceKind string + Persistent bool + NodeGroups []DataNodeGroup +} + +type DataNodeGroup struct { + Name string + Count int + VCPUs int + Memory int + Disk int +} + type awsKubernetesTerraformCasting struct { logger *slog.Logger } @@ -24,6 +42,11 @@ func (c *awsKubernetesTerraformCasting) Enricher(ctx context.Context, config *in } func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { + data, err := newData(config) + if err != nil { + return nil, err + } + items := []struct { template *domain.Template path string @@ -36,7 +59,7 @@ func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infras materials := make([]domain.Material, 0, len(items)) for _, item := range items { - material, err := item.template.Render(config, filepath.Join(rootcasting.DeploymentDir, item.path)) + material, err := item.template.Render(data, filepath.Join(rootcasting.DeploymentDir, item.path)) if err != nil { return nil, err } @@ -50,3 +73,39 @@ func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrast c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(poursPath, rootcasting.DeploymentDir))) return nil } + +// newData resolves the resource requirement document into the values the +// templates render. +func newData(config infrastructure.Casting) (*Data, error) { + doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] + if doc == "" { + return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "resource config %q is missing from the resource status", resourcemolding.ResourceConfigName) + } + + resourceConfig := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(doc), resourceConfig); err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config") + } + + data := &Data{ + Name: config.Metadata.Name, + ResourceKind: config.Spec.Resource.Kind.String(), + Persistent: resourceConfig.Storage.Persistent != nil && *resourceConfig.Storage.Persistent, + } + + for _, group := range resourceConfig.NodeGroups { + if group.Count == nil || group.VCPUs == nil || group.Memory == nil || group.Disk == nil { + return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "node group %q in resource config is incomplete", group.Name) + } + + data.NodeGroups = append(data.NodeGroups, DataNodeGroup{ + Name: group.Name, + Count: *group.Count, + VCPUs: *group.VCPUs, + Memory: *group.Memory, + Disk: *group.Disk, + }) + } + + return data, nil +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go index d3840de3..c6981c5e 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go @@ -3,15 +3,19 @@ package awskubernetesterraformcasting import ( "testing" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" ) func TestTemplates_RenderValidJSON(t *testing.T) { - config := infrastructure.Default() - config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation - config.Spec.Resource.Spec.Name = "signoz" + data := &Data{ + Name: "signoz", + ResourceKind: "Installation", + Persistent: true, + NodeGroups: []DataNodeGroup{ + {Name: "default", Count: 2, VCPUs: 2, Memory: 8, Disk: 50}, + }, + } testCases := []struct { name string @@ -25,7 +29,7 @@ func TestTemplates_RenderValidJSON(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - material, err := tc.template.Render(*config, "out.tf.json") + material, err := tc.template.Render(data, "out.tf.json") assert.NoError(t, err) assert.NotEmpty(t, material.FmtContents()) }) diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl index 85757305..e794054a 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl @@ -1,12 +1,29 @@ { "locals": { - "name": "{{ .Metadata.Name }}" + "name": "{{ .Name }}" + {{- range $group := .NodeGroups }}, + "node_{{ $group.Name }}_instance_type": "${var.node_{{ $group.Name }}_instance_type != \"\" ? var.node_{{ $group.Name }}_instance_type : sort(data.aws_ec2_instance_types.{{ $group.Name }}.instance_types)[0]}" + {{- end }} }, "data": { "aws_availability_zones": { "available": { "state": "available" } + }, + "aws_ec2_instance_types": { + {{- range $i, $group := .NodeGroups }}{{ if $i }},{{ end }} + "{{ $group.Name }}": { + "filter": [ + {"name": "instance-type", "values": ["m*", "c*"]}, + {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, + {"name": "current-generation", "values": ["true"]}, + {"name": "burstable-performance-supported", "values": ["false"]}, + {"name": "vcpu-info.default-vcpus", "values": ["{{ $group.VCPUs }}"]}, + {"name": "memory-info.size-in-mib", "values": ["{{ mul $group.Memory 1024 }}"]} + ] + } + {{- end }} } }, "resource": { @@ -17,8 +34,8 @@ "enable_dns_support": true, "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-vpc", "kubernetes.io/cluster/${local.name}": "shared" } @@ -32,8 +49,8 @@ "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-private-${count.index}", "kubernetes.io/cluster/${local.name}": "shared", "kubernetes.io/role/internal-elb": "1" @@ -47,8 +64,8 @@ "map_public_ip_on_launch": true, "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-public-${count.index}", "kubernetes.io/cluster/${local.name}": "shared", "kubernetes.io/role/elb": "1" @@ -60,8 +77,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-igw" } } @@ -72,8 +89,8 @@ "domain": "vpc", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-nat-eip-${count.index}" } } @@ -85,8 +102,8 @@ "subnet_id": "${aws_subnet.public[count.index].id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-nat-${count.index}" }, "depends_on": ["aws_internet_gateway.main"] @@ -97,8 +114,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-public-rt" } }, @@ -107,8 +124,8 @@ "vpc_id": "${aws_vpc.main.id}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", "Name": "${local.name}-private-rt-${count.index}" } } @@ -144,8 +161,8 @@ "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" } }, "eks_node_group": { @@ -153,8 +170,8 @@ "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" } } }, @@ -174,11 +191,11 @@ "eks_container_registry": { "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", "role": "${aws_iam_role.eks_node_group.name}" - }, + }{{ if .Persistent }}, "ebs_csi_driver": { "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", "role": "${aws_iam_role.eks_node_group.name}" - } + }{{ end }} }, "aws_eks_cluster": { "main": { @@ -192,30 +209,31 @@ }], "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}" + "foundry.signoz.io/name": "{{ .Name }}", + "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" }, "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] } }, "aws_eks_node_group": { - "main": { + {{- range $i, $group := .NodeGroups }}{{ if $i }},{{ end }} + "{{ $group.Name }}": { "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}", + "node_group_name": "${local.name}-{{ $group.Name }}", "node_role_arn": "${aws_iam_role.eks_node_group.arn}", "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${var.node_instance_type}"], + "instance_types": ["${local.node_{{ $group.Name }}_instance_type}"], "scaling_config": [{ - "desired_size": "${var.node_desired_size}", - "min_size": "${var.node_min_size}", - "max_size": "${var.node_max_size}" + "desired_size": "${var.node_{{ $group.Name }}_count}", + "min_size": "${var.node_{{ $group.Name }}_count}", + "max_size": "${var.node_{{ $group.Name }}_count}" }], - "disk_size": "${var.node_disk_size}", + "disk_size": "${var.node_{{ $group.Name }}_disk_size}", "tags": { "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Metadata.Name }}", - "foundry.signoz.io/resource-kind": "{{ .Spec.Resource.Kind }}", - "Name": "${local.name}" + "foundry.signoz.io/name": "{{ $.Name }}", + "foundry.signoz.io/resource-kind": "{{ $.ResourceKind }}", + "Name": "${local.name}-{{ $group.Name }}" }, "depends_on": [ "aws_iam_role_policy_attachment.eks_worker_node_policy", @@ -223,13 +241,14 @@ "aws_iam_role_policy_attachment.eks_container_registry" ] } - }, + {{- end }} + }{{ if .Persistent }}, "aws_eks_addon": { "ebs_csi_driver": { "cluster_name": "${aws_eks_cluster.main.name}", "addon_name": "aws-ebs-csi-driver", - "depends_on": ["aws_eks_node_group.main"] + "depends_on": [{{ range $i, $group := .NodeGroups }}{{ if $i }}, {{ end }}"aws_eks_node_group.{{ $group.Name }}"{{ end }}] } - } + }{{ end }} } } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl index f9be3cec..1baed1b9 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl @@ -28,14 +28,16 @@ "public_subnet_ids": { "description": "IDs of the public subnets", "value": "${aws_subnet.public[*].id}" - }, - "node_group_arn": { - "description": "ARN of the node group", - "value": "${aws_eks_node_group.main.arn}" - }, - "node_group_status": { - "description": "Status of the node group", - "value": "${aws_eks_node_group.main.status}" } + {{- range $group := .NodeGroups }}, + "node_group_{{ $group.Name }}_arn": { + "description": "ARN of the {{ $group.Name }} node group", + "value": "${aws_eks_node_group.{{ $group.Name }}.arn}" + }, + "node_group_{{ $group.Name }}_status": { + "description": "Status of the {{ $group.Name }} node group", + "value": "${aws_eks_node_group.{{ $group.Name }}.status}" + } + {{- end }} } } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl index 18ae1b16..84e36a37 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl @@ -18,37 +18,29 @@ "name": { "description": "The name of the deployment", "type": "string", - "default": "{{ .Metadata.Name }}" + "default": "{{ .Name }}" }, "kubernetes_version": { "description": "Kubernetes version for the EKS cluster", "type": "string", "default": "1.33" - }, - "node_instance_type": { - "description": "EC2 instance type for the node group", + } + {{- range $group := .NodeGroups }}, + "node_{{ $group.Name }}_instance_type": { + "description": "EC2 instance type for the {{ $group.Name }} node group; empty resolves the declared criteria against the platform's instance catalog", "type": "string", - "default": "t3.large" - }, - "node_desired_size": { - "description": "Desired number of nodes in the node group", - "type": "number", - "default": 2 + "default": "" }, - "node_min_size": { - "description": "Minimum number of nodes in the node group", + "node_{{ $group.Name }}_count": { + "description": "Number of nodes in the {{ $group.Name }} node group; elastic bounds belong with a cluster autoscaler", "type": "number", - "default": 2 - }, - "node_max_size": { - "description": "Maximum number of nodes in the node group; raise together with a cluster autoscaler, bounds alone add no elasticity", - "type": "number", - "default": 2 + "default": {{ $group.Count }} }, - "node_disk_size": { - "description": "Root volume size (GB) for the nodes", + "node_{{ $group.Name }}_disk_size": { + "description": "Root volume size (GB) for the {{ $group.Name }} nodes", "type": "number", - "default": 50 + "default": {{ $group.Disk }} } + {{- end }} } } diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index b53401e1..4d7f09cc 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -6,10 +6,22 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) +// The edge conventions of the SigNoz resource kinds. +var ( + otlpGRPCAddress = domain.MustNewAddress("tcp", "0.0.0.0", 4317).String() + otlpHTTPAddress = domain.MustNewAddress("tcp", "0.0.0.0", 4318).String() + apiServerAddress = domain.MustNewAddress("tcp", "0.0.0.0", 8080).String() +) + +// ResourceConfigName is the config document carrying the requirements a +// substrate shaped for the resource kind must satisfy beyond its edge. +const ResourceConfigName = "resource.yaml" + var _ infrastructuremolding.Molding = (*resourceMolding)(nil) type resourceMolding struct { @@ -24,11 +36,54 @@ func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindResource } +// MoldV1Alpha1 writes the kind-level requirement set into the resource +// status: baseline first, preserving entries an enricher has already +// contributed. func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { + status := &config.Spec.Resource.Status + + var baseline *infrastructure.ResourceConfig switch config.Spec.Resource.Kind { - case infrastructure.ResourceKindInstallation, infrastructure.ResourceKindCollectionAgent: - return nil + case infrastructure.ResourceKindInstallation: + status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) + status.Addresses.APIServer = append([]string{apiServerAddress}, status.Addresses.APIServer...) + baseline = &infrastructure.ResourceConfig{ + Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(true)}, + NodeGroups: []infrastructure.ResourceConfigNodeGroup{ + {Name: "default", Count: v1alpha1.IntPtr(2), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + }, + } + case infrastructure.ResourceKindCollectionAgent: + status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) + baseline = &infrastructure.ResourceConfig{ + Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(false)}, + NodeGroups: []infrastructure.ResourceConfigNodeGroup{ + {Name: "default", Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + }, + } default: return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported resource kind %q", config.Spec.Resource.Kind) } + + if overrides := status.Config.Data[ResourceConfigName]; overrides != "" { + overrideConfig := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(overrides), overrideConfig); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config overrides") + } + if err := v1alpha1.Merge(baseline, overrideConfig); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to merge resource config overrides") + } + } + + doc, err := domain.MarshalYAML(baseline) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal resource config") + } + + if status.Config.Data == nil { + status.Config.Data = make(map[string]string) + } + status.Config.Data[ResourceConfigName] = string(doc) + + return nil } diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index 2a123eee..839011a3 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -5,25 +5,40 @@ import ( "log/slog" "testing" + "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" ) func TestMoldV1Alpha1(t *testing.T) { tests := []struct { - name string - kind infrastructure.ResourceKind - pass bool + name string + kind infrastructure.ResourceKind + pass bool + expected infrastructure.ResourceConfig }{ { - name: "InstallationResource_Supported", + name: "InstallationResource_PersistentStorageAndDefaultNodeGroup", kind: infrastructure.ResourceKindInstallation, pass: true, + expected: infrastructure.ResourceConfig{ + Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(true)}, + NodeGroups: []infrastructure.ResourceConfigNodeGroup{ + {Name: "default", Count: v1alpha1.IntPtr(2), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + }, + }, }, { - name: "CollectionAgentResource_Supported", + name: "CollectionAgentResource_EphemeralStorageAndDefaultNodeGroup", kind: infrastructure.ResourceKindCollectionAgent, pass: true, + expected: infrastructure.ResourceConfig{ + Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(false)}, + NodeGroups: []infrastructure.ResourceConfigNodeGroup{ + {Name: "default", Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + }, + }, }, { name: "UnknownResourceKind_Unsupported", @@ -43,6 +58,53 @@ func TestMoldV1Alpha1(t *testing.T) { return } assert.NoError(t, err) + + got := infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[ResourceConfigName]), &got)) + assert.Equal(t, tt.expected, got) }) } } + +func TestMoldV1Alpha1_AddressesBaseline(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation + + err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + assert.NoError(t, err) + assert.Equal(t, []string{"tcp://0.0.0.0:4317", "tcp://0.0.0.0:4318"}, config.Spec.Resource.Status.Addresses.OTLP) + assert.Equal(t, []string{"tcp://0.0.0.0:8080"}, config.Spec.Resource.Status.Addresses.APIServer) +} + +func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation + config.Spec.Resource.Status.Addresses.OTLP = []string{"tcp://0.0.0.0:9411"} + config.Spec.Resource.Status.Config.Data = map[string]string{ + ResourceConfigName: "nodeGroups:\n- name: default\n count: 4\n- name: keeper\n count: 3\n vcpus: 2\n memory: 8\n disk: 100\n", + } + + err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + assert.NoError(t, err) + assert.Equal(t, []string{"tcp://0.0.0.0:4317", "tcp://0.0.0.0:4318", "tcp://0.0.0.0:9411"}, config.Spec.Resource.Status.Addresses.OTLP) + + got := infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[ResourceConfigName]), &got)) + + assert.Equal(t, v1alpha1.BoolPtr(true), got.Storage.Persistent) + assert.Len(t, got.NodeGroups, 2) + for _, group := range got.NodeGroups { + switch group.Name { + case "default": + assert.Equal(t, v1alpha1.IntPtr(4), group.Count) + assert.Equal(t, v1alpha1.IntPtr(2), group.VCPUs) + assert.Equal(t, v1alpha1.IntPtr(8), group.Memory) + assert.Equal(t, v1alpha1.IntPtr(50), group.Disk) + case "keeper": + assert.Equal(t, v1alpha1.IntPtr(3), group.Count) + assert.Equal(t, v1alpha1.IntPtr(100), group.Disk) + default: + t.Fatalf("unexpected node group %q", group.Name) + } + } +} From cdc1476949a721856ece3ff8a8cd9a85fbaba344 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 16 Jul 2026 16:16:56 +0530 Subject: [PATCH 18/38] refactor(infrastructure): finalize the casting layout and add the example --- api/v1alpha1/infrastructure/casting.go | 1 + .../infrastructure/resource_config.go | 24 +- .../aws/kubernetes/terraform/README.md | 132 ++++++++++ .../aws/kubernetes/terraform/casting.yaml | 13 + .../kubernetes/terraform/casting.yaml.lock | 32 +++ .../pours/infrastructure/main.tf.json | 248 ++++++++++++++++++ .../pours/infrastructure/outputs.tf.json | 41 +++ .../pours/infrastructure/providers.tf.json | 14 + .../pours/infrastructure/variables.tf.json | 44 ++++ .../awskubernetesterraformcasting/casting.go | 9 +- .../infrastructure/{ => casting}/casting.go | 6 +- .../ecsec2terraformcasting/casting.go | 32 --- .../ecsec2terraformcasting/enricher.go | 20 -- internal/casting/infrastructure/planner.go | 3 +- internal/casting/infrastructure/registry.go | 14 +- 15 files changed, 556 insertions(+), 77 deletions(-) create mode 100644 docs/examples/aws/kubernetes/terraform/README.md create mode 100644 docs/examples/aws/kubernetes/terraform/casting.yaml create mode 100644 docs/examples/aws/kubernetes/terraform/casting.yaml.lock create mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json create mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json create mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json create mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json rename internal/casting/infrastructure/{ => casting}/casting.go (75%) delete mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/casting.go delete mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/enricher.go diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index 96135fe9..fee5cb37 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -71,6 +71,7 @@ func (c *Casting) TrackableProperties() domain.Properties { return domain.NewProperties(). Set("kind", v1alpha1.KindInfrastructure.String()). Set("platform", c.Spec.Deployment.Platform.String()). + Set("mode", c.Spec.Deployment.Mode.String()). Set("flavor", c.Spec.Deployment.Flavor.String()). Set("resource_kind", c.Spec.Resource.Kind.String()). Set("patches_count", len(c.Spec.Patches)) diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index 89472400..e217b519 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -3,36 +3,42 @@ package infrastructure // ResourceConfig is the resource requirement document (resource.yaml): the // canonical internal representation of what a substrate shaped for the // resource kind must provide. It speaks criteria only; platform vocabulary -// never enters it (machines are resolved by castings). +// never enters it (machines are resolved by the platform). type ResourceConfig struct { // Storage the resource requires from the substrate. - Storage ResourceConfigStorage `json:"storage"` + Storage ResourceConfigStorage `json:"storage" description:"Storage the resource requires from the substrate"` // Node groups the resource requires from the substrate. - NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name"` + NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name" description:"Node groups the resource requires from the substrate"` + + _ struct{} `additionalProperties:"false"` } // ResourceConfigStorage describes the storage requirement. type ResourceConfigStorage struct { // Whether the resource persists data. - Persistent *bool `json:"persistent,omitempty"` + Persistent *bool `json:"persistent,omitempty" description:"Whether the resource persists data"` + + _ struct{} `additionalProperties:"false"` } // ResourceConfigNodeGroup sizes a pool of nodes as criteria, never as // machine types. type ResourceConfigNodeGroup struct { // Name of the node group. - Name string `json:"name"` + Name string `json:"name" description:"Name of the node group"` // Count of nodes. - Count *int `json:"count,omitempty"` + Count *int `json:"count,omitempty" description:"Count of nodes"` // VCPUs per node. - VCPUs *int `json:"vcpus,omitempty"` + VCPUs *int `json:"vcpus,omitempty" description:"VCPUs per node"` // Memory per node in GiB. - Memory *int `json:"memory,omitempty"` + Memory *int `json:"memory,omitempty" description:"Memory per node in GiB"` // Disk per node in GiB. - Disk *int `json:"disk,omitempty"` + Disk *int `json:"disk,omitempty" description:"Disk per node in GiB"` + + _ struct{} `additionalProperties:"false"` } diff --git a/docs/examples/aws/kubernetes/terraform/README.md b/docs/examples/aws/kubernetes/terraform/README.md new file mode 100644 index 00000000..1c0a0903 --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/README.md @@ -0,0 +1,132 @@ +# AWS Kubernetes with Terraform (Infrastructure) + +| Field | Value | +| --- | --- | +| **Kind** | `Infrastructure` | +| **Platform** | `aws` | +| **Mode** | `kubernetes` | +| **Flavor** | `terraform` | + +## Overview + +Provisions an EKS substrate shaped for a SigNoz Installation. The infrastructure never reads the installation's casting: the resource declaration names what the substrate is shaped for, and foundry's own kind-level knowledge (the requirement set) drives what gets provisioned. + +Resources: +- VPC with public and private subnets across two availability zones, internet and NAT gateways +- EKS cluster with IAM roles for the control plane and nodes +- One managed node group sized from the requirement set, in private subnets +- EBS CSI driver addon (the installation's components request storage through PVCs) + +## Prerequisites + +- AWS credentials with permissions to create VPC, EKS, and IAM resources +- [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.0 + +## Configuration + +```yaml +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: kubernetes + flavor: terraform + resource: + kind: Installation + spec: + name: signoz +``` + +`spec.resource` declares the kind of resource the substrate serves and the name of the casting embodying it. It is a declaration, not a reference: nothing is read from the installation's casting. + +## Deploy + +```bash +# 1. Generate Terraform files +foundryctl forge -f casting.yaml + +# 2. Initialize and apply Terraform +cd pours/infrastructure +terraform init +terraform apply +``` + +## Generated output + +```text +pours/infrastructure/ + providers.tf.json + main.tf.json + variables.tf.json + outputs.tf.json +``` + +## Customization + +To pin an exact instance type instead of resolving the criteria, set the instance type variable through `spec.patches`: + +```yaml +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: kubernetes + flavor: terraform + resource: + kind: Installation + spec: + name: signoz + patches: + - target: "infrastructure/variables.tf.json" + operations: + - op: replace + path: /variable/node_default_instance_type/default + value: t3.large +``` + +Any generated value can be changed the same way; run `foundryctl forge` and inspect the files under `pours/infrastructure/` to identify the JSON paths. + +## Platform details + +### Variables + +| Variable | Default | Description | +| --- | --- | --- | +| `aws_region` | `us-east-1` | AWS region | +| `vpc_cidr` | `10.0.0.0/16` | CIDR block for the VPC | +| `az_count` | `2` | Number of availability zones | +| `name` | `signoz` | Name of the deployment | +| `kubernetes_version` | `1.33` | Kubernetes version for the EKS cluster | +| `node_default_instance_type` | `""` | Instance type; empty resolves the declared criteria against the platform's instance catalog | +| `node_default_count` | `2` | Number of nodes; elastic bounds belong with a cluster autoscaler | +| `node_default_disk_size` | `50` | Root volume size (GB) for the nodes | + +### Outputs + +| Output | Description | +| --- | --- | +| `cluster_name` | Name of the EKS cluster | +| `cluster_endpoint` | EKS API server endpoint | +| `cluster_ca_certificate` | Base64-encoded CA data (sensitive) | +| `cluster_version` | Kubernetes version of the cluster | +| `vpc_id` | ID of the VPC | +| `private_subnet_ids` | IDs of the private subnets | +| `public_subnet_ids` | IDs of the public subnets | +| `node_group_default_arn` | ARN of the default node group | +| `node_group_default_status` | Status of the default node group | + +### Tags + +Every resource carries the discovery tags, so consumers can find the substrate by name: + +| Tag | Value | +| --- | --- | +| `app.kubernetes.io/managed-by` | `foundry` | +| `foundry.signoz.io/name` | `signoz` | +| `foundry.signoz.io/resource-kind` | `Installation` | diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml b/docs/examples/aws/kubernetes/terraform/casting.yaml new file mode 100644 index 00000000..3aa215b3 --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/casting.yaml @@ -0,0 +1,13 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: kubernetes + flavor: terraform + resource: + kind: Installation + spec: + name: signoz diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock new file mode 100644 index 00000000..1110957e --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock @@ -0,0 +1,32 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + flavor: terraform + mode: kubernetes + platform: aws + resource: + kind: Installation + spec: + apiVersion: v1alpha1 + name: signoz + status: + addresses: + apiserver: + - tcp://0.0.0.0:8080 + otlp: + - tcp://0.0.0.0:4317 + - tcp://0.0.0.0:4318 + config: + data: + resource.yaml: | + nodeGroups: + - count: 2 + disk: 50 + memory: 8 + name: default + vcpus: 2 + storage: + persistent: true diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json new file mode 100644 index 00000000..67076ed7 --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json @@ -0,0 +1,248 @@ +{ + "locals": { + "name": "signoz", + "node_default_instance_type": "${var.node_default_instance_type != \"\" ? var.node_default_instance_type : sort(data.aws_ec2_instance_types.default.instance_types)[0]}" + }, + "data": { + "aws_availability_zones": { + "available": { + "state": "available" + } + }, + "aws_ec2_instance_types": { + "default": { + "filter": [ + {"name": "instance-type", "values": ["m*", "c*"]}, + {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, + {"name": "current-generation", "values": ["true"]}, + {"name": "burstable-performance-supported", "values": ["false"]}, + {"name": "vcpu-info.default-vcpus", "values": ["2"]}, + {"name": "memory-info.size-in-mib", "values": ["8192"]} + ] + } + } + }, + "resource": { + "aws_vpc": { + "main": { + "cidr_block": "${var.vpc_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-vpc", + "kubernetes.io/cluster/${local.name}": "shared" + } + } + }, + "aws_subnet": { + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-private-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/internal-elb": "1" + } + }, + "public": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", + "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", + "map_public_ip_on_launch": true, + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-public-${count.index}", + "kubernetes.io/cluster/${local.name}": "shared", + "kubernetes.io/role/elb": "1" + } + } + }, + "aws_internet_gateway": { + "main": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-igw" + } + } + }, + "aws_eip": { + "nat": { + "count": "${var.az_count}", + "domain": "vpc", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-nat-eip-${count.index}" + } + } + }, + "aws_nat_gateway": { + "main": { + "count": "${var.az_count}", + "allocation_id": "${aws_eip.nat[count.index].id}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-nat-${count.index}" + }, + "depends_on": ["aws_internet_gateway.main"] + } + }, + "aws_route_table": { + "public": { + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-public-rt" + } + }, + "private": { + "count": "${var.az_count}", + "vpc_id": "${aws_vpc.main.id}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-private-rt-${count.index}" + } + } + }, + "aws_route": { + "public_internet": { + "route_table_id": "${aws_route_table.public.id}", + "destination_cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + }, + "private_nat": { + "count": "${var.az_count}", + "route_table_id": "${aws_route_table.private[count.index].id}", + "destination_cidr_block": "0.0.0.0/0", + "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" + } + }, + "aws_route_table_association": { + "public": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.public[count.index].id}", + "route_table_id": "${aws_route_table.public.id}" + }, + "private": { + "count": "${var.az_count}", + "subnet_id": "${aws_subnet.private[count.index].id}", + "route_table_id": "${aws_route_table.private[count.index].id}" + } + }, + "aws_iam_role": { + "eks_cluster": { + "name": "${local.name}-eks-cluster-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation" + } + }, + "eks_node_group": { + "name": "${local.name}-eks-node-group-role", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation" + } + } + }, + "aws_iam_role_policy_attachment": { + "eks_cluster_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", + "role": "${aws_iam_role.eks_cluster.name}" + }, + "eks_worker_node_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_cni_policy": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "eks_container_registry": { + "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", + "role": "${aws_iam_role.eks_node_group.name}" + }, + "ebs_csi_driver": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", + "role": "${aws_iam_role.eks_node_group.name}" + } + }, + "aws_eks_cluster": { + "main": { + "name": "${local.name}", + "role_arn": "${aws_iam_role.eks_cluster.arn}", + "version": "${var.kubernetes_version}", + "vpc_config": [{ + "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", + "endpoint_private_access": true, + "endpoint_public_access": true + }], + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation" + }, + "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] + } + }, + "aws_eks_node_group": { + "default": { + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-default", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${local.node_default_instance_type}"], + "scaling_config": [{ + "desired_size": "${var.node_default_count}", + "min_size": "${var.node_default_count}", + "max_size": "${var.node_default_count}" + }], + "disk_size": "${var.node_default_disk_size}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-default" + }, + "depends_on": [ + "aws_iam_role_policy_attachment.eks_worker_node_policy", + "aws_iam_role_policy_attachment.eks_cni_policy", + "aws_iam_role_policy_attachment.eks_container_registry" + ] + } + }, + "aws_eks_addon": { + "ebs_csi_driver": { + "cluster_name": "${aws_eks_cluster.main.name}", + "addon_name": "aws-ebs-csi-driver", + "depends_on": ["aws_eks_node_group.default"] + } + } + } +} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json new file mode 100644 index 00000000..de5a8cea --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json @@ -0,0 +1,41 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the EKS cluster", + "value": "${aws_eks_cluster.main.name}" + }, + "cluster_endpoint": { + "description": "Endpoint for the EKS cluster API server", + "value": "${aws_eks_cluster.main.endpoint}" + }, + "cluster_ca_certificate": { + "description": "Base64-encoded certificate authority data for the EKS cluster", + "value": "${aws_eks_cluster.main.certificate_authority[0].data}", + "sensitive": true + }, + "cluster_version": { + "description": "Kubernetes version of the EKS cluster", + "value": "${aws_eks_cluster.main.version}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${aws_vpc.main.id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets", + "value": "${aws_subnet.private[*].id}" + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": "${aws_subnet.public[*].id}" + }, + "node_group_default_arn": { + "description": "ARN of the default node group", + "value": "${aws_eks_node_group.default.arn}" + }, + "node_group_default_status": { + "description": "Status of the default node group", + "value": "${aws_eks_node_group.default.status}" + } + } +} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json new file mode 100644 index 00000000..b367d253 --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json @@ -0,0 +1,14 @@ +{ + "terraform": { + "required_version": ">= 1.0.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + }, + "provider": { + "aws": [{}] + } +} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json new file mode 100644 index 00000000..e2da7960 --- /dev/null +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json @@ -0,0 +1,44 @@ +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources", + "type": "string", + "default": "us-east-1" + }, + "vpc_cidr": { + "description": "CIDR block for the VPC", + "type": "string", + "default": "10.0.0.0/16" + }, + "az_count": { + "description": "Number of availability zones to use", + "type": "number", + "default": 2 + }, + "name": { + "description": "The name of the deployment", + "type": "string", + "default": "signoz" + }, + "kubernetes_version": { + "description": "Kubernetes version for the EKS cluster", + "type": "string", + "default": "1.33" + }, + "node_default_instance_type": { + "description": "EC2 instance type for the default node group; empty resolves the declared criteria against the platform's instance catalog", + "type": "string", + "default": "" + }, + "node_default_count": { + "description": "Number of nodes in the default node group; elastic bounds belong with a cluster autoscaler", + "type": "number", + "default": 2 + }, + "node_default_disk_size": { + "description": "Root volume size (GB) for the default nodes", + "type": "number", + "default": 50 + } + } +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index a3515e84..46ac5232 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -6,7 +6,7 @@ import ( "path/filepath" "github.com/signoz/foundry/api/v1alpha1/infrastructure" - rootcasting "github.com/signoz/foundry/internal/casting" + infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" @@ -21,6 +21,7 @@ type Data struct { NodeGroups []DataNodeGroup } +// DataNodeGroup carries one node group's criteria for the templates. type DataNodeGroup struct { Name string Count int @@ -29,6 +30,8 @@ type DataNodeGroup struct { Disk int } +var _ infrastructurecasting.Casting = (*awsKubernetesTerraformCasting)(nil) + type awsKubernetesTerraformCasting struct { logger *slog.Logger } @@ -59,7 +62,7 @@ func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infras materials := make([]domain.Material, 0, len(items)) for _, item := range items { - material, err := item.template.Render(data, filepath.Join(rootcasting.DeploymentDir, item.path)) + material, err := item.template.Render(data, filepath.Join(infrastructurecasting.InfrastructureDir, item.path)) if err != nil { return nil, err } @@ -70,7 +73,7 @@ func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infras } func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { - c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(poursPath, rootcasting.DeploymentDir))) + c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(poursPath, infrastructurecasting.InfrastructureDir))) return nil } diff --git a/internal/casting/infrastructure/casting.go b/internal/casting/infrastructure/casting/casting.go similarity index 75% rename from internal/casting/infrastructure/casting.go rename to internal/casting/infrastructure/casting/casting.go index 52a2d3a5..7299a05a 100644 --- a/internal/casting/infrastructure/casting.go +++ b/internal/casting/infrastructure/casting/casting.go @@ -1,4 +1,4 @@ -package infrastructure +package casting import ( "context" @@ -8,6 +8,10 @@ import ( infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) +// InfrastructureDir is the subdirectory within the pours directory where +// infrastructure materials are written. +const InfrastructureDir = "infrastructure" + type Casting interface { Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/casting.go b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go deleted file mode 100644 index d9725908..00000000 --- a/internal/casting/infrastructure/ecsec2terraformcasting/casting.go +++ /dev/null @@ -1,32 +0,0 @@ -package ecsec2terraformcasting - -import ( - "context" - "log/slog" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - "github.com/signoz/foundry/internal/domain" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -type ecsEC2TerraformCasting struct { - logger *slog.Logger -} - -func New(logger *slog.Logger) *ecsEC2TerraformCasting { - return &ecsEC2TerraformCasting{logger: logger} -} - -func (c *ecsEC2TerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { - return &enricher{logger: c.logger}, nil -} - -func (c *ecsEC2TerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { - c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, no materials generated") - return nil, nil -} - -func (c *ecsEC2TerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { - c.logger.WarnContext(ctx, "the infrastructure kind is not implemented yet, nothing to cast") - return nil -} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go deleted file mode 100644 index a3355888..00000000 --- a/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go +++ /dev/null @@ -1,20 +0,0 @@ -package ecsec2terraformcasting - -import ( - "context" - "log/slog" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) - -type enricher struct { - logger *slog.Logger -} - -func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { - return nil -} diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 215a5b4c..37f6b912 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -6,6 +6,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" @@ -22,7 +23,7 @@ var _ planner.Planner = (*Planner)(nil) type Planner struct { config *infrastructure.Casting logger *slog.Logger - casting Casting + casting infrastructurecasting.Casting toolers []tooler.Tooler enricher infrastructuremolding.MoldingEnricher moldings []infrastructuremolding.Molding diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index d3486bb5..3e9971e8 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -5,14 +5,14 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/internal/casting/infrastructure/awskubernetesterraformcasting" - "github.com/signoz/foundry/internal/casting/infrastructure/ecsec2terraformcasting" + infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" "github.com/signoz/foundry/internal/tooler/terraformtooler" ) type CastingItem struct { - Casting Casting + Casting infrastructurecasting.Casting Toolers []tooler.Tooler } @@ -23,14 +23,6 @@ type Registry struct { func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ castings: map[v1alpha1.TypeDeployment]CastingItem{ - { - Platform: v1alpha1.PlatformECS, - Mode: v1alpha1.ModeEC2, - Flavor: v1alpha1.FlavorTerraform, - }: { - Casting: ecsec2terraformcasting.New(logger), - Toolers: []tooler.Tooler{terraformtooler.New()}, - }, { Platform: v1alpha1.PlatformAWS, Mode: v1alpha1.ModeKubernetes, @@ -50,7 +42,7 @@ func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingIte return item, ok } -func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (Casting, error) { +func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (infrastructurecasting.Casting, error) { item, ok := registry.lookup(deployment) if !ok { return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "infrastructure deployment '%+v' is not supported", deployment) From 9de5f382878ed29f3574c597d693f4bfc0f3ef3c Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 21 Jul 2026 17:34:30 +0530 Subject: [PATCH 19/38] refactor(infrastructure): declare the resource by kind alone --- api/v1alpha1/casting_ref.go | 25 -------------- api/v1alpha1/infrastructure/casting.go | 10 +----- .../infrastructure/casting.schema.json | 34 +------------------ api/v1alpha1/infrastructure/resource.go | 14 ++------ api/v1alpha1/infrastructure/schema_test.go | 31 +---------------- .../aws/kubernetes/terraform/README.md | 4 +-- .../aws/kubernetes/terraform/casting.yaml | 2 -- .../kubernetes/terraform/casting.yaml.lock | 3 -- internal/config/yamlconfig/config_test.go | 9 +---- 9 files changed, 8 insertions(+), 124 deletions(-) delete mode 100644 api/v1alpha1/casting_ref.go diff --git a/api/v1alpha1/casting_ref.go b/api/v1alpha1/casting_ref.go deleted file mode 100644 index 78da586a..00000000 --- a/api/v1alpha1/casting_ref.go +++ /dev/null @@ -1,25 +0,0 @@ -package v1alpha1 - -import "encoding/json" - -// TypeCastingRefSpec is the identity a casting is referenced by. -type TypeCastingRefSpec struct { - TypeVersion `json:",inline" yaml:",inline"` - - Name string `json:"name" yaml:"name" required:"true" nullable:"false" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the referenced casting."` - _ struct{} `additionalProperties:"false"` -} - -// MarshalJSON implements json.Marshaler. It manually omits empty fields -// so that the strategic merge patch doesn't overwrite defaults with empty -// values. -func (spec TypeCastingRefSpec) MarshalJSON() ([]byte, error) { - m := map[string]any{} - if spec.APIVersion != "" { - m["apiVersion"] = spec.APIVersion - } - if spec.Name != "" { - m["name"] = spec.Name - } - return json.Marshal(m) -} diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index fee5cb37..a56a9f9e 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -30,15 +30,7 @@ func Default() *Casting { Kind: v1alpha1.KindInfrastructure, Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, }, - Spec: Spec{ - Resource: Resource{ - Spec: ResourceSpec{ - TypeCastingRefSpec: v1alpha1.TypeCastingRefSpec{ - TypeVersion: v1alpha1.TypeVersion{APIVersion: "v1alpha1"}, - }, - }, - }, - }, + Spec: Spec{}, } } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 7dd8e77d..165e119b 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -9,8 +9,7 @@ "definitions": { "InfrastructureResource": { "required": [ - "kind", - "spec" + "kind" ], "additionalProperties": false, "properties": { @@ -21,10 +20,6 @@ "Installation" ] }, - "spec": { - "$ref": "#/definitions/InfrastructureResourceSpec", - "description": "Specification for the resource" - }, "status": { "$ref": "#/definitions/InfrastructureResourceStatus", "description": "Status of the resource" @@ -39,33 +34,6 @@ ], "type": "string" }, - "InfrastructureResourceSpec": { - "required": [ - "apiVersion", - "name" - ], - "additionalProperties": false, - "properties": { - "apiVersion": { - "description": "API Version of the configuration schema.", - "default": "v1alpha1", - "examples": [ - "v1alpha1" - ], - "enum": [ - "v1alpha1" - ], - "type": "string" - }, - "name": { - "description": "Name of the referenced casting.", - "maxLength": 63, - "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", - "type": "string" - } - }, - "type": "object" - }, "InfrastructureResourceStatus": { "additionalProperties": false, "properties": { diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index 8ef7edd3..14258837 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -2,27 +2,19 @@ package infrastructure import "github.com/signoz/foundry/api/v1alpha1" -// Resource declares the resource this infrastructure is shaped for. +// Resource declares the kind of resource this infrastructure is shaped for. +// It is a declaration, not a reference: the consumer owns the binding and +// declares it on its own casting. type Resource struct { // Kind of the resource this infrastructure serves. Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` - // Specification for the resource. - Spec ResourceSpec `json:"spec" yaml:"spec" required:"true" description:"Specification for the resource"` - // Status of the resource. Status ResourceStatus `json:"status" yaml:"status,omitempty" description:"Status of the resource"` _ struct{} `additionalProperties:"false"` } -// ResourceSpec carries the identity of the casting embodying the resource. -type ResourceSpec struct { - v1alpha1.TypeCastingRefSpec `json:",inline" yaml:",inline"` - - _ struct{} `additionalProperties:"false"` -} - // ResourceStatus carries the requirement set a substrate shaped for the // resource kind must satisfy. type ResourceStatus struct { diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index 4edd19ab..a9113758 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "testing" - "github.com/signoz/foundry/api/v1alpha1" "github.com/stretchr/testify/assert" ) @@ -22,7 +21,6 @@ func TestSchemaValidate(t *testing.T) { name: "InstallationResource_Valid", mutate: func(casting *Casting) { casting.Spec.Resource.Kind = ResourceKindInstallation - casting.Spec.Resource.Spec.Name = "signoz" }, pass: true, }, @@ -30,41 +28,14 @@ func TestSchemaValidate(t *testing.T) { name: "CollectionAgentResource_Valid", mutate: func(casting *Casting) { casting.Spec.Resource.Kind = ResourceKindCollectionAgent - casting.Spec.Resource.Spec.Name = "signoz-gateway" }, pass: true, }, { - name: "ResourceMissing_Invalid", + name: "ResourceKindMissing_Invalid", mutate: func(casting *Casting) {}, pass: false, }, - { - name: "ResourceKindMissing_Invalid", - mutate: func(casting *Casting) { - casting.Spec.Resource.Spec.Name = "signoz" - }, - pass: false, - }, - { - name: "ResourceNameMissing_Invalid", - mutate: func(casting *Casting) { - casting.Spec.Resource.Kind = ResourceKindInstallation - }, - pass: false, - }, - { - name: "ResourceAPIVersionMissing_Invalid", - mutate: func(casting *Casting) { - casting.Spec.Resource = Resource{ - Kind: ResourceKindInstallation, - Spec: ResourceSpec{ - TypeCastingRefSpec: v1alpha1.TypeCastingRefSpec{Name: "signoz"}, - }, - } - }, - pass: false, - }, } for _, tt := range tests { diff --git a/docs/examples/aws/kubernetes/terraform/README.md b/docs/examples/aws/kubernetes/terraform/README.md index 1c0a0903..074b42f9 100644 --- a/docs/examples/aws/kubernetes/terraform/README.md +++ b/docs/examples/aws/kubernetes/terraform/README.md @@ -36,11 +36,9 @@ spec: flavor: terraform resource: kind: Installation - spec: - name: signoz ``` -`spec.resource` declares the kind of resource the substrate serves and the name of the casting embodying it. It is a declaration, not a reference: nothing is read from the installation's casting. +`spec.resource` declares the kind of resource the substrate serves. It is a declaration, not a reference: the installation owns the binding and nothing is read from its casting. ## Deploy diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml b/docs/examples/aws/kubernetes/terraform/casting.yaml index 3aa215b3..e82d6d38 100644 --- a/docs/examples/aws/kubernetes/terraform/casting.yaml +++ b/docs/examples/aws/kubernetes/terraform/casting.yaml @@ -9,5 +9,3 @@ spec: flavor: terraform resource: kind: Installation - spec: - name: signoz diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock index 1110957e..54eb162a 100644 --- a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock +++ b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock @@ -9,9 +9,6 @@ spec: platform: aws resource: kind: Installation - spec: - apiVersion: v1alpha1 - name: signoz status: addresses: apiserver: diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go index 973311b2..3e18cc52 100644 --- a/internal/config/yamlconfig/config_test.go +++ b/internal/config/yamlconfig/config_test.go @@ -393,8 +393,6 @@ spec: flavor: terraform resource: kind: Installation - spec: - name: signoz `, expectedResource: infrastructure.ResourceKindInstallation, pass: true, @@ -428,8 +426,6 @@ spec: flavor: terraform resource: kind: Infrastructure - spec: - name: signoz `, pass: false, }, @@ -445,9 +441,7 @@ spec: platform: ecs mode: ec2 flavor: terraform - resource: - spec: - name: signoz + resource: {} `, pass: false, }, @@ -473,7 +467,6 @@ spec: } assert.Equal(t, v1alpha1.KindInfrastructure, casting.Kind()) assert.Equal(t, tt.expectedResource, casting.Spec.Resource.Kind) - assert.Equal(t, "signoz", casting.Spec.Resource.Spec.Name) }) } } From 5e1319c10e64444f155f02fdebc6af6a29caf356 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 21 Jul 2026 17:56:06 +0530 Subject: [PATCH 20/38] feat(installation): bind to the infrastructure casting by name --- api/v1alpha1/installation/casting.go | 3 +- api/v1alpha1/installation/casting.schema.json | 13 +- api/v1alpha1/installation/infrastructure.go | 35 +- internal/casting/installation/planner.go | 7 +- internal/foundry/forge.go | 38 +- internal/foundry/foundry.go | 6 - internal/infrastructure/compute_type.go | 91 --- internal/infrastructure/generator.go | 20 - internal/infrastructure/resolve.go | 61 -- internal/infrastructure/terraform/embed.go | 57 -- .../infrastructure/terraform/generator.go | 130 ----- .../templates/aws/ec2/main.tf.json.gotmpl | 456 --------------- .../templates/aws/ec2/outputs.tf.json.gotmpl | 64 --- .../aws/ec2/variables.tf.json.gotmpl | 79 --- .../templates/aws/eks/main.tf.json.gotmpl | 327 ----------- .../templates/aws/eks/outputs.tf.json.gotmpl | 73 --- .../aws/eks/variables.tf.json.gotmpl | 79 --- .../templates/azure/aks/main.tf.json.gotmpl | 149 ----- .../azure/aks/outputs.tf.json.gotmpl | 53 -- .../azure/aks/variables.tf.json.gotmpl | 83 --- .../templates/azure/vm/main.tf.json.gotmpl | 537 ------------------ .../templates/azure/vm/outputs.tf.json.gotmpl | 68 --- .../azure/vm/variables.tf.json.gotmpl | 87 --- .../templates/gcp/gce/main.tf.json.gotmpl | 261 --------- .../templates/gcp/gce/outputs.tf.json.gotmpl | 64 --- .../gcp/gce/variables.tf.json.gotmpl | 78 --- .../templates/gcp/gke/main.tf.json.gotmpl | 224 -------- .../templates/gcp/gke/outputs.tf.json.gotmpl | 46 -- .../gcp/gke/variables.tf.json.gotmpl | 98 ---- .../templates/providers.tf.json.gotmpl | 32 -- 30 files changed, 14 insertions(+), 3305 deletions(-) delete mode 100644 internal/infrastructure/compute_type.go delete mode 100644 internal/infrastructure/generator.go delete mode 100644 internal/infrastructure/resolve.go delete mode 100644 internal/infrastructure/terraform/embed.go delete mode 100644 internal/infrastructure/terraform/generator.go delete mode 100644 internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl delete mode 100644 internal/infrastructure/terraform/templates/providers.tf.json.gotmpl diff --git a/api/v1alpha1/installation/casting.go b/api/v1alpha1/installation/casting.go index fe7f3f5f..103132ad 100644 --- a/api/v1alpha1/installation/casting.go +++ b/api/v1alpha1/installation/casting.go @@ -40,7 +40,6 @@ func Default(declared *Casting) *Casting { Metadata: v1alpha1.TypeMetadata{Name: "signoz"}, }, Spec: Spec{ - Infrastructure: DefaultInfrastructure(), Signoz: DefaultSigNoz(), TelemetryStore: DefaultTelemetryStore(), TelemetryKeeper: DefaultTelemetryKeeper(declared.Spec.TelemetryKeeper.Kind), @@ -99,7 +98,7 @@ func (c *Casting) TrackableProperties() domain.Properties { Set("mode", c.Spec.Deployment.Mode.String()). Set("flavor", c.Spec.Deployment.Flavor.String()). Set("patches_count", len(c.Spec.Patches)). - Set("infrastructure_enabled", c.Spec.Infrastructure.Enabled). + Set("infrastructure_bound", c.Spec.Infrastructure.Name != ""). Set("metastore_kind", c.Spec.MetaStore.Kind.String()). Set("telemetrystore_kind", c.Spec.TelemetryStore.Kind.String()). Set("telemetrykeeper_kind", c.Spec.TelemetryKeeper.Kind.String()). diff --git a/api/v1alpha1/installation/casting.schema.json b/api/v1alpha1/installation/casting.schema.json index 01a00b04..c41aa1f1 100644 --- a/api/v1alpha1/installation/casting.schema.json +++ b/api/v1alpha1/installation/casting.schema.json @@ -10,14 +10,11 @@ "InstallationInfrastructure": { "additionalProperties": false, "properties": { - "enabled": { - "type": "boolean" - }, - "status": { - "additionalProperties": { - "type": "string" - }, - "type": "object" + "name": { + "description": "Name of the infrastructure casting this installation runs on", + "maxLength": 63, + "pattern": "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", + "type": "string" } }, "type": "object" diff --git a/api/v1alpha1/installation/infrastructure.go b/api/v1alpha1/installation/infrastructure.go index 7baa07a8..6eef5b73 100644 --- a/api/v1alpha1/installation/infrastructure.go +++ b/api/v1alpha1/installation/infrastructure.go @@ -1,36 +1,11 @@ package installation -import "encoding/json" - -// Infrastructure holds the configuration for infrastructure manifest generation (e.g., Terraform). -// The cloud provider is resolved automatically from spec.deployment.platform — no provider field -// is needed here. +// Infrastructure is the installation's binding to the infrastructure casting +// it runs on. The consumer owns the binding: it orders the casts +// (infrastructure first) and names the substrate for by-name lookups. type Infrastructure struct { - // Whether infrastructure manifest generation is enabled - Enabled bool `json:"enabled" yaml:"enabled"` - - // Status holds the generated IaC file contents keyed by filename (e.g. "main.tf.json"). - // This is populated by foundry after generation and written to the lock file. - Status map[string]string `json:"status,omitempty" yaml:"status,omitempty"` + // Name of the infrastructure casting. + Name string `json:"name,omitempty" yaml:"name,omitempty" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the infrastructure casting this installation runs on"` _ struct{} `additionalProperties:"false"` } - -// MarshalJSON implements json.Marshaler. It manually omits Status when zero -// so that the strategic merge patch doesn't overwrite defaults with empty values. -func (i Infrastructure) MarshalJSON() ([]byte, error) { - m := map[string]any{ - "enabled": i.Enabled, - } - if len(i.Status) > 0 { - m["status"] = i.Status - } - return json.Marshal(m) -} - -// DefaultInfrastructure returns the default Infrastructure configuration. -func DefaultInfrastructure() Infrastructure { - return Infrastructure{ - Enabled: false, - } -} diff --git a/internal/casting/installation/planner.go b/internal/casting/installation/planner.go index 2a2faa44..c6cc0294 100644 --- a/internal/casting/installation/planner.go +++ b/internal/casting/installation/planner.go @@ -18,7 +18,6 @@ import ( "github.com/signoz/foundry/internal/molding/telemetrystoremolding" "github.com/signoz/foundry/internal/planner" "github.com/signoz/foundry/internal/tooler" - "github.com/signoz/foundry/internal/tooler/terraformtooler" ) var _ planner.Planner = (*Planner)(nil) @@ -109,9 +108,5 @@ func (p *Planner) Cast(ctx context.Context, poursPath string) error { } func (p *Planner) Toolers() []tooler.Tooler { - toolers := p.toolers - if p.config.Spec.Infrastructure.Enabled { - toolers = append(toolers, terraformtooler.New()) - } - return toolers + return p.toolers } diff --git a/internal/foundry/forge.go b/internal/foundry/forge.go index 54270cd0..81dc4823 100644 --- a/internal/foundry/forge.go +++ b/internal/foundry/forge.go @@ -3,11 +3,8 @@ package foundry import ( "context" "log/slog" - "path/filepath" "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/installation" - "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/writer" ) @@ -52,37 +49,13 @@ func (foundry *Foundry) Forge(ctx context.Context, machinery v1alpha1.Machinery, } } - // Generate infrastructure-as-code manifests if enabled, before writing the lock file - // so that the generated file contents are captured in the lock's infrastructure.status. - // Gated to installation.Casting - var infraMaterials []domain.Material - if config, ok := machinery.(*installation.Casting); ok && config.Spec.Infrastructure.Enabled { - spec := &config.Spec - foundry.Logger.InfoContext(ctx, "generating infrastructure manifests", - slog.String("casting.metadata.name", config.Metadata.Name), - slog.String("deployment.platform", spec.Deployment.Platform.String())) - - infraMaterials, err = foundry.InfrastructureGenerator.Generate(ctx, *config) - if err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to generate infrastructure manifests") - } - - // Populate infrastructure status with generated file contents keyed by filename. - if len(infraMaterials) > 0 { - spec.Infrastructure.Status = make(map[string]string, len(infraMaterials)) - for _, m := range infraMaterials { - spec.Infrastructure.Status[filepath.Base(m.Path())] = string(m.FmtContents()) - } - } - } - - // writing the merged config (including infrastructure status) to the lock file + // writing the merged config to the lock file foundry.Logger.InfoContext(ctx, "writing lock file") if err := foundry.Config.CreateV1Alpha1Lock(ctx, p.Machinery(), path); err != nil { return err } - if len(materials) == 0 && len(infraMaterials) == 0 { + if len(materials) == 0 { foundry.Logger.WarnContext(ctx, "casting did not generate any materials for writing") return nil } @@ -92,13 +65,6 @@ func (foundry *Foundry) Forge(ctx context.Context, machinery v1alpha1.Machinery, return err } - if len(infraMaterials) > 0 { - foundry.Logger.InfoContext(ctx, "writing infrastructure materials", slog.Int("count", len(infraMaterials))) - if err := poursWriter.WriteMany(ctx, infraMaterials...); err != nil { - return err - } - } - foundry.Logger.InfoContext(ctx, "writing materials") if err := poursWriter.WriteMany(ctx, materials...); err != nil { return err diff --git a/internal/foundry/foundry.go b/internal/foundry/foundry.go index b4f4ec7b..b92ccca9 100644 --- a/internal/foundry/foundry.go +++ b/internal/foundry/foundry.go @@ -14,8 +14,6 @@ import ( "github.com/signoz/foundry/internal/config" "github.com/signoz/foundry/internal/config/yamlconfig" foundryerrors "github.com/signoz/foundry/internal/errors" - "github.com/signoz/foundry/internal/infrastructure" - terraformgenerator "github.com/signoz/foundry/internal/infrastructure/terraform" "github.com/signoz/foundry/internal/patch" "github.com/signoz/foundry/internal/patch/jsonpatch" "github.com/signoz/foundry/internal/planner" @@ -35,9 +33,6 @@ type Foundry struct { // Planners for the different casting kinds. Planners map[v1alpha1.Kind]plannerCtor - - // InfrastructureGenerator for generating infrastructure-as-code manifests. - InfrastructureGenerator infrastructure.Generator } func New(logger *slog.Logger) (*Foundry, error) { @@ -58,7 +53,6 @@ func New(logger *slog.Logger) (*Foundry, error) { return infrastructurecasting.NewPlanner(ctx, m.(*infrastructurev1alpha1.Casting), logger) }, }, - InfrastructureGenerator: terraformgenerator.New(logger), }, nil } diff --git a/internal/infrastructure/compute_type.go b/internal/infrastructure/compute_type.go deleted file mode 100644 index 37e77f7f..00000000 --- a/internal/infrastructure/compute_type.go +++ /dev/null @@ -1,91 +0,0 @@ -package infrastructure - -import ( - "encoding/json" - "errors" - "fmt" - - "go.yaml.in/yaml/v3" -) - -var _ yaml.Marshaler = (*ComputeType)(nil) -var _ yaml.Unmarshaler = (*ComputeType)(nil) -var _ json.Marshaler = (*ComputeType)(nil) -var _ json.Unmarshaler = (*ComputeType)(nil) -var _ fmt.Stringer = (*ComputeType)(nil) - -var ( - // AWS compute types. - ComputeTypeEC2 ComputeType = ComputeType{s: "ec2"} - ComputeTypeEKS ComputeType = ComputeType{s: "eks"} - // GCP compute types. - ComputeTypeGCE ComputeType = ComputeType{s: "gce"} - ComputeTypeGKE ComputeType = ComputeType{s: "gke"} - // Azure compute types. - ComputeTypeVM ComputeType = ComputeType{s: "vm"} - ComputeTypeAKS ComputeType = ComputeType{s: "aks"} -) - -// ComputeType identifies the compute resource type for a given cloud provider. -// It is an internal type resolved from the provider + deployment combination — -// users do not set this directly. -type ComputeType struct { - s string -} - -func (c ComputeType) String() string { - return c.s -} - -func (c ComputeType) IsZero() bool { - return c.s == "" -} - -func ComputeTypes() []ComputeType { - return []ComputeType{ - ComputeTypeEC2, - ComputeTypeEKS, - ComputeTypeGCE, - ComputeTypeGKE, - ComputeTypeVM, - ComputeTypeAKS, - } -} - -func (c ComputeType) MarshalJSON() ([]byte, error) { - return json.Marshal(c.String()) -} - -func (c *ComputeType) UnmarshalJSON(text []byte) error { - var str string - if err := json.Unmarshal(text, &str); err != nil { - return err - } - return c.UnmarshalText([]byte(str)) -} - -func (c *ComputeType) UnmarshalText(text []byte) error { - for _, available := range ComputeTypes() { - if available.String() == string(text) { - *c = available - return nil - } - } - if len(text) == 0 { - *c = ComputeType{s: ""} - return nil - } - return errors.New("invalid infrastructure compute type: " + string(text)) -} - -func (c ComputeType) MarshalText() ([]byte, error) { - return []byte(c.String()), nil -} - -func (c *ComputeType) UnmarshalYAML(node *yaml.Node) error { - return c.UnmarshalText([]byte(node.Value)) -} - -func (c ComputeType) MarshalYAML() (any, error) { - return c.String(), nil -} diff --git a/internal/infrastructure/generator.go b/internal/infrastructure/generator.go deleted file mode 100644 index 9e7983dd..00000000 --- a/internal/infrastructure/generator.go +++ /dev/null @@ -1,20 +0,0 @@ -package infrastructure - -import ( - "context" - - "github.com/signoz/foundry/api/v1alpha1/installation" - "github.com/signoz/foundry/internal/domain" -) - -// Generator is the interface for infrastructure-as-code generators. -// Implementations produce IaC manifests (e.g., Terraform, Pulumi) from a casting configuration -// and can validate the generated output using the underlying tool. -type Generator interface { - // Generate produces IaC materials from the casting configuration. - Generate(ctx context.Context, config installation.Casting) ([]domain.Material, error) - - // Validate runs the IaC tool's built-in validation (e.g., terraform validate) - // against the manifests written to poursPath. - Validate(ctx context.Context, poursPath string) error -} diff --git a/internal/infrastructure/resolve.go b/internal/infrastructure/resolve.go deleted file mode 100644 index d8324ccc..00000000 --- a/internal/infrastructure/resolve.go +++ /dev/null @@ -1,61 +0,0 @@ -package infrastructure - -import ( - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/internal/errors" -) - -// ResolveProvider normalizes a deployment platform to the cloud platform that -// hosts it. Only the cloud platforms (aws, gcp, azure) and ECS resolve; -// managed platforms (render, coolify, railway) have no IaC backing. -func ResolveProvider(platform v1alpha1.Platform) (v1alpha1.Platform, error) { - switch platform { - case v1alpha1.PlatformAWS, v1alpha1.PlatformECS: - return v1alpha1.PlatformAWS, nil - case v1alpha1.PlatformGCP: - return v1alpha1.PlatformGCP, nil - case v1alpha1.PlatformAzure: - return v1alpha1.PlatformAzure, nil - case v1alpha1.Platform{}: - return v1alpha1.Platform{}, errors.Newf(errors.TypeInvalidInput, "no platform specified in deployment.platform: infrastructure generation requires aws, gcp, or azure") - default: - return v1alpha1.Platform{}, errors.Newf(errors.TypeUnsupported, "unsupported platform for infrastructure generation: %q (must be aws, gcp, azure, or ecs)", platform) - } -} - -// ResolveComputeType derives the appropriate ComputeType from a cloud platform -// and deployment configuration. Users do not specify the compute type directly -// — foundry resolves it automatically using this matrix: -// -// AWS + kubernetes (any flavor) → EKS -// AWS + anything else → EC2 -// GCP + kubernetes (any flavor) → GKE -// GCP + anything else → GCE -// Azure + kubernetes (any flavor) → AKS -// Azure + anything else → VM -func ResolveComputeType(provider v1alpha1.Platform, deployment v1alpha1.TypeDeployment) (ComputeType, error) { - isKubernetes := deployment.Mode == v1alpha1.ModeKubernetes - - switch provider { - case v1alpha1.PlatformAWS: - if isKubernetes { - return ComputeTypeEKS, nil - } - return ComputeTypeEC2, nil - - case v1alpha1.PlatformGCP: - if isKubernetes { - return ComputeTypeGKE, nil - } - return ComputeTypeGCE, nil - - case v1alpha1.PlatformAzure: - if isKubernetes { - return ComputeTypeAKS, nil - } - return ComputeTypeVM, nil - - default: - return ComputeType{}, errors.Newf(errors.TypeUnsupported, "unsupported infrastructure platform: %s", provider) - } -} diff --git a/internal/infrastructure/terraform/embed.go b/internal/infrastructure/terraform/embed.go deleted file mode 100644 index 914ab8dc..00000000 --- a/internal/infrastructure/terraform/embed.go +++ /dev/null @@ -1,57 +0,0 @@ -package terraform - -import ( - "embed" - - "github.com/signoz/foundry/internal/domain" -) - -//go:embed templates/*.gotmpl templates/aws/ec2/*.gotmpl templates/aws/eks/*.gotmpl templates/gcp/gce/*.gotmpl templates/gcp/gke/*.gotmpl templates/azure/vm/*.gotmpl templates/azure/aks/*.gotmpl -var templates embed.FS - -// Common templates. -var ( - providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) -) - -// AWS EC2 templates. -var ( - awsEC2MainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/main.tf.json.gotmpl", domain.FormatJSON) - awsEC2VariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/variables.tf.json.gotmpl", domain.FormatJSON) - awsEC2OutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/ec2/outputs.tf.json.gotmpl", domain.FormatJSON) -) - -// AWS EKS templates. -var ( - awsEKSMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/main.tf.json.gotmpl", domain.FormatJSON) - awsEKSVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/variables.tf.json.gotmpl", domain.FormatJSON) - awsEKSOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/aws/eks/outputs.tf.json.gotmpl", domain.FormatJSON) -) - -// GCP GCE templates. -var ( - gcpGCEMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/main.tf.json.gotmpl", domain.FormatJSON) - gcpGCEVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/variables.tf.json.gotmpl", domain.FormatJSON) - gcpGCEOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gce/outputs.tf.json.gotmpl", domain.FormatJSON) -) - -// GCP GKE templates. -var ( - gcpGKEMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/main.tf.json.gotmpl", domain.FormatJSON) - gcpGKEVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/variables.tf.json.gotmpl", domain.FormatJSON) - gcpGKEOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/gcp/gke/outputs.tf.json.gotmpl", domain.FormatJSON) -) - -// Azure VM templates. -var ( - azureVMMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/main.tf.json.gotmpl", domain.FormatJSON) - azureVMVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/variables.tf.json.gotmpl", domain.FormatJSON) - azureVMOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/vm/outputs.tf.json.gotmpl", domain.FormatJSON) -) - -// Azure AKS templates. -var ( - azureAKSMainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/main.tf.json.gotmpl", domain.FormatJSON) - azureAKSVariablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/variables.tf.json.gotmpl", domain.FormatJSON) - azureAKSOutputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/azure/aks/outputs.tf.json.gotmpl", domain.FormatJSON) -) diff --git a/internal/infrastructure/terraform/generator.go b/internal/infrastructure/terraform/generator.go deleted file mode 100644 index 5b7a7be8..00000000 --- a/internal/infrastructure/terraform/generator.go +++ /dev/null @@ -1,130 +0,0 @@ -package terraform - -import ( - "context" - "log/slog" - "os/exec" - "path/filepath" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/installation" - "github.com/signoz/foundry/internal/domain" - "github.com/signoz/foundry/internal/errors" - "github.com/signoz/foundry/internal/infrastructure" -) - -var _ infrastructure.Generator = (*Generator)(nil) - -const infrastructureDir = "infrastructure" - -// Generator generates Terraform manifests for infrastructure deployment. -type Generator struct { - logger *slog.Logger -} - -type templateData struct { - installation.Casting - Provider v1alpha1.Platform - ComputeType infrastructure.ComputeType -} - -// New creates a new Terraform Generator. -func New(logger *slog.Logger) *Generator { - return &Generator{ - logger: logger, - } -} - -// Generate creates Terraform manifests based on the casting configuration. -// The compute type is resolved automatically from the provider and deployment mode. -func (g *Generator) Generate(ctx context.Context, config installation.Casting) ([]domain.Material, error) { - if !config.Spec.Infrastructure.Enabled { - return nil, nil - } - - provider, err := infrastructure.ResolveProvider(config.Spec.Deployment.Platform) - if err != nil { - return nil, err - } - computeType, err := infrastructure.ResolveComputeType(provider, config.Spec.Deployment) - if err != nil { - return nil, err - } - - g.logger.InfoContext(ctx, "generating terraform manifests", - slog.String("provider", provider.String()), - slog.String("computeType", computeType.String()), - ) - - data := templateData{ - Casting: config, - Provider: provider, - ComputeType: computeType, - } - - mainTemplate, varsTemplate, outputsTemplate, err := g.templatesFor(provider, computeType) - if err != nil { - return nil, err - } - - materials := make([]domain.Material, 0, 4) - for _, item := range []struct { - tmpl *domain.Template - path string - }{ - {mainTemplate, "main.tf.json"}, - {varsTemplate, "variables.tf.json"}, - {providersTFTemplate, "providers.tf.json"}, - {outputsTemplate, "outputs.tf.json"}, - } { - m, err := item.tmpl.Render(data, filepath.Join(infrastructureDir, item.path)) - if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to render %s", item.path) - } - materials = append(materials, m) - } - - return materials, nil -} - -// Validate runs `terraform validate` against the manifests in poursPath/infrastructure. -func (g *Generator) Validate(ctx context.Context, poursPath string) error { - infraDir := filepath.Join(poursPath, infrastructureDir) - g.logger.InfoContext(ctx, "validating terraform manifests", slog.String("path", infraDir)) - - cmd := exec.CommandContext(ctx, "terraform", "validate") - cmd.Dir = infraDir - out, err := cmd.CombinedOutput() - if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "terraform validate failed\n%s", out) - } - return nil -} - -// templatesFor returns the provider+compute-type specific templates. -func (g *Generator) templatesFor(provider v1alpha1.Platform, computeType infrastructure.ComputeType) (main, vars, outputs *domain.Template, err error) { - switch provider { - case v1alpha1.PlatformAWS: - switch computeType { - case infrastructure.ComputeTypeEC2: - return awsEC2MainTFTemplate, awsEC2VariablesTFTemplate, awsEC2OutputsTFTemplate, nil - case infrastructure.ComputeTypeEKS: - return awsEKSMainTFTemplate, awsEKSVariablesTFTemplate, awsEKSOutputsTFTemplate, nil - } - case v1alpha1.PlatformGCP: - switch computeType { - case infrastructure.ComputeTypeGCE: - return gcpGCEMainTFTemplate, gcpGCEVariablesTFTemplate, gcpGCEOutputsTFTemplate, nil - case infrastructure.ComputeTypeGKE: - return gcpGKEMainTFTemplate, gcpGKEVariablesTFTemplate, gcpGKEOutputsTFTemplate, nil - } - case v1alpha1.PlatformAzure: - switch computeType { - case infrastructure.ComputeTypeVM: - return azureVMMainTFTemplate, azureVMVariablesTFTemplate, azureVMOutputsTFTemplate, nil - case infrastructure.ComputeTypeAKS: - return azureAKSMainTFTemplate, azureAKSVariablesTFTemplate, azureAKSOutputsTFTemplate, nil - } - } - return nil, nil, nil, errors.Newf(errors.TypeUnsupported, "unsupported provider %q / compute type %q combination", provider, computeType) -} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl deleted file mode 100644 index 3ede88d5..00000000 --- a/internal/infrastructure/terraform/templates/aws/ec2/main.tf.json.gotmpl +++ /dev/null @@ -1,456 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "data": { - "aws_availability_zones": { - "available": { - "state": "available" - } - }, - "aws_ami": { - "ubuntu": { - "most_recent": true, - "owners": ["099720109477"], - "filter": [ - { - "name": "name", - "values": ["ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*"] - }, - { - "name": "virtualization-type", - "values": ["hvm"] - } - ] - } - } - }, - "resource": { - "aws_vpc": { - "main": { - "cidr_block": "${var.vpc_cidr}", - "enable_dns_hostnames": true, - "enable_dns_support": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-vpc" - } - } - }, - "aws_subnet": { - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-private-${count.index}" - } - }, - "public": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "map_public_ip_on_launch": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-public-${count.index}" - } - } - }, - "aws_internet_gateway": { - "main": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-igw" - } - } - }, - "aws_eip": { - "nat": { - "count": "${var.az_count}", - "domain": "vpc", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-nat-eip-${count.index}" - } - } - }, - "aws_nat_gateway": { - "main": { - "count": "${var.az_count}", - "allocation_id": "${aws_eip.nat[count.index].id}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-nat-${count.index}" - }, - "depends_on": ["${aws_internet_gateway.main}"] - } - }, - "aws_route_table": { - "public": { - "vpc_id": "${aws_vpc.main.id}", - "route": [ - { - "cidr_block": "0.0.0.0/0", - "gateway_id": "${aws_internet_gateway.main.id}" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-public-rt" - } - }, - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "route": [ - { - "cidr_block": "0.0.0.0/0", - "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-private-rt-${count.index}" - } - } - }, - "aws_route_table_association": { - "public": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "route_table_id": "${aws_route_table.public.id}" - }, - "private": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.private[count.index].id}", - "route_table_id": "${aws_route_table.private[count.index].id}" - } - }, - "aws_security_group": { - "telemetrykeeper": { - "name": "${local.name}-telemetrykeeper-sg", - "description": "Security group for TelemetryKeeper (ClickHouse Keeper)", - "vpc_id": "${aws_vpc.main.id}", - "ingress": [ - { - "from_port": 9181, - "to_port": 9181, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "ClickHouse Keeper client port" - }, - { - "from_port": 9234, - "to_port": 9234, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "ClickHouse Keeper raft port" - }, - { - "from_port": 22, - "to_port": 22, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SSH access from VPC" - } - ], - "egress": [ - { - "from_port": 0, - "to_port": 0, - "protocol": "-1", - "cidr_blocks": ["0.0.0.0/0"] - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrykeeper-sg" - } - }, - "telemetrystore": { - "name": "${local.name}-telemetrystore-sg", - "description": "Security group for TelemetryStore (ClickHouse)", - "vpc_id": "${aws_vpc.main.id}", - "ingress": [ - { - "from_port": 9000, - "to_port": 9000, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "ClickHouse native port" - }, - { - "from_port": 8123, - "to_port": 8123, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "ClickHouse HTTP port" - }, - { - "from_port": 22, - "to_port": 22, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SSH access from VPC" - } - ], - "egress": [ - { - "from_port": 0, - "to_port": 0, - "protocol": "-1", - "cidr_blocks": ["0.0.0.0/0"] - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrystore-sg" - } - }, - "metastore": { - "name": "${local.name}-metastore-sg", - "description": "Security group for MetaStore (PostgreSQL)", - "vpc_id": "${aws_vpc.main.id}", - "ingress": [ - { - "from_port": 5432, - "to_port": 5432, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "PostgreSQL port" - }, - { - "from_port": 22, - "to_port": 22, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SSH access from VPC" - } - ], - "egress": [ - { - "from_port": 0, - "to_port": 0, - "protocol": "-1", - "cidr_blocks": ["0.0.0.0/0"] - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-metastore-sg" - } - }, - "ingester": { - "name": "${local.name}-ingester-sg", - "description": "Security group for Ingester (OpenTelemetry Collector)", - "vpc_id": "${aws_vpc.main.id}", - "ingress": [ - { - "from_port": 4317, - "to_port": 4317, - "protocol": "tcp", - "cidr_blocks": ["0.0.0.0/0"], - "description": "OTLP gRPC port" - }, - { - "from_port": 4318, - "to_port": 4318, - "protocol": "tcp", - "cidr_blocks": ["0.0.0.0/0"], - "description": "OTLP HTTP port" - }, - { - "from_port": 22, - "to_port": 22, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SSH access from VPC" - } - ], - "egress": [ - { - "from_port": 0, - "to_port": 0, - "protocol": "-1", - "cidr_blocks": ["0.0.0.0/0"] - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-ingester-sg" - } - }, - "signoz": { - "name": "${local.name}-signoz-sg", - "description": "Security group for SigNoz", - "vpc_id": "${aws_vpc.main.id}", - "ingress": [ - { - "from_port": 8080, - "to_port": 8080, - "protocol": "tcp", - "cidr_blocks": ["0.0.0.0/0"], - "description": "SigNoz UI port" - }, - { - "from_port": 3301, - "to_port": 3301, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SigNoz API port" - }, - { - "from_port": 22, - "to_port": 22, - "protocol": "tcp", - "cidr_blocks": ["${var.vpc_cidr}"], - "description": "SSH access from VPC" - } - ], - "egress": [ - { - "from_port": 0, - "to_port": 0, - "protocol": "-1", - "cidr_blocks": ["0.0.0.0/0"] - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-signoz-sg" - } - } - }, - "aws_key_pair": { - "main": { - "count": "${var.ssh_public_key != \"\" ? 1 : 0}", - "key_name": "${local.name}-key", - "public_key": "${var.ssh_public_key}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "aws_instance": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "ami": "${data.aws_ami.ubuntu.id}", - "instance_type": "${var.telemetrykeeper_instance_type}", - "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", - "vpc_security_group_ids": ["${aws_security_group.telemetrykeeper.id}"], - "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", - "root_block_device": [{ - "volume_size": "${var.telemetrykeeper_volume_size}", - "volume_type": "gp3", - "encrypted": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrykeeper-${count.index}", - "Role": "telemetrykeeper" - } - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "ami": "${data.aws_ami.ubuntu.id}", - "instance_type": "${var.telemetrystore_instance_type}", - "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", - "vpc_security_group_ids": ["${aws_security_group.telemetrystore.id}"], - "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", - "root_block_device": [{ - "volume_size": "${var.telemetrystore_volume_size}", - "volume_type": "gp3", - "encrypted": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrystore-${count.index}", - "Role": "telemetrystore" - } - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "ami": "${data.aws_ami.ubuntu.id}", - "instance_type": "${var.metastore_instance_type}", - "subnet_id": "${aws_subnet.private[count.index % var.az_count].id}", - "vpc_security_group_ids": ["${aws_security_group.metastore.id}"], - "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", - "root_block_device": [{ - "volume_size": "${var.metastore_volume_size}", - "volume_type": "gp3", - "encrypted": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-metastore-${count.index}", - "Role": "metastore" - } - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "ami": "${data.aws_ami.ubuntu.id}", - "instance_type": "${var.ingester_instance_type}", - "subnet_id": "${aws_subnet.public[count.index % var.az_count].id}", - "vpc_security_group_ids": ["${aws_security_group.ingester.id}"], - "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", - "root_block_device": [{ - "volume_size": "${var.ingester_volume_size}", - "volume_type": "gp3", - "encrypted": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-ingester-${count.index}", - "Role": "ingester" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "ami": "${data.aws_ami.ubuntu.id}", - "instance_type": "${var.signoz_instance_type}", - "subnet_id": "${aws_subnet.public[count.index % var.az_count].id}", - "vpc_security_group_ids": ["${aws_security_group.signoz.id}"], - "key_name": "${var.ssh_public_key != \"\" ? aws_key_pair.main[0].key_name : null}", - "root_block_device": [{ - "volume_size": "${var.signoz_volume_size}", - "volume_type": "gp3", - "encrypted": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-signoz-${count.index}", - "Role": "signoz" - } - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl deleted file mode 100644 index 38d7d0b6..00000000 --- a/internal/infrastructure/terraform/templates/aws/ec2/outputs.tf.json.gotmpl +++ /dev/null @@ -1,64 +0,0 @@ -{ - "output": { - "vpc_id": { - "description": "ID of the VPC", - "value": "${aws_vpc.main.id}" - }, - "private_subnet_ids": { - "description": "IDs of the private subnets", - "value": "${aws_subnet.private[*].id}" - }, - "public_subnet_ids": { - "description": "IDs of the public subnets", - "value": "${aws_subnet.public[*].id}" - }, - "telemetrykeeper_instance_ids": { - "description": "IDs of the TelemetryKeeper EC2 instances", - "value": "${aws_instance.telemetrykeeper[*].id}" - }, - "telemetrykeeper_private_ips": { - "description": "Private IP addresses of the TelemetryKeeper EC2 instances", - "value": "${aws_instance.telemetrykeeper[*].private_ip}" - }, - "telemetrystore_instance_ids": { - "description": "IDs of the TelemetryStore EC2 instances", - "value": "${aws_instance.telemetrystore[*].id}" - }, - "telemetrystore_private_ips": { - "description": "Private IP addresses of the TelemetryStore EC2 instances", - "value": "${aws_instance.telemetrystore[*].private_ip}" - }, - "metastore_instance_ids": { - "description": "IDs of the MetaStore EC2 instances", - "value": "${aws_instance.metastore[*].id}" - }, - "metastore_private_ips": { - "description": "Private IP addresses of the MetaStore EC2 instances", - "value": "${aws_instance.metastore[*].private_ip}" - }, - "ingester_instance_ids": { - "description": "IDs of the Ingester EC2 instances", - "value": "${aws_instance.ingester[*].id}" - }, - "ingester_public_ips": { - "description": "Public IP addresses of the Ingester EC2 instances", - "value": "${aws_instance.ingester[*].public_ip}" - }, - "ingester_private_ips": { - "description": "Private IP addresses of the Ingester EC2 instances", - "value": "${aws_instance.ingester[*].private_ip}" - }, - "signoz_instance_ids": { - "description": "IDs of the SigNoz EC2 instances", - "value": "${aws_instance.signoz[*].id}" - }, - "signoz_public_ips": { - "description": "Public IP addresses of the SigNoz EC2 instances", - "value": "${aws_instance.signoz[*].public_ip}" - }, - "signoz_private_ips": { - "description": "Private IP addresses of the SigNoz EC2 instances", - "value": "${aws_instance.signoz[*].private_ip}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl deleted file mode 100644 index 1d1061d2..00000000 --- a/internal/infrastructure/terraform/templates/aws/ec2/variables.tf.json.gotmpl +++ /dev/null @@ -1,79 +0,0 @@ -{ - "variable": { - "aws_region": { - "description": "AWS region to deploy resources", - "type": "string", - "default": "us-east-1" - }, - "vpc_cidr": { - "description": "CIDR block for the VPC", - "type": "string", - "default": "10.0.0.0/16" - }, - "az_count": { - "description": "Number of availability zones to use", - "type": "number", - "default": 2 - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "ssh_public_key": { - "description": "SSH public key for EC2 instance access. Leave empty to disable SSH key pair creation.", - "type": "string", - "default": "" - }, - "telemetrykeeper_instance_type": { - "description": "EC2 instance type for TelemetryKeeper", - "type": "string", - "default": "t3.small" - }, - "telemetrykeeper_volume_size": { - "description": "Root volume size (GB) for TelemetryKeeper instances", - "type": "number", - "default": 20 - }, - "telemetrystore_instance_type": { - "description": "EC2 instance type for TelemetryStore", - "type": "string", - "default": "r6i.xlarge" - }, - "telemetrystore_volume_size": { - "description": "Root volume size (GB) for TelemetryStore instances", - "type": "number", - "default": 100 - }, - "metastore_instance_type": { - "description": "EC2 instance type for MetaStore", - "type": "string", - "default": "t3.small" - }, - "metastore_volume_size": { - "description": "Root volume size (GB) for MetaStore instances", - "type": "number", - "default": 20 - }, - "ingester_instance_type": { - "description": "EC2 instance type for Ingester", - "type": "string", - "default": "t3.medium" - }, - "ingester_volume_size": { - "description": "Root volume size (GB) for Ingester instances", - "type": "number", - "default": 50 - }, - "signoz_instance_type": { - "description": "EC2 instance type for SigNoz", - "type": "string", - "default": "t3.medium" - }, - "signoz_volume_size": { - "description": "Root volume size (GB) for SigNoz instances", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl deleted file mode 100644 index 5376c049..00000000 --- a/internal/infrastructure/terraform/templates/aws/eks/main.tf.json.gotmpl +++ /dev/null @@ -1,327 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "data": { - "aws_availability_zones": { - "available": { - "state": "available" - } - } - }, - "resource": { - "aws_vpc": { - "main": { - "cidr_block": "${var.vpc_cidr}", - "enable_dns_hostnames": true, - "enable_dns_support": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-vpc", - "kubernetes.io/cluster/${local.name}": "shared" - } - } - }, - "aws_subnet": { - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-private-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/internal-elb": "1" - } - }, - "public": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "map_public_ip_on_launch": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-public-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/elb": "1" - } - } - }, - "aws_internet_gateway": { - "main": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-igw" - } - } - }, - "aws_eip": { - "nat": { - "count": "${var.az_count}", - "domain": "vpc", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-nat-eip-${count.index}" - } - } - }, - "aws_nat_gateway": { - "main": { - "count": "${var.az_count}", - "allocation_id": "${aws_eip.nat[count.index].id}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-nat-${count.index}" - }, - "depends_on": ["${aws_internet_gateway.main}"] - } - }, - "aws_route_table": { - "public": { - "vpc_id": "${aws_vpc.main.id}", - "route": [ - { - "cidr_block": "0.0.0.0/0", - "gateway_id": "${aws_internet_gateway.main.id}" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-public-rt" - } - }, - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "route": [ - { - "cidr_block": "0.0.0.0/0", - "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-private-rt-${count.index}" - } - } - }, - "aws_route_table_association": { - "public": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "route_table_id": "${aws_route_table.public.id}" - }, - "private": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.private[count.index].id}", - "route_table_id": "${aws_route_table.private[count.index].id}" - } - }, - "aws_iam_role": { - "eks_cluster": { - "name": "${local.name}-eks-cluster-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "eks_node_group": { - "name": "${local.name}-eks-node-group-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "aws_iam_role_policy_attachment": { - "eks_cluster_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", - "role": "${aws_iam_role.eks_cluster.name}" - }, - "eks_worker_node_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_cni_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_container_registry": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", - "role": "${aws_iam_role.eks_node_group.name}" - } - }, - "aws_eks_cluster": { - "main": { - "name": "${local.name}", - "role_arn": "${aws_iam_role.eks_cluster.arn}", - "version": "${var.kubernetes_version}", - "vpc_config": [{ - "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", - "endpoint_private_access": true, - "endpoint_public_access": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - }, - "depends_on": ["${aws_iam_role_policy_attachment.eks_cluster_policy}"] - } - }, - "aws_eks_node_group": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-telemetrykeeper", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${var.telemetrykeeper_instance_type}"], - "scaling_config": [{ - "desired_size": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, - "min_size": 1, - "max_size": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }} - }], - "disk_size": "${var.telemetrykeeper_volume_size}", - "labels": { - "role": "telemetrykeeper" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrykeeper" - }, - "depends_on": [ - "${aws_iam_role_policy_attachment.eks_worker_node_policy}", - "${aws_iam_role_policy_attachment.eks_cni_policy}", - "${aws_iam_role_policy_attachment.eks_container_registry}" - ] - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-telemetrystore", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${var.telemetrystore_instance_type}"], - "scaling_config": [{ - "desired_size": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, - "min_size": 1, - "max_size": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }} - }], - "disk_size": "${var.telemetrystore_volume_size}", - "labels": { - "role": "telemetrystore" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-telemetrystore" - }, - "depends_on": [ - "${aws_iam_role_policy_attachment.eks_worker_node_policy}", - "${aws_iam_role_policy_attachment.eks_cni_policy}", - "${aws_iam_role_policy_attachment.eks_container_registry}" - ] - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-metastore", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${var.metastore_instance_type}"], - "scaling_config": [{ - "desired_size": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, - "min_size": 1, - "max_size": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }} - }], - "disk_size": "${var.metastore_volume_size}", - "labels": { - "role": "metastore" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-metastore" - }, - "depends_on": [ - "${aws_iam_role_policy_attachment.eks_worker_node_policy}", - "${aws_iam_role_policy_attachment.eks_cni_policy}", - "${aws_iam_role_policy_attachment.eks_container_registry}" - ] - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-ingester", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.public[*].id}", - "instance_types": ["${var.ingester_instance_type}"], - "scaling_config": [{ - "desired_size": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, - "min_size": 1, - "max_size": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }} - }], - "disk_size": "${var.ingester_volume_size}", - "labels": { - "role": "ingester" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-ingester" - }, - "depends_on": [ - "${aws_iam_role_policy_attachment.eks_worker_node_policy}", - "${aws_iam_role_policy_attachment.eks_cni_policy}", - "${aws_iam_role_policy_attachment.eks_container_registry}" - ] - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-signoz", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.public[*].id}", - "instance_types": ["${var.signoz_instance_type}"], - "scaling_config": [{ - "desired_size": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, - "min_size": 1, - "max_size": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }} - }], - "disk_size": "${var.signoz_volume_size}", - "labels": { - "role": "signoz" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Name": "${local.name}-signoz" - }, - "depends_on": [ - "${aws_iam_role_policy_attachment.eks_worker_node_policy}", - "${aws_iam_role_policy_attachment.eks_cni_policy}", - "${aws_iam_role_policy_attachment.eks_container_registry}" - ] - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl deleted file mode 100644 index e071577c..00000000 --- a/internal/infrastructure/terraform/templates/aws/eks/outputs.tf.json.gotmpl +++ /dev/null @@ -1,73 +0,0 @@ -{ - "output": { - "cluster_name": { - "description": "Name of the EKS cluster", - "value": "${aws_eks_cluster.main.name}" - }, - "cluster_endpoint": { - "description": "Endpoint for the EKS cluster API server", - "value": "${aws_eks_cluster.main.endpoint}" - }, - "cluster_ca_certificate": { - "description": "Base64-encoded certificate authority data for the EKS cluster", - "value": "${aws_eks_cluster.main.certificate_authority[0].data}", - "sensitive": true - }, - "cluster_version": { - "description": "Kubernetes version of the EKS cluster", - "value": "${aws_eks_cluster.main.version}" - }, - "vpc_id": { - "description": "ID of the VPC", - "value": "${aws_vpc.main.id}" - }, - "private_subnet_ids": { - "description": "IDs of the private subnets", - "value": "${aws_subnet.private[*].id}" - }, - "public_subnet_ids": { - "description": "IDs of the public subnets", - "value": "${aws_subnet.public[*].id}" - }, - "telemetrykeeper_node_group_arn": { - "description": "ARN of the TelemetryKeeper node group", - "value": "${aws_eks_node_group.telemetrykeeper[*].arn}" - }, - "telemetrykeeper_node_group_status": { - "description": "Status of the TelemetryKeeper node group", - "value": "${aws_eks_node_group.telemetrykeeper[*].status}" - }, - "telemetrystore_node_group_arn": { - "description": "ARN of the TelemetryStore node group", - "value": "${aws_eks_node_group.telemetrystore[*].arn}" - }, - "telemetrystore_node_group_status": { - "description": "Status of the TelemetryStore node group", - "value": "${aws_eks_node_group.telemetrystore[*].status}" - }, - "metastore_node_group_arn": { - "description": "ARN of the MetaStore node group", - "value": "${aws_eks_node_group.metastore[*].arn}" - }, - "metastore_node_group_status": { - "description": "Status of the MetaStore node group", - "value": "${aws_eks_node_group.metastore[*].status}" - }, - "ingester_node_group_arn": { - "description": "ARN of the Ingester node group", - "value": "${aws_eks_node_group.ingester[*].arn}" - }, - "ingester_node_group_status": { - "description": "Status of the Ingester node group", - "value": "${aws_eks_node_group.ingester[*].status}" - }, - "signoz_node_group_arn": { - "description": "ARN of the SigNoz node group", - "value": "${aws_eks_node_group.signoz[*].arn}" - }, - "signoz_node_group_status": { - "description": "Status of the SigNoz node group", - "value": "${aws_eks_node_group.signoz[*].status}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl deleted file mode 100644 index 47cfe41f..00000000 --- a/internal/infrastructure/terraform/templates/aws/eks/variables.tf.json.gotmpl +++ /dev/null @@ -1,79 +0,0 @@ -{ - "variable": { - "aws_region": { - "description": "AWS region to deploy resources", - "type": "string", - "default": "us-east-1" - }, - "vpc_cidr": { - "description": "CIDR block for the VPC", - "type": "string", - "default": "10.0.0.0/16" - }, - "az_count": { - "description": "Number of availability zones to use", - "type": "number", - "default": 2 - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "kubernetes_version": { - "description": "Kubernetes version for the EKS cluster", - "type": "string", - "default": "1.30" - }, - "telemetrykeeper_instance_type": { - "description": "EC2 instance type for TelemetryKeeper node group", - "type": "string", - "default": "t3.small" - }, - "telemetrykeeper_volume_size": { - "description": "Root volume size (GB) for TelemetryKeeper nodes", - "type": "number", - "default": 20 - }, - "telemetrystore_instance_type": { - "description": "EC2 instance type for TelemetryStore node group", - "type": "string", - "default": "r6i.xlarge" - }, - "telemetrystore_volume_size": { - "description": "Root volume size (GB) for TelemetryStore nodes", - "type": "number", - "default": 100 - }, - "metastore_instance_type": { - "description": "EC2 instance type for MetaStore node group", - "type": "string", - "default": "t3.small" - }, - "metastore_volume_size": { - "description": "Root volume size (GB) for MetaStore nodes", - "type": "number", - "default": 20 - }, - "ingester_instance_type": { - "description": "EC2 instance type for Ingester node group", - "type": "string", - "default": "t3.medium" - }, - "ingester_volume_size": { - "description": "Root volume size (GB) for Ingester nodes", - "type": "number", - "default": 50 - }, - "signoz_instance_type": { - "description": "EC2 instance type for SigNoz node group", - "type": "string", - "default": "t3.medium" - }, - "signoz_volume_size": { - "description": "Root volume size (GB) for SigNoz nodes", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl deleted file mode 100644 index f0410ed2..00000000 --- a/internal/infrastructure/terraform/templates/azure/aks/main.tf.json.gotmpl +++ /dev/null @@ -1,149 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "resource": { - "azurerm_resource_group": { - "main": { - "name": "${var.resource_group_name}", - "location": "${var.location}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_virtual_network": { - "main": { - "name": "${local.name}-vnet", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "address_space": ["${var.vnet_cidr}"], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_subnet": { - "aks": { - "name": "${local.name}-aks", - "resource_group_name": "${azurerm_resource_group.main.name}", - "virtual_network_name": "${azurerm_virtual_network.main.name}", - "address_prefixes": ["${var.aks_subnet_cidr}"] - } - }, - "azurerm_kubernetes_cluster": { - "main": { - "name": "${local.name}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "dns_prefix": "${local.name}", - "kubernetes_version": "${var.kubernetes_version}", - "default_node_pool": [{ - "name": "system", - "node_count": 1, - "vm_size": "Standard_D2s_v3", - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "upgrade_settings": [{"max_surge": "10%"}] - }], - "identity": [{"type": "SystemAssigned"}], - "network_profile": [{ - "network_plugin": "azure", - "load_balancer_sku": "standard", - "outbound_type": "loadBalancer" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_kubernetes_cluster_node_pool": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "tkeepr", - "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", - "vm_size": "${var.telemetrykeeper_vm_size}", - "node_count": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "os_disk_size_gb": "${var.telemetrykeeper_disk_size}", - "node_labels": { - "role": "telemetrykeeper" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "tstore", - "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", - "vm_size": "${var.telemetrystore_vm_size}", - "node_count": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "os_disk_size_gb": "${var.telemetrystore_disk_size}", - "node_labels": { - "role": "telemetrystore" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "metastore", - "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", - "vm_size": "${var.metastore_vm_size}", - "node_count": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "os_disk_size_gb": "${var.metastore_disk_size}", - "node_labels": { - "role": "metastore" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "ingester", - "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", - "vm_size": "${var.ingester_vm_size}", - "node_count": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "os_disk_size_gb": "${var.ingester_disk_size}", - "node_labels": { - "role": "ingester" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "signoz", - "kubernetes_cluster_id": "${azurerm_kubernetes_cluster.main.id}", - "vm_size": "${var.signoz_vm_size}", - "node_count": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, - "vnet_subnet_id": "${azurerm_subnet.aks.id}", - "os_disk_size_gb": "${var.signoz_disk_size}", - "node_labels": { - "role": "signoz" - }, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl deleted file mode 100644 index 612cb035..00000000 --- a/internal/infrastructure/terraform/templates/azure/aks/outputs.tf.json.gotmpl +++ /dev/null @@ -1,53 +0,0 @@ -{ - "output": { - "cluster_name": { - "description": "Name of the AKS cluster", - "value": "${azurerm_kubernetes_cluster.main.name}" - }, - "cluster_fqdn": { - "description": "FQDN of the AKS cluster", - "value": "${azurerm_kubernetes_cluster.main.fqdn}" - }, - "kube_config": { - "description": "Raw kubeconfig for the AKS cluster", - "value": "${azurerm_kubernetes_cluster.main.kube_config_raw}", - "sensitive": true - }, - "cluster_identity_principal_id": { - "description": "Principal ID of the AKS cluster managed identity", - "value": "${azurerm_kubernetes_cluster.main.identity[0].principal_id}" - }, - "resource_group_name": { - "description": "Name of the Azure resource group", - "value": "${azurerm_resource_group.main.name}" - }, - "vnet_id": { - "description": "ID of the virtual network", - "value": "${azurerm_virtual_network.main.id}" - }, - "aks_subnet_id": { - "description": "ID of the AKS subnet", - "value": "${azurerm_subnet.aks.id}" - }, - "telemetrykeeper_node_pool_id": { - "description": "ID of the TelemetryKeeper node pool", - "value": "${azurerm_kubernetes_cluster_node_pool.telemetrykeeper[*].id}" - }, - "telemetrystore_node_pool_id": { - "description": "ID of the TelemetryStore node pool", - "value": "${azurerm_kubernetes_cluster_node_pool.telemetrystore[*].id}" - }, - "metastore_node_pool_id": { - "description": "ID of the MetaStore node pool", - "value": "${azurerm_kubernetes_cluster_node_pool.metastore[*].id}" - }, - "ingester_node_pool_id": { - "description": "ID of the Ingester node pool", - "value": "${azurerm_kubernetes_cluster_node_pool.ingester[*].id}" - }, - "signoz_node_pool_id": { - "description": "ID of the SigNoz node pool", - "value": "${azurerm_kubernetes_cluster_node_pool.signoz[*].id}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl deleted file mode 100644 index af39836b..00000000 --- a/internal/infrastructure/terraform/templates/azure/aks/variables.tf.json.gotmpl +++ /dev/null @@ -1,83 +0,0 @@ -{ - "variable": { - "resource_group_name": { - "description": "Azure resource group name", - "type": "string" - }, - "location": { - "description": "Azure region to deploy resources", - "type": "string", - "default": "eastus" - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "vnet_cidr": { - "description": "CIDR block for the virtual network", - "type": "string", - "default": "10.0.0.0/8" - }, - "aks_subnet_cidr": { - "description": "CIDR block for the AKS subnet", - "type": "string", - "default": "10.240.0.0/16" - }, - "kubernetes_version": { - "description": "Kubernetes version for the AKS cluster (leave empty for latest)", - "type": "string", - "default": "" - }, - "telemetrykeeper_vm_size": { - "description": "Azure VM size for TelemetryKeeper node pool", - "type": "string", - "default": "Standard_B2s" - }, - "telemetrykeeper_disk_size": { - "description": "OS disk size (GB) for TelemetryKeeper nodes", - "type": "number", - "default": 20 - }, - "telemetrystore_vm_size": { - "description": "Azure VM size for TelemetryStore node pool", - "type": "string", - "default": "Standard_E4s_v3" - }, - "telemetrystore_disk_size": { - "description": "OS disk size (GB) for TelemetryStore nodes", - "type": "number", - "default": 100 - }, - "metastore_vm_size": { - "description": "Azure VM size for MetaStore node pool", - "type": "string", - "default": "Standard_B2s" - }, - "metastore_disk_size": { - "description": "OS disk size (GB) for MetaStore nodes", - "type": "number", - "default": 20 - }, - "ingester_vm_size": { - "description": "Azure VM size for Ingester node pool", - "type": "string", - "default": "Standard_B4ms" - }, - "ingester_disk_size": { - "description": "OS disk size (GB) for Ingester nodes", - "type": "number", - "default": 50 - }, - "signoz_vm_size": { - "description": "Azure VM size for SigNoz node pool", - "type": "string", - "default": "Standard_B4ms" - }, - "signoz_disk_size": { - "description": "OS disk size (GB) for SigNoz nodes", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl deleted file mode 100644 index c23f8c69..00000000 --- a/internal/infrastructure/terraform/templates/azure/vm/main.tf.json.gotmpl +++ /dev/null @@ -1,537 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "resource": { - "azurerm_resource_group": { - "main": { - "name": "${var.resource_group_name}", - "location": "${var.location}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_virtual_network": { - "main": { - "name": "${local.name}-vnet", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "address_space": ["${var.vnet_cidr}"], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_subnet": { - "private": { - "name": "${local.name}-private", - "resource_group_name": "${azurerm_resource_group.main.name}", - "virtual_network_name": "${azurerm_virtual_network.main.name}", - "address_prefixes": ["${var.private_subnet_cidr}"] - }, - "public": { - "name": "${local.name}-public", - "resource_group_name": "${azurerm_resource_group.main.name}", - "virtual_network_name": "${azurerm_virtual_network.main.name}", - "address_prefixes": ["${var.public_subnet_cidr}"] - } - }, - "azurerm_network_security_group": { - "telemetrykeeper": { - "name": "${local.name}-telemetrykeeper-nsg", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "security_rule": [ - { - "name": "allow-keeper-client", - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "9181", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-keeper-raft", - "priority": 110, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "9234", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-ssh", - "priority": 120, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "22", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "telemetrystore": { - "name": "${local.name}-telemetrystore-nsg", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "security_rule": [ - { - "name": "allow-clickhouse-native", - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "9000", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-clickhouse-http", - "priority": 110, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "8123", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-ssh", - "priority": 120, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "22", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "metastore": { - "name": "${local.name}-metastore-nsg", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "security_rule": [ - { - "name": "allow-postgres", - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "5432", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-ssh", - "priority": 110, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "22", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "ingester": { - "name": "${local.name}-ingester-nsg", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "security_rule": [ - { - "name": "allow-otlp-grpc", - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "4317", - "source_address_prefix": "*", - "destination_address_prefix": "*" - }, - { - "name": "allow-otlp-http", - "priority": 110, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "4318", - "source_address_prefix": "*", - "destination_address_prefix": "*" - }, - { - "name": "allow-ssh", - "priority": 120, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "22", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "signoz": { - "name": "${local.name}-signoz-nsg", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "security_rule": [ - { - "name": "allow-ui", - "priority": 100, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "8080", - "source_address_prefix": "*", - "destination_address_prefix": "*" - }, - { - "name": "allow-api", - "priority": 110, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "3301", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - }, - { - "name": "allow-ssh", - "priority": 120, - "direction": "Inbound", - "access": "Allow", - "protocol": "Tcp", - "source_port_range": "*", - "destination_port_range": "22", - "source_address_prefix": "${var.vnet_cidr}", - "destination_address_prefix": "*" - } - ], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_network_interface": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrykeeper-nic-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "ip_configuration": [{ - "name": "internal", - "subnet_id": "${azurerm_subnet.private.id}", - "private_ip_address_allocation": "Dynamic" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrystore-nic-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "ip_configuration": [{ - "name": "internal", - "subnet_id": "${azurerm_subnet.private.id}", - "private_ip_address_allocation": "Dynamic" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-metastore-nic-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "ip_configuration": [{ - "name": "internal", - "subnet_id": "${azurerm_subnet.private.id}", - "private_ip_address_allocation": "Dynamic" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-ingester-nic-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "ip_configuration": [{ - "name": "external", - "subnet_id": "${azurerm_subnet.public.id}", - "private_ip_address_allocation": "Dynamic", - "public_ip_address_id": "${azurerm_public_ip.ingester[count.index].id}" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-signoz-nic-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "ip_configuration": [{ - "name": "external", - "subnet_id": "${azurerm_subnet.public.id}", - "private_ip_address_allocation": "Dynamic", - "public_ip_address_id": "${azurerm_public_ip.signoz[count.index].id}" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_network_interface_security_group_association": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "network_interface_id": "${azurerm_network_interface.telemetrykeeper[count.index].id}", - "network_security_group_id": "${azurerm_network_security_group.telemetrykeeper.id}" - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "network_interface_id": "${azurerm_network_interface.telemetrystore[count.index].id}", - "network_security_group_id": "${azurerm_network_security_group.telemetrystore.id}" - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "network_interface_id": "${azurerm_network_interface.metastore[count.index].id}", - "network_security_group_id": "${azurerm_network_security_group.metastore.id}" - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "network_interface_id": "${azurerm_network_interface.ingester[count.index].id}", - "network_security_group_id": "${azurerm_network_security_group.ingester.id}" - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "network_interface_id": "${azurerm_network_interface.signoz[count.index].id}", - "network_security_group_id": "${azurerm_network_security_group.signoz.id}" - } - }, - "azurerm_public_ip": { - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-ingester-pip-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "allocation_method": "Static", - "sku": "Standard", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-signoz-pip-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "allocation_method": "Static", - "sku": "Standard", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}" - } - } - }, - "azurerm_linux_virtual_machine": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrykeeper-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "size": "${var.telemetrykeeper_vm_size}", - "admin_username": "ubuntu", - "network_interface_ids": ["${azurerm_network_interface.telemetrykeeper[count.index].id}"], - "admin_ssh_key": [{ - "username": "ubuntu", - "public_key": "${var.ssh_public_key}" - }], - "os_disk": [{ - "caching": "ReadWrite", - "storage_account_type": "Premium_LRS", - "disk_size_gb": "${var.telemetrykeeper_disk_size}" - }], - "source_image_reference": [{ - "publisher": "Canonical", - "offer": "0001-com-ubuntu-server-jammy", - "sku": "22_04-lts", - "version": "latest" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Role": "telemetrykeeper" - } - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrystore-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "size": "${var.telemetrystore_vm_size}", - "admin_username": "ubuntu", - "network_interface_ids": ["${azurerm_network_interface.telemetrystore[count.index].id}"], - "admin_ssh_key": [{ - "username": "ubuntu", - "public_key": "${var.ssh_public_key}" - }], - "os_disk": [{ - "caching": "ReadWrite", - "storage_account_type": "Premium_LRS", - "disk_size_gb": "${var.telemetrystore_disk_size}" - }], - "source_image_reference": [{ - "publisher": "Canonical", - "offer": "0001-com-ubuntu-server-jammy", - "sku": "22_04-lts", - "version": "latest" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Role": "telemetrystore" - } - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-metastore-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "size": "${var.metastore_vm_size}", - "admin_username": "ubuntu", - "network_interface_ids": ["${azurerm_network_interface.metastore[count.index].id}"], - "admin_ssh_key": [{ - "username": "ubuntu", - "public_key": "${var.ssh_public_key}" - }], - "os_disk": [{ - "caching": "ReadWrite", - "storage_account_type": "Premium_LRS", - "disk_size_gb": "${var.metastore_disk_size}" - }], - "source_image_reference": [{ - "publisher": "Canonical", - "offer": "0001-com-ubuntu-server-jammy", - "sku": "22_04-lts", - "version": "latest" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Role": "metastore" - } - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-ingester-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "size": "${var.ingester_vm_size}", - "admin_username": "ubuntu", - "network_interface_ids": ["${azurerm_network_interface.ingester[count.index].id}"], - "admin_ssh_key": [{ - "username": "ubuntu", - "public_key": "${var.ssh_public_key}" - }], - "os_disk": [{ - "caching": "ReadWrite", - "storage_account_type": "Premium_LRS", - "disk_size_gb": "${var.ingester_disk_size}" - }], - "source_image_reference": [{ - "publisher": "Canonical", - "offer": "0001-com-ubuntu-server-jammy", - "sku": "22_04-lts", - "version": "latest" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Role": "ingester" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-signoz-${count.index}", - "location": "${azurerm_resource_group.main.location}", - "resource_group_name": "${azurerm_resource_group.main.name}", - "size": "${var.signoz_vm_size}", - "admin_username": "ubuntu", - "network_interface_ids": ["${azurerm_network_interface.signoz[count.index].id}"], - "admin_ssh_key": [{ - "username": "ubuntu", - "public_key": "${var.ssh_public_key}" - }], - "os_disk": [{ - "caching": "ReadWrite", - "storage_account_type": "Premium_LRS", - "disk_size_gb": "${var.signoz_disk_size}" - }], - "source_image_reference": [{ - "publisher": "Canonical", - "offer": "0001-com-ubuntu-server-jammy", - "sku": "22_04-lts", - "version": "latest" - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "app.kubernetes.io/name": "{{ .Metadata.Name }}", - "Role": "signoz" - } - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl deleted file mode 100644 index 152e95aa..00000000 --- a/internal/infrastructure/terraform/templates/azure/vm/outputs.tf.json.gotmpl +++ /dev/null @@ -1,68 +0,0 @@ -{ - "output": { - "resource_group_name": { - "description": "Name of the Azure resource group", - "value": "${azurerm_resource_group.main.name}" - }, - "vnet_id": { - "description": "ID of the virtual network", - "value": "${azurerm_virtual_network.main.id}" - }, - "private_subnet_id": { - "description": "ID of the private subnet", - "value": "${azurerm_subnet.private.id}" - }, - "public_subnet_id": { - "description": "ID of the public subnet", - "value": "${azurerm_subnet.public.id}" - }, - "telemetrykeeper_vm_ids": { - "description": "IDs of the TelemetryKeeper VMs", - "value": "${azurerm_linux_virtual_machine.telemetrykeeper[*].id}" - }, - "telemetrykeeper_private_ips": { - "description": "Private IP addresses of the TelemetryKeeper VMs", - "value": "${azurerm_network_interface.telemetrykeeper[*].private_ip_address}" - }, - "telemetrystore_vm_ids": { - "description": "IDs of the TelemetryStore VMs", - "value": "${azurerm_linux_virtual_machine.telemetrystore[*].id}" - }, - "telemetrystore_private_ips": { - "description": "Private IP addresses of the TelemetryStore VMs", - "value": "${azurerm_network_interface.telemetrystore[*].private_ip_address}" - }, - "metastore_vm_ids": { - "description": "IDs of the MetaStore VMs", - "value": "${azurerm_linux_virtual_machine.metastore[*].id}" - }, - "metastore_private_ips": { - "description": "Private IP addresses of the MetaStore VMs", - "value": "${azurerm_network_interface.metastore[*].private_ip_address}" - }, - "ingester_vm_ids": { - "description": "IDs of the Ingester VMs", - "value": "${azurerm_linux_virtual_machine.ingester[*].id}" - }, - "ingester_public_ips": { - "description": "Public IP addresses of the Ingester VMs", - "value": "${azurerm_public_ip.ingester[*].ip_address}" - }, - "ingester_private_ips": { - "description": "Private IP addresses of the Ingester VMs", - "value": "${azurerm_network_interface.ingester[*].private_ip_address}" - }, - "signoz_vm_ids": { - "description": "IDs of the SigNoz VMs", - "value": "${azurerm_linux_virtual_machine.signoz[*].id}" - }, - "signoz_public_ips": { - "description": "Public IP addresses of the SigNoz VMs", - "value": "${azurerm_public_ip.signoz[*].ip_address}" - }, - "signoz_private_ips": { - "description": "Private IP addresses of the SigNoz VMs", - "value": "${azurerm_network_interface.signoz[*].private_ip_address}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl deleted file mode 100644 index 53267376..00000000 --- a/internal/infrastructure/terraform/templates/azure/vm/variables.tf.json.gotmpl +++ /dev/null @@ -1,87 +0,0 @@ -{ - "variable": { - "resource_group_name": { - "description": "Azure resource group name", - "type": "string" - }, - "location": { - "description": "Azure region to deploy resources", - "type": "string", - "default": "eastus" - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "vnet_cidr": { - "description": "CIDR block for the virtual network", - "type": "string", - "default": "10.0.0.0/16" - }, - "private_subnet_cidr": { - "description": "CIDR block for the private subnet", - "type": "string", - "default": "10.0.1.0/24" - }, - "public_subnet_cidr": { - "description": "CIDR block for the public subnet", - "type": "string", - "default": "10.0.2.0/24" - }, - "ssh_public_key": { - "description": "SSH public key for VM access", - "type": "string" - }, - "telemetrykeeper_vm_size": { - "description": "Azure VM size for TelemetryKeeper", - "type": "string", - "default": "Standard_B2s" - }, - "telemetrykeeper_disk_size": { - "description": "OS disk size (GB) for TelemetryKeeper VMs", - "type": "number", - "default": 20 - }, - "telemetrystore_vm_size": { - "description": "Azure VM size for TelemetryStore", - "type": "string", - "default": "Standard_E4s_v3" - }, - "telemetrystore_disk_size": { - "description": "OS disk size (GB) for TelemetryStore VMs", - "type": "number", - "default": 100 - }, - "metastore_vm_size": { - "description": "Azure VM size for MetaStore", - "type": "string", - "default": "Standard_B2s" - }, - "metastore_disk_size": { - "description": "OS disk size (GB) for MetaStore VMs", - "type": "number", - "default": 20 - }, - "ingester_vm_size": { - "description": "Azure VM size for Ingester", - "type": "string", - "default": "Standard_B4ms" - }, - "ingester_disk_size": { - "description": "OS disk size (GB) for Ingester VMs", - "type": "number", - "default": 50 - }, - "signoz_vm_size": { - "description": "Azure VM size for SigNoz", - "type": "string", - "default": "Standard_B4ms" - }, - "signoz_disk_size": { - "description": "OS disk size (GB) for SigNoz VMs", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl deleted file mode 100644 index b0f52724..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gce/main.tf.json.gotmpl +++ /dev/null @@ -1,261 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}" - } - }, - "data": { - "google_compute_image": { - "ubuntu": { - "family": "ubuntu-2204-lts", - "project": "ubuntu-os-cloud" - } - } - }, - "resource": { - "google_compute_network": { - "main": { - "name": "${local.name}-vpc", - "auto_create_subnetworks": false, - "project": "${var.project_id}" - } - }, - "google_compute_subnetwork": { - "private": { - "name": "${local.name}-private", - "ip_cidr_range": "${var.private_subnet_cidr}", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "private_ip_google_access": true - }, - "public": { - "name": "${local.name}-public", - "ip_cidr_range": "${var.public_subnet_cidr}", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}" - } - }, - "google_compute_router": { - "main": { - "name": "${local.name}-router", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}" - } - }, - "google_compute_router_nat": { - "main": { - "name": "${local.name}-nat", - "router": "${google_compute_router.main.name}", - "region": "${var.region}", - "project": "${var.project_id}", - "nat_ip_allocate_option": "AUTO_ONLY", - "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", - "subnetwork": [{ - "name": "${google_compute_subnetwork.private.id}", - "source_ip_ranges_to_nat": ["ALL_IP_RANGES"] - }] - } - }, - "google_compute_firewall": { - "telemetrykeeper": { - "name": "${local.name}-telemetrykeeper", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "allow": [ - {"protocol": "tcp", "ports": ["9181", "9234"]}, - {"protocol": "tcp", "ports": ["22"]} - ], - "source_ranges": ["${var.private_subnet_cidr}"], - "target_tags": ["telemetrykeeper"], - "description": "Allow TelemetryKeeper (ClickHouse Keeper) traffic" - }, - "telemetrystore": { - "name": "${local.name}-telemetrystore", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "allow": [ - {"protocol": "tcp", "ports": ["9000", "8123"]}, - {"protocol": "tcp", "ports": ["22"]} - ], - "source_ranges": ["${var.private_subnet_cidr}"], - "target_tags": ["telemetrystore"], - "description": "Allow TelemetryStore (ClickHouse) traffic" - }, - "metastore": { - "name": "${local.name}-metastore", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "allow": [ - {"protocol": "tcp", "ports": ["5432"]}, - {"protocol": "tcp", "ports": ["22"]} - ], - "source_ranges": ["${var.private_subnet_cidr}"], - "target_tags": ["metastore"], - "description": "Allow MetaStore (PostgreSQL) traffic" - }, - "ingester": { - "name": "${local.name}-ingester", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "allow": [ - {"protocol": "tcp", "ports": ["4317", "4318"]}, - {"protocol": "tcp", "ports": ["22"]} - ], - "source_ranges": ["0.0.0.0/0"], - "target_tags": ["ingester"], - "description": "Allow Ingester (OpenTelemetry Collector) traffic" - }, - "signoz": { - "name": "${local.name}-signoz", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "allow": [ - {"protocol": "tcp", "ports": ["8080", "3301"]}, - {"protocol": "tcp", "ports": ["22"]} - ], - "source_ranges": ["0.0.0.0/0"], - "target_tags": ["signoz"], - "description": "Allow SigNoz UI and API traffic" - } - }, - "google_compute_instance": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}{{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrykeeper-${count.index}", - "machine_type": "${var.telemetrykeeper_machine_type}", - "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", - "project": "${var.project_id}", - "tags": ["telemetrykeeper"], - "boot_disk": [{ - "initialize_params": [{ - "image": "${data.google_compute_image.ubuntu.self_link}", - "size": "${var.telemetrykeeper_disk_size}", - "type": "pd-ssd" - }] - }], - "network_interface": [{ - "subnetwork": "${google_compute_subnetwork.private.id}" - }], - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "telemetrykeeper" - }, - "metadata": { - "enable-oslogin": "TRUE" - } - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}{{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-telemetrystore-${count.index}", - "machine_type": "${var.telemetrystore_machine_type}", - "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", - "project": "${var.project_id}", - "tags": ["telemetrystore"], - "boot_disk": [{ - "initialize_params": [{ - "image": "${data.google_compute_image.ubuntu.self_link}", - "size": "${var.telemetrystore_disk_size}", - "type": "pd-ssd" - }] - }], - "network_interface": [{ - "subnetwork": "${google_compute_subnetwork.private.id}" - }], - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "telemetrystore" - }, - "metadata": { - "enable-oslogin": "TRUE" - } - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}{{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-metastore-${count.index}", - "machine_type": "${var.metastore_machine_type}", - "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", - "project": "${var.project_id}", - "tags": ["metastore"], - "boot_disk": [{ - "initialize_params": [{ - "image": "${data.google_compute_image.ubuntu.self_link}", - "size": "${var.metastore_disk_size}", - "type": "pd-ssd" - }] - }], - "network_interface": [{ - "subnetwork": "${google_compute_subnetwork.private.id}" - }], - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "metastore" - }, - "metadata": { - "enable-oslogin": "TRUE" - } - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}{{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-ingester-${count.index}", - "machine_type": "${var.ingester_machine_type}", - "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", - "project": "${var.project_id}", - "tags": ["ingester"], - "boot_disk": [{ - "initialize_params": [{ - "image": "${data.google_compute_image.ubuntu.self_link}", - "size": "${var.ingester_disk_size}", - "type": "pd-ssd" - }] - }], - "network_interface": [{ - "subnetwork": "${google_compute_subnetwork.public.id}", - "access_config": [{}] - }], - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "ingester" - }, - "metadata": { - "enable-oslogin": "TRUE" - } - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}{{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}{{ else }}0{{ end }}, - "name": "${local.name}-signoz-${count.index}", - "machine_type": "${var.signoz_machine_type}", - "zone": "${var.region}-${element([\"a\", \"b\", \"c\"], count.index)}", - "project": "${var.project_id}", - "tags": ["signoz"], - "boot_disk": [{ - "initialize_params": [{ - "image": "${data.google_compute_image.ubuntu.self_link}", - "size": "${var.signoz_disk_size}", - "type": "pd-ssd" - }] - }], - "network_interface": [{ - "subnetwork": "${google_compute_subnetwork.public.id}", - "access_config": [{}] - }], - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "signoz" - }, - "metadata": { - "enable-oslogin": "TRUE" - } - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl deleted file mode 100644 index abe80056..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gce/outputs.tf.json.gotmpl +++ /dev/null @@ -1,64 +0,0 @@ -{ - "output": { - "network_id": { - "description": "ID of the VPC network", - "value": "${google_compute_network.main.id}" - }, - "private_subnetwork_id": { - "description": "ID of the private subnetwork", - "value": "${google_compute_subnetwork.private.id}" - }, - "public_subnetwork_id": { - "description": "ID of the public subnetwork", - "value": "${google_compute_subnetwork.public.id}" - }, - "telemetrykeeper_instance_ids": { - "description": "IDs of the TelemetryKeeper GCE instances", - "value": "${google_compute_instance.telemetrykeeper[*].id}" - }, - "telemetrykeeper_private_ips": { - "description": "Private IP addresses of the TelemetryKeeper GCE instances", - "value": "${[for i in google_compute_instance.telemetrykeeper : i.network_interface[0].network_ip]}" - }, - "telemetrystore_instance_ids": { - "description": "IDs of the TelemetryStore GCE instances", - "value": "${google_compute_instance.telemetrystore[*].id}" - }, - "telemetrystore_private_ips": { - "description": "Private IP addresses of the TelemetryStore GCE instances", - "value": "${[for i in google_compute_instance.telemetrystore : i.network_interface[0].network_ip]}" - }, - "metastore_instance_ids": { - "description": "IDs of the MetaStore GCE instances", - "value": "${google_compute_instance.metastore[*].id}" - }, - "metastore_private_ips": { - "description": "Private IP addresses of the MetaStore GCE instances", - "value": "${[for i in google_compute_instance.metastore : i.network_interface[0].network_ip]}" - }, - "ingester_instance_ids": { - "description": "IDs of the Ingester GCE instances", - "value": "${google_compute_instance.ingester[*].id}" - }, - "ingester_public_ips": { - "description": "Public IP addresses of the Ingester GCE instances", - "value": "${[for i in google_compute_instance.ingester : i.network_interface[0].access_config[0].nat_ip]}" - }, - "ingester_private_ips": { - "description": "Private IP addresses of the Ingester GCE instances", - "value": "${[for i in google_compute_instance.ingester : i.network_interface[0].network_ip]}" - }, - "signoz_instance_ids": { - "description": "IDs of the SigNoz GCE instances", - "value": "${google_compute_instance.signoz[*].id}" - }, - "signoz_public_ips": { - "description": "Public IP addresses of the SigNoz GCE instances", - "value": "${[for i in google_compute_instance.signoz : i.network_interface[0].access_config[0].nat_ip]}" - }, - "signoz_private_ips": { - "description": "Private IP addresses of the SigNoz GCE instances", - "value": "${[for i in google_compute_instance.signoz : i.network_interface[0].network_ip]}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl deleted file mode 100644 index c94e3079..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gce/variables.tf.json.gotmpl +++ /dev/null @@ -1,78 +0,0 @@ -{ - "variable": { - "project_id": { - "description": "GCP project ID", - "type": "string" - }, - "region": { - "description": "GCP region to deploy resources", - "type": "string", - "default": "us-central1" - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "private_subnet_cidr": { - "description": "CIDR block for the private subnet", - "type": "string", - "default": "10.0.1.0/24" - }, - "public_subnet_cidr": { - "description": "CIDR block for the public subnet", - "type": "string", - "default": "10.0.2.0/24" - }, - "telemetrykeeper_machine_type": { - "description": "GCE machine type for TelemetryKeeper", - "type": "string", - "default": "n2-standard-2" - }, - "telemetrykeeper_disk_size": { - "description": "Boot disk size (GB) for TelemetryKeeper instances", - "type": "number", - "default": 20 - }, - "telemetrystore_machine_type": { - "description": "GCE machine type for TelemetryStore", - "type": "string", - "default": "n2-highmem-4" - }, - "telemetrystore_disk_size": { - "description": "Boot disk size (GB) for TelemetryStore instances", - "type": "number", - "default": 100 - }, - "metastore_machine_type": { - "description": "GCE machine type for MetaStore", - "type": "string", - "default": "n2-standard-2" - }, - "metastore_disk_size": { - "description": "Boot disk size (GB) for MetaStore instances", - "type": "number", - "default": 20 - }, - "ingester_machine_type": { - "description": "GCE machine type for Ingester", - "type": "string", - "default": "n2-standard-4" - }, - "ingester_disk_size": { - "description": "Boot disk size (GB) for Ingester instances", - "type": "number", - "default": 50 - }, - "signoz_machine_type": { - "description": "GCE machine type for SigNoz", - "type": "string", - "default": "n2-standard-4" - }, - "signoz_disk_size": { - "description": "Boot disk size (GB) for SigNoz instances", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl deleted file mode 100644 index 9a4a60d7..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gke/main.tf.json.gotmpl +++ /dev/null @@ -1,224 +0,0 @@ -{ - "locals": { - "name": "{{ .Metadata.Name }}", - "common_labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}" - } - }, - "resource": { - "google_compute_network": { - "main": { - "name": "${local.name}-vpc", - "auto_create_subnetworks": false, - "project": "${var.project_id}" - } - }, - "google_compute_subnetwork": { - "private": { - "name": "${local.name}-private", - "ip_cidr_range": "${var.private_subnet_cidr}", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}", - "private_ip_google_access": true, - "secondary_ip_range": [ - { - "range_name": "${local.name}-pods", - "ip_cidr_range": "${var.pods_cidr}" - }, - { - "range_name": "${local.name}-services", - "ip_cidr_range": "${var.services_cidr}" - } - ] - }, - "public": { - "name": "${local.name}-public", - "ip_cidr_range": "${var.public_subnet_cidr}", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}" - } - }, - "google_compute_router": { - "main": { - "name": "${local.name}-router", - "region": "${var.region}", - "network": "${google_compute_network.main.id}", - "project": "${var.project_id}" - } - }, - "google_compute_router_nat": { - "main": { - "name": "${local.name}-nat", - "router": "${google_compute_router.main.name}", - "region": "${var.region}", - "project": "${var.project_id}", - "nat_ip_allocate_option": "AUTO_ONLY", - "source_subnetwork_ip_ranges_to_nat": "LIST_OF_SUBNETWORKS", - "subnetwork": [{ - "name": "${google_compute_subnetwork.private.id}", - "source_ip_ranges_to_nat": ["ALL_IP_RANGES"] - }] - } - }, - "google_container_cluster": { - "main": { - "name": "${local.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "network": "${google_compute_network.main.id}", - "subnetwork": "${google_compute_subnetwork.private.id}", - "remove_default_node_pool": true, - "initial_node_count": 1, - "ip_allocation_policy": [{ - "cluster_secondary_range_name": "${local.name}-pods", - "services_secondary_range_name": "${local.name}-services" - }], - "private_cluster_config": [{ - "enable_private_nodes": true, - "enable_private_endpoint": false, - "master_ipv4_cidr_block": "${var.master_cidr}" - }], - "master_auth": [{ - "client_certificate_config": [{ - "issue_client_certificate": false - }] - }], - "workload_identity_config": [{ - "workload_pool": "${var.project_id}.svc.id.goog" - }], - "release_channel": [{ - "channel": "REGULAR" - }], - "resource_labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}" - } - } - }, - "google_container_node_pool": { - "telemetrykeeper": { - "count": {{ if .Spec.TelemetryKeeper.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "${local.name}-telemetrykeeper", - "cluster": "${google_container_cluster.main.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "node_count": {{ derefInt .Spec.TelemetryKeeper.Spec.Cluster.Replicas }}, - "node_config": [{ - "machine_type": "${var.telemetrykeeper_machine_type}", - "disk_size_gb": "${var.telemetrykeeper_disk_size}", - "disk_type": "pd-ssd", - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "telemetrykeeper" - }, - "workload_metadata_config": [{"mode": "GKE_METADATA"}], - "shielded_instance_config": [{"enable_secure_boot": true}] - }], - "management": [{ - "auto_repair": true, - "auto_upgrade": true - }] - }, - "telemetrystore": { - "count": {{ if .Spec.TelemetryStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "${local.name}-telemetrystore", - "cluster": "${google_container_cluster.main.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "node_count": {{ derefInt .Spec.TelemetryStore.Spec.Cluster.Replicas }}, - "node_config": [{ - "machine_type": "${var.telemetrystore_machine_type}", - "disk_size_gb": "${var.telemetrystore_disk_size}", - "disk_type": "pd-ssd", - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "telemetrystore" - }, - "workload_metadata_config": [{"mode": "GKE_METADATA"}], - "shielded_instance_config": [{"enable_secure_boot": true}] - }], - "management": [{ - "auto_repair": true, - "auto_upgrade": true - }] - }, - "metastore": { - "count": {{ if .Spec.MetaStore.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "${local.name}-metastore", - "cluster": "${google_container_cluster.main.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "node_count": {{ derefInt .Spec.MetaStore.Spec.Cluster.Replicas }}, - "node_config": [{ - "machine_type": "${var.metastore_machine_type}", - "disk_size_gb": "${var.metastore_disk_size}", - "disk_type": "pd-ssd", - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "metastore" - }, - "workload_metadata_config": [{"mode": "GKE_METADATA"}], - "shielded_instance_config": [{"enable_secure_boot": true}] - }], - "management": [{ - "auto_repair": true, - "auto_upgrade": true - }] - }, - "ingester": { - "count": {{ if .Spec.Ingester.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "${local.name}-ingester", - "cluster": "${google_container_cluster.main.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "node_count": {{ derefInt .Spec.Ingester.Spec.Cluster.Replicas }}, - "node_config": [{ - "machine_type": "${var.ingester_machine_type}", - "disk_size_gb": "${var.ingester_disk_size}", - "disk_type": "pd-ssd", - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "ingester" - }, - "workload_metadata_config": [{"mode": "GKE_METADATA"}], - "shielded_instance_config": [{"enable_secure_boot": true}] - }], - "management": [{ - "auto_repair": true, - "auto_upgrade": true - }] - }, - "signoz": { - "count": {{ if .Spec.Signoz.Spec.Enabled }}1{{ else }}0{{ end }}, - "name": "${local.name}-signoz", - "cluster": "${google_container_cluster.main.name}", - "location": "${var.region}", - "project": "${var.project_id}", - "node_count": {{ derefInt .Spec.Signoz.Spec.Cluster.Replicas }}, - "node_config": [{ - "machine_type": "${var.signoz_machine_type}", - "disk_size_gb": "${var.signoz_disk_size}", - "disk_type": "pd-ssd", - "labels": { - "app-kubernetes-io-managed-by": "foundry", - "app-kubernetes-io-name": "{{ .Metadata.Name }}", - "role": "signoz" - }, - "workload_metadata_config": [{"mode": "GKE_METADATA"}], - "shielded_instance_config": [{"enable_secure_boot": true}] - }], - "management": [{ - "auto_repair": true, - "auto_upgrade": true - }] - } - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl deleted file mode 100644 index e68728ae..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gke/outputs.tf.json.gotmpl +++ /dev/null @@ -1,46 +0,0 @@ -{ - "output": { - "cluster_name": { - "description": "Name of the GKE cluster", - "value": "${google_container_cluster.main.name}" - }, - "cluster_endpoint": { - "description": "Endpoint for the GKE cluster master", - "value": "${google_container_cluster.main.endpoint}", - "sensitive": true - }, - "cluster_ca_certificate": { - "description": "Base64-encoded public certificate of the cluster's certificate authority", - "value": "${google_container_cluster.main.master_auth[0].cluster_ca_certificate}", - "sensitive": true - }, - "network_id": { - "description": "ID of the VPC network", - "value": "${google_compute_network.main.id}" - }, - "private_subnetwork_id": { - "description": "ID of the private subnetwork", - "value": "${google_compute_subnetwork.private.id}" - }, - "telemetrykeeper_node_pool_id": { - "description": "ID of the TelemetryKeeper node pool", - "value": "${google_container_node_pool.telemetrykeeper[*].id}" - }, - "telemetrystore_node_pool_id": { - "description": "ID of the TelemetryStore node pool", - "value": "${google_container_node_pool.telemetrystore[*].id}" - }, - "metastore_node_pool_id": { - "description": "ID of the MetaStore node pool", - "value": "${google_container_node_pool.metastore[*].id}" - }, - "ingester_node_pool_id": { - "description": "ID of the Ingester node pool", - "value": "${google_container_node_pool.ingester[*].id}" - }, - "signoz_node_pool_id": { - "description": "ID of the SigNoz node pool", - "value": "${google_container_node_pool.signoz[*].id}" - } - } -} diff --git a/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl b/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl deleted file mode 100644 index 6138d3cd..00000000 --- a/internal/infrastructure/terraform/templates/gcp/gke/variables.tf.json.gotmpl +++ /dev/null @@ -1,98 +0,0 @@ -{ - "variable": { - "project_id": { - "description": "GCP project ID", - "type": "string" - }, - "region": { - "description": "GCP region to deploy resources", - "type": "string", - "default": "us-central1" - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Metadata.Name }}" - }, - "private_subnet_cidr": { - "description": "CIDR block for the private subnet", - "type": "string", - "default": "10.0.1.0/24" - }, - "public_subnet_cidr": { - "description": "CIDR block for the public subnet", - "type": "string", - "default": "10.0.2.0/24" - }, - "pods_cidr": { - "description": "Secondary CIDR block for GKE pods", - "type": "string", - "default": "10.1.0.0/16" - }, - "services_cidr": { - "description": "Secondary CIDR block for GKE services", - "type": "string", - "default": "10.2.0.0/20" - }, - "master_cidr": { - "description": "CIDR block for the GKE master nodes (must be /28)", - "type": "string", - "default": "172.16.0.0/28" - }, - "kubernetes_version": { - "description": "Minimum Kubernetes version for the GKE cluster (leave empty for latest)", - "type": "string", - "default": "" - }, - "telemetrykeeper_machine_type": { - "description": "GCE machine type for TelemetryKeeper node pool", - "type": "string", - "default": "n2-standard-2" - }, - "telemetrykeeper_disk_size": { - "description": "Boot disk size (GB) for TelemetryKeeper nodes", - "type": "number", - "default": 20 - }, - "telemetrystore_machine_type": { - "description": "GCE machine type for TelemetryStore node pool", - "type": "string", - "default": "n2-highmem-4" - }, - "telemetrystore_disk_size": { - "description": "Boot disk size (GB) for TelemetryStore nodes", - "type": "number", - "default": 100 - }, - "metastore_machine_type": { - "description": "GCE machine type for MetaStore node pool", - "type": "string", - "default": "n2-standard-2" - }, - "metastore_disk_size": { - "description": "Boot disk size (GB) for MetaStore nodes", - "type": "number", - "default": 20 - }, - "ingester_machine_type": { - "description": "GCE machine type for Ingester node pool", - "type": "string", - "default": "n2-standard-4" - }, - "ingester_disk_size": { - "description": "Boot disk size (GB) for Ingester nodes", - "type": "number", - "default": 50 - }, - "signoz_machine_type": { - "description": "GCE machine type for SigNoz node pool", - "type": "string", - "default": "n2-standard-4" - }, - "signoz_disk_size": { - "description": "Boot disk size (GB) for SigNoz nodes", - "type": "number", - "default": 50 - } - } -} diff --git a/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl b/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl deleted file mode 100644 index cdcb200b..00000000 --- a/internal/infrastructure/terraform/templates/providers.tf.json.gotmpl +++ /dev/null @@ -1,32 +0,0 @@ -{ - "terraform": { - "required_version": ">= 1.0.0", - "required_providers": { - {{- if eq .Provider.String "aws" }} - "aws": { - "source": "hashicorp/aws", - "version": "~> 5.0" - } - {{- else if eq .Provider.String "gcp" }} - "google": { - "source": "hashicorp/google", - "version": "~> 5.0" - } - {{- else if eq .Provider.String "azure" }} - "azurerm": { - "source": "hashicorp/azurerm", - "version": "~> 3.0" - } - {{- end }} - } - }, - "provider": { - {{- if eq .Provider.String "aws" }} - "aws": [{}] - {{- else if eq .Provider.String "gcp" }} - "google": [{}] - {{- else if eq .Provider.String "azure" }} - "azurerm": [{"features": [{}]}] - {{- end }} - } -} From ce84e5a04e7e403395834d9249cbc03759c55b07 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 29 Jul 2026 18:55:25 +0530 Subject: [PATCH 21/38] feat(infrastructure): resolve the requirement document in the resource molding Persistence moves onto the node groups: each group declares whether its nodes persist data, and the standalone storage block goes away. The molding now merges contributions (enricher deltas, operator overrides) at the document level, so casting-specific keys survive, and validates the shared shape of the resolved document before storing it. A contribution owns the node groups it states. --- .../infrastructure/resource_config.go | 14 +---- .../awskubernetesterraformcasting/casting.go | 5 +- .../resourcemolding/resource.go | 54 ++++++++++++----- .../resourcemolding/resource_test.go | 60 ++++++++++++++----- 4 files changed, 90 insertions(+), 43 deletions(-) diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index e217b519..ddde45fa 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -5,29 +5,21 @@ package infrastructure // resource kind must provide. It speaks criteria only; platform vocabulary // never enters it (machines are resolved by the platform). type ResourceConfig struct { - // Storage the resource requires from the substrate. - Storage ResourceConfigStorage `json:"storage" description:"Storage the resource requires from the substrate"` - // Node groups the resource requires from the substrate. NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name" description:"Node groups the resource requires from the substrate"` _ struct{} `additionalProperties:"false"` } -// ResourceConfigStorage describes the storage requirement. -type ResourceConfigStorage struct { - // Whether the resource persists data. - Persistent *bool `json:"persistent,omitempty" description:"Whether the resource persists data"` - - _ struct{} `additionalProperties:"false"` -} - // ResourceConfigNodeGroup sizes a pool of nodes as criteria, never as // machine types. type ResourceConfigNodeGroup struct { // Name of the node group. Name string `json:"name" description:"Name of the node group"` + // Whether this group's nodes persist data. + Persistent *bool `json:"persistent,omitempty" description:"Whether this group's nodes persist data"` + // Count of nodes. Count *int `json:"count,omitempty" description:"Count of nodes"` diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index 46ac5232..31efb8e3 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -93,7 +93,6 @@ func newData(config infrastructure.Casting) (*Data, error) { data := &Data{ Name: config.Metadata.Name, ResourceKind: config.Spec.Resource.Kind.String(), - Persistent: resourceConfig.Storage.Persistent != nil && *resourceConfig.Storage.Persistent, } for _, group := range resourceConfig.NodeGroups { @@ -101,6 +100,10 @@ func newData(config infrastructure.Casting) (*Data, error) { return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "node group %q in resource config is incomplete", group.Name) } + if group.Persistent != nil && *group.Persistent { + data.Persistent = true + } + data.NodeGroups = append(data.NodeGroups, DataNodeGroup{ Name: group.Name, Count: *group.Count, diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index 4d7f09cc..80fddd3a 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -48,42 +48,66 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) status.Addresses.APIServer = append([]string{apiServerAddress}, status.Addresses.APIServer...) baseline = &infrastructure.ResourceConfig{ - Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(true)}, NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "default", Count: v1alpha1.IntPtr(2), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + // Three persistent nodes cover the default installation's + // stateful set: one keeper, the metadata node, one store node. + {Name: "persistent", Persistent: v1alpha1.BoolPtr(true), Count: v1alpha1.IntPtr(3), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, }, } case infrastructure.ResourceKindCollectionAgent: status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) baseline = &infrastructure.ResourceConfig{ - Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(false)}, NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "default", Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, }, } default: return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported resource kind %q", config.Spec.Resource.Kind) } - if overrides := status.Config.Data[ResourceConfigName]; overrides != "" { - overrideConfig := &infrastructure.ResourceConfig{} - if err := domain.UnmarshalYAML([]byte(overrides), overrideConfig); err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config overrides") - } - if err := v1alpha1.Merge(baseline, overrideConfig); err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to merge resource config overrides") + baselineDoc, err := domain.MarshalYAML(baseline) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal resource config") + } + + doc := string(baselineDoc) + + // Contributions (enricher deltas, operator overrides) merge at the + // document level so casting-specific keys survive; a contribution owns + // the node groups it states. + if contribution := status.Config.Data[ResourceConfigName]; contribution != "" { + doc, err = domain.StrategicMergeYAML(doc, contribution, nil) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to merge resource config contribution") } } - doc, err := domain.MarshalYAML(baseline) - if err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal resource config") + if err := validate(doc); err != nil { + return err } if status.Config.Data == nil { status.Config.Data = make(map[string]string) } - status.Config.Data[ResourceConfigName] = string(doc) + status.Config.Data[ResourceConfigName] = doc + + return nil +} + +// validate checks the shared shape of the resolved document; casting-specific +// keys pass through unchecked. +func validate(doc string) error { + config := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(doc), config); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to unmarshal resolved resource config") + } + + for _, group := range config.NodeGroups { + if group.Count == nil || group.VCPUs == nil || group.Memory == nil || group.Disk == nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q in resource config is incomplete", group.Name) + } + } return nil } diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index 839011a3..8478bc76 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -19,24 +19,23 @@ func TestMoldV1Alpha1(t *testing.T) { expected infrastructure.ResourceConfig }{ { - name: "InstallationResource_PersistentStorageAndDefaultNodeGroup", + name: "InstallationResource_PersistentAndEphemeralNodeGroups", kind: infrastructure.ResourceKindInstallation, pass: true, expected: infrastructure.ResourceConfig{ - Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(true)}, NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "default", Count: v1alpha1.IntPtr(2), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + {Name: "persistent", Persistent: v1alpha1.BoolPtr(true), Count: v1alpha1.IntPtr(3), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, + {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, }, }, }, { - name: "CollectionAgentResource_EphemeralStorageAndDefaultNodeGroup", + name: "CollectionAgentResource_EphemeralNodeGroup", kind: infrastructure.ResourceKindCollectionAgent, pass: true, expected: infrastructure.ResourceConfig{ - Storage: infrastructure.ResourceConfigStorage{Persistent: v1alpha1.BoolPtr(false)}, NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "default", Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, }, }, }, @@ -81,30 +80,59 @@ func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation config.Spec.Resource.Status.Addresses.OTLP = []string{"tcp://0.0.0.0:9411"} config.Spec.Resource.Status.Config.Data = map[string]string{ - ResourceConfigName: "nodeGroups:\n- name: default\n count: 4\n- name: keeper\n count: 3\n vcpus: 2\n memory: 8\n disk: 100\n", + ResourceConfigName: `nodeGroups: +- name: persistent + persistent: true + count: 4 + vcpus: 2 + memory: 8 + disk: 50 + nodes: [{ordinal: 0}, {ordinal: 1}, {ordinal: 2}, {ordinal: 3}] +- name: ephemeral + persistent: false + count: 2 + vcpus: 2 + memory: 4 + disk: 20 +`, } err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) assert.NoError(t, err) assert.Equal(t, []string{"tcp://0.0.0.0:4317", "tcp://0.0.0.0:4318", "tcp://0.0.0.0:9411"}, config.Spec.Resource.Status.Addresses.OTLP) + doc := config.Spec.Resource.Status.Config.Data[ResourceConfigName] + + // Casting-specific keys survive the merge untouched. + assert.Contains(t, doc, "ordinal") + got := infrastructure.ResourceConfig{} - assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[ResourceConfigName]), &got)) + assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) - assert.Equal(t, v1alpha1.BoolPtr(true), got.Storage.Persistent) + // The contribution owns the node groups it states: the list replaces the + // baseline wholesale. assert.Len(t, got.NodeGroups, 2) for _, group := range got.NodeGroups { switch group.Name { - case "default": + case "persistent": + assert.Equal(t, v1alpha1.BoolPtr(true), group.Persistent) assert.Equal(t, v1alpha1.IntPtr(4), group.Count) - assert.Equal(t, v1alpha1.IntPtr(2), group.VCPUs) - assert.Equal(t, v1alpha1.IntPtr(8), group.Memory) - assert.Equal(t, v1alpha1.IntPtr(50), group.Disk) - case "keeper": - assert.Equal(t, v1alpha1.IntPtr(3), group.Count) - assert.Equal(t, v1alpha1.IntPtr(100), group.Disk) + case "ephemeral": + assert.Equal(t, v1alpha1.BoolPtr(false), group.Persistent) + assert.Equal(t, v1alpha1.IntPtr(2), group.Count) default: t.Fatalf("unexpected node group %q", group.Name) } } } + +func TestMoldV1Alpha1_IncompleteContributionFails(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation + config.Spec.Resource.Status.Config.Data = map[string]string{ + ResourceConfigName: "nodeGroups:\n- name: keeper\n persistent: true\n count: 3\n", + } + + err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + assert.Error(t, err) +} From 6c5729281d483faf54a14e1363275d2a0cafc410 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 29 Jul 2026 18:57:24 +0530 Subject: [PATCH 22/38] refactor(infrastructure): adopt the pourer casting contract Castings deposit their outputs into a pourer instead of returning materials, matching the CollectionAgent contract: the planner builds the pourer for the kind's output directory, the casting adds content under relative paths, and the planner pours the staged entries into materials. Enrichers follow the established shape: a stateless named type with a constructor. --- .../kubernetes/terraform/casting.yaml.lock | 13 +++-- .../pours/infrastructure/main.tf.json | 57 +++++++++++++++---- .../pours/infrastructure/outputs.tf.json | 20 +++++-- .../pours/infrastructure/variables.tf.json | 29 +++++++--- .../awskubernetesterraformcasting/casting.go | 24 ++++---- .../awskubernetesterraformcasting/enricher.go | 11 ++-- .../casting/infrastructure/casting/casting.go | 10 +--- internal/casting/infrastructure/planner.go | 10 +++- 8 files changed, 121 insertions(+), 53 deletions(-) diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock index 54eb162a..aa150461 100644 --- a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock +++ b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock @@ -20,10 +20,15 @@ spec: data: resource.yaml: | nodeGroups: - - count: 2 + - count: 3 disk: 50 memory: 8 - name: default - vcpus: 2 - storage: + name: persistent persistent: true + vcpus: 2 + - count: 1 + disk: 20 + memory: 4 + name: ephemeral + persistent: false + vcpus: 2 diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json index 67076ed7..9a93f362 100644 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json @@ -1,7 +1,8 @@ { "locals": { "name": "signoz", - "node_default_instance_type": "${var.node_default_instance_type != \"\" ? var.node_default_instance_type : sort(data.aws_ec2_instance_types.default.instance_types)[0]}" + "node_persistent_instance_type": "${var.node_persistent_instance_type != \"\" ? var.node_persistent_instance_type : sort(data.aws_ec2_instance_types.persistent.instance_types)[0]}", + "node_ephemeral_instance_type": "${var.node_ephemeral_instance_type != \"\" ? var.node_ephemeral_instance_type : sort(data.aws_ec2_instance_types.ephemeral.instance_types)[0]}" }, "data": { "aws_availability_zones": { @@ -10,7 +11,7 @@ } }, "aws_ec2_instance_types": { - "default": { + "persistent": { "filter": [ {"name": "instance-type", "values": ["m*", "c*"]}, {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, @@ -19,6 +20,16 @@ {"name": "vcpu-info.default-vcpus", "values": ["2"]}, {"name": "memory-info.size-in-mib", "values": ["8192"]} ] + }, + "ephemeral": { + "filter": [ + {"name": "instance-type", "values": ["m*", "c*"]}, + {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, + {"name": "current-generation", "values": ["true"]}, + {"name": "burstable-performance-supported", "values": ["false"]}, + {"name": "vcpu-info.default-vcpus", "values": ["2"]}, + {"name": "memory-info.size-in-mib", "values": ["4096"]} + ] } } }, @@ -212,23 +223,47 @@ } }, "aws_eks_node_group": { - "default": { + "persistent": { + "cluster_name": "${aws_eks_cluster.main.name}", + "node_group_name": "${local.name}-persistent", + "node_role_arn": "${aws_iam_role.eks_node_group.arn}", + "subnet_ids": "${aws_subnet.private[*].id}", + "instance_types": ["${local.node_persistent_instance_type}"], + "scaling_config": [{ + "desired_size": "${var.node_persistent_count}", + "min_size": "${var.node_persistent_count}", + "max_size": "${var.node_persistent_count}" + }], + "disk_size": "${var.node_persistent_disk_size}", + "tags": { + "app.kubernetes.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + "foundry.signoz.io/resource-kind": "Installation", + "Name": "${local.name}-persistent" + }, + "depends_on": [ + "aws_iam_role_policy_attachment.eks_worker_node_policy", + "aws_iam_role_policy_attachment.eks_cni_policy", + "aws_iam_role_policy_attachment.eks_container_registry" + ] + }, + "ephemeral": { "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-default", + "node_group_name": "${local.name}-ephemeral", "node_role_arn": "${aws_iam_role.eks_node_group.arn}", "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${local.node_default_instance_type}"], + "instance_types": ["${local.node_ephemeral_instance_type}"], "scaling_config": [{ - "desired_size": "${var.node_default_count}", - "min_size": "${var.node_default_count}", - "max_size": "${var.node_default_count}" + "desired_size": "${var.node_ephemeral_count}", + "min_size": "${var.node_ephemeral_count}", + "max_size": "${var.node_ephemeral_count}" }], - "disk_size": "${var.node_default_disk_size}", + "disk_size": "${var.node_ephemeral_disk_size}", "tags": { "app.kubernetes.io/managed-by": "foundry", "foundry.signoz.io/name": "signoz", "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-default" + "Name": "${local.name}-ephemeral" }, "depends_on": [ "aws_iam_role_policy_attachment.eks_worker_node_policy", @@ -241,7 +276,7 @@ "ebs_csi_driver": { "cluster_name": "${aws_eks_cluster.main.name}", "addon_name": "aws-ebs-csi-driver", - "depends_on": ["aws_eks_node_group.default"] + "depends_on": ["aws_eks_node_group.persistent", "aws_eks_node_group.ephemeral"] } } } diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json index de5a8cea..d278004f 100644 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json @@ -29,13 +29,21 @@ "description": "IDs of the public subnets", "value": "${aws_subnet.public[*].id}" }, - "node_group_default_arn": { - "description": "ARN of the default node group", - "value": "${aws_eks_node_group.default.arn}" + "node_group_persistent_arn": { + "description": "ARN of the persistent node group", + "value": "${aws_eks_node_group.persistent.arn}" }, - "node_group_default_status": { - "description": "Status of the default node group", - "value": "${aws_eks_node_group.default.status}" + "node_group_persistent_status": { + "description": "Status of the persistent node group", + "value": "${aws_eks_node_group.persistent.status}" + }, + "node_group_ephemeral_arn": { + "description": "ARN of the ephemeral node group", + "value": "${aws_eks_node_group.ephemeral.arn}" + }, + "node_group_ephemeral_status": { + "description": "Status of the ephemeral node group", + "value": "${aws_eks_node_group.ephemeral.status}" } } } diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json index e2da7960..dceecf61 100644 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json +++ b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json @@ -25,20 +25,35 @@ "type": "string", "default": "1.33" }, - "node_default_instance_type": { - "description": "EC2 instance type for the default node group; empty resolves the declared criteria against the platform's instance catalog", + "node_persistent_instance_type": { + "description": "EC2 instance type for the persistent node group; empty resolves the declared criteria against the platform's instance catalog", "type": "string", "default": "" }, - "node_default_count": { - "description": "Number of nodes in the default node group; elastic bounds belong with a cluster autoscaler", + "node_persistent_count": { + "description": "Number of nodes in the persistent node group; elastic bounds belong with a cluster autoscaler", "type": "number", - "default": 2 + "default": 3 }, - "node_default_disk_size": { - "description": "Root volume size (GB) for the default nodes", + "node_persistent_disk_size": { + "description": "Root volume size (GB) for the persistent nodes", "type": "number", "default": 50 + }, + "node_ephemeral_instance_type": { + "description": "EC2 instance type for the ephemeral node group; empty resolves the declared criteria against the platform's instance catalog", + "type": "string", + "default": "" + }, + "node_ephemeral_count": { + "description": "Number of nodes in the ephemeral node group; elastic bounds belong with a cluster autoscaler", + "type": "number", + "default": 1 + }, + "node_ephemeral_disk_size": { + "description": "Root volume size (GB) for the ephemeral nodes", + "type": "number", + "default": 20 } } } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index 31efb8e3..bf7b1181 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -1,6 +1,7 @@ package awskubernetesterraformcasting import ( + "bytes" "context" "log/slog" "path/filepath" @@ -11,6 +12,7 @@ import ( foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/signoz/foundry/internal/pourer" ) // Data carries the resolved values the templates render. @@ -41,13 +43,13 @@ func New(logger *slog.Logger) *awsKubernetesTerraformCasting { } func (c *awsKubernetesTerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { - return &enricher{logger: c.logger}, nil + return newAwsKubernetesTerraformMoldingEnricher(), nil } -func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) { +func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, p *pourer.Pourer) error { data, err := newData(config) if err != nil { - return nil, err + return err } items := []struct { @@ -60,20 +62,20 @@ func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infras {outputsTFTemplate, "outputs.tf.json"}, } - materials := make([]domain.Material, 0, len(items)) for _, item := range items { - material, err := item.template.Render(data, filepath.Join(infrastructurecasting.InfrastructureDir, item.path)) - if err != nil { - return nil, err + buf := bytes.NewBuffer(nil) + if err := item.template.Execute(buf, data); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute %s template", item.path) } - materials = append(materials, material) + + p.AddJSON(buf.Bytes(), item.path) } - return materials, nil + return nil } -func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error { - c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(poursPath, infrastructurecasting.InfrastructureDir))) +func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer) error { + c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(outputPath, p.Dir()))) return nil } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go b/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go index 2587707a..ea363d6b 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go @@ -2,19 +2,20 @@ package awskubernetesterraformcasting import ( "context" - "log/slog" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) -var _ infrastructuremolding.MoldingEnricher = (*enricher)(nil) +var _ infrastructuremolding.MoldingEnricher = (*awsKubernetesTerraformMoldingEnricher)(nil) -type enricher struct { - logger *slog.Logger +type awsKubernetesTerraformMoldingEnricher struct{} + +func newAwsKubernetesTerraformMoldingEnricher() *awsKubernetesTerraformMoldingEnricher { + return &awsKubernetesTerraformMoldingEnricher{} } -func (e *enricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { +func (e *awsKubernetesTerraformMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { return nil } diff --git a/internal/casting/infrastructure/casting/casting.go b/internal/casting/infrastructure/casting/casting.go index 7299a05a..805ed92b 100644 --- a/internal/casting/infrastructure/casting/casting.go +++ b/internal/casting/infrastructure/casting/casting.go @@ -4,16 +4,12 @@ import ( "context" "github.com/signoz/foundry/api/v1alpha1/infrastructure" - "github.com/signoz/foundry/internal/domain" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/pourer" ) -// InfrastructureDir is the subdirectory within the pours directory where -// infrastructure materials are written. -const InfrastructureDir = "infrastructure" - type Casting interface { Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) - Forge(ctx context.Context, config infrastructure.Casting, poursPath string) ([]domain.Material, error) - Cast(ctx context.Context, config infrastructure.Casting, poursPath string) error + Forge(ctx context.Context, config infrastructure.Casting, p *pourer.Pourer) error + Cast(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer) error } diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 37f6b912..f0f8afa0 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -3,6 +3,7 @@ package infrastructure import ( "context" "log/slog" + "strings" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" @@ -12,6 +13,7 @@ import ( infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" "github.com/signoz/foundry/internal/planner" + "github.com/signoz/foundry/internal/pourer" "github.com/signoz/foundry/internal/tooler" ) @@ -90,11 +92,15 @@ func (p *Planner) MergeStatusIntoSpec() error { } func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, error) { - return p.casting.Forge(ctx, *p.config, target) + pr := pourer.New(strings.ToLower(p.config.Kind().String())) + if err := p.casting.Forge(ctx, *p.config, pr); err != nil { + return nil, err + } + return pr.Pour() } func (p *Planner) Cast(ctx context.Context, poursPath string) error { - return p.casting.Cast(ctx, *p.config, poursPath) + return p.casting.Cast(ctx, *p.config, poursPath, pourer.New(strings.ToLower(p.config.Kind().String()))) } func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } From 1bad275a55d14e8ad9beb96694eefa174ce4ea2a Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 4 Aug 2026 18:18:53 +0530 Subject: [PATCH 23/38] feat(domain): merge lists of maps by a key field ListTypeSet and ListTypeOrdered both degrade to atomic on a list of maps, so nothing could merge a list of objects: an override stating one element replaced the whole list and silently dropped the rest. ListTypeMap(key) mirrors Kubernetes' listType: map with a listMapKey. Elements match on the key, a matched pair merges as a document so the override states only what it changes, and override-only elements append. It degrades to atomic when an element is not a map or lacks the key. --- internal/domain/merge.go | 99 +++++++++++++++++++++++++++++++++++ internal/domain/merge_test.go | 32 +++++++++++ 2 files changed, 131 insertions(+) diff --git a/internal/domain/merge.go b/internal/domain/merge.go index f4b33afc..7872f684 100644 --- a/internal/domain/merge.go +++ b/internal/domain/merge.go @@ -32,6 +32,18 @@ var ( ListTypeOrdered = ListType{name: "ordered", merge: mergeOrdered} ) +// ListTypeMap merges a list of maps by a key field, mirroring Kubernetes' +// listType: map with a listMapKey: elements are matched on the key, a matched +// pair merges as a document so the override states only what it changes, and +// override-only elements append. Degrades to Atomic if either list holds an +// element that is not a map or is missing the key. +func ListTypeMap(key string) ListType { + return ListType{ + name: "map:" + key, + merge: func(base, override []any) []any { return mergeByKey(key, base, override) }, + } +} + // ListTypes declares the list types of a document's paths: dotted keys with // "*" matching any single segment, e.g. "service.pipelines.*.receivers". // Undeclared paths are ListTypeAtomic. @@ -192,6 +204,93 @@ func mergeOrdered(base, override []any) []any { return unionScalars(out, nil) } +// mergeByKey matches elements on key, merges each matched pair as a document, +// and appends the override's new elements. +func mergeByKey(key string, base, override []any) []any { + overrides := make(map[any]map[string]any, len(override)) + order := make([]any, 0, len(override)) + for _, elem := range override { + keyed, ok := keyedMap(elem, key) + if !ok { + return override + } + + overrides[keyed[key]] = keyed + order = append(order, keyed[key]) + } + + out := make([]any, 0, len(base)+len(override)) + merged := make(map[any]struct{}, len(override)) + for _, elem := range base { + keyed, ok := keyedMap(elem, key) + if !ok { + return override + } + + patch, matched := overrides[keyed[key]] + if !matched { + out = append(out, elem) + continue + } + + document, err := mergeDocument(keyed, patch) + if err != nil { + return override + } + + merged[keyed[key]] = struct{}{} + out = append(out, document) + } + + for _, id := range order { + if _, done := merged[id]; !done { + out = append(out, overrides[id]) + } + } + + return out +} + +// keyedMap returns the element as a map carrying the key. +func keyedMap(elem any, key string) (map[string]any, bool) { + document, ok := elem.(map[string]any) + if !ok { + return nil, false + } + + if _, ok := document[key]; !ok { + return nil, false + } + + return document, true +} + +// mergeDocument applies override onto base with RFC 7386 semantics, the same +// rule StrategicMergeYAML lands at the document level. +func mergeDocument(base, override map[string]any) (map[string]any, error) { + baseJSON, err := json.Marshal(base) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal list element") + } + + overrideJSON, err := json.Marshal(override) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal list element override") + } + + mergedJSON, err := jsonpatchv5.MergePatch(baseJSON, overrideJSON) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge list element") + } + + out := map[string]any{} + if err := json.Unmarshal(mergedJSON, &out); err != nil { + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal merged list element") + } + + return out, nil +} + // scalarList reports whether every element is a comparable scalar. func scalarList(list []any) bool { for _, v := range list { diff --git a/internal/domain/merge_test.go b/internal/domain/merge_test.go index 7a353463..1c46d199 100644 --- a/internal/domain/merge_test.go +++ b/internal/domain/merge_test.go @@ -89,6 +89,38 @@ func TestStrategicMergeYAML(t *testing.T) { override: "a: [b\n", pass: false, }, + { + name: "MapList_MergesMatchedElementByKey", + base: "groups:\n- name: a\n size: 1\n cpu: 2\n- name: b\n size: 9\n", + override: "groups:\n- name: a\n size: 4\n", + listTypes: ListTypes{"groups": ListTypeMap("name")}, + pass: true, + expected: "groups:\n- cpu: 2\n name: a\n size: 4\n- name: b\n size: 9\n", + }, + { + name: "MapList_AppendsUnmatchedElement", + base: "groups:\n- name: a\n size: 1\n", + override: "groups:\n- name: b\n size: 2\n", + listTypes: ListTypes{"groups": ListTypeMap("name")}, + pass: true, + expected: "groups:\n- name: a\n size: 1\n- name: b\n size: 2\n", + }, + { + name: "MapList_MergesNestedObjectWithinElement", + base: "groups:\n- name: a\n volume:\n size: 20\n type: gp3\n", + override: "groups:\n- name: a\n volume:\n size: 50\n", + listTypes: ListTypes{"groups": ListTypeMap("name")}, + pass: true, + expected: "groups:\n- name: a\n volume:\n size: 50\n type: gp3\n", + }, + { + name: "MapList_MissingKeyDegradesToAtomic", + base: "groups:\n- name: a\n size: 1\n", + override: "groups:\n- size: 2\n", + listTypes: ListTypes{"groups": ListTypeMap("name")}, + pass: true, + expected: "groups:\n- size: 2\n", + }, } for _, tt := range tests { From 49089497fb5458487c284d2ab6d57c5c97d41e3c Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 4 Aug 2026 18:19:11 +0530 Subject: [PATCH 24/38] refactor(infrastructure): shape node groups in the node-pool vocabulary The requirement document invented its own words for a machine pool, and one of them meant two different things: `disk` was the data volume on a persistent group and the root volume on an ephemeral one, with different minimums. There was also no way to name a machine type in the casting, only in a terraform variable. Adopt the vocabulary every node-pool abstraction already uses. kOps is the closest precedent -- the one machine-pool shape that spans AWS, GCE, Azure and DigitalOcean -- and where it had to choose it picked machineType, minSize and maxSize, which overlap with GKE's and eksctl's terms too. Narrowed to what foundry has to understand: nodeGroups: - name: persistent storage: persistent minSize: 3 maxSize: 3 machineType: "" # empty resolves from cpu/memory cpu: 2 memory: 8 rootVolume: {size: 30} dataVolume: {size: 50} cpu and memory stay as the portable criteria so the document does not name provider machine types, with machineType as the escape hatch. Volumes are nested objects rather than flat scalars so volume type or encryption can arrive without new top-level keys, and splitting them lets the AMI snapshot floor apply to every root disk unconditionally. dataVolume is singular, not a list as kOps has it: the claim controller assumes one volume per node, and a list would advertise what foundry cannot honour. StorageClass is a value object rather than a string with an enum tag, so an unknown class fails at unmarshal instead of deep in the molding, and each class carries what it implies -- whether it needs a data volume, whether the group is pinned. Adding a class is one var entry rather than another branch wherever groups are checked. Node groups now merge by name, so a contribution or an operator override states only the group and the fields it changes. Before this, stating one group deleted the others. `spec.resource.spec.config.data` merges last, so an operator's own document beats the casting's contribution. --- .../infrastructure/casting.schema.json | 75 ++++++++++++ api/v1alpha1/infrastructure/resource.go | 3 + .../infrastructure/resource_config.go | 51 ++++++--- api/v1alpha1/infrastructure/storage_class.go | 108 ++++++++++++++++++ .../infrastructure/storage_class_test.go | 67 +++++++++++ .../awskubernetesterraformcasting/casting.go | 47 +++++--- .../embed_test.go | 2 +- .../templates/main.tf.json.gotmpl | 8 +- .../templates/variables.tf.json.gotmpl | 15 ++- .../resourcemolding/resource.go | 88 ++++++++++++-- .../resourcemolding/resource_test.go | 60 +++++++--- 11 files changed, 456 insertions(+), 68 deletions(-) create mode 100644 api/v1alpha1/infrastructure/storage_class.go create mode 100644 api/v1alpha1/infrastructure/storage_class_test.go diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 165e119b..157c9efa 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -20,6 +20,9 @@ "Installation" ] }, + "spec": { + "$ref": "#/definitions/V1Alpha1MoldingSpec" + }, "status": { "$ref": "#/definitions/InfrastructureResourceStatus", "description": "Status of the resource" @@ -142,6 +145,50 @@ ], "type": "string" }, + "V1Alpha1MoldingSpec": { + "additionalProperties": false, + "properties": { + "cluster": { + "$ref": "#/definitions/V1Alpha1TypeCluster", + "description": "Cluster configuration for the molding" + }, + "config": { + "$ref": "#/definitions/V1Alpha1TypeConfig", + "description": "Configuration for the molding" + }, + "enabled": { + "description": "Whether the molding is enabled", + "default": true, + "type": [ + "null", + "boolean" + ] + }, + "env": { + "description": "Environment variables for the molding", + "additionalProperties": { + "type": "string" + }, + "type": "object" + }, + "image": { + "description": "Container image of the molding", + "examples": [ + "signoz/signoz:latest" + ], + "pattern": "^[a-z0-9]+([._-][a-z0-9]+)*(/[a-z0-9]+([._-][a-z0-9]+)*)*(:[a-zA-Z0-9._-]+)?(@sha256:[a-f0-9]{64})?$", + "type": "string" + }, + "version": { + "description": "The version of the molding to use", + "examples": [ + "latest" + ], + "type": "string" + } + }, + "type": "object" + }, "V1Alpha1PatchEntry": { "required": [ "target", @@ -249,6 +296,34 @@ }, "type": "object" }, + "V1Alpha1TypeCluster": { + "additionalProperties": false, + "properties": { + "replicas": { + "description": "Number of replicas for the molding.", + "examples": [ + 1 + ], + "minimum": 0, + "type": [ + "null", + "integer" + ] + }, + "shards": { + "description": "Number of shards for the molding", + "examples": [ + 1 + ], + "minimum": 1, + "type": [ + "null", + "integer" + ] + } + }, + "type": "object" + }, "V1Alpha1TypeConfig": { "additionalProperties": false, "properties": { diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index 14258837..d6dcf981 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -9,6 +9,9 @@ type Resource struct { // Kind of the resource this infrastructure serves. Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` + // Specification for the resource. + Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" jsonschema:"description=Specification for the resource"` + // Status of the resource. Status ResourceStatus `json:"status" yaml:"status,omitempty" description:"Status of the resource"` diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index ddde45fa..8c0ee119 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -2,8 +2,7 @@ package infrastructure // ResourceConfig is the resource requirement document (resource.yaml): the // canonical internal representation of what a substrate shaped for the -// resource kind must provide. It speaks criteria only; platform vocabulary -// never enters it (machines are resolved by the platform). +// resource kind must provide. type ResourceConfig struct { // Node groups the resource requires from the substrate. NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name" description:"Node groups the resource requires from the substrate"` @@ -11,26 +10,50 @@ type ResourceConfig struct { _ struct{} `additionalProperties:"false"` } -// ResourceConfigNodeGroup sizes a pool of nodes as criteria, never as -// machine types. +// ResourceConfigNodeGroup sizes a pool of nodes. The vocabulary is the one +// every node-pool abstraction already uses -- machineType, minSize, maxSize +// and per-volume sizes are kOps', GKE's and eksctl's terms -- narrowed to what +// foundry has to understand. Capacity may be stated as criteria (cpu, memory) +// so the document stays portable across providers, with machineType as the +// escape hatch when a concrete type is wanted. type ResourceConfigNodeGroup struct { // Name of the node group. Name string `json:"name" description:"Name of the node group"` - // Whether this group's nodes persist data. - Persistent *bool `json:"persistent,omitempty" description:"Whether this group's nodes persist data"` + // Storage class of the group's nodes. + Storage StorageClass `json:"storage,omitzero" description:"Durability of the group's storage" examples:"[\"persistent\"]"` - // Count of nodes. - Count *int `json:"count,omitempty" description:"Count of nodes"` + // MinSize is the smallest the group may be. A pinned group states the + // same value for both bounds. + MinSize *int `json:"minSize,omitempty" minimum:"0" description:"Minimum number of nodes in the group"` - // VCPUs per node. - VCPUs *int `json:"vcpus,omitempty" description:"VCPUs per node"` + // MaxSize is the largest the group may grow to. + MaxSize *int `json:"maxSize,omitempty" minimum:"0" description:"Maximum number of nodes in the group"` - // Memory per node in GiB. - Memory *int `json:"memory,omitempty" description:"Memory per node in GiB"` + // MachineType names the provider's machine type outright; empty resolves + // one from cpu and memory against the provider's catalog. + MachineType string `json:"machineType,omitempty" description:"Provider machine type; empty resolves one from cpu and memory" example:"m5.large"` - // Disk per node in GiB. - Disk *int `json:"disk,omitempty" description:"Disk per node in GiB"` + // CPU per node. + CPU *int `json:"cpu,omitempty" minimum:"1" description:"CPUs per node, used when machineType is not stated"` + + // Memory per node in GB. + Memory *int `json:"memory,omitempty" minimum:"1" description:"Memory per node in GB, used when machineType is not stated"` + + // RootVolume is the disk each node boots from. + RootVolume ResourceConfigVolume `json:"rootVolume,omitzero" description:"The disk each node boots from"` + + // DataVolume outlives the node it is attached to. Absent on a group whose + // nodes keep nothing. + DataVolume *ResourceConfigVolume `json:"dataVolume,omitempty" description:"Volume attached to each node that outlives it; persistent storage class only"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigVolume sizes one volume. +type ResourceConfigVolume struct { + // Size of the volume in GB. + Size *int `json:"size,omitempty" minimum:"1" description:"Size of the volume in GB"` _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/infrastructure/storage_class.go b/api/v1alpha1/infrastructure/storage_class.go new file mode 100644 index 00000000..ddd7f165 --- /dev/null +++ b/api/v1alpha1/infrastructure/storage_class.go @@ -0,0 +1,108 @@ +package infrastructure + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*StorageClass)(nil) +var _ yaml.Unmarshaler = (*StorageClass)(nil) +var _ json.Marshaler = (*StorageClass)(nil) +var _ json.Unmarshaler = (*StorageClass)(nil) +var _ fmt.Stringer = (*StorageClass)(nil) +var _ jsonschema.Enum = (*StorageClass)(nil) + +var ( + // StorageClassPersistent nodes each carry a volume that outlives them. + // Stateful identities claim those volumes, which is what pins the group: + // a node cannot be swapped for another without moving a claim. + StorageClassPersistent = StorageClass{s: "persistent", data: true, pinned: true} + + // StorageClassEphemeral nodes are interchangeable and keep nothing. + StorageClassEphemeral = StorageClass{s: "ephemeral"} +) + +// StorageClass is the durability of a node group's storage. Each class carries +// what it implies, so a new class is one var entry rather than another branch +// wherever groups are checked. +type StorageClass struct { + s string + data bool + pinned bool +} + +func (class StorageClass) String() string { + return class.s +} + +// RequiresDataVolume reports whether nodes of this class must declare a volume +// that outlives them, and conversely that other classes must not. +func (class StorageClass) RequiresDataVolume() bool { + return class.data +} + +// IsPinned reports whether the group's size is fixed. Every node in a pinned +// group owns a claimed volume, so there is nothing to scale between bounds. +func (class StorageClass) IsPinned() bool { + return class.pinned +} + +func StorageClasses() []StorageClass { + return []StorageClass{StorageClassPersistent, StorageClassEphemeral} +} + +func (class StorageClass) MarshalJSON() ([]byte, error) { + return json.Marshal(class.String()) +} + +func (class *StorageClass) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return class.UnmarshalText([]byte(str)) +} + +func (class *StorageClass) UnmarshalText(text []byte) error { + for _, availableClass := range StorageClasses() { + if availableClass.String() == string(text) { + *class = availableClass + return nil + } + } + + // A nil slice is an absent value, which leaves the zero class; an empty + // string is a stated value that names no class, and falls through. + if text == nil { + *class = StorageClass{} + return nil + } + + return errors.New("invalid storage class: " + string(text)) +} + +func (class StorageClass) MarshalText() ([]byte, error) { + return []byte(class.String()), nil +} + +func (class *StorageClass) UnmarshalYAML(node *yaml.Node) error { + return class.UnmarshalText([]byte(node.Value)) +} + +func (class StorageClass) MarshalYAML() (any, error) { + return class.String(), nil +} + +func (class StorageClass) Enum() []any { + classes := []any{} + for _, class := range StorageClasses() { + classes = append(classes, class.String()) + } + + return classes +} diff --git a/api/v1alpha1/infrastructure/storage_class_test.go b/api/v1alpha1/infrastructure/storage_class_test.go new file mode 100644 index 00000000..5b251bff --- /dev/null +++ b/api/v1alpha1/infrastructure/storage_class_test.go @@ -0,0 +1,67 @@ +package infrastructure + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestStorageClassUnmarshalText(t *testing.T) { + tests := []struct { + name string + input string + pass bool + expected StorageClass + }{ + {name: "Persistent_Valid", input: "persistent", pass: true, expected: StorageClassPersistent}, + {name: "Ephemeral_Valid", input: "ephemeral", pass: true, expected: StorageClassEphemeral}, + // An absent key never reaches the unmarshaler; an explicitly + // empty one is a stated value that names no class. + {name: "Empty_Invalid", input: "", pass: false}, + {name: "Unknown_Invalid", input: "durable", pass: false}, + {name: "Capitalised_Invalid", input: "Persistent", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + class := StorageClass{} + err := class.UnmarshalText([]byte(tt.input)) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expected, class) + + // Round-trip: what the class renders unmarshals back to itself. + roundTripped := StorageClass{} + assert.NoError(t, roundTripped.UnmarshalText([]byte(class.String()))) + assert.Equal(t, class, roundTripped) + }) + } +} + +func TestStorageClassImplications(t *testing.T) { + tests := []struct { + name string + class StorageClass + expectedRequiresDataVolume bool + expectedPinned bool + }{ + {name: "Persistent_CarriesDataAndIsPinned", class: StorageClassPersistent, expectedRequiresDataVolume: true, expectedPinned: true}, + {name: "Ephemeral_CarriesNothingAndScales", class: StorageClassEphemeral, expectedRequiresDataVolume: false, expectedPinned: false}, + {name: "Unset_CarriesNothingAndScales", class: StorageClass{}, expectedRequiresDataVolume: false, expectedPinned: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedRequiresDataVolume, tt.class.RequiresDataVolume()) + assert.Equal(t, tt.expectedPinned, tt.class.IsPinned()) + }) + } +} + +func TestStorageClassEnum(t *testing.T) { + assert.Equal(t, []any{"persistent", "ephemeral"}, StorageClassPersistent.Enum()) +} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go index bf7b1181..a9b525a6 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go @@ -25,11 +25,15 @@ type Data struct { // DataNodeGroup carries one node group's criteria for the templates. type DataNodeGroup struct { - Name string - Count int - VCPUs int - Memory int - Disk int + Name string + Storage string + MinSize int + MaxSize int + MachineType string + CPU int + Memory int + RootVolumeSize int + DataVolumeSize int } var _ infrastructurecasting.Casting = (*awsKubernetesTerraformCasting)(nil) @@ -97,22 +101,35 @@ func newData(config infrastructure.Casting) (*Data, error) { ResourceKind: config.Spec.Resource.Kind.String(), } + // The resource molding validates the resolved document, and moldings run + // before any casting forges, so every field dereferenced here is present. + // A missing one is a foundry bug, not bad input: recoverRunE turns the + // panic into TypeFatal with the stack, which points at the line rather + // than reporting a group as vaguely "incomplete". for _, group := range resourceConfig.NodeGroups { - if group.Count == nil || group.VCPUs == nil || group.Memory == nil || group.Disk == nil { - return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "node group %q in resource config is incomplete", group.Name) + node := DataNodeGroup{ + Name: group.Name, + Storage: group.Storage.String(), + MinSize: *group.MinSize, + MaxSize: *group.MaxSize, + MachineType: group.MachineType, + RootVolumeSize: *group.RootVolume.Size, } - if group.Persistent != nil && *group.Persistent { + if group.CPU != nil { + node.CPU = *group.CPU + } + + if group.Memory != nil { + node.Memory = *group.Memory + } + + if group.Storage.RequiresDataVolume() { data.Persistent = true + node.DataVolumeSize = *group.DataVolume.Size } - data.NodeGroups = append(data.NodeGroups, DataNodeGroup{ - Name: group.Name, - Count: *group.Count, - VCPUs: *group.VCPUs, - Memory: *group.Memory, - Disk: *group.Disk, - }) + data.NodeGroups = append(data.NodeGroups, node) } return data, nil diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go index c6981c5e..5ed7e85e 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go @@ -13,7 +13,7 @@ func TestTemplates_RenderValidJSON(t *testing.T) { ResourceKind: "Installation", Persistent: true, NodeGroups: []DataNodeGroup{ - {Name: "default", Count: 2, VCPUs: 2, Memory: 8, Disk: 50}, + {Name: "default", Storage: "persistent", MinSize: 2, MaxSize: 2, CPU: 2, Memory: 8, RootVolumeSize: 30, DataVolumeSize: 50}, }, } diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl index e794054a..dda4a76f 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl @@ -19,7 +19,7 @@ {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, {"name": "current-generation", "values": ["true"]}, {"name": "burstable-performance-supported", "values": ["false"]}, - {"name": "vcpu-info.default-vcpus", "values": ["{{ $group.VCPUs }}"]}, + {"name": "vcpu-info.default-vcpus", "values": ["{{ $group.CPU }}"]}, {"name": "memory-info.size-in-mib", "values": ["{{ mul $group.Memory 1024 }}"]} ] } @@ -224,9 +224,9 @@ "subnet_ids": "${aws_subnet.private[*].id}", "instance_types": ["${local.node_{{ $group.Name }}_instance_type}"], "scaling_config": [{ - "desired_size": "${var.node_{{ $group.Name }}_count}", - "min_size": "${var.node_{{ $group.Name }}_count}", - "max_size": "${var.node_{{ $group.Name }}_count}" + "desired_size": "${var.node_{{ $group.Name }}_min_size}", + "min_size": "${var.node_{{ $group.Name }}_min_size}", + "max_size": "${var.node_{{ $group.Name }}_max_size}" }], "disk_size": "${var.node_{{ $group.Name }}_disk_size}", "tags": { diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl index 84e36a37..6b8040de 100644 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl +++ b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl @@ -29,17 +29,22 @@ "node_{{ $group.Name }}_instance_type": { "description": "EC2 instance type for the {{ $group.Name }} node group; empty resolves the declared criteria against the platform's instance catalog", "type": "string", - "default": "" + "default": "{{ $group.MachineType }}" }, - "node_{{ $group.Name }}_count": { - "description": "Number of nodes in the {{ $group.Name }} node group; elastic bounds belong with a cluster autoscaler", + "node_{{ $group.Name }}_min_size": { + "description": "Smallest the {{ $group.Name }} node group may be", "type": "number", - "default": {{ $group.Count }} + "default": {{ $group.MinSize }} + }, + "node_{{ $group.Name }}_max_size": { + "description": "Largest the {{ $group.Name }} node group may grow to", + "type": "number", + "default": {{ $group.MaxSize }} }, "node_{{ $group.Name }}_disk_size": { "description": "Root volume size (GB) for the {{ $group.Name }} nodes", "type": "number", - "default": {{ $group.Disk }} + "default": {{ $group.RootVolumeSize }} } {{- end }} } diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index 80fddd3a..73ecb319 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -22,6 +22,11 @@ var ( // substrate shaped for the resource kind must satisfy beyond its edge. const ResourceConfigName = "resource.yaml" +// Node groups merge by name so a contribution or an operator override states +// only the group and the fields it changes, instead of restating every group +// to avoid deleting the ones it left out. +var resourceConfigListTypes = domain.ListTypes{"nodeGroups": domain.ListTypeMap("name")} + var _ infrastructuremolding.Molding = (*resourceMolding)(nil) type resourceMolding struct { @@ -51,15 +56,43 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras NodeGroups: []infrastructure.ResourceConfigNodeGroup{ // Three persistent nodes cover the default installation's // stateful set: one keeper, the metadata node, one store node. - {Name: "persistent", Persistent: v1alpha1.BoolPtr(true), Count: v1alpha1.IntPtr(3), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, - {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + // A group holding stateful identities is pinned, so its bounds + // are equal -- there is nothing to autoscale when every node + // owns a claimed volume. + { + Name: infrastructure.StorageClassPersistent.String(), + Storage: infrastructure.StorageClassPersistent, + MinSize: v1alpha1.IntPtr(3), + MaxSize: v1alpha1.IntPtr(3), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(8), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, + }, + { + Name: infrastructure.StorageClassEphemeral.String(), + Storage: infrastructure.StorageClassEphemeral, + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(4), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + }, }, } case infrastructure.ResourceKindCollectionAgent: status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) baseline = &infrastructure.ResourceConfig{ NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + { + Name: infrastructure.StorageClassEphemeral.String(), + Storage: infrastructure.StorageClassEphemeral, + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(4), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + }, }, } default: @@ -73,13 +106,20 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras doc := string(baselineDoc) - // Contributions (enricher deltas, operator overrides) merge at the - // document level so casting-specific keys survive; a contribution owns - // the node groups it states. - if contribution := status.Config.Data[ResourceConfigName]; contribution != "" { - doc, err = domain.StrategicMergeYAML(doc, contribution, nil) + // Contributions (enricher deltas) merge first so casting-specific keys + // survive, then the operator's own spec, which wins: spec beats status + // wherever they disagree. + for _, override := range []string{ + status.Config.Data[ResourceConfigName], + config.Spec.Resource.Spec.Config.Data[ResourceConfigName], + } { + if override == "" { + continue + } + + doc, err = domain.StrategicMergeYAML(doc, override, resourceConfigListTypes) if err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to merge resource config contribution") + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to merge resource config override") } } @@ -104,9 +144,37 @@ func validate(doc string) error { } for _, group := range config.NodeGroups { - if group.Count == nil || group.VCPUs == nil || group.Memory == nil || group.Disk == nil { + if group.MinSize == nil || group.MaxSize == nil || group.RootVolume.Size == nil { return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q in resource config is incomplete", group.Name) } + + if *group.MaxSize < *group.MinSize { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q has maxSize below minSize", group.Name) + } + + // A machine is named outright or resolved from criteria; one of the + // two has to be stated or there is nothing to launch. + if group.MachineType == "" && (group.CPU == nil || group.Memory == nil) { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states neither machineType nor cpu and memory", group.Name) + } + + // An unknown class cannot reach here: it fails at unmarshal. What is + // left is a group that named none at all. + if group.Storage.String() == "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states no storage class", group.Name) + } + + if group.Storage.RequiresDataVolume() { + if group.DataVolume == nil || group.DataVolume.Size == nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so it must state a dataVolume size", group.Name, group.Storage) + } + } else if group.DataVolume != nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so it cannot state a dataVolume", group.Name, group.Storage) + } + + if group.Storage.IsPinned() && *group.MinSize != *group.MaxSize { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so minSize and maxSize must be equal", group.Name, group.Storage) + } } return nil diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index 8478bc76..aa697e35 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -24,8 +24,25 @@ func TestMoldV1Alpha1(t *testing.T) { pass: true, expected: infrastructure.ResourceConfig{ NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "persistent", Persistent: v1alpha1.BoolPtr(true), Count: v1alpha1.IntPtr(3), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(8), Disk: v1alpha1.IntPtr(50)}, - {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + { + Name: "persistent", + Storage: infrastructure.StorageClassPersistent, + MinSize: v1alpha1.IntPtr(3), + MaxSize: v1alpha1.IntPtr(3), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(8), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, + }, + { + Name: "ephemeral", + Storage: infrastructure.StorageClassEphemeral, + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(4), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + }, }, }, }, @@ -35,7 +52,15 @@ func TestMoldV1Alpha1(t *testing.T) { pass: true, expected: infrastructure.ResourceConfig{ NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - {Name: "ephemeral", Persistent: v1alpha1.BoolPtr(false), Count: v1alpha1.IntPtr(1), VCPUs: v1alpha1.IntPtr(2), Memory: v1alpha1.IntPtr(4), Disk: v1alpha1.IntPtr(20)}, + { + Name: "ephemeral", + Storage: infrastructure.StorageClassEphemeral, + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + CPU: v1alpha1.IntPtr(2), + Memory: v1alpha1.IntPtr(4), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + }, }, }, }, @@ -82,18 +107,12 @@ func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { config.Spec.Resource.Status.Config.Data = map[string]string{ ResourceConfigName: `nodeGroups: - name: persistent - persistent: true - count: 4 - vcpus: 2 - memory: 8 - disk: 50 + minSize: 4 + maxSize: 4 nodes: [{ordinal: 0}, {ordinal: 1}, {ordinal: 2}, {ordinal: 3}] - name: ephemeral - persistent: false - count: 2 - vcpus: 2 - memory: 4 - disk: 20 + minSize: 2 + maxSize: 2 `, } @@ -109,17 +128,20 @@ func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { got := infrastructure.ResourceConfig{} assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) - // The contribution owns the node groups it states: the list replaces the - // baseline wholesale. + // Node groups merge by name: the contribution states only the sizes it + // changes and the baseline's other fields survive. assert.Len(t, got.NodeGroups, 2) for _, group := range got.NodeGroups { switch group.Name { case "persistent": - assert.Equal(t, v1alpha1.BoolPtr(true), group.Persistent) - assert.Equal(t, v1alpha1.IntPtr(4), group.Count) + assert.Equal(t, v1alpha1.IntPtr(4), group.MinSize) + assert.Equal(t, infrastructure.StorageClassPersistent, group.Storage) + assert.Equal(t, v1alpha1.IntPtr(8), group.Memory) + assert.Equal(t, v1alpha1.IntPtr(50), group.DataVolume.Size) case "ephemeral": - assert.Equal(t, v1alpha1.BoolPtr(false), group.Persistent) - assert.Equal(t, v1alpha1.IntPtr(2), group.Count) + assert.Equal(t, v1alpha1.IntPtr(2), group.MinSize) + assert.Equal(t, infrastructure.StorageClassEphemeral, group.Storage) + assert.Equal(t, v1alpha1.IntPtr(4), group.Memory) default: t.Fatalf("unexpected node group %q", group.Name) } From 74718a27cf248a0bd5ef53f553da6fe14f048781 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Tue, 4 Aug 2026 18:43:41 +0530 Subject: [PATCH 25/38] chore(infrastructure): drop the aws kubernetes terraform casting The scaffolding provides the Kind, the casting contract, the resource molding and the registry; a concrete casting is a separate change with its own review. Carrying one here made the scaffolding PR argue for an EKS substrate at the same time as the machinery that hosts it. The registry is left empty. The first casting to register lands with the platform it serves. --- .../aws/kubernetes/terraform/README.md | 130 -------- .../aws/kubernetes/terraform/casting.yaml | 11 - .../kubernetes/terraform/casting.yaml.lock | 34 --- .../pours/infrastructure/main.tf.json | 283 ------------------ .../pours/infrastructure/outputs.tf.json | 49 --- .../pours/infrastructure/providers.tf.json | 14 - .../pours/infrastructure/variables.tf.json | 59 ---- .../awskubernetesterraformcasting/casting.go | 136 --------- .../awskubernetesterraformcasting/embed.go | 17 -- .../embed_test.go | 37 --- .../awskubernetesterraformcasting/enricher.go | 21 -- .../templates/main.tf.json.gotmpl | 254 ---------------- .../templates/outputs.tf.json.gotmpl | 43 --- .../templates/providers.tf.json.gotmpl | 14 - .../templates/variables.tf.json.gotmpl | 51 ---- internal/casting/infrastructure/registry.go | 13 +- 16 files changed, 1 insertion(+), 1165 deletions(-) delete mode 100644 docs/examples/aws/kubernetes/terraform/README.md delete mode 100644 docs/examples/aws/kubernetes/terraform/casting.yaml delete mode 100644 docs/examples/aws/kubernetes/terraform/casting.yaml.lock delete mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json delete mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json delete mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json delete mode 100644 docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/casting.go delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/embed.go delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl delete mode 100644 internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl diff --git a/docs/examples/aws/kubernetes/terraform/README.md b/docs/examples/aws/kubernetes/terraform/README.md deleted file mode 100644 index 074b42f9..00000000 --- a/docs/examples/aws/kubernetes/terraform/README.md +++ /dev/null @@ -1,130 +0,0 @@ -# AWS Kubernetes with Terraform (Infrastructure) - -| Field | Value | -| --- | --- | -| **Kind** | `Infrastructure` | -| **Platform** | `aws` | -| **Mode** | `kubernetes` | -| **Flavor** | `terraform` | - -## Overview - -Provisions an EKS substrate shaped for a SigNoz Installation. The infrastructure never reads the installation's casting: the resource declaration names what the substrate is shaped for, and foundry's own kind-level knowledge (the requirement set) drives what gets provisioned. - -Resources: -- VPC with public and private subnets across two availability zones, internet and NAT gateways -- EKS cluster with IAM roles for the control plane and nodes -- One managed node group sized from the requirement set, in private subnets -- EBS CSI driver addon (the installation's components request storage through PVCs) - -## Prerequisites - -- AWS credentials with permissions to create VPC, EKS, and IAM resources -- [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.0 - -## Configuration - -```yaml -apiVersion: v1alpha1 -kind: Infrastructure -metadata: - name: signoz -spec: - deployment: - platform: aws - mode: kubernetes - flavor: terraform - resource: - kind: Installation -``` - -`spec.resource` declares the kind of resource the substrate serves. It is a declaration, not a reference: the installation owns the binding and nothing is read from its casting. - -## Deploy - -```bash -# 1. Generate Terraform files -foundryctl forge -f casting.yaml - -# 2. Initialize and apply Terraform -cd pours/infrastructure -terraform init -terraform apply -``` - -## Generated output - -```text -pours/infrastructure/ - providers.tf.json - main.tf.json - variables.tf.json - outputs.tf.json -``` - -## Customization - -To pin an exact instance type instead of resolving the criteria, set the instance type variable through `spec.patches`: - -```yaml -apiVersion: v1alpha1 -kind: Infrastructure -metadata: - name: signoz -spec: - deployment: - platform: aws - mode: kubernetes - flavor: terraform - resource: - kind: Installation - spec: - name: signoz - patches: - - target: "infrastructure/variables.tf.json" - operations: - - op: replace - path: /variable/node_default_instance_type/default - value: t3.large -``` - -Any generated value can be changed the same way; run `foundryctl forge` and inspect the files under `pours/infrastructure/` to identify the JSON paths. - -## Platform details - -### Variables - -| Variable | Default | Description | -| --- | --- | --- | -| `aws_region` | `us-east-1` | AWS region | -| `vpc_cidr` | `10.0.0.0/16` | CIDR block for the VPC | -| `az_count` | `2` | Number of availability zones | -| `name` | `signoz` | Name of the deployment | -| `kubernetes_version` | `1.33` | Kubernetes version for the EKS cluster | -| `node_default_instance_type` | `""` | Instance type; empty resolves the declared criteria against the platform's instance catalog | -| `node_default_count` | `2` | Number of nodes; elastic bounds belong with a cluster autoscaler | -| `node_default_disk_size` | `50` | Root volume size (GB) for the nodes | - -### Outputs - -| Output | Description | -| --- | --- | -| `cluster_name` | Name of the EKS cluster | -| `cluster_endpoint` | EKS API server endpoint | -| `cluster_ca_certificate` | Base64-encoded CA data (sensitive) | -| `cluster_version` | Kubernetes version of the cluster | -| `vpc_id` | ID of the VPC | -| `private_subnet_ids` | IDs of the private subnets | -| `public_subnet_ids` | IDs of the public subnets | -| `node_group_default_arn` | ARN of the default node group | -| `node_group_default_status` | Status of the default node group | - -### Tags - -Every resource carries the discovery tags, so consumers can find the substrate by name: - -| Tag | Value | -| --- | --- | -| `app.kubernetes.io/managed-by` | `foundry` | -| `foundry.signoz.io/name` | `signoz` | -| `foundry.signoz.io/resource-kind` | `Installation` | diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml b/docs/examples/aws/kubernetes/terraform/casting.yaml deleted file mode 100644 index e82d6d38..00000000 --- a/docs/examples/aws/kubernetes/terraform/casting.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: v1alpha1 -kind: Infrastructure -metadata: - name: signoz -spec: - deployment: - platform: aws - mode: kubernetes - flavor: terraform - resource: - kind: Installation diff --git a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock b/docs/examples/aws/kubernetes/terraform/casting.yaml.lock deleted file mode 100644 index aa150461..00000000 --- a/docs/examples/aws/kubernetes/terraform/casting.yaml.lock +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1alpha1 -kind: Infrastructure -metadata: - name: signoz -spec: - deployment: - flavor: terraform - mode: kubernetes - platform: aws - resource: - kind: Installation - status: - addresses: - apiserver: - - tcp://0.0.0.0:8080 - otlp: - - tcp://0.0.0.0:4317 - - tcp://0.0.0.0:4318 - config: - data: - resource.yaml: | - nodeGroups: - - count: 3 - disk: 50 - memory: 8 - name: persistent - persistent: true - vcpus: 2 - - count: 1 - disk: 20 - memory: 4 - name: ephemeral - persistent: false - vcpus: 2 diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json deleted file mode 100644 index 9a93f362..00000000 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/main.tf.json +++ /dev/null @@ -1,283 +0,0 @@ -{ - "locals": { - "name": "signoz", - "node_persistent_instance_type": "${var.node_persistent_instance_type != \"\" ? var.node_persistent_instance_type : sort(data.aws_ec2_instance_types.persistent.instance_types)[0]}", - "node_ephemeral_instance_type": "${var.node_ephemeral_instance_type != \"\" ? var.node_ephemeral_instance_type : sort(data.aws_ec2_instance_types.ephemeral.instance_types)[0]}" - }, - "data": { - "aws_availability_zones": { - "available": { - "state": "available" - } - }, - "aws_ec2_instance_types": { - "persistent": { - "filter": [ - {"name": "instance-type", "values": ["m*", "c*"]}, - {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, - {"name": "current-generation", "values": ["true"]}, - {"name": "burstable-performance-supported", "values": ["false"]}, - {"name": "vcpu-info.default-vcpus", "values": ["2"]}, - {"name": "memory-info.size-in-mib", "values": ["8192"]} - ] - }, - "ephemeral": { - "filter": [ - {"name": "instance-type", "values": ["m*", "c*"]}, - {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, - {"name": "current-generation", "values": ["true"]}, - {"name": "burstable-performance-supported", "values": ["false"]}, - {"name": "vcpu-info.default-vcpus", "values": ["2"]}, - {"name": "memory-info.size-in-mib", "values": ["4096"]} - ] - } - } - }, - "resource": { - "aws_vpc": { - "main": { - "cidr_block": "${var.vpc_cidr}", - "enable_dns_hostnames": true, - "enable_dns_support": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-vpc", - "kubernetes.io/cluster/${local.name}": "shared" - } - } - }, - "aws_subnet": { - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-private-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/internal-elb": "1" - } - }, - "public": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "map_public_ip_on_launch": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-public-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/elb": "1" - } - } - }, - "aws_internet_gateway": { - "main": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-igw" - } - } - }, - "aws_eip": { - "nat": { - "count": "${var.az_count}", - "domain": "vpc", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-nat-eip-${count.index}" - } - } - }, - "aws_nat_gateway": { - "main": { - "count": "${var.az_count}", - "allocation_id": "${aws_eip.nat[count.index].id}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-nat-${count.index}" - }, - "depends_on": ["aws_internet_gateway.main"] - } - }, - "aws_route_table": { - "public": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-public-rt" - } - }, - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-private-rt-${count.index}" - } - } - }, - "aws_route": { - "public_internet": { - "route_table_id": "${aws_route_table.public.id}", - "destination_cidr_block": "0.0.0.0/0", - "gateway_id": "${aws_internet_gateway.main.id}" - }, - "private_nat": { - "count": "${var.az_count}", - "route_table_id": "${aws_route_table.private[count.index].id}", - "destination_cidr_block": "0.0.0.0/0", - "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" - } - }, - "aws_route_table_association": { - "public": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "route_table_id": "${aws_route_table.public.id}" - }, - "private": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.private[count.index].id}", - "route_table_id": "${aws_route_table.private[count.index].id}" - } - }, - "aws_iam_role": { - "eks_cluster": { - "name": "${local.name}-eks-cluster-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation" - } - }, - "eks_node_group": { - "name": "${local.name}-eks-node-group-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation" - } - } - }, - "aws_iam_role_policy_attachment": { - "eks_cluster_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", - "role": "${aws_iam_role.eks_cluster.name}" - }, - "eks_worker_node_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_cni_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_container_registry": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "ebs_csi_driver": { - "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", - "role": "${aws_iam_role.eks_node_group.name}" - } - }, - "aws_eks_cluster": { - "main": { - "name": "${local.name}", - "role_arn": "${aws_iam_role.eks_cluster.arn}", - "version": "${var.kubernetes_version}", - "vpc_config": [{ - "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", - "endpoint_private_access": true, - "endpoint_public_access": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation" - }, - "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] - } - }, - "aws_eks_node_group": { - "persistent": { - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-persistent", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${local.node_persistent_instance_type}"], - "scaling_config": [{ - "desired_size": "${var.node_persistent_count}", - "min_size": "${var.node_persistent_count}", - "max_size": "${var.node_persistent_count}" - }], - "disk_size": "${var.node_persistent_disk_size}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-persistent" - }, - "depends_on": [ - "aws_iam_role_policy_attachment.eks_worker_node_policy", - "aws_iam_role_policy_attachment.eks_cni_policy", - "aws_iam_role_policy_attachment.eks_container_registry" - ] - }, - "ephemeral": { - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-ephemeral", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${local.node_ephemeral_instance_type}"], - "scaling_config": [{ - "desired_size": "${var.node_ephemeral_count}", - "min_size": "${var.node_ephemeral_count}", - "max_size": "${var.node_ephemeral_count}" - }], - "disk_size": "${var.node_ephemeral_disk_size}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "signoz", - "foundry.signoz.io/resource-kind": "Installation", - "Name": "${local.name}-ephemeral" - }, - "depends_on": [ - "aws_iam_role_policy_attachment.eks_worker_node_policy", - "aws_iam_role_policy_attachment.eks_cni_policy", - "aws_iam_role_policy_attachment.eks_container_registry" - ] - } - }, - "aws_eks_addon": { - "ebs_csi_driver": { - "cluster_name": "${aws_eks_cluster.main.name}", - "addon_name": "aws-ebs-csi-driver", - "depends_on": ["aws_eks_node_group.persistent", "aws_eks_node_group.ephemeral"] - } - } - } -} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json deleted file mode 100644 index d278004f..00000000 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/outputs.tf.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "output": { - "cluster_name": { - "description": "Name of the EKS cluster", - "value": "${aws_eks_cluster.main.name}" - }, - "cluster_endpoint": { - "description": "Endpoint for the EKS cluster API server", - "value": "${aws_eks_cluster.main.endpoint}" - }, - "cluster_ca_certificate": { - "description": "Base64-encoded certificate authority data for the EKS cluster", - "value": "${aws_eks_cluster.main.certificate_authority[0].data}", - "sensitive": true - }, - "cluster_version": { - "description": "Kubernetes version of the EKS cluster", - "value": "${aws_eks_cluster.main.version}" - }, - "vpc_id": { - "description": "ID of the VPC", - "value": "${aws_vpc.main.id}" - }, - "private_subnet_ids": { - "description": "IDs of the private subnets", - "value": "${aws_subnet.private[*].id}" - }, - "public_subnet_ids": { - "description": "IDs of the public subnets", - "value": "${aws_subnet.public[*].id}" - }, - "node_group_persistent_arn": { - "description": "ARN of the persistent node group", - "value": "${aws_eks_node_group.persistent.arn}" - }, - "node_group_persistent_status": { - "description": "Status of the persistent node group", - "value": "${aws_eks_node_group.persistent.status}" - }, - "node_group_ephemeral_arn": { - "description": "ARN of the ephemeral node group", - "value": "${aws_eks_node_group.ephemeral.arn}" - }, - "node_group_ephemeral_status": { - "description": "Status of the ephemeral node group", - "value": "${aws_eks_node_group.ephemeral.status}" - } - } -} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json deleted file mode 100644 index b367d253..00000000 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/providers.tf.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "terraform": { - "required_version": ">= 1.0.0", - "required_providers": { - "aws": { - "source": "hashicorp/aws", - "version": "~> 5.0" - } - } - }, - "provider": { - "aws": [{}] - } -} diff --git a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json b/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json deleted file mode 100644 index dceecf61..00000000 --- a/docs/examples/aws/kubernetes/terraform/pours/infrastructure/variables.tf.json +++ /dev/null @@ -1,59 +0,0 @@ -{ - "variable": { - "aws_region": { - "description": "AWS region to deploy resources", - "type": "string", - "default": "us-east-1" - }, - "vpc_cidr": { - "description": "CIDR block for the VPC", - "type": "string", - "default": "10.0.0.0/16" - }, - "az_count": { - "description": "Number of availability zones to use", - "type": "number", - "default": 2 - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "signoz" - }, - "kubernetes_version": { - "description": "Kubernetes version for the EKS cluster", - "type": "string", - "default": "1.33" - }, - "node_persistent_instance_type": { - "description": "EC2 instance type for the persistent node group; empty resolves the declared criteria against the platform's instance catalog", - "type": "string", - "default": "" - }, - "node_persistent_count": { - "description": "Number of nodes in the persistent node group; elastic bounds belong with a cluster autoscaler", - "type": "number", - "default": 3 - }, - "node_persistent_disk_size": { - "description": "Root volume size (GB) for the persistent nodes", - "type": "number", - "default": 50 - }, - "node_ephemeral_instance_type": { - "description": "EC2 instance type for the ephemeral node group; empty resolves the declared criteria against the platform's instance catalog", - "type": "string", - "default": "" - }, - "node_ephemeral_count": { - "description": "Number of nodes in the ephemeral node group; elastic bounds belong with a cluster autoscaler", - "type": "number", - "default": 1 - }, - "node_ephemeral_disk_size": { - "description": "Root volume size (GB) for the ephemeral nodes", - "type": "number", - "default": 20 - } - } -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go b/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go deleted file mode 100644 index a9b525a6..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/casting.go +++ /dev/null @@ -1,136 +0,0 @@ -package awskubernetesterraformcasting - -import ( - "bytes" - "context" - "log/slog" - "path/filepath" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" - "github.com/signoz/foundry/internal/domain" - foundryerrors "github.com/signoz/foundry/internal/errors" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" - "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" - "github.com/signoz/foundry/internal/pourer" -) - -// Data carries the resolved values the templates render. -type Data struct { - Name string - ResourceKind string - Persistent bool - NodeGroups []DataNodeGroup -} - -// DataNodeGroup carries one node group's criteria for the templates. -type DataNodeGroup struct { - Name string - Storage string - MinSize int - MaxSize int - MachineType string - CPU int - Memory int - RootVolumeSize int - DataVolumeSize int -} - -var _ infrastructurecasting.Casting = (*awsKubernetesTerraformCasting)(nil) - -type awsKubernetesTerraformCasting struct { - logger *slog.Logger -} - -func New(logger *slog.Logger) *awsKubernetesTerraformCasting { - return &awsKubernetesTerraformCasting{logger: logger} -} - -func (c *awsKubernetesTerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { - return newAwsKubernetesTerraformMoldingEnricher(), nil -} - -func (c *awsKubernetesTerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, p *pourer.Pourer) error { - data, err := newData(config) - if err != nil { - return err - } - - items := []struct { - template *domain.Template - path string - }{ - {providersTFTemplate, "providers.tf.json"}, - {mainTFTemplate, "main.tf.json"}, - {variablesTFTemplate, "variables.tf.json"}, - {outputsTFTemplate, "outputs.tf.json"}, - } - - for _, item := range items { - buf := bytes.NewBuffer(nil) - if err := item.template.Execute(buf, data); err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute %s template", item.path) - } - - p.AddJSON(buf.Bytes(), item.path) - } - - return nil -} - -func (c *awsKubernetesTerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer) error { - c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(outputPath, p.Dir()))) - return nil -} - -// newData resolves the resource requirement document into the values the -// templates render. -func newData(config infrastructure.Casting) (*Data, error) { - doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] - if doc == "" { - return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "resource config %q is missing from the resource status", resourcemolding.ResourceConfigName) - } - - resourceConfig := &infrastructure.ResourceConfig{} - if err := domain.UnmarshalYAML([]byte(doc), resourceConfig); err != nil { - return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config") - } - - data := &Data{ - Name: config.Metadata.Name, - ResourceKind: config.Spec.Resource.Kind.String(), - } - - // The resource molding validates the resolved document, and moldings run - // before any casting forges, so every field dereferenced here is present. - // A missing one is a foundry bug, not bad input: recoverRunE turns the - // panic into TypeFatal with the stack, which points at the line rather - // than reporting a group as vaguely "incomplete". - for _, group := range resourceConfig.NodeGroups { - node := DataNodeGroup{ - Name: group.Name, - Storage: group.Storage.String(), - MinSize: *group.MinSize, - MaxSize: *group.MaxSize, - MachineType: group.MachineType, - RootVolumeSize: *group.RootVolume.Size, - } - - if group.CPU != nil { - node.CPU = *group.CPU - } - - if group.Memory != nil { - node.Memory = *group.Memory - } - - if group.Storage.RequiresDataVolume() { - data.Persistent = true - node.DataVolumeSize = *group.DataVolume.Size - } - - data.NodeGroups = append(data.NodeGroups, node) - } - - return data, nil -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go deleted file mode 100644 index f43cf6e4..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/embed.go +++ /dev/null @@ -1,17 +0,0 @@ -package awskubernetesterraformcasting - -import ( - "embed" - - "github.com/signoz/foundry/internal/domain" -) - -//go:embed templates/*.gotmpl -var templates embed.FS - -var ( - providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) - mainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) - variablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) - outputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/outputs.tf.json.gotmpl", domain.FormatJSON) -) diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go b/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go deleted file mode 100644 index 5ed7e85e..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/embed_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package awskubernetesterraformcasting - -import ( - "testing" - - "github.com/signoz/foundry/internal/domain" - "github.com/stretchr/testify/assert" -) - -func TestTemplates_RenderValidJSON(t *testing.T) { - data := &Data{ - Name: "signoz", - ResourceKind: "Installation", - Persistent: true, - NodeGroups: []DataNodeGroup{ - {Name: "default", Storage: "persistent", MinSize: 2, MaxSize: 2, CPU: 2, Memory: 8, RootVolumeSize: 30, DataVolumeSize: 50}, - }, - } - - testCases := []struct { - name string - template *domain.Template - }{ - {name: "ProvidersTemplate_RendersValidJSON", template: providersTFTemplate}, - {name: "MainTemplate_RendersValidJSON", template: mainTFTemplate}, - {name: "VariablesTemplate_RendersValidJSON", template: variablesTFTemplate}, - {name: "OutputsTemplate_RendersValidJSON", template: outputsTFTemplate}, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - material, err := tc.template.Render(data, "out.tf.json") - assert.NoError(t, err) - assert.NotEmpty(t, material.FmtContents()) - }) - } -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go b/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go deleted file mode 100644 index ea363d6b..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/enricher.go +++ /dev/null @@ -1,21 +0,0 @@ -package awskubernetesterraformcasting - -import ( - "context" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" -) - -var _ infrastructuremolding.MoldingEnricher = (*awsKubernetesTerraformMoldingEnricher)(nil) - -type awsKubernetesTerraformMoldingEnricher struct{} - -func newAwsKubernetesTerraformMoldingEnricher() *awsKubernetesTerraformMoldingEnricher { - return &awsKubernetesTerraformMoldingEnricher{} -} - -func (e *awsKubernetesTerraformMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { - return nil -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl deleted file mode 100644 index dda4a76f..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/main.tf.json.gotmpl +++ /dev/null @@ -1,254 +0,0 @@ -{ - "locals": { - "name": "{{ .Name }}" - {{- range $group := .NodeGroups }}, - "node_{{ $group.Name }}_instance_type": "${var.node_{{ $group.Name }}_instance_type != \"\" ? var.node_{{ $group.Name }}_instance_type : sort(data.aws_ec2_instance_types.{{ $group.Name }}.instance_types)[0]}" - {{- end }} - }, - "data": { - "aws_availability_zones": { - "available": { - "state": "available" - } - }, - "aws_ec2_instance_types": { - {{- range $i, $group := .NodeGroups }}{{ if $i }},{{ end }} - "{{ $group.Name }}": { - "filter": [ - {"name": "instance-type", "values": ["m*", "c*"]}, - {"name": "processor-info.supported-architecture", "values": ["x86_64"]}, - {"name": "current-generation", "values": ["true"]}, - {"name": "burstable-performance-supported", "values": ["false"]}, - {"name": "vcpu-info.default-vcpus", "values": ["{{ $group.CPU }}"]}, - {"name": "memory-info.size-in-mib", "values": ["{{ mul $group.Memory 1024 }}"]} - ] - } - {{- end }} - } - }, - "resource": { - "aws_vpc": { - "main": { - "cidr_block": "${var.vpc_cidr}", - "enable_dns_hostnames": true, - "enable_dns_support": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-vpc", - "kubernetes.io/cluster/${local.name}": "shared" - } - } - }, - "aws_subnet": { - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-private-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/internal-elb": "1" - } - }, - "public": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "cidr_block": "${cidrsubnet(var.vpc_cidr, 8, count.index + var.az_count)}", - "availability_zone": "${data.aws_availability_zones.available.names[count.index]}", - "map_public_ip_on_launch": true, - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-public-${count.index}", - "kubernetes.io/cluster/${local.name}": "shared", - "kubernetes.io/role/elb": "1" - } - } - }, - "aws_internet_gateway": { - "main": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-igw" - } - } - }, - "aws_eip": { - "nat": { - "count": "${var.az_count}", - "domain": "vpc", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-nat-eip-${count.index}" - } - } - }, - "aws_nat_gateway": { - "main": { - "count": "${var.az_count}", - "allocation_id": "${aws_eip.nat[count.index].id}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-nat-${count.index}" - }, - "depends_on": ["aws_internet_gateway.main"] - } - }, - "aws_route_table": { - "public": { - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-public-rt" - } - }, - "private": { - "count": "${var.az_count}", - "vpc_id": "${aws_vpc.main.id}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}", - "Name": "${local.name}-private-rt-${count.index}" - } - } - }, - "aws_route": { - "public_internet": { - "route_table_id": "${aws_route_table.public.id}", - "destination_cidr_block": "0.0.0.0/0", - "gateway_id": "${aws_internet_gateway.main.id}" - }, - "private_nat": { - "count": "${var.az_count}", - "route_table_id": "${aws_route_table.private[count.index].id}", - "destination_cidr_block": "0.0.0.0/0", - "nat_gateway_id": "${aws_nat_gateway.main[count.index].id}" - } - }, - "aws_route_table_association": { - "public": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.public[count.index].id}", - "route_table_id": "${aws_route_table.public.id}" - }, - "private": { - "count": "${var.az_count}", - "subnet_id": "${aws_subnet.private[count.index].id}", - "route_table_id": "${aws_route_table.private[count.index].id}" - } - }, - "aws_iam_role": { - "eks_cluster": { - "name": "${local.name}-eks-cluster-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"eks.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" - } - }, - "eks_node_group": { - "name": "${local.name}-eks-node-group-role", - "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" - } - } - }, - "aws_iam_role_policy_attachment": { - "eks_cluster_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", - "role": "${aws_iam_role.eks_cluster.name}" - }, - "eks_worker_node_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_cni_policy": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", - "role": "${aws_iam_role.eks_node_group.name}" - }, - "eks_container_registry": { - "policy_arn": "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", - "role": "${aws_iam_role.eks_node_group.name}" - }{{ if .Persistent }}, - "ebs_csi_driver": { - "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy", - "role": "${aws_iam_role.eks_node_group.name}" - }{{ end }} - }, - "aws_eks_cluster": { - "main": { - "name": "${local.name}", - "role_arn": "${aws_iam_role.eks_cluster.arn}", - "version": "${var.kubernetes_version}", - "vpc_config": [{ - "subnet_ids": "${concat(aws_subnet.private[*].id, aws_subnet.public[*].id)}", - "endpoint_private_access": true, - "endpoint_public_access": true - }], - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ .Name }}", - "foundry.signoz.io/resource-kind": "{{ .ResourceKind }}" - }, - "depends_on": ["aws_iam_role_policy_attachment.eks_cluster_policy"] - } - }, - "aws_eks_node_group": { - {{- range $i, $group := .NodeGroups }}{{ if $i }},{{ end }} - "{{ $group.Name }}": { - "cluster_name": "${aws_eks_cluster.main.name}", - "node_group_name": "${local.name}-{{ $group.Name }}", - "node_role_arn": "${aws_iam_role.eks_node_group.arn}", - "subnet_ids": "${aws_subnet.private[*].id}", - "instance_types": ["${local.node_{{ $group.Name }}_instance_type}"], - "scaling_config": [{ - "desired_size": "${var.node_{{ $group.Name }}_min_size}", - "min_size": "${var.node_{{ $group.Name }}_min_size}", - "max_size": "${var.node_{{ $group.Name }}_max_size}" - }], - "disk_size": "${var.node_{{ $group.Name }}_disk_size}", - "tags": { - "app.kubernetes.io/managed-by": "foundry", - "foundry.signoz.io/name": "{{ $.Name }}", - "foundry.signoz.io/resource-kind": "{{ $.ResourceKind }}", - "Name": "${local.name}-{{ $group.Name }}" - }, - "depends_on": [ - "aws_iam_role_policy_attachment.eks_worker_node_policy", - "aws_iam_role_policy_attachment.eks_cni_policy", - "aws_iam_role_policy_attachment.eks_container_registry" - ] - } - {{- end }} - }{{ if .Persistent }}, - "aws_eks_addon": { - "ebs_csi_driver": { - "cluster_name": "${aws_eks_cluster.main.name}", - "addon_name": "aws-ebs-csi-driver", - "depends_on": [{{ range $i, $group := .NodeGroups }}{{ if $i }}, {{ end }}"aws_eks_node_group.{{ $group.Name }}"{{ end }}] - } - }{{ end }} - } -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl deleted file mode 100644 index 1baed1b9..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/outputs.tf.json.gotmpl +++ /dev/null @@ -1,43 +0,0 @@ -{ - "output": { - "cluster_name": { - "description": "Name of the EKS cluster", - "value": "${aws_eks_cluster.main.name}" - }, - "cluster_endpoint": { - "description": "Endpoint for the EKS cluster API server", - "value": "${aws_eks_cluster.main.endpoint}" - }, - "cluster_ca_certificate": { - "description": "Base64-encoded certificate authority data for the EKS cluster", - "value": "${aws_eks_cluster.main.certificate_authority[0].data}", - "sensitive": true - }, - "cluster_version": { - "description": "Kubernetes version of the EKS cluster", - "value": "${aws_eks_cluster.main.version}" - }, - "vpc_id": { - "description": "ID of the VPC", - "value": "${aws_vpc.main.id}" - }, - "private_subnet_ids": { - "description": "IDs of the private subnets", - "value": "${aws_subnet.private[*].id}" - }, - "public_subnet_ids": { - "description": "IDs of the public subnets", - "value": "${aws_subnet.public[*].id}" - } - {{- range $group := .NodeGroups }}, - "node_group_{{ $group.Name }}_arn": { - "description": "ARN of the {{ $group.Name }} node group", - "value": "${aws_eks_node_group.{{ $group.Name }}.arn}" - }, - "node_group_{{ $group.Name }}_status": { - "description": "Status of the {{ $group.Name }} node group", - "value": "${aws_eks_node_group.{{ $group.Name }}.status}" - } - {{- end }} - } -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl deleted file mode 100644 index b367d253..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/providers.tf.json.gotmpl +++ /dev/null @@ -1,14 +0,0 @@ -{ - "terraform": { - "required_version": ">= 1.0.0", - "required_providers": { - "aws": { - "source": "hashicorp/aws", - "version": "~> 5.0" - } - } - }, - "provider": { - "aws": [{}] - } -} diff --git a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl deleted file mode 100644 index 6b8040de..00000000 --- a/internal/casting/infrastructure/awskubernetesterraformcasting/templates/variables.tf.json.gotmpl +++ /dev/null @@ -1,51 +0,0 @@ -{ - "variable": { - "aws_region": { - "description": "AWS region to deploy resources", - "type": "string", - "default": "us-east-1" - }, - "vpc_cidr": { - "description": "CIDR block for the VPC", - "type": "string", - "default": "10.0.0.0/16" - }, - "az_count": { - "description": "Number of availability zones to use", - "type": "number", - "default": 2 - }, - "name": { - "description": "The name of the deployment", - "type": "string", - "default": "{{ .Name }}" - }, - "kubernetes_version": { - "description": "Kubernetes version for the EKS cluster", - "type": "string", - "default": "1.33" - } - {{- range $group := .NodeGroups }}, - "node_{{ $group.Name }}_instance_type": { - "description": "EC2 instance type for the {{ $group.Name }} node group; empty resolves the declared criteria against the platform's instance catalog", - "type": "string", - "default": "{{ $group.MachineType }}" - }, - "node_{{ $group.Name }}_min_size": { - "description": "Smallest the {{ $group.Name }} node group may be", - "type": "number", - "default": {{ $group.MinSize }} - }, - "node_{{ $group.Name }}_max_size": { - "description": "Largest the {{ $group.Name }} node group may grow to", - "type": "number", - "default": {{ $group.MaxSize }} - }, - "node_{{ $group.Name }}_disk_size": { - "description": "Root volume size (GB) for the {{ $group.Name }} nodes", - "type": "number", - "default": {{ $group.RootVolumeSize }} - } - {{- end }} - } -} diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 3e9971e8..48c2d2be 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,11 +4,9 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" - "github.com/signoz/foundry/internal/casting/infrastructure/awskubernetesterraformcasting" infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" - "github.com/signoz/foundry/internal/tooler/terraformtooler" ) type CastingItem struct { @@ -22,16 +20,7 @@ type Registry struct { func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ - castings: map[v1alpha1.TypeDeployment]CastingItem{ - { - Platform: v1alpha1.PlatformAWS, - Mode: v1alpha1.ModeKubernetes, - Flavor: v1alpha1.FlavorTerraform, - }: { - Casting: awskubernetesterraformcasting.New(logger), - Toolers: []tooler.Tooler{terraformtooler.New()}, - }, - }, + castings: map[v1alpha1.TypeDeployment]CastingItem{}, } } From 5feb1106b5d78857fd6c9d23b7f263705315f062 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 5 Aug 2026 16:10:47 +0530 Subject: [PATCH 26/38] feat(convention): derive substrate names and tags Foundry generates rather than reconciles, so it can never ask a platform what it created. A consuming casting finds a producing casting's resources only by deriving the same names and filtering the same tags, with nothing between them to reconcile a mismatch. Derive both from one description of a resource, so a fact stated once cannot render two ways: a subnet's visibility reaches its name and its tag from the same field, and the tags a consumer filters on are a subset of the tags the producer stamps rather than a parallel list. Each resource type declares its own ordered qualifiers, so adding a resource is one var entry. Add domain.MetadataPrefix for the foundry key namespace, shared by labels, annotations and tags. --- internal/convention/identity.go | 130 ++++++++++++++ internal/convention/identity_test.go | 188 ++++++++++++++++++++ internal/convention/ownership.go | 26 +++ internal/convention/ownership_test.go | 31 ++++ internal/convention/resource.go | 153 ++++++++++++++++ internal/convention/resource_test.go | 206 ++++++++++++++++++++++ internal/convention/resource_type.go | 45 +++++ internal/convention/resource_type_test.go | 79 +++++++++ internal/convention/role.go | 17 ++ internal/convention/role_test.go | 37 ++++ internal/convention/selection.go | 56 ++++++ internal/convention/selection_test.go | 138 +++++++++++++++ internal/convention/substrate.go | 60 +++++++ internal/convention/substrate_test.go | 40 +++++ internal/convention/tag.go | 44 +++++ internal/convention/tag_test.go | 44 +++++ internal/convention/visibility.go | 21 +++ internal/convention/visibility_test.go | 35 ++++ internal/convention/zone.go | 46 +++++ internal/convention/zone_test.go | 40 +++++ internal/domain/metadata.go | 10 ++ 21 files changed, 1446 insertions(+) create mode 100644 internal/convention/identity.go create mode 100644 internal/convention/identity_test.go create mode 100644 internal/convention/ownership.go create mode 100644 internal/convention/ownership_test.go create mode 100644 internal/convention/resource.go create mode 100644 internal/convention/resource_test.go create mode 100644 internal/convention/resource_type.go create mode 100644 internal/convention/resource_type_test.go create mode 100644 internal/convention/role.go create mode 100644 internal/convention/role_test.go create mode 100644 internal/convention/selection.go create mode 100644 internal/convention/selection_test.go create mode 100644 internal/convention/substrate.go create mode 100644 internal/convention/substrate_test.go create mode 100644 internal/convention/tag.go create mode 100644 internal/convention/tag_test.go create mode 100644 internal/convention/visibility.go create mode 100644 internal/convention/visibility_test.go create mode 100644 internal/convention/zone.go create mode 100644 internal/convention/zone_test.go create mode 100644 internal/domain/metadata.go diff --git a/internal/convention/identity.go b/internal/convention/identity.go new file mode 100644 index 00000000..8dbcc14d --- /dev/null +++ b/internal/convention/identity.go @@ -0,0 +1,130 @@ +package convention + +import ( + "slices" + "strconv" + "strings" + + "github.com/signoz/foundry/internal/errors" +) + +const identitySeparator = "," + +// Identity is a stateful seat that claims a volume and keeps its data across an +// instance replacement: a component and its ordinals, "telemetrystore-0-0". It +// carries no substrate prefix, because deployed claims are spelled this way. +type Identity struct { + s string +} + +func NewIdentity(component string, ordinals ...int) (Identity, error) { + if component == "" { + return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity: component is empty") + } + + // The separator carries the encoding, so one inside a component would split + // into two identities on the way back. + if strings.Contains(component, identitySeparator) { + return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity from %q: component contains %q", component, identitySeparator) + } + + parts := make([]string, 0, len(ordinals)+1) + parts = append(parts, component) + + for _, ordinal := range ordinals { + if ordinal < 0 { + return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity from %q: ordinal %d is negative", component, ordinal) + } + + parts = append(parts, strconv.Itoa(ordinal)) + } + + return Identity{s: strings.Join(parts, "-")}, nil +} + +func MustNewIdentity(component string, ordinals ...int) Identity { + identity, err := NewIdentity(component, ordinals...) + if err != nil { + panic(err) + } + + return identity +} + +// ParseIdentity reads back a claimed identity. Trailing numeric segments are the +// ordinals; the rest is the component, which may itself be hyphenated. +func ParseIdentity(value string) (Identity, error) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity from %q: identity is empty", value) + } + + segments := strings.Split(trimmed, "-") + + ordinals := make([]int, 0, len(segments)) + boundary := len(segments) + + for boundary > 1 { + ordinal, err := strconv.Atoi(segments[boundary-1]) + if err != nil { + break + } + + ordinals = append([]int{ordinal}, ordinals...) + boundary-- + } + + return NewIdentity(strings.Join(segments[:boundary], "-"), ordinals...) +} + +func (identity Identity) String() string { + return identity.s +} + +// Identities is the claim record one volume carries, encoded as a single tag +// value: sorted and separator-joined, matching Terraform's join and split. +// Sorting keeps the value stable so an unchanged claim set produces no diff. +// +// Only a platform with no stateful identity primitive of its own needs a claim +// record. Kubernetes binds a pod to its volume through the StatefulSet +// controller, compose and swarm by name in the generated file, systemd by host +// path. Empty is therefore the norm, and stamps no tag. +type Identities []Identity + +func (identities Identities) String() string { + parts := make([]string, 0, len(identities)) + for _, identity := range identities.sorted() { + parts = append(parts, identity.s) + } + + return strings.Join(parts, identitySeparator) +} + +// ParseIdentities is the counterpart of String, validating through ParseIdentity +// so there is one path into the type. +func ParseIdentities(value string) (Identities, error) { + if strings.TrimSpace(value) == "" { + return nil, nil + } + + parts := strings.Split(value, identitySeparator) + + identities := make(Identities, 0, len(parts)) + for _, part := range parts { + identity, err := ParseIdentity(part) + if err != nil { + return nil, errors.Wrapf(err, errors.TypeInvalidInput, "failed to create identities from %q", value) + } + + identities = append(identities, identity) + } + + return identities.sorted(), nil +} + +func (identities Identities) sorted() Identities { + out := slices.Clone(identities) + slices.SortFunc(out, func(a, b Identity) int { return strings.Compare(a.s, b.s) }) + + return out +} diff --git a/internal/convention/identity_test.go b/internal/convention/identity_test.go new file mode 100644 index 00000000..78cd2a77 --- /dev/null +++ b/internal/convention/identity_test.go @@ -0,0 +1,188 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewIdentity(t *testing.T) { + tests := []struct { + name string + component string + ordinals []int + pass bool + expectedIdentity string + }{ + {name: "ShardAndReplica_Valid", component: "telemetrystore", ordinals: []int{0, 0}, pass: true, expectedIdentity: "telemetrystore-0-0"}, + {name: "SingleOrdinal_Valid", component: "signoz", ordinals: []int{0}, pass: true, expectedIdentity: "signoz-0"}, + {name: "NoOrdinal_Valid", component: "metastore", pass: true, expectedIdentity: "metastore"}, + {name: "Empty_Invalid", component: "", ordinals: []int{0}, pass: false}, + {name: "ComponentWithSeparator_Invalid", component: "telemetry,store", ordinals: []int{0}, pass: false}, + {name: "NegativeOrdinal_Invalid", component: "signoz", ordinals: []int{-1}, pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + identity, err := NewIdentity(tt.component, tt.ordinals...) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedIdentity, identity.String()) + }) + } +} + +func TestParseIdentity(t *testing.T) { + tests := []struct { + name string + value string + pass bool + expectedIdentity Identity + }{ + {name: "ShardAndReplica_Valid", value: "telemetrystore-0-0", pass: true, expectedIdentity: MustNewIdentity("telemetrystore", 0, 0)}, + {name: "SingleOrdinal_Valid", value: "signoz-0", pass: true, expectedIdentity: MustNewIdentity("signoz", 0)}, + {name: "NoOrdinal_Valid", value: "metastore", pass: true, expectedIdentity: MustNewIdentity("metastore")}, + {name: "HyphenatedComponent_Valid", value: "store-pool-1-2", pass: true, expectedIdentity: MustNewIdentity("store-pool", 1, 2)}, + {name: "Spaced_Trimmed", value: " keeper-1 ", pass: true, expectedIdentity: MustNewIdentity("keeper", 1)}, + {name: "Empty_Invalid", value: "", pass: false}, + {name: "Blank_Invalid", value: " ", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + identity, err := ParseIdentity(tt.value) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedIdentity, identity) + }) + } +} + +// Parsing goes through NewIdentity, so a value a claim record could not have +// held is rejected on the way back in rather than becoming an identity nothing +// can match. +func TestParseIdentityDelegatesValidation(t *testing.T) { + _, direct := NewIdentity("telemetry,store", 0) + _, parsed := ParseIdentity("telemetry,store-0") + + assert.Error(t, direct) + assert.Error(t, parsed) +} + +func TestIdentitiesString(t *testing.T) { + tests := []struct { + name string + identities Identities + expectedValue string + }{ + { + name: "Empty_RendersEmpty", + identities: Identities{}, + expectedValue: "", + }, + { + name: "Single_RendersBare", + identities: Identities{MustNewIdentity("signoz", 0)}, + expectedValue: "signoz-0", + }, + { + name: "Several_JoinsSorted", + identities: Identities{ + MustNewIdentity("telemetrystore", 0, 1), + MustNewIdentity("metastore", 0), + MustNewIdentity("telemetrystore", 0, 0), + }, + expectedValue: "metastore-0,telemetrystore-0-0,telemetrystore-0-1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedValue, tt.identities.String()) + }) + } +} + +// Sorting is the whole reason the value is stable: the same claims stated in a +// different order have to render identically, or every plan shows a tag diff. +func TestIdentitiesRenderIndependentOfOrder(t *testing.T) { + forward := Identities{MustNewIdentity("keeper", 0), MustNewIdentity("keeper", 1), MustNewIdentity("keeper", 2)} + reversed := Identities{forward[2], forward[1], forward[0]} + + assert.Equal(t, forward.String(), reversed.String()) +} + +func TestParseIdentities(t *testing.T) { + tests := []struct { + name string + value string + pass bool + expectedIdentities Identities + }{ + { + name: "Empty_YieldsNone", + value: "", + pass: true, + expectedIdentities: nil, + }, + { + name: "Single_YieldsOne", + value: "signoz-0", + pass: true, + expectedIdentities: Identities{MustNewIdentity("signoz", 0)}, + }, + { + name: "Several_YieldsSorted", + value: "telemetrystore-0-1,metastore-0", + pass: true, + expectedIdentities: Identities{MustNewIdentity("metastore", 0), MustNewIdentity("telemetrystore", 0, 1)}, + }, + { + name: "Spaced_TrimsEntries", + value: "keeper-0, keeper-1", + pass: true, + expectedIdentities: Identities{MustNewIdentity("keeper", 0), MustNewIdentity("keeper", 1)}, + }, + {name: "TrailingSeparator_Invalid", value: "keeper-0,", pass: false}, + {name: "DoubledSeparator_Invalid", value: "keeper-0,,keeper-1", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + identities, err := ParseIdentities(tt.value) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedIdentities, identities) + }) + } +} + +// The tag value written here is read back by Terraform's split(), so the two +// halves of the encoding have to agree. Round-tripping in Go is the only place +// that can be asserted. +func TestIdentitiesRoundTrip(t *testing.T) { + identities := Identities{ + MustNewIdentity("telemetrykeeper", 0), + MustNewIdentity("telemetrystore", 0, 0), + MustNewIdentity("metastore", 0), + MustNewIdentity("signoz", 0), + } + + parsed, err := ParseIdentities(identities.String()) + + assert.NoError(t, err) + assert.Equal(t, identities.String(), parsed.String()) + assert.Len(t, parsed, len(identities)) +} diff --git a/internal/convention/ownership.go b/internal/convention/ownership.go new file mode 100644 index 00000000..c3334a2a --- /dev/null +++ b/internal/convention/ownership.go @@ -0,0 +1,26 @@ +package convention + +// Ownership is whether the substrate created a resource or adopted one it must +// never delete. +type Ownership struct { + s string + shared bool +} + +var ( + OwnershipOwned = Ownership{s: "owned"} + OwnershipShared = Ownership{s: "shared", shared: true} +) + +// String resolves the zero value to owned, which is what saying nothing means. +func (ownership Ownership) String() string { + if ownership.s == "" { + return OwnershipOwned.s + } + + return ownership.s +} + +func (ownership Ownership) IsShared() bool { + return ownership.shared +} diff --git a/internal/convention/ownership_test.go b/internal/convention/ownership_test.go new file mode 100644 index 00000000..c3da2189 --- /dev/null +++ b/internal/convention/ownership_test.go @@ -0,0 +1,31 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestOwnership(t *testing.T) { + tests := []struct { + name string + ownership Ownership + expectedWord string + expectedShared bool + }{ + {name: "Owned_NotShared", ownership: OwnershipOwned, expectedWord: "owned", expectedShared: false}, + {name: "Shared_IsShared", ownership: OwnershipShared, expectedWord: "shared", expectedShared: true}, + + // A caller says nothing when the substrate created the resource itself, + // which is the common case, so the zero value has to mean owned in both + // renderings rather than only in one. + {name: "ZeroValue_OwnedInBothForms", ownership: Ownership{}, expectedWord: "owned", expectedShared: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedWord, tt.ownership.String()) + assert.Equal(t, tt.expectedShared, tt.ownership.IsShared()) + }) + } +} diff --git a/internal/convention/resource.go b/internal/convention/resource.go new file mode 100644 index 00000000..039a676f --- /dev/null +++ b/internal/convention/resource.go @@ -0,0 +1,153 @@ +package convention + +import ( + "strings" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" +) + +// Resource is one thing a substrate provisions, described once. Its name, its +// tags, and the selection that finds it are all derived from that description, so +// a fact stated once cannot render two ways. +// +// Callers use the constructor for what they are provisioning; it supplies the +// resource type. +type Resource struct { + substrate Substrate + resourceType resourceType + + visibility Visibility + storage infrastructure.StorageClass + zone Zone + role Role + ordinal int + + ownership Ownership + kind infrastructure.ResourceKind + identities Identities +} + +func (s Substrate) Cluster() Resource { + return Resource{substrate: s, resourceType: typeCluster} +} + +func (s Substrate) VPC() Resource { + return Resource{substrate: s, resourceType: typeVPC} +} + +func (s Substrate) InternetGateway() Resource { + return Resource{substrate: s, resourceType: typeInternetGateway} +} + +func (s Substrate) Subnet(visibility Visibility, zone Zone) Resource { + return Resource{substrate: s, resourceType: typeSubnet, visibility: visibility, zone: zone} +} + +// RouteTable is a table shared across zones, which one internet gateway serves. +func (s Substrate) RouteTable(visibility Visibility) Resource { + return Resource{substrate: s, resourceType: typeRouteTable, visibility: visibility} +} + +// RouteTableInZone is a table per zone, each routing to that zone's NAT gateway. +func (s Substrate) RouteTableInZone(visibility Visibility, zone Zone) Resource { + return Resource{substrate: s, resourceType: typeRouteTable, visibility: visibility, zone: zone} +} + +func (s Substrate) NATGateway(zone Zone) Resource { + return Resource{substrate: s, resourceType: typeNATGateway, zone: zone} +} + +func (s Substrate) SecurityGroup(role Role) Resource { + return Resource{substrate: s, resourceType: typeSecurityGroup, role: role} +} + +func (s Substrate) Role(role Role) Resource { + return Resource{substrate: s, resourceType: typeRole, role: role} +} + +// Node and Volume take the storage class, which is a node group's whole +// identity: a consuming casting selects by class and cannot name a group. +func (s Substrate) Node(storage infrastructure.StorageClass, ordinal int) Resource { + return Resource{substrate: s, resourceType: typeNode, storage: storage, ordinal: ordinal} +} + +func (s Substrate) Volume(storage infrastructure.StorageClass, ordinal int) Resource { + return Resource{substrate: s, resourceType: typeVolume, storage: storage, ordinal: ordinal} +} + +// WithOwnership marks a resource adopted rather than created. It changes no name. +func (r Resource) WithOwnership(ownership Ownership) Resource { + r.ownership = ownership + + return r +} + +// WithKind records the Kind the substrate is provisioned for. +func (r Resource) WithKind(kind infrastructure.ResourceKind) Resource { + r.kind = kind + + return r +} + +// WithClaims records the identities holding a volume. See Identities. +func (r Resource) WithClaims(identities Identities) Resource { + r.identities = identities + + return r +} + +// Name is -[-...], broad to narrow so a substrate's +// resources share a prefix and sort together. It fills a provider's name argument +// where one exists, and the display tag always -- an instance or a volume has no +// name of its own. Which qualifiers apply is the resource type's declaration. +func (r Resource) Name() string { + parts := make([]string, 0, len(r.resourceType.qualifiers)+2) + parts = append(parts, r.substrate.name, r.resourceType.String()) + + for _, qualifier := range r.resourceType.qualifiers { + if segment := qualifier.of(r); segment != "" { + parts = append(parts, segment) + } + } + + return strings.Join(parts, "-") +} + +// Selection is the set that finds exactly this resource. +func (r Resource) Selection() Selection { + return Selection{substrate: r.substrate, storage: r.storage, identities: r.identities} +} + +// stamp is the selection's tags plus the provenance nothing reads back. Each +// check is on whether an axis applies, not on which resource this is. +func (r Resource) stamp() Tags { + tags := r.Selection().match() + + tags = append(tags, Tag{Key: TagKeyOwner, Value: r.ownership.String()}) + + // An adopted resource keeps the name it already had. + if !r.ownership.IsShared() { + tags = append(tags, Tag{Key: TagKeyDisplayName, Value: r.Name()}) + } + + if r.kind != (infrastructure.ResourceKind{}) { + tags = append(tags, Tag{Key: TagKeyResourceKind, Value: r.kind.String()}) + } + + if r.visibility != (Visibility{}) { + tags = append(tags, Tag{Key: TagKeyVisibility, Value: r.visibility.String()}) + } + + return tags +} + +// Tags is every tag this resource carries. Ownership labels are a separate +// family: a casting merges CastingMeta.Labels() in alongside these. +func (r Resource) Tags() map[string]string { + return r.stamp().Map() +} + +// Filter is the tag match that finds this resource. +func (r Resource) Filter() map[string]string { + return r.Selection().Filter() +} diff --git a/internal/convention/resource_test.go b/internal/convention/resource_test.go new file mode 100644 index 00000000..454131a6 --- /dev/null +++ b/internal/convention/resource_test.go @@ -0,0 +1,206 @@ +package convention + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/stretchr/testify/assert" +) + +func TestResourceName(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + persistent := infrastructure.StorageClassPersistent + ephemeral := infrastructure.StorageClassEphemeral + + tests := []struct { + name string + resource Resource + expectedName string + }{ + {name: "Cluster_Unqualified", resource: substrate.Cluster(), expectedName: "foundry-cls"}, + {name: "VPC_Unqualified", resource: substrate.VPC(), expectedName: "foundry-vpc"}, + {name: "InternetGateway_Unqualified", resource: substrate.InternetGateway(), expectedName: "foundry-igw"}, + {name: "PrivateSubnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPrivate, zone), expectedName: "foundry-sub-prv-east1a"}, + {name: "PublicSubnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPublic, zone), expectedName: "foundry-sub-pub-east1a"}, + {name: "PrivateRouteTable_PerZone", resource: substrate.RouteTableInZone(VisibilityPrivate, zone), expectedName: "foundry-rt-prv-east1a"}, + {name: "PublicRouteTable_ZoneShared", resource: substrate.RouteTable(VisibilityPublic), expectedName: "foundry-rt-pub"}, + {name: "NATGateway_PerZone", resource: substrate.NATGateway(zone), expectedName: "foundry-nat-east1a"}, + {name: "TaskSecurityGroup_Role", resource: substrate.SecurityGroup(RoleTask), expectedName: "foundry-sg-task"}, + {name: "ExecRole_Role", resource: substrate.Role(RoleExec), expectedName: "foundry-iam-exec"}, + {name: "Node_ClassAndOrdinal", resource: substrate.Node(persistent, 0), expectedName: "foundry-node-persistent-0"}, + {name: "Volume_ClassAndOrdinal", resource: substrate.Volume(persistent, 2), expectedName: "foundry-vol-persistent-2"}, + {name: "EphemeralNode_ClassAndOrdinal", resource: substrate.Node(ephemeral, 1), expectedName: "foundry-node-ephemeral-1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedName, tt.resource.Name()) + }) + } +} + +// Roles are the shortest derivation because they are the only names near a +// provider cap. This package does not know that cap -- a casting measures the +// derived name against its own provider's limit -- so what is fixed here is the +// overhead a caller has to budget for. +func TestRoleNameOverheadIsBounded(t *testing.T) { + const maxRoleSuffix = len("-iam-exec") + + for _, name := range []string{"a", "foundry", "signoz-prod-eu-central"} { + substrate := MustNewSubstrate(name) + assert.LessOrEqual(t, len(substrate.Role(RoleExec).Name())-len(name), maxRoleSuffix) + } +} + +// Adopting a resource must not rename it: the name belongs to whoever created it. +func TestSharedResourceKeepsItsName(t *testing.T) { + substrate := MustNewSubstrate("foundry") + shared := substrate.VPC().WithOwnership(OwnershipShared) + + assert.Equal(t, substrate.VPC().Name(), shared.Name()) + assert.NotContains(t, shared.Tags(), TagKeyDisplayName.String()) + assert.Equal(t, "shared", shared.Tags()[TagKeyOwner.String()]) +} + +func TestResourceTags(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + persistent := infrastructure.StorageClassPersistent + + tests := []struct { + name string + resource Resource + expectedPresent map[string]string + expectedAbsent []TagKey + }{ + { + name: "Cluster_CarriesIdentityAndOwner", + resource: substrate.Cluster(), + expectedPresent: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyOwner.String(): "owned", + TagKeyDisplayName.String(): "foundry-cls", + }, + expectedAbsent: []TagKey{TagKeyVisibility, TagKeyStorage, TagKeyIdentities, TagKeyResourceKind}, + }, + { + name: "PrivateSubnet_CarriesVisibilitySpelledOut", + resource: substrate.Subnet(VisibilityPrivate, zone).WithKind(infrastructure.ResourceKindInstallation), + expectedPresent: map[string]string{ + TagKeyDisplayName.String(): "foundry-sub-prv-east1a", + TagKeyVisibility.String(): "private", + TagKeyResourceKind.String(): "Installation", + }, + expectedAbsent: []TagKey{TagKeyStorage}, + }, + { + name: "PersistentNode_CarriesStorageFromItsGroup", + resource: substrate.Node(persistent, 0), + expectedPresent: map[string]string{ + TagKeyDisplayName.String(): "foundry-node-persistent-0", + TagKeyStorage.String(): "persistent", + }, + expectedAbsent: []TagKey{TagKeyVisibility, TagKeyIdentities}, + }, + { + name: "ClaimedVolume_CarriesIdentities", + resource: substrate.Volume(persistent, 0).WithClaims(Identities{ + MustNewIdentity("telemetrystore", 0, 0), + MustNewIdentity("metastore", 0), + }), + expectedPresent: map[string]string{ + TagKeyDisplayName.String(): "foundry-vol-persistent-0", + TagKeyStorage.String(): "persistent", + TagKeyIdentities.String(): "metastore-0,telemetrystore-0-0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tags := tt.resource.Tags() + + for key, expected := range tt.expectedPresent { + assert.Equal(t, expected, tags[key], "tag %s", key) + } + + for _, key := range tt.expectedAbsent { + assert.NotContains(t, tags, key.String()) + } + }) + } +} + +func TestResourceFilter(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + persistent := infrastructure.StorageClassPersistent + + tests := []struct { + name string + resource Resource + expectedFilter map[string]string + }{ + { + name: "VPC_SelectsIdentityOnly", + resource: substrate.VPC(), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + }, + }, + { + name: "ProvenanceOnly_IsNotSelectedOn", + resource: substrate.Subnet(VisibilityPrivate, zone).WithKind(infrastructure.ResourceKindInstallation), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + }, + }, + { + name: "PersistentNode_SelectsIdentityAndStorage", + resource: substrate.Node(persistent, 0), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyStorage.String(): "persistent", + }, + }, + { + name: "ClaimedVolume_SelectsTheClaim", + resource: substrate.Volume(persistent, 0).WithClaims(Identities{MustNewIdentity("signoz", 0)}), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyStorage.String(): "persistent", + TagKeyIdentities.String(): "signoz-0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedFilter, tt.resource.Filter()) + }) + } +} + +// The reason a Resource exists. A provider offers two places to state one fact -- +// a subnet named "private" and tagged "visibility: private" -- and stating it +// twice is what lets the two drift. Here it is stated once, so the name's short +// form and the tag's word are asserted to correspond for every enum value. +func TestNameAndTagsAgreeOnTheSameFact(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + + for _, visibility := range []Visibility{VisibilityPrivate, VisibilityPublic} { + subnet := substrate.Subnet(visibility, zone) + + assert.Contains(t, subnet.Name(), visibility.Short()) + assert.Equal(t, visibility.String(), subnet.Tags()[TagKeyVisibility.String()]) + } + + for _, storage := range []infrastructure.StorageClass{infrastructure.StorageClassPersistent, infrastructure.StorageClassEphemeral} { + node := substrate.Node(storage, 0) + + assert.Contains(t, node.Name(), storage.String()) + assert.Equal(t, storage.String(), node.Tags()[TagKeyStorage.String()]) + } +} diff --git a/internal/convention/resource_type.go b/internal/convention/resource_type.go new file mode 100644 index 00000000..f7e003a6 --- /dev/null +++ b/internal/convention/resource_type.go @@ -0,0 +1,45 @@ +package convention + +import ( + "strconv" +) + +// resourceType is what a derived name says the thing is, and the ordered +// qualifiers that narrow it. Adding a resource is one var entry below. +type resourceType struct { + short string + qualifiers []qualifier +} + +var ( + typeCluster = resourceType{short: "cls"} + typeVPC = resourceType{short: "vpc"} + typeInternetGateway = resourceType{short: "igw"} + typeSubnet = resourceType{short: "sub", qualifiers: []qualifier{qualifierVisibility, qualifierZone}} + typeRouteTable = resourceType{short: "rt", qualifiers: []qualifier{qualifierVisibility, qualifierZone}} + typeNATGateway = resourceType{short: "nat", qualifiers: []qualifier{qualifierZone}} + typeSecurityGroup = resourceType{short: "sg", qualifiers: []qualifier{qualifierRole}} + typeRole = resourceType{short: "iam", qualifiers: []qualifier{qualifierRole}} + typeNode = resourceType{short: "node", qualifiers: []qualifier{qualifierStorage, qualifierOrdinal}} + typeVolume = resourceType{short: "vol", qualifiers: []qualifier{qualifierStorage, qualifierOrdinal}} +) + +func (resource resourceType) String() string { + return resource.short +} + +// qualifier renders one axis into a name segment. An empty string drops the +// segment, so one route table declaration serves both the zonal and shared forms. +type qualifier struct { + of func(Resource) string +} + +var ( + qualifierVisibility = qualifier{of: func(r Resource) string { return r.visibility.Short() }} + qualifierZone = qualifier{of: func(r Resource) string { return r.zone.Short() }} + qualifierRole = qualifier{of: func(r Resource) string { return r.role.String() }} + qualifierStorage = qualifier{of: func(r Resource) string { return r.storage.String() }} + + // Only types that have an ordinal declare it, so zero renders as "0". + qualifierOrdinal = qualifier{of: func(r Resource) string { return strconv.Itoa(r.ordinal) }} +) diff --git a/internal/convention/resource_type_test.go b/internal/convention/resource_type_test.go new file mode 100644 index 00000000..b4e42c0b --- /dev/null +++ b/internal/convention/resource_type_test.go @@ -0,0 +1,79 @@ +package convention + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/stretchr/testify/assert" +) + +// Two resource types sharing a short form would derive the same name shape, and +// a consumer reading a console could not tell them apart. +func TestResourceTypeShortFormsAreDistinct(t *testing.T) { + resourceTypes := []resourceType{ + typeCluster, typeVPC, typeInternetGateway, typeSubnet, typeRouteTable, + typeNATGateway, typeSecurityGroup, typeRole, typeNode, typeVolume, + } + + seen := make(map[string]struct{}, len(resourceTypes)) + for _, resource := range resourceTypes { + assert.NotContains(t, seen, resource.String()) + seen[resource.String()] = struct{}{} + } +} + +// A qualifier that renders empty drops out, which is what lets one route table +// declaration serve both the zonal private tables and the zone-shared public +// one. Without it they would be two variants for one AWS resource type. +func TestUnsetQualifierDropsFromTheName(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + + assert.Equal(t, "foundry-rt-pub", substrate.RouteTable(VisibilityPublic).Name()) + assert.Equal(t, "foundry-rt-prv-east1a", substrate.RouteTableInZone(VisibilityPrivate, zone).Name()) +} + +// Every declared qualifier has to contribute for the constructor that uses the +// type, or a name silently loses a segment that was meant to distinguish it. +func TestEveryDeclaredQualifierContributes(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + persistent := infrastructure.StorageClassPersistent + + tests := []struct { + name string + resource Resource + expectedSegments int + }{ + {name: "VPC_NoQualifier", resource: substrate.VPC(), expectedSegments: 0}, + {name: "Subnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPrivate, zone), expectedSegments: 2}, + {name: "NATGateway_Zone", resource: substrate.NATGateway(zone), expectedSegments: 1}, + {name: "IAMRole_Role", resource: substrate.Role(RoleExec), expectedSegments: 1}, + {name: "Node_ClassAndOrdinal", resource: substrate.Node(persistent, 0), expectedSegments: 2}, + {name: "Volume_ClassAndOrdinal", resource: substrate.Volume(persistent, 0), expectedSegments: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rendered := 0 + for _, qualifier := range tt.resource.resourceType.qualifiers { + if qualifier.of(tt.resource) != "" { + rendered++ + } + } + + assert.Equal(t, tt.expectedSegments, rendered) + }) + } +} + +// Ordinal zero is a real ordinal, not an absent one, so only the types that have +// one declare the qualifier -- otherwise every other resource would render a +// stray "0". +func TestOrdinalZeroRenders(t *testing.T) { + substrate := MustNewSubstrate("foundry") + persistent := infrastructure.StorageClassPersistent + + assert.Equal(t, "foundry-node-persistent-0", substrate.Node(persistent, 0).Name()) + assert.Equal(t, "foundry-vpc", substrate.VPC().Name()) +} diff --git a/internal/convention/role.go b/internal/convention/role.go new file mode 100644 index 00000000..1e0d65e4 --- /dev/null +++ b/internal/convention/role.go @@ -0,0 +1,17 @@ +package convention + +// Role is what a security group or an IAM role is attached to. IAM roles use all +// three; a security group uses node and task. +type Role struct { + s string +} + +var ( + RoleNode = Role{s: "node"} + RoleTask = Role{s: "task"} + RoleExec = Role{s: "exec"} +) + +func (role Role) String() string { + return role.s +} diff --git a/internal/convention/role_test.go b/internal/convention/role_test.go new file mode 100644 index 00000000..7d63ee67 --- /dev/null +++ b/internal/convention/role_test.go @@ -0,0 +1,37 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRole(t *testing.T) { + tests := []struct { + name string + role Role + expectedWord string + }{ + {name: "Node_Rendered", role: RoleNode, expectedWord: "node"}, + {name: "Task_Rendered", role: RoleTask, expectedWord: "task"}, + {name: "Exec_Rendered", role: RoleExec, expectedWord: "exec"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedWord, tt.role.String()) + }) + } +} + +// Two roles sharing a rendering would give an IAM role and a security group the +// same derived name. +func TestRolesAreDistinct(t *testing.T) { + roles := []Role{RoleNode, RoleTask, RoleExec} + + seen := make(map[string]struct{}, len(roles)) + for _, role := range roles { + assert.NotContains(t, seen, role.String()) + seen[role.String()] = struct{}{} + } +} diff --git a/internal/convention/selection.go b/internal/convention/selection.go new file mode 100644 index 00000000..88e6490a --- /dev/null +++ b/internal/convention/selection.go @@ -0,0 +1,56 @@ +package convention + +import ( + "github.com/signoz/foundry/api/v1alpha1/infrastructure" +) + +// Selection is what a consuming casting looks for: a substrate, narrowed by the +// facts it knows. A consumer knows a class rather than an instance, and cannot +// know the group names the producer chose, so no name or resource type appears +// here -- neither reaches a tag. +type Selection struct { + substrate Substrate + storage infrastructure.StorageClass + identities Identities +} + +func (s Substrate) Select() Selection { + return Selection{substrate: s} +} + +func (selection Selection) WithStorage(storage infrastructure.StorageClass) Selection { + selection.storage = storage + + return selection +} + +// WithClaims narrows to the resource holding these identities. See Identities: +// only a platform with no stateful identity primitive of its own needs this. +func (selection Selection) WithClaims(identities Identities) Selection { + selection.identities = identities + + return selection +} + +// match is the only place the tags a consumer depends on are decided. Resource +// stamps these plus provenance, so the two cannot disagree. +func (selection Selection) match() Tags { + tags := Tags{ + {Key: TagKeyName, Value: selection.substrate.name}, + } + + if selection.storage != (infrastructure.StorageClass{}) { + tags = append(tags, Tag{Key: TagKeyStorage, Value: selection.storage.String()}) + } + + if len(selection.identities) > 0 { + tags = append(tags, Tag{Key: TagKeyIdentities, Value: selection.identities.String()}) + } + + return tags +} + +// Filter is the tag match a consuming casting writes into a data source. +func (selection Selection) Filter() map[string]string { + return selection.match().Map() +} diff --git a/internal/convention/selection_test.go b/internal/convention/selection_test.go new file mode 100644 index 00000000..5e94b824 --- /dev/null +++ b/internal/convention/selection_test.go @@ -0,0 +1,138 @@ +package convention + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/stretchr/testify/assert" +) + +func TestSelectionFilter(t *testing.T) { + substrate := MustNewSubstrate("foundry") + + tests := []struct { + name string + selection Selection + expectedFilter map[string]string + }{ + { + name: "Substrate_MatchesEverythingItOwns", + selection: substrate.Select(), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + }, + }, + { + name: "PersistentClass_MatchesTheClass", + selection: substrate.Select().WithStorage(infrastructure.StorageClassPersistent), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyStorage.String(): "persistent", + }, + }, + { + name: "EphemeralClass_MatchesTheClass", + selection: substrate.Select().WithStorage(infrastructure.StorageClassEphemeral), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyStorage.String(): "ephemeral", + }, + }, + { + name: "Claim_MatchesTheHolder", + selection: substrate.Select(). + WithStorage(infrastructure.StorageClassPersistent). + WithClaims(Identities{MustNewIdentity("telemetrystore", 0, 0)}), + expectedFilter: map[string]string{ + TagKeyName.String(): "foundry", + TagKeyStorage.String(): "persistent", + TagKeyIdentities.String(): "telemetrystore-0-0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedFilter, tt.selection.Filter()) + }) + } +} + +// A consumer states a class and gets what the producer stamped on every node of +// it, at any ordinal. The class is the whole identity, so there is nothing else +// for the two sides to agree on. +func TestClassFilterMatchesWhatTheProducerStamped(t *testing.T) { + substrate := MustNewSubstrate("foundry") + consumer := substrate.Select().WithStorage(infrastructure.StorageClassPersistent).Filter() + + for ordinal := range 3 { + stamped := substrate.Node(infrastructure.StorageClassPersistent, ordinal).Tags() + + for key, value := range consumer { + assert.Equal(t, value, stamped[key], "node %d does not match the class filter on %s", ordinal, key) + } + } +} + +// Claims are optional: a casting whose platform tracks the identity-to-disk +// binding itself never mentions them, and must then see no claim tag stamped and +// no claim key in its filter. Every casting but ECS is in that position, so this +// is the common path, not the edge. +func TestClaimsAreOptional(t *testing.T) { + substrate := MustNewSubstrate("foundry") + persistent := infrastructure.StorageClassPersistent + + volume := substrate.Volume(persistent, 0) + assert.NotContains(t, volume.Tags(), TagKeyIdentities.String()) + assert.NotContains(t, volume.Filter(), TagKeyIdentities.String()) + + selection := substrate.Select().WithStorage(infrastructure.StorageClassPersistent) + assert.NotContains(t, selection.Filter(), TagKeyIdentities.String()) + + // And an empty claim set is the same as never mentioning them. + assert.Equal(t, volume.Tags(), substrate.Volume(persistent, 0).WithClaims(Identities{}).Tags()) +} + +// A resource's contract tags are not a parallel list that has to agree with the +// consumer's filter -- they are that filter. Asserted so the composition cannot +// be unwound back into two lists. +func TestResourceContractTagsAreItsSelection(t *testing.T) { + substrate := MustNewSubstrate("foundry") + zone := MustParseZone("us-east-1a") + persistent := infrastructure.StorageClassPersistent + + resources := []Resource{ + substrate.Cluster(), + substrate.VPC().WithKind(infrastructure.ResourceKindCollectionAgent), + substrate.Subnet(VisibilityPrivate, zone), + substrate.NATGateway(zone), + substrate.Role(RoleExec), + substrate.Node(persistent, 0), + substrate.Volume(persistent, 1).WithClaims(Identities{MustNewIdentity("signoz", 0)}), + } + + for _, resource := range resources { + tags := resource.Tags() + + for key, value := range resource.Selection().Filter() { + assert.Equal(t, value, tags[key], "the selection filters on %s differently to how it is stamped", key) + } + } +} + +// A filter's keys are the only ones a consumer depends on the exact spelling of. +// Renaming one leaves live infrastructure unmatched, and that failure has no +// checkpoint: the filter returns nothing, after foundry has exited. Provenance +// keys carry no such constraint, which is why they are absent here. +func TestFilterKeysMatchDeployedSpelling(t *testing.T) { + filter := MustNewSubstrate("foundry").Select(). + WithStorage(infrastructure.StorageClassPersistent). + WithClaims(Identities{MustNewIdentity("signoz", 0)}). + Filter() + + assert.Equal(t, map[string]string{ + "foundry.signoz.io/name": "foundry", + "foundry.signoz.io/storage": "persistent", + "foundry.signoz.io/identities": "signoz-0", + }, filter) +} diff --git a/internal/convention/substrate.go b/internal/convention/substrate.go new file mode 100644 index 00000000..cb3bb9ac --- /dev/null +++ b/internal/convention/substrate.go @@ -0,0 +1,60 @@ +// Package convention derives the names and tags a provisioned substrate is +// identified by. +// +// Foundry generates rather than reconciles, so it can never ask a platform what +// it created. A consuming casting finds a producing casting's resources only by +// deriving the same names and filtering the same tags. Both sides are derived +// here so neither can drift. +// +// Provider limits are absent: a length cap belongs to the platform enforcing it, +// so a casting measures what it derives against its own provider's limits. +package convention + +import ( + "regexp" + + "github.com/signoz/foundry/internal/errors" +) + +// namePattern is what the strictest provider accepts as a name segment, and is +// shared by every name a caller supplies. +var namePattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) + +// maxNameLength matches the metadata.name cap in the casting schema. +const maxNameLength = 63 + +// Substrate is the infrastructure an installation runs on, known by the +// provisioning casting's metadata.name. Every resource is named and tagged from +// it, and it is the only fact a consumer needs to find them. +type Substrate struct { + name string +} + +func NewSubstrate(name string) (Substrate, error) { + if name == "" { + return Substrate{}, errors.Newf(errors.TypeInvalidInput, "failed to create substrate from %q: name is empty", name) + } + + if len(name) > maxNameLength { + return Substrate{}, errors.Newf(errors.TypeInvalidInput, "failed to create substrate from %q: name is longer than %d characters", name, maxNameLength) + } + + if !namePattern.MatchString(name) { + return Substrate{}, errors.Newf(errors.TypeInvalidInput, "failed to create substrate from %q: name is not lowercase alphanumeric with interior hyphens", name) + } + + return Substrate{name: name}, nil +} + +func MustNewSubstrate(name string) Substrate { + substrate, err := NewSubstrate(name) + if err != nil { + panic(err) + } + + return substrate +} + +func (s Substrate) String() string { + return s.name +} diff --git a/internal/convention/substrate_test.go b/internal/convention/substrate_test.go new file mode 100644 index 00000000..b590c777 --- /dev/null +++ b/internal/convention/substrate_test.go @@ -0,0 +1,40 @@ +package convention + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewSubstrate(t *testing.T) { + tests := []struct { + name string + input string + pass bool + expectedName string + }{ + {name: "Lowercase_Valid", input: "foundry", pass: true, expectedName: "foundry"}, + {name: "InteriorHyphens_Valid", input: "signoz-prod-eu", pass: true, expectedName: "signoz-prod-eu"}, + {name: "Digits_Valid", input: "signoz2", pass: true, expectedName: "signoz2"}, + {name: "Empty_Invalid", input: "", pass: false}, + {name: "Uppercase_Invalid", input: "Foundry", pass: false}, + {name: "LeadingHyphen_Invalid", input: "-foundry", pass: false}, + {name: "TrailingHyphen_Invalid", input: "foundry-", pass: false}, + {name: "Underscore_Invalid", input: "foundry_prod", pass: false}, + {name: "TooLong_Invalid", input: strings.Repeat("a", maxNameLength+1), pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + substrate, err := NewSubstrate(tt.input) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedName, substrate.String()) + }) + } +} diff --git a/internal/convention/tag.go b/internal/convention/tag.go new file mode 100644 index 00000000..36246f2a --- /dev/null +++ b/internal/convention/tag.go @@ -0,0 +1,44 @@ +package convention + +import ( + "github.com/signoz/foundry/internal/domain" +) + +// TagKey is a tag's key. Which of these a consumer filters on is decided by +// Selection, not declared here. +type TagKey struct { + key string +} + +var ( + TagKeyName = TagKey{key: domain.MetadataPrefix + "name"} + TagKeyStorage = TagKey{key: domain.MetadataPrefix + "storage"} + TagKeyIdentities = TagKey{key: domain.MetadataPrefix + "identities"} + TagKeyResourceKind = TagKey{key: domain.MetadataPrefix + "resource-kind"} + TagKeyOwner = TagKey{key: domain.MetadataPrefix + "owner"} + TagKeyVisibility = TagKey{key: domain.MetadataPrefix + "visibility"} + + // TagKeyDisplayName is unprefixed: "Name" is the provider's own convention + // for what a console shows. + TagKeyDisplayName = TagKey{key: "Name"} +) + +func (tagKey TagKey) String() string { + return tagKey.key +} + +type Tag struct { + Key TagKey + Value string +} + +type Tags []Tag + +func (tags Tags) Map() map[string]string { + out := make(map[string]string, len(tags)) + for _, tag := range tags { + out[tag.Key.String()] = tag.Value + } + + return out +} diff --git a/internal/convention/tag_test.go b/internal/convention/tag_test.go new file mode 100644 index 00000000..38b58a64 --- /dev/null +++ b/internal/convention/tag_test.go @@ -0,0 +1,44 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTagKeys(t *testing.T) { + tests := []struct { + name string + tagKey TagKey + expectedKey string + }{ + {name: "Name_Prefixed", tagKey: TagKeyName, expectedKey: "foundry.signoz.io/name"}, + {name: "Storage_Prefixed", tagKey: TagKeyStorage, expectedKey: "foundry.signoz.io/storage"}, + {name: "Identities_Prefixed", tagKey: TagKeyIdentities, expectedKey: "foundry.signoz.io/identities"}, + {name: "ResourceKind_Prefixed", tagKey: TagKeyResourceKind, expectedKey: "foundry.signoz.io/resource-kind"}, + {name: "Owner_Prefixed", tagKey: TagKeyOwner, expectedKey: "foundry.signoz.io/owner"}, + {name: "Visibility_Prefixed", tagKey: TagKeyVisibility, expectedKey: "foundry.signoz.io/visibility"}, + {name: "DisplayName_ProviderNative", tagKey: TagKeyDisplayName, expectedKey: "Name"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedKey, tt.tagKey.String()) + }) + } +} + +// Two keys sharing a string would collide on one resource, the second silently +// overwriting the first. +func TestTagKeysAreDistinct(t *testing.T) { + tagKeys := []TagKey{ + TagKeyName, TagKeyStorage, TagKeyIdentities, + TagKeyResourceKind, TagKeyOwner, TagKeyVisibility, TagKeyDisplayName, + } + + seen := make(map[string]struct{}, len(tagKeys)) + for _, tagKey := range tagKeys { + assert.NotContains(t, seen, tagKey.String()) + seen[tagKey.String()] = struct{}{} + } +} diff --git a/internal/convention/visibility.go b/internal/convention/visibility.go new file mode 100644 index 00000000..714acbd6 --- /dev/null +++ b/internal/convention/visibility.go @@ -0,0 +1,21 @@ +package convention + +// Visibility is whether a network resource faces the internet. String is the +// form a tag value carries; Short is the form a name carries. +type Visibility struct { + s string + short string +} + +var ( + VisibilityPrivate = Visibility{s: "private", short: "prv"} + VisibilityPublic = Visibility{s: "public", short: "pub"} +) + +func (visibility Visibility) String() string { + return visibility.s +} + +func (visibility Visibility) Short() string { + return visibility.short +} diff --git a/internal/convention/visibility_test.go b/internal/convention/visibility_test.go new file mode 100644 index 00000000..2abc67ad --- /dev/null +++ b/internal/convention/visibility_test.go @@ -0,0 +1,35 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// Names are length-constrained and tag values are not, so visibility renders +// compact in a name and spelled out in a tag. +func TestVisibility(t *testing.T) { + tests := []struct { + name string + visibility Visibility + expectedWord string + expectedShort string + }{ + {name: "Private_BothForms", visibility: VisibilityPrivate, expectedWord: "private", expectedShort: "prv"}, + {name: "Public_BothForms", visibility: VisibilityPublic, expectedWord: "public", expectedShort: "pub"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedWord, tt.visibility.String()) + assert.Equal(t, tt.expectedShort, tt.visibility.Short()) + }) + } +} + +// The zero value renders nothing, which is how a resource with no network face +// drops the qualifier and the tag rather than carrying them empty. +func TestVisibilityZeroValueRendersNothing(t *testing.T) { + assert.Empty(t, Visibility{}.String()) + assert.Empty(t, Visibility{}.Short()) +} diff --git a/internal/convention/zone.go b/internal/convention/zone.go new file mode 100644 index 00000000..ef28270b --- /dev/null +++ b/internal/convention/zone.go @@ -0,0 +1,46 @@ +package convention + +import ( + "strings" + + "github.com/signoz/foundry/internal/errors" +) + +// Zone is an availability zone. String is the provider's identifier, kept +// verbatim; Short is the form a name carries, derived from it. +type Zone struct { + s string + short string +} + +// ParseZone drops the leading locale segment and joins the rest for the short +// form: "us-east-1a" becomes "east1a", "asia-south2-c" becomes "south2c". +func ParseZone(zone string) (Zone, error) { + if zone == "" { + return Zone{}, errors.Newf(errors.TypeInvalidInput, "failed to create zone from %q: zone is empty", zone) + } + + segments := strings.Split(zone, "-") + if len(segments) < 2 { + return Zone{}, errors.Newf(errors.TypeInvalidInput, "failed to create zone from %q: zone has no locale and suffix segments", zone) + } + + return Zone{s: zone, short: strings.Join(segments[1:], "")}, nil +} + +func MustParseZone(zone string) Zone { + parsed, err := ParseZone(zone) + if err != nil { + panic(err) + } + + return parsed +} + +func (zone Zone) String() string { + return zone.s +} + +func (zone Zone) Short() string { + return zone.short +} diff --git a/internal/convention/zone_test.go b/internal/convention/zone_test.go new file mode 100644 index 00000000..b57867c9 --- /dev/null +++ b/internal/convention/zone_test.go @@ -0,0 +1,40 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseZone(t *testing.T) { + tests := []struct { + name string + input string + pass bool + expectedZone string + expectedShort string + }{ + {name: "AWSZone_Valid", input: "us-east-1a", pass: true, expectedZone: "us-east-1a", expectedShort: "east1a"}, + {name: "GCPZone_Valid", input: "asia-south2-c", pass: true, expectedZone: "asia-south2-c", expectedShort: "south2c"}, + {name: "GCPRegionZone_Valid", input: "us-central1-b", pass: true, expectedZone: "us-central1-b", expectedShort: "central1b"}, + {name: "GovCloudZone_Valid", input: "us-gov-east-1a", pass: true, expectedZone: "us-gov-east-1a", expectedShort: "goveast1a"}, + {name: "Empty_Invalid", input: "", pass: false}, + {name: "NoSeparator_Invalid", input: "useast1a", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + zone, err := ParseZone(tt.input) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + + // The provider form is kept verbatim; only the short form is derived. + assert.Equal(t, tt.expectedZone, zone.String()) + assert.Equal(t, tt.expectedShort, zone.Short()) + }) + } +} diff --git a/internal/domain/metadata.go b/internal/domain/metadata.go new file mode 100644 index 00000000..097ea203 --- /dev/null +++ b/internal/domain/metadata.go @@ -0,0 +1,10 @@ +package domain + +// MetadataPrefix namespaces every key foundry stamps onto, or reads from, +// something it generates: labels on a workload, annotations a user writes, tags +// on a cloud resource. Declaring it once is what keeps the three families in one +// namespace even though nothing compares them. +// +// Keys are one segment deep by convention -- foundry.signoz.io/managed-by, not +// foundry.signoz.io/ecs/cluster-id -- so the namespace stays flat and greppable. +const MetadataPrefix = "foundry.signoz.io/" From 267bdb21064078539b6a16b3eaf18a7c56ba73b0 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 5 Aug 2026 16:11:30 +0530 Subject: [PATCH 27/38] refactor(infrastructure): key node groups by storage class A node group's only identity is its storage class. A consuming casting selects nodes by class and has no way to name a group, so a second group of the same class is unreachable: nothing can be steered onto it, and the claim controller spreads stateful identities across every volume of that class by id, which reads as replication while leaving copies on one disk. Key nodeGroups by class and drop the name and storage fields. One group per class becomes structurally impossible to violate rather than a rule to check, and the class already supplies every identifier the generated Terraform needs. This also removes the reason the keyed list merge was added: a map merges under plain document semantics, so revert internal/domain to its three list types. --- .../infrastructure/resource_config.go | 12 +- internal/domain/merge.go | 99 --------------- internal/domain/merge_test.go | 32 ----- .../resourcemolding/resource.go | 62 ++++----- .../resourcemolding/resource_test.go | 119 +++++++++++------- 5 files changed, 105 insertions(+), 219 deletions(-) diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index 8c0ee119..6f46668c 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -4,8 +4,10 @@ package infrastructure // canonical internal representation of what a substrate shaped for the // resource kind must provide. type ResourceConfig struct { - // Node groups the resource requires from the substrate. - NodeGroups []ResourceConfigNodeGroup `json:"nodeGroups" patchStrategy:"merge" patchMergeKey:"name" description:"Node groups the resource requires from the substrate"` + // NodeGroups is keyed by storage class, which is the only identity a group + // has: a consuming casting selects nodes by class and has no way to name a + // group, so a second group of the same class would be unreachable. + NodeGroups map[StorageClass]ResourceConfigNodeGroup `json:"nodeGroups" description:"Node groups the resource requires from the substrate, keyed by storage class"` _ struct{} `additionalProperties:"false"` } @@ -17,12 +19,6 @@ type ResourceConfig struct { // so the document stays portable across providers, with machineType as the // escape hatch when a concrete type is wanted. type ResourceConfigNodeGroup struct { - // Name of the node group. - Name string `json:"name" description:"Name of the node group"` - - // Storage class of the group's nodes. - Storage StorageClass `json:"storage,omitzero" description:"Durability of the group's storage" examples:"[\"persistent\"]"` - // MinSize is the smallest the group may be. A pinned group states the // same value for both bounds. MinSize *int `json:"minSize,omitempty" minimum:"0" description:"Minimum number of nodes in the group"` diff --git a/internal/domain/merge.go b/internal/domain/merge.go index 7872f684..f4b33afc 100644 --- a/internal/domain/merge.go +++ b/internal/domain/merge.go @@ -32,18 +32,6 @@ var ( ListTypeOrdered = ListType{name: "ordered", merge: mergeOrdered} ) -// ListTypeMap merges a list of maps by a key field, mirroring Kubernetes' -// listType: map with a listMapKey: elements are matched on the key, a matched -// pair merges as a document so the override states only what it changes, and -// override-only elements append. Degrades to Atomic if either list holds an -// element that is not a map or is missing the key. -func ListTypeMap(key string) ListType { - return ListType{ - name: "map:" + key, - merge: func(base, override []any) []any { return mergeByKey(key, base, override) }, - } -} - // ListTypes declares the list types of a document's paths: dotted keys with // "*" matching any single segment, e.g. "service.pipelines.*.receivers". // Undeclared paths are ListTypeAtomic. @@ -204,93 +192,6 @@ func mergeOrdered(base, override []any) []any { return unionScalars(out, nil) } -// mergeByKey matches elements on key, merges each matched pair as a document, -// and appends the override's new elements. -func mergeByKey(key string, base, override []any) []any { - overrides := make(map[any]map[string]any, len(override)) - order := make([]any, 0, len(override)) - for _, elem := range override { - keyed, ok := keyedMap(elem, key) - if !ok { - return override - } - - overrides[keyed[key]] = keyed - order = append(order, keyed[key]) - } - - out := make([]any, 0, len(base)+len(override)) - merged := make(map[any]struct{}, len(override)) - for _, elem := range base { - keyed, ok := keyedMap(elem, key) - if !ok { - return override - } - - patch, matched := overrides[keyed[key]] - if !matched { - out = append(out, elem) - continue - } - - document, err := mergeDocument(keyed, patch) - if err != nil { - return override - } - - merged[keyed[key]] = struct{}{} - out = append(out, document) - } - - for _, id := range order { - if _, done := merged[id]; !done { - out = append(out, overrides[id]) - } - } - - return out -} - -// keyedMap returns the element as a map carrying the key. -func keyedMap(elem any, key string) (map[string]any, bool) { - document, ok := elem.(map[string]any) - if !ok { - return nil, false - } - - if _, ok := document[key]; !ok { - return nil, false - } - - return document, true -} - -// mergeDocument applies override onto base with RFC 7386 semantics, the same -// rule StrategicMergeYAML lands at the document level. -func mergeDocument(base, override map[string]any) (map[string]any, error) { - baseJSON, err := json.Marshal(base) - if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal list element") - } - - overrideJSON, err := json.Marshal(override) - if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to marshal list element override") - } - - mergedJSON, err := jsonpatchv5.MergePatch(baseJSON, overrideJSON) - if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to merge list element") - } - - out := map[string]any{} - if err := json.Unmarshal(mergedJSON, &out); err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to unmarshal merged list element") - } - - return out, nil -} - // scalarList reports whether every element is a comparable scalar. func scalarList(list []any) bool { for _, v := range list { diff --git a/internal/domain/merge_test.go b/internal/domain/merge_test.go index 1c46d199..7a353463 100644 --- a/internal/domain/merge_test.go +++ b/internal/domain/merge_test.go @@ -89,38 +89,6 @@ func TestStrategicMergeYAML(t *testing.T) { override: "a: [b\n", pass: false, }, - { - name: "MapList_MergesMatchedElementByKey", - base: "groups:\n- name: a\n size: 1\n cpu: 2\n- name: b\n size: 9\n", - override: "groups:\n- name: a\n size: 4\n", - listTypes: ListTypes{"groups": ListTypeMap("name")}, - pass: true, - expected: "groups:\n- cpu: 2\n name: a\n size: 4\n- name: b\n size: 9\n", - }, - { - name: "MapList_AppendsUnmatchedElement", - base: "groups:\n- name: a\n size: 1\n", - override: "groups:\n- name: b\n size: 2\n", - listTypes: ListTypes{"groups": ListTypeMap("name")}, - pass: true, - expected: "groups:\n- name: a\n size: 1\n- name: b\n size: 2\n", - }, - { - name: "MapList_MergesNestedObjectWithinElement", - base: "groups:\n- name: a\n volume:\n size: 20\n type: gp3\n", - override: "groups:\n- name: a\n volume:\n size: 50\n", - listTypes: ListTypes{"groups": ListTypeMap("name")}, - pass: true, - expected: "groups:\n- name: a\n volume:\n size: 50\n type: gp3\n", - }, - { - name: "MapList_MissingKeyDegradesToAtomic", - base: "groups:\n- name: a\n size: 1\n", - override: "groups:\n- size: 2\n", - listTypes: ListTypes{"groups": ListTypeMap("name")}, - pass: true, - expected: "groups:\n- size: 2\n", - }, } for _, tt := range tests { diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index 73ecb319..126b89e9 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -22,11 +22,6 @@ var ( // substrate shaped for the resource kind must satisfy beyond its edge. const ResourceConfigName = "resource.yaml" -// Node groups merge by name so a contribution or an operator override states -// only the group and the fields it changes, instead of restating every group -// to avoid deleting the ones it left out. -var resourceConfigListTypes = domain.ListTypes{"nodeGroups": domain.ListTypeMap("name")} - var _ infrastructuremolding.Molding = (*resourceMolding)(nil) type resourceMolding struct { @@ -53,15 +48,15 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) status.Addresses.APIServer = append([]string{apiServerAddress}, status.Addresses.APIServer...) baseline = &infrastructure.ResourceConfig{ - NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - // Three persistent nodes cover the default installation's - // stateful set: one keeper, the metadata node, one store node. - // A group holding stateful identities is pinned, so its bounds - // are equal -- there is nothing to autoscale when every node - // owns a claimed volume. - { - Name: infrastructure.StorageClassPersistent.String(), - Storage: infrastructure.StorageClassPersistent, + NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ + // Three persistent nodes cover the default topology's stateful + // set: one keeper, the metadata node, one store node. A scaled + // installation needs more and must state them, because + // Infrastructure provisions for a Kind and never reads the + // Installation casting. A pinned group's bounds are equal -- + // there is nothing to autoscale when every node owns a claimed + // volume. + infrastructure.StorageClassPersistent: { MinSize: v1alpha1.IntPtr(3), MaxSize: v1alpha1.IntPtr(3), CPU: v1alpha1.IntPtr(2), @@ -69,9 +64,7 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, }, - { - Name: infrastructure.StorageClassEphemeral.String(), - Storage: infrastructure.StorageClassEphemeral, + infrastructure.StorageClassEphemeral: { MinSize: v1alpha1.IntPtr(1), MaxSize: v1alpha1.IntPtr(1), CPU: v1alpha1.IntPtr(2), @@ -83,10 +76,8 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras case infrastructure.ResourceKindCollectionAgent: status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) baseline = &infrastructure.ResourceConfig{ - NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - { - Name: infrastructure.StorageClassEphemeral.String(), - Storage: infrastructure.StorageClassEphemeral, + NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ + infrastructure.StorageClassEphemeral: { MinSize: v1alpha1.IntPtr(1), MaxSize: v1alpha1.IntPtr(1), CPU: v1alpha1.IntPtr(2), @@ -108,7 +99,8 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras // Contributions (enricher deltas) merge first so casting-specific keys // survive, then the operator's own spec, which wins: spec beats status - // wherever they disagree. + // wherever they disagree. Groups are keyed by class, so an override states + // only the class and the fields it changes. for _, override := range []string{ status.Config.Data[ResourceConfigName], config.Spec.Resource.Spec.Config.Data[ResourceConfigName], @@ -117,7 +109,7 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras continue } - doc, err = domain.StrategicMergeYAML(doc, override, resourceConfigListTypes) + doc, err = domain.MergeYAML(doc, override) if err != nil { return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to merge resource config override") } @@ -143,37 +135,31 @@ func validate(doc string) error { return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to unmarshal resolved resource config") } - for _, group := range config.NodeGroups { + for storage, group := range config.NodeGroups { if group.MinSize == nil || group.MaxSize == nil || group.RootVolume.Size == nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q in resource config is incomplete", group.Name) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q in resource config is incomplete", storage) } if *group.MaxSize < *group.MinSize { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q has maxSize below minSize", group.Name) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q has maxSize below minSize", storage) } // A machine is named outright or resolved from criteria; one of the // two has to be stated or there is nothing to launch. if group.MachineType == "" && (group.CPU == nil || group.Memory == nil) { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states neither machineType nor cpu and memory", group.Name) - } - - // An unknown class cannot reach here: it fails at unmarshal. What is - // left is a group that named none at all. - if group.Storage.String() == "" { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states no storage class", group.Name) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states neither machineType nor cpu and memory", storage) } - if group.Storage.RequiresDataVolume() { + if storage.RequiresDataVolume() { if group.DataVolume == nil || group.DataVolume.Size == nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so it must state a dataVolume size", group.Name, group.Storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q must state a dataVolume size", storage) } } else if group.DataVolume != nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so it cannot state a dataVolume", group.Name, group.Storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q cannot state a dataVolume", storage) } - if group.Storage.IsPinned() && *group.MinSize != *group.MaxSize { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is %s, so minSize and maxSize must be equal", group.Name, group.Storage) + if storage.IsPinned() && *group.MinSize != *group.MaxSize { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is pinned, so minSize and maxSize must be equal", storage) } } diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index aa697e35..4e837c1b 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -23,10 +23,8 @@ func TestMoldV1Alpha1(t *testing.T) { kind: infrastructure.ResourceKindInstallation, pass: true, expected: infrastructure.ResourceConfig{ - NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - { - Name: "persistent", - Storage: infrastructure.StorageClassPersistent, + NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ + infrastructure.StorageClassPersistent: { MinSize: v1alpha1.IntPtr(3), MaxSize: v1alpha1.IntPtr(3), CPU: v1alpha1.IntPtr(2), @@ -34,9 +32,7 @@ func TestMoldV1Alpha1(t *testing.T) { RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, }, - { - Name: "ephemeral", - Storage: infrastructure.StorageClassEphemeral, + infrastructure.StorageClassEphemeral: { MinSize: v1alpha1.IntPtr(1), MaxSize: v1alpha1.IntPtr(1), CPU: v1alpha1.IntPtr(2), @@ -51,10 +47,8 @@ func TestMoldV1Alpha1(t *testing.T) { kind: infrastructure.ResourceKindCollectionAgent, pass: true, expected: infrastructure.ResourceConfig{ - NodeGroups: []infrastructure.ResourceConfigNodeGroup{ - { - Name: "ephemeral", - Storage: infrastructure.StorageClassEphemeral, + NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ + infrastructure.StorageClassEphemeral: { MinSize: v1alpha1.IntPtr(1), MaxSize: v1alpha1.IntPtr(1), CPU: v1alpha1.IntPtr(2), @@ -106,13 +100,13 @@ func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { config.Spec.Resource.Status.Addresses.OTLP = []string{"tcp://0.0.0.0:9411"} config.Spec.Resource.Status.Config.Data = map[string]string{ ResourceConfigName: `nodeGroups: -- name: persistent - minSize: 4 - maxSize: 4 - nodes: [{ordinal: 0}, {ordinal: 1}, {ordinal: 2}, {ordinal: 3}] -- name: ephemeral - minSize: 2 - maxSize: 2 + persistent: + minSize: 4 + maxSize: 4 + nodes: [{ordinal: 0}, {ordinal: 1}, {ordinal: 2}, {ordinal: 3}] + ephemeral: + minSize: 2 + maxSize: 2 `, } @@ -128,33 +122,74 @@ func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { got := infrastructure.ResourceConfig{} assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) - // Node groups merge by name: the contribution states only the sizes it - // changes and the baseline's other fields survive. + // Groups are keyed by class, so the contribution states only the sizes it + // changes and the baseline's other fields survive -- under plain document + // merge, with no list strategy. assert.Len(t, got.NodeGroups, 2) - for _, group := range got.NodeGroups { - switch group.Name { - case "persistent": - assert.Equal(t, v1alpha1.IntPtr(4), group.MinSize) - assert.Equal(t, infrastructure.StorageClassPersistent, group.Storage) - assert.Equal(t, v1alpha1.IntPtr(8), group.Memory) - assert.Equal(t, v1alpha1.IntPtr(50), group.DataVolume.Size) - case "ephemeral": - assert.Equal(t, v1alpha1.IntPtr(2), group.MinSize) - assert.Equal(t, infrastructure.StorageClassEphemeral, group.Storage) - assert.Equal(t, v1alpha1.IntPtr(4), group.Memory) - default: - t.Fatalf("unexpected node group %q", group.Name) - } - } + + persistent := got.NodeGroups[infrastructure.StorageClassPersistent] + assert.Equal(t, v1alpha1.IntPtr(4), persistent.MinSize) + assert.Equal(t, v1alpha1.IntPtr(8), persistent.Memory) + assert.Equal(t, v1alpha1.IntPtr(50), persistent.DataVolume.Size) + + ephemeral := got.NodeGroups[infrastructure.StorageClassEphemeral] + assert.Equal(t, v1alpha1.IntPtr(2), ephemeral.MinSize) + assert.Equal(t, v1alpha1.IntPtr(4), ephemeral.Memory) } -func TestMoldV1Alpha1_IncompleteContributionFails(t *testing.T) { - config := infrastructure.Default() - config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation - config.Spec.Resource.Status.Config.Data = map[string]string{ - ResourceConfigName: "nodeGroups:\n- name: keeper\n persistent: true\n count: 3\n", +func TestMoldV1Alpha1_ContributionValidity(t *testing.T) { + tests := []struct { + name string + contribution string + pass bool + }{ + { + // A partial override is the point of keying by class: the baseline + // supplies everything it does not mention. + name: "PartialOverride_Valid", + contribution: "nodeGroups:\n persistent:\n minSize: 3\n", + pass: true, + }, + { + // An unnameable group cannot be reached by any consumer, so an + // unknown key fails at unmarshal rather than provisioning nodes + // nothing will be placed on. + name: "UnknownClass_Invalid", + contribution: "nodeGroups:\n keeper:\n minSize: 3\n", + pass: false, + }, + { + name: "PinnedGroupWithUnequalBounds_Invalid", + contribution: "nodeGroups:\n persistent:\n minSize: 3\n maxSize: 5\n", + pass: false, + }, + { + // A null deletes the key under RFC 7386, so this removes the + // baseline's data volume from a class that requires one. + name: "PersistentWithoutDataVolume_Invalid", + contribution: "nodeGroups:\n persistent:\n dataVolume: null\n", + pass: false, + }, + { + name: "EphemeralWithDataVolume_Invalid", + contribution: "nodeGroups:\n ephemeral:\n dataVolume:\n size: 20\n", + pass: false, + }, } - err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) - assert.Error(t, err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation + config.Spec.Resource.Status.Config.Data = map[string]string{ResourceConfigName: tt.contribution} + + err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + }) + } } From ef9c7219fa6ce6532fed4cfc700ff10f87b025d2 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 5 Aug 2026 18:52:20 +0530 Subject: [PATCH 28/38] docs(convention): state the constraint, not the reasoning Doc comments carried design discussion rather than documentation: why a decision was reached, what a reviewer should notice, how the code is structured. Cut to what a reader cannot infer -- formats, invariants, and the reason a check exists. Field comments in the api package restated their own description tag, which is the published documentation and strictly more informative. Drop them and keep the tag, matching the tag-only style in meta.go. Fix Selection's doc, which still explained itself in terms of node group names after the storage class became a group's only identity. --- .../infrastructure/resource_config.go | 34 ++++++------------- api/v1alpha1/installation/infrastructure.go | 8 ++--- internal/convention/identity.go | 5 ++- internal/convention/identity_test.go | 14 ++++---- internal/convention/resource.go | 5 ++- internal/convention/resource_test.go | 11 ++---- internal/convention/resource_type_test.go | 16 ++++----- internal/convention/role_test.go | 3 +- internal/convention/selection.go | 6 ++-- internal/convention/selection_test.go | 22 +++++------- internal/convention/tag_test.go | 3 +- internal/convention/visibility_test.go | 8 ++--- .../resourcemolding/resource.go | 13 ++++--- 13 files changed, 56 insertions(+), 92 deletions(-) diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index 6f46668c..c35bf792 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -1,54 +1,40 @@ package infrastructure -// ResourceConfig is the resource requirement document (resource.yaml): the -// canonical internal representation of what a substrate shaped for the -// resource kind must provide. +// ResourceConfig is the resource requirement document, written as resource.yaml: +// what a substrate shaped for the resource kind must provide. type ResourceConfig struct { - // NodeGroups is keyed by storage class, which is the only identity a group - // has: a consuming casting selects nodes by class and has no way to name a - // group, so a second group of the same class would be unreachable. + // NodeGroups is keyed by storage class, a group's only identity: a consuming + // casting selects nodes by class and cannot name a group, so a second group + // of the same class would be unreachable. NodeGroups map[StorageClass]ResourceConfigNodeGroup `json:"nodeGroups" description:"Node groups the resource requires from the substrate, keyed by storage class"` _ struct{} `additionalProperties:"false"` } -// ResourceConfigNodeGroup sizes a pool of nodes. The vocabulary is the one -// every node-pool abstraction already uses -- machineType, minSize, maxSize -// and per-volume sizes are kOps', GKE's and eksctl's terms -- narrowed to what -// foundry has to understand. Capacity may be stated as criteria (cpu, memory) -// so the document stays portable across providers, with machineType as the -// escape hatch when a concrete type is wanted. +// ResourceConfigNodeGroup sizes a pool of nodes in the vocabulary every node-pool +// abstraction already uses -- kOps', GKE's and eksctl's terms -- narrowed to what +// foundry has to understand. Capacity is criteria so the document stays portable, +// with machineType as the escape hatch when a concrete type is wanted. type ResourceConfigNodeGroup struct { - // MinSize is the smallest the group may be. A pinned group states the - // same value for both bounds. + // MinSize and MaxSize are equal on a pinned group. MinSize *int `json:"minSize,omitempty" minimum:"0" description:"Minimum number of nodes in the group"` - // MaxSize is the largest the group may grow to. MaxSize *int `json:"maxSize,omitempty" minimum:"0" description:"Maximum number of nodes in the group"` - // MachineType names the provider's machine type outright; empty resolves - // one from cpu and memory against the provider's catalog. MachineType string `json:"machineType,omitempty" description:"Provider machine type; empty resolves one from cpu and memory" example:"m5.large"` - // CPU per node. CPU *int `json:"cpu,omitempty" minimum:"1" description:"CPUs per node, used when machineType is not stated"` - // Memory per node in GB. Memory *int `json:"memory,omitempty" minimum:"1" description:"Memory per node in GB, used when machineType is not stated"` - // RootVolume is the disk each node boots from. RootVolume ResourceConfigVolume `json:"rootVolume,omitzero" description:"The disk each node boots from"` - // DataVolume outlives the node it is attached to. Absent on a group whose - // nodes keep nothing. DataVolume *ResourceConfigVolume `json:"dataVolume,omitempty" description:"Volume attached to each node that outlives it; persistent storage class only"` _ struct{} `additionalProperties:"false"` } -// ResourceConfigVolume sizes one volume. type ResourceConfigVolume struct { - // Size of the volume in GB. Size *int `json:"size,omitempty" minimum:"1" description:"Size of the volume in GB"` _ struct{} `additionalProperties:"false"` diff --git a/api/v1alpha1/installation/infrastructure.go b/api/v1alpha1/installation/infrastructure.go index 6eef5b73..918b2585 100644 --- a/api/v1alpha1/installation/infrastructure.go +++ b/api/v1alpha1/installation/infrastructure.go @@ -1,10 +1,10 @@ package installation -// Infrastructure is the installation's binding to the infrastructure casting -// it runs on. The consumer owns the binding: it orders the casts -// (infrastructure first) and names the substrate for by-name lookups. +// Infrastructure is the installation's binding to the substrate it runs on. The +// consumer owns the binding because the two castings share no state: naming the +// substrate is what lets a casting derive the tag filter that finds its +// resources. Only a casting resolves it. type Infrastructure struct { - // Name of the infrastructure casting. Name string `json:"name,omitempty" yaml:"name,omitempty" pattern:"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" maxLength:"63" description:"Name of the infrastructure casting this installation runs on"` _ struct{} `additionalProperties:"false"` diff --git a/internal/convention/identity.go b/internal/convention/identity.go index 8dbcc14d..d08160c5 100644 --- a/internal/convention/identity.go +++ b/internal/convention/identity.go @@ -86,9 +86,8 @@ func (identity Identity) String() string { // Sorting keeps the value stable so an unchanged claim set produces no diff. // // Only a platform with no stateful identity primitive of its own needs a claim -// record. Kubernetes binds a pod to its volume through the StatefulSet -// controller, compose and swarm by name in the generated file, systemd by host -// path. Empty is therefore the norm, and stamps no tag. +// record; Kubernetes, compose, swarm and systemd each bind an identity to its +// disk themselves. Empty is the norm and stamps no tag. type Identities []Identity func (identities Identities) String() string { diff --git a/internal/convention/identity_test.go b/internal/convention/identity_test.go index 78cd2a77..61560b47 100644 --- a/internal/convention/identity_test.go +++ b/internal/convention/identity_test.go @@ -66,9 +66,8 @@ func TestParseIdentity(t *testing.T) { } } -// Parsing goes through NewIdentity, so a value a claim record could not have -// held is rejected on the way back in rather than becoming an identity nothing -// can match. +// Parsing validates through NewIdentity, so a value the encoder could not have +// produced is rejected. func TestParseIdentityDelegatesValidation(t *testing.T) { _, direct := NewIdentity("telemetry,store", 0) _, parsed := ParseIdentity("telemetry,store-0") @@ -111,8 +110,8 @@ func TestIdentitiesString(t *testing.T) { } } -// Sorting is the whole reason the value is stable: the same claims stated in a -// different order have to render identically, or every plan shows a tag diff. +// The same claims in a different order must render identically, or every plan +// shows a tag diff. func TestIdentitiesRenderIndependentOfOrder(t *testing.T) { forward := Identities{MustNewIdentity("keeper", 0), MustNewIdentity("keeper", 1), MustNewIdentity("keeper", 2)} reversed := Identities{forward[2], forward[1], forward[0]} @@ -169,9 +168,8 @@ func TestParseIdentities(t *testing.T) { } } -// The tag value written here is read back by Terraform's split(), so the two -// halves of the encoding have to agree. Round-tripping in Go is the only place -// that can be asserted. +// Terraform reads this value back with split(), so the encoding has to +// round-trip. func TestIdentitiesRoundTrip(t *testing.T) { identities := Identities{ MustNewIdentity("telemetrykeeper", 0), diff --git a/internal/convention/resource.go b/internal/convention/resource.go index 039a676f..60dc43f8 100644 --- a/internal/convention/resource.go +++ b/internal/convention/resource.go @@ -99,7 +99,7 @@ func (r Resource) WithClaims(identities Identities) Resource { // Name is -[-...], broad to narrow so a substrate's // resources share a prefix and sort together. It fills a provider's name argument // where one exists, and the display tag always -- an instance or a volume has no -// name of its own. Which qualifiers apply is the resource type's declaration. +// name of its own. func (r Resource) Name() string { parts := make([]string, 0, len(r.resourceType.qualifiers)+2) parts = append(parts, r.substrate.name, r.resourceType.String()) @@ -118,8 +118,7 @@ func (r Resource) Selection() Selection { return Selection{substrate: r.substrate, storage: r.storage, identities: r.identities} } -// stamp is the selection's tags plus the provenance nothing reads back. Each -// check is on whether an axis applies, not on which resource this is. +// stamp is the selection's tags plus the provenance nothing reads back. func (r Resource) stamp() Tags { tags := r.Selection().match() diff --git a/internal/convention/resource_test.go b/internal/convention/resource_test.go index 454131a6..57f8142f 100644 --- a/internal/convention/resource_test.go +++ b/internal/convention/resource_test.go @@ -40,10 +40,8 @@ func TestResourceName(t *testing.T) { } } -// Roles are the shortest derivation because they are the only names near a -// provider cap. This package does not know that cap -- a casting measures the -// derived name against its own provider's limit -- so what is fixed here is the -// overhead a caller has to budget for. +// A role name is the longest suffix a caller has to budget for against its +// provider's cap, which this package does not know. func TestRoleNameOverheadIsBounded(t *testing.T) { const maxRoleSuffix = len("-iam-exec") @@ -182,10 +180,7 @@ func TestResourceFilter(t *testing.T) { } } -// The reason a Resource exists. A provider offers two places to state one fact -- -// a subnet named "private" and tagged "visibility: private" -- and stating it -// twice is what lets the two drift. Here it is stated once, so the name's short -// form and the tag's word are asserted to correspond for every enum value. +// A fact stated once must render the same way in the name and in the tag. func TestNameAndTagsAgreeOnTheSameFact(t *testing.T) { substrate := MustNewSubstrate("foundry") zone := MustParseZone("us-east-1a") diff --git a/internal/convention/resource_type_test.go b/internal/convention/resource_type_test.go index b4e42c0b..bc4ad504 100644 --- a/internal/convention/resource_type_test.go +++ b/internal/convention/resource_type_test.go @@ -7,8 +7,7 @@ import ( "github.com/stretchr/testify/assert" ) -// Two resource types sharing a short form would derive the same name shape, and -// a consumer reading a console could not tell them apart. +// Two types sharing a short form would derive the same name shape. func TestResourceTypeShortFormsAreDistinct(t *testing.T) { resourceTypes := []resourceType{ typeCluster, typeVPC, typeInternetGateway, typeSubnet, typeRouteTable, @@ -22,9 +21,8 @@ func TestResourceTypeShortFormsAreDistinct(t *testing.T) { } } -// A qualifier that renders empty drops out, which is what lets one route table -// declaration serve both the zonal private tables and the zone-shared public -// one. Without it they would be two variants for one AWS resource type. +// An empty qualifier drops its segment, so one route table declaration serves +// both the zonal and the zone-shared form. func TestUnsetQualifierDropsFromTheName(t *testing.T) { substrate := MustNewSubstrate("foundry") zone := MustParseZone("us-east-1a") @@ -33,8 +31,8 @@ func TestUnsetQualifierDropsFromTheName(t *testing.T) { assert.Equal(t, "foundry-rt-prv-east1a", substrate.RouteTableInZone(VisibilityPrivate, zone).Name()) } -// Every declared qualifier has to contribute for the constructor that uses the -// type, or a name silently loses a segment that was meant to distinguish it. +// A declared qualifier that renders nothing would silently drop a segment meant +// to distinguish the name. func TestEveryDeclaredQualifierContributes(t *testing.T) { substrate := MustNewSubstrate("foundry") zone := MustParseZone("us-east-1a") @@ -67,9 +65,7 @@ func TestEveryDeclaredQualifierContributes(t *testing.T) { } } -// Ordinal zero is a real ordinal, not an absent one, so only the types that have -// one declare the qualifier -- otherwise every other resource would render a -// stray "0". +// Ordinal zero is a real ordinal, so only types that have one declare it. func TestOrdinalZeroRenders(t *testing.T) { substrate := MustNewSubstrate("foundry") persistent := infrastructure.StorageClassPersistent diff --git a/internal/convention/role_test.go b/internal/convention/role_test.go index 7d63ee67..8371348d 100644 --- a/internal/convention/role_test.go +++ b/internal/convention/role_test.go @@ -24,8 +24,7 @@ func TestRole(t *testing.T) { } } -// Two roles sharing a rendering would give an IAM role and a security group the -// same derived name. +// Two roles sharing a rendering would collide in a derived name. func TestRolesAreDistinct(t *testing.T) { roles := []Role{RoleNode, RoleTask, RoleExec} diff --git a/internal/convention/selection.go b/internal/convention/selection.go index 88e6490a..4598c67f 100644 --- a/internal/convention/selection.go +++ b/internal/convention/selection.go @@ -5,9 +5,9 @@ import ( ) // Selection is what a consuming casting looks for: a substrate, narrowed by the -// facts it knows. A consumer knows a class rather than an instance, and cannot -// know the group names the producer chose, so no name or resource type appears -// here -- neither reaches a tag. +// facts it knows. Neither a name nor a resource type appears here because +// neither reaches a tag -- a filter for instances and one for volumes are the +// same tags, and the data source decides which it returns. type Selection struct { substrate Substrate storage infrastructure.StorageClass diff --git a/internal/convention/selection_test.go b/internal/convention/selection_test.go index 5e94b824..35ef831b 100644 --- a/internal/convention/selection_test.go +++ b/internal/convention/selection_test.go @@ -58,9 +58,8 @@ func TestSelectionFilter(t *testing.T) { } } -// A consumer states a class and gets what the producer stamped on every node of -// it, at any ordinal. The class is the whole identity, so there is nothing else -// for the two sides to agree on. +// A consumer states a class; the producer stamps it per node. The filter has to +// match at any ordinal. func TestClassFilterMatchesWhatTheProducerStamped(t *testing.T) { substrate := MustNewSubstrate("foundry") consumer := substrate.Select().WithStorage(infrastructure.StorageClassPersistent).Filter() @@ -74,10 +73,8 @@ func TestClassFilterMatchesWhatTheProducerStamped(t *testing.T) { } } -// Claims are optional: a casting whose platform tracks the identity-to-disk -// binding itself never mentions them, and must then see no claim tag stamped and -// no claim key in its filter. Every casting but ECS is in that position, so this -// is the common path, not the edge. +// A platform that tracks the identity-to-disk binding itself stamps no claim tag +// and filters on none. func TestClaimsAreOptional(t *testing.T) { substrate := MustNewSubstrate("foundry") persistent := infrastructure.StorageClassPersistent @@ -93,9 +90,8 @@ func TestClaimsAreOptional(t *testing.T) { assert.Equal(t, volume.Tags(), substrate.Volume(persistent, 0).WithClaims(Identities{}).Tags()) } -// A resource's contract tags are not a parallel list that has to agree with the -// consumer's filter -- they are that filter. Asserted so the composition cannot -// be unwound back into two lists. +// A resource's filter is a subset of its tags, not a parallel list that has to +// agree with them. func TestResourceContractTagsAreItsSelection(t *testing.T) { substrate := MustNewSubstrate("foundry") zone := MustParseZone("us-east-1a") @@ -120,10 +116,8 @@ func TestResourceContractTagsAreItsSelection(t *testing.T) { } } -// A filter's keys are the only ones a consumer depends on the exact spelling of. -// Renaming one leaves live infrastructure unmatched, and that failure has no -// checkpoint: the filter returns nothing, after foundry has exited. Provenance -// keys carry no such constraint, which is why they are absent here. +// A filter's keys are the only ones whose spelling live infrastructure depends +// on: renaming one leaves it unmatched, with no checkpoint to catch it. func TestFilterKeysMatchDeployedSpelling(t *testing.T) { filter := MustNewSubstrate("foundry").Select(). WithStorage(infrastructure.StorageClassPersistent). diff --git a/internal/convention/tag_test.go b/internal/convention/tag_test.go index 38b58a64..0f20dd0b 100644 --- a/internal/convention/tag_test.go +++ b/internal/convention/tag_test.go @@ -28,8 +28,7 @@ func TestTagKeys(t *testing.T) { } } -// Two keys sharing a string would collide on one resource, the second silently -// overwriting the first. +// Two keys sharing a string would collide on one resource. func TestTagKeysAreDistinct(t *testing.T) { tagKeys := []TagKey{ TagKeyName, TagKeyStorage, TagKeyIdentities, diff --git a/internal/convention/visibility_test.go b/internal/convention/visibility_test.go index 2abc67ad..5c95fa94 100644 --- a/internal/convention/visibility_test.go +++ b/internal/convention/visibility_test.go @@ -6,8 +6,8 @@ import ( "github.com/stretchr/testify/assert" ) -// Names are length-constrained and tag values are not, so visibility renders -// compact in a name and spelled out in a tag. +// A name is length-constrained where a tag value is not, so the two renderings +// differ. func TestVisibility(t *testing.T) { tests := []struct { name string @@ -27,8 +27,8 @@ func TestVisibility(t *testing.T) { } } -// The zero value renders nothing, which is how a resource with no network face -// drops the qualifier and the tag rather than carrying them empty. +// The zero value renders nothing, so a resource with no network face carries +// neither the qualifier nor the tag. func TestVisibilityZeroValueRendersNothing(t *testing.T) { assert.Empty(t, Visibility{}.String()) assert.Empty(t, Visibility{}.Short()) diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index 126b89e9..5c6bbcef 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -49,13 +49,12 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras status.Addresses.APIServer = append([]string{apiServerAddress}, status.Addresses.APIServer...) baseline = &infrastructure.ResourceConfig{ NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ - // Three persistent nodes cover the default topology's stateful - // set: one keeper, the metadata node, one store node. A scaled - // installation needs more and must state them, because - // Infrastructure provisions for a Kind and never reads the - // Installation casting. A pinned group's bounds are equal -- - // there is nothing to autoscale when every node owns a claimed - // volume. + // Three persistent nodes cover the default topology: one + // keeper, the metadata node, one store node. A scaled + // installation must state its own, because Infrastructure + // provisions for a Kind and never reads the Installation + // casting. A pinned group's bounds are equal -- there is + // nothing to autoscale when every node owns a claimed volume. infrastructure.StorageClassPersistent: { MinSize: v1alpha1.IntPtr(3), MaxSize: v1alpha1.IntPtr(3), From 8b9c09a4d3c3b24f9511d7cce957be02ac41559b Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 5 Aug 2026 18:52:30 +0530 Subject: [PATCH 29/38] docs(infrastructure): add the concept page for the kind Covers what an Infrastructure casting declares, the requirement document it produces, the order the two castings apply in, and the naming and tagging convention that joins them. The requirement document and every derived name in it are the molding's and the convention's real output rather than hand-written examples. States plainly that no platform is registered yet, so forging reports an unsupported deployment until the castings land, and records the limits that would otherwise be found the hard way: one node group per storage class, and a substrate that does not grow when SigNoz is scaled. --- docs/concepts/infrastructure.md | 282 ++++++++++++++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 docs/concepts/infrastructure.md diff --git a/docs/concepts/infrastructure.md b/docs/concepts/infrastructure.md new file mode 100644 index 00000000..0925b4cb --- /dev/null +++ b/docs/concepts/infrastructure.md @@ -0,0 +1,282 @@ +# Infrastructure + +An Installation casting deploys SigNoz. An Infrastructure casting provisions what it runs on: the network, the machines, and the disks. + +They are separate kinds, forged separately and applied separately. Infrastructure never reads your Installation, and the Installation never reads Infrastructure's state. They find each other through the names and tags Foundry puts on every resource. + +## Declaring a substrate + +An Infrastructure casting says which Kind it is provisioning for, and nothing about that Kind's internals: + +```yaml +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: ec2 + flavor: terraform + resource: + kind: Installation +``` + +`spec.resource.kind` is either `Installation` or `CollectionAgent`. That single field is the whole input: Foundry knows what a default SigNoz installation needs, so it can size a substrate for one without being told anything about your components. + +## Requirements + +Forging turns that declaration into a requirement document, written to `casting.yaml.lock` under `spec.resource.status`. For an `Installation`: + +```yaml +nodeGroups: + ephemeral: + cpu: 2 + maxSize: 1 + memory: 4 + minSize: 1 + rootVolume: + size: 30 + persistent: + cpu: 2 + dataVolume: + size: 50 + maxSize: 3 + memory: 8 + minSize: 3 + rootVolume: + size: 30 +``` + +Plus the ports the substrate has to admit at its edge: `4317` and `4318` for OTLP, and `8080` for the API server. A `CollectionAgent` gets the ephemeral group and the OTLP ports only, because it stores nothing. + +Capacity is stated as CPU and memory rather than as `m5.large`, so the same document works on any provider. The casting resolves it to a real machine type at plan time. Set `machineType` yourself when you want an exact one. + +### Node group fields + +| Field | Meaning | +|---|---| +| `minSize` | Smallest the group may be | +| `maxSize` | Largest the group may grow to | +| `cpu` | CPUs per node | +| `memory` | Memory per node, in GB | +| `machineType` | Provider machine type; when set, `cpu` and `memory` are ignored | +| `rootVolume.size` | Boot disk per node, in GB | +| `dataVolume.size` | Disk that outlives the node, in GB. Persistent groups only | + +### Storage classes + +Node groups are keyed by storage class, and the class decides how the group behaves: + +| Class | Data | Size | Used by | +|---|---|---|---| +| `persistent` | Each node carries a disk that outlives it | Fixed: `minSize` and `maxSize` must match | ClickHouse, Keeper, PostgreSQL | +| `ephemeral` | Keeps nothing | Scales between the bounds | Collector, MCP, UI | + +A persistent node cannot be swapped for another, because a component's data is on the disk attached to it. That is why its bounds are pinned: there is nothing to autoscale when every node owns a claimed disk. + +There is one group per class. Two persistent groups would be indistinguishable to the Installation, which selects nodes by class, so anything scheduled would land on either one at random. + +### Overriding the defaults + +Put your own values under `spec.resource.spec.config.data`, keyed by class. You only state what you are changing; everything else comes from the defaults above. + +```yaml +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: ec2 + flavor: terraform + resource: + kind: Installation + spec: + config: + data: + resource.yaml: | + nodeGroups: + persistent: + minSize: 6 + maxSize: 6 + machineType: m5.xlarge + ephemeral: + minSize: 2 + maxSize: 4 +``` + +**If you scale SigNoz, you have to scale the persistent group yourself.** The default is three persistent nodes, which covers one Keeper, the metadata store, and one ClickHouse node. Three Keeper replicas and two ClickHouse shards need more, and Infrastructure cannot work that out for you because it never reads your Installation. + +## Order of operations + +``` + casting.yaml (Infrastructure) casting.yaml (Installation) + | | + | forge | forge + v v + pours/infrastructure/ pours/deployment/ + | | + | terraform apply | terraform apply + v v + +------------------------------------------------------------------+ + | the provider | + | network, machines, disks, tagged as Foundry names them | + +------------------------------------------------------------------+ + stamps tags ---------------------> reads them back +``` + +Infrastructure first. The Installation's lookups return nothing until the substrate exists, so applying it early produces a plan that places nothing. + +The two runs keep separate Terraform state. Neither reads the other's, and Foundry passes nothing between them. + +## The channel between castings + +Your Installation names the substrate it runs on: + +```yaml +apiVersion: v1alpha1 +kind: Installation +metadata: + name: signoz +spec: + deployment: + platform: aws + mode: ec2 + flavor: terraform + infrastructure: + name: signoz +``` + +That is the whole binding. Everything else travels as tags on the resources themselves: the Installation searches for `foundry.signoz.io/name` and `foundry.signoz.io/storage` to find machines and disks, and reads `foundry.signoz.io/identities` off a disk to learn which component owns it. + +No outputs are wired between the two, and no state file is shared. Foundry generates files and exits; it never calls a cloud API, so it cannot ask the provider what it created a moment ago. Both sides have to work the names and tags out the same way, which is why they are derived rather than configured. + +## Conventions + +Foundry derives every name and every tag from the substrate's name and a closed set of values. + +### Names + +``` +-[-...] +``` + +Broad to narrow, so everything belonging to one deployment shares a prefix and sorts together. A qualifier that does not apply is left out rather than padded, which is why a zone-shared route table has no zone in its name. + +| Resource | Type | Qualifiers | Example | +|---|---|---|---| +| Cluster | `cls` | | `signoz-cls` | +| VPC | `vpc` | | `signoz-vpc` | +| Internet gateway | `igw` | | `signoz-igw` | +| Subnet | `sub` | visibility, zone | `signoz-sub-prv-east1a` | +| Route table, per zone | `rt` | visibility, zone | `signoz-rt-prv-east1a` | +| Route table, zone-shared | `rt` | visibility | `signoz-rt-pub` | +| NAT gateway | `nat` | zone | `signoz-nat-east1a` | +| Security group | `sg` | role | `signoz-sg-task` | +| IAM role | `iam` | role | `signoz-iam-exec` | +| Node | `node` | storage class, ordinal | `signoz-node-persistent-0` | +| Volume | `vol` | storage class, ordinal | `signoz-vol-persistent-0` | + +### Values + +| Axis | Values | In a name | In a tag | +|---|---|---|---| +| Visibility | private, public | `prv`, `pub` | `private`, `public` | +| Storage class | persistent, ephemeral | `persistent`, `ephemeral` | same | +| Role | node, task, exec | `node`, `task`, `exec` | not tagged | +| Zone | the provider's zone | locale dropped: `us-east-1a` becomes `east1a` | provider's own form | +| Ordinal | position in a group | zero-based: `0`, `1`, `2` | not tagged | + +A value the Installation matches on is never abbreviated, because the string has to be identical on both sides. A value only a person reads can be short where space is tight, which is why visibility has two forms and the storage class has one. + +Length caps belong to the platform. IAM role names cap at 64 characters on AWS, which is why roles are the shortest derivation above; the substrate name is capped at 63. + +### Tags + +Every tag lives under `foundry.signoz.io/` and is one segment deep, so a single filter finds everything Foundry touched in an account. + +| Tag | Value | Read by | +|---|---|---| +| `foundry.signoz.io/name` | The substrate's name | The Installation, to find this substrate's resources | +| `foundry.signoz.io/storage` | `persistent` or `ephemeral` | The Installation, to pick which nodes a component runs on | +| `foundry.signoz.io/identities` | Which components claim a disk | The Installation, to keep a component on its own data | +| `foundry.signoz.io/resource-kind` | The Kind the substrate serves | People | +| `foundry.signoz.io/owner` | `owned` or `shared` | People, to tell what Foundry may delete | +| `foundry.signoz.io/visibility` | `private` or `public` | People | +| `Name` | The derived name | Cloud consoles, which show this tag by convention | + +The first three are how the Installation finds anything, so they are fixed. The rest describe a resource and are free to change. + +### Identities + +A component that keeps data has an identity, written `--` with zero-based ordinals: `telemetrystore-0-0`, `telemetrykeeper-1`, `metastore-0`. An identity claims a disk and stays with it, so a component keeps its data when the machine under it is replaced. + +Claims are recorded on the disk in `foundry.signoz.io/identities`, comma-joined and sorted, so the value is stable between runs. + +## Dependencies + +### What the substrate contains + +``` + vpc + | + +-- internet gateway + | + +-- subnet (one per zone, private and public) + | | + | +-- nat gateway (public subnet, one per zone) + | +-- route table (private: per zone, via nat) + | (public: shared, via igw) + +-- security group + + iam role -- instance profile + + per persistent ordinal: + instance --> subnet in zone N, instance profile, security group + volume --> zone N + attachment --> instance + volume + + per ephemeral group: + launch template --> subnets, security group, instance profile + autoscaling group --> launch template +``` + +A persistent node and its disk are placed in the same zone, because a disk can only attach to a machine in its own zone. Ordinal 0 goes in the first zone, 1 in the second, and so on. + +Persistent nodes are individual machines rather than an autoscaling group. An autoscaler replacing a machine would move a disk out from under whatever component owns it. + +### How a component reaches its data + +``` + task --pinned to--> instance --currently holds--> volume --claimed by--> identity +``` + +Read it right to left. An identity such as `telemetrystore-0-0` claims a disk. The disk is attached to some machine. The task is pinned to that machine, so it starts on top of its own data. + +The claim is recorded on the **disk**, not the machine. Machines get replaced routinely, by a resize, an image update, or a failure. A claim written on the machine would be lost every time one was replaced. Written on the disk it survives, and each plan works out which machine currently holds it. + +### Effects of a change + +| Change | What follows | +|---|---| +| Add a ClickHouse replica | A new identity appears and claims a free disk. Its task is pinned to whichever machine holds that disk. | +| Resize a machine | The machine is replaced. Its disk detaches and reattaches, the claim is untouched, and the task re-pins to the new machine. | +| Change a disk's size | The disk is grown in place. Nothing else moves. | +| Remove a replica | The identity goes away. Its disk keeps the old claim tag until something claims it again. | +| Destroy a disk | The data and the claim are both gone. The identity claims a different disk and starts empty. | + +## Adopting resources you already have + +A resource tagged `foundry.signoz.io/owner: shared` keeps the name it already had and is never deleted. This is how an existing VPC gets used rather than replaced. + +You can point an Installation at a network, subnets and a cluster you already run. Persistent components are the exception: they need disks discovered by tag, so those have to come from an Infrastructure casting. + +## Limits + +**One node group per storage class.** There is no way to put Keeper on cheaper machines than ClickHouse, because the Installation selects nodes by class and has no vocabulary for naming a group. + +**More stateful components than persistent nodes.** Two identities end up on one disk. Both components run, both write to the same volume, and it looks like replication without being replication. Count one persistent node per identity: one per Keeper replica, one per ClickHouse node, one for the metadata store. + +**A claimed disk attached to nothing.** The task stays pending rather than starting empty somewhere else, which is deliberate: starting empty would look like it worked. \ No newline at end of file From e46c3d080a45f46ca5aaad95ea51a0ce200ef585 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Wed, 5 Aug 2026 18:54:38 +0530 Subject: [PATCH 30/38] fix: fix the infrastructure package layout --- internal/casting/infrastructure/{casting => }/casting.go | 2 +- internal/casting/infrastructure/planner.go | 3 +-- internal/casting/infrastructure/registry.go | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) rename internal/casting/infrastructure/{casting => }/casting.go (95%) diff --git a/internal/casting/infrastructure/casting/casting.go b/internal/casting/infrastructure/casting.go similarity index 95% rename from internal/casting/infrastructure/casting/casting.go rename to internal/casting/infrastructure/casting.go index 805ed92b..fdbf1de2 100644 --- a/internal/casting/infrastructure/casting/casting.go +++ b/internal/casting/infrastructure/casting.go @@ -1,4 +1,4 @@ -package casting +package infrastructure import ( "context" diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index f0f8afa0..8257615d 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -7,7 +7,6 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" - infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" @@ -25,7 +24,7 @@ var _ planner.Planner = (*Planner)(nil) type Planner struct { config *infrastructure.Casting logger *slog.Logger - casting infrastructurecasting.Casting + casting Casting toolers []tooler.Tooler enricher infrastructuremolding.MoldingEnricher moldings []infrastructuremolding.Molding diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 48c2d2be..110c1b53 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,13 +4,12 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" - infrastructurecasting "github.com/signoz/foundry/internal/casting/infrastructure/casting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" ) type CastingItem struct { - Casting infrastructurecasting.Casting + Casting Casting Toolers []tooler.Tooler } @@ -31,7 +30,7 @@ func (registry *Registry) lookup(deployment v1alpha1.TypeDeployment) (CastingIte return item, ok } -func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (infrastructurecasting.Casting, error) { +func (registry *Registry) Casting(deployment v1alpha1.TypeDeployment) (Casting, error) { item, ok := registry.lookup(deployment) if !ok { return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "infrastructure deployment '%+v' is not supported", deployment) From 5e2b75f97460931554b53de3ba27cbf1b56be92a Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 17:47:05 +0530 Subject: [PATCH 31/38] refactor(infrastructure): make resource.yaml canonical and derive per deployment The requirement document now has two halves. The top is a declaration in kOps' vocabulary: networking with keyed subnets, instanceGroups, cloudLabels. The bottom is derived from it once the declaration settles, and holds every name and tag the substrate stamps. A casting interpolates the derived half rather than assembling names of its own, which is what keeps a producing and a consuming casting from spelling the same string two ways. Derivation walks a provider's topology, so the molding takes a Deriver the registry supplies per deployment instead of reaching for one cloud's. The convention package splits to match: Substrate, Key, NodeGroup, Selection, Identity and the tag facts stay neutral, and the AWS noun table and topology walk land beside the casting that uses them. TagKey names a fact and no longer carries a prefix. A GCP label key rejects the dot and the slash and an Azure tag name rejects the slash, so the spelling is rendered where the resource is created. StorageClass and SubnetType move to api/v1alpha1. An installation casting selects on both and should not import the Infrastructure kind's API to do it. Drops resource.kind. Once workload identity moved to the workload castings it selected only a node-group baseline, and one of its two values had no consumer: a collection agent lands on capacity that already exists and asks a substrate for nothing. One baseline now covers every substrate, and a substrate that keeps nothing drops the persistent group with a null. Drops the edge addresses, which the molding recorded and no template read. --- api/v1alpha1/infrastructure/casting.go | 1 - .../infrastructure/casting.schema.json | 47 --- api/v1alpha1/infrastructure/resource.go | 24 +- .../infrastructure/resource_config.go | 200 +++++++++- api/v1alpha1/infrastructure/resource_kind.go | 83 ---- api/v1alpha1/infrastructure/schema_test.go | 26 +- .../{infrastructure => }/storage_class.go | 2 +- .../storage_class_test.go | 2 +- api/v1alpha1/subnet_type.go | 101 +++++ api/v1alpha1/subnet_type_test.go | 65 +++ internal/casting/infrastructure/planner.go | 7 +- internal/casting/infrastructure/registry.go | 13 + internal/config/yamlconfig/config_test.go | 42 +- internal/convention/identity.go | 23 +- internal/convention/key.go | 37 ++ internal/convention/key_test.go | 50 +++ internal/convention/node_group.go | 24 ++ internal/convention/node_group_test.go | 25 ++ internal/convention/ownership.go | 2 +- internal/convention/resource.go | 152 ------- internal/convention/resource_test.go | 201 ---------- internal/convention/resource_type.go | 45 --- internal/convention/resource_type_test.go | 75 ---- internal/convention/role.go | 17 - internal/convention/role_test.go | 36 -- internal/convention/selection.go | 46 ++- internal/convention/selection_test.go | 121 ++---- internal/convention/substrate.go | 19 +- internal/convention/tag.go | 44 +- internal/convention/tag_test.go | 21 +- internal/convention/visibility.go | 21 - internal/convention/visibility_test.go | 35 -- internal/convention/zone.go | 46 --- internal/convention/zone_test.go | 40 -- internal/domain/metadata.go | 5 +- internal/molding/infrastructure/molding.go | 6 + .../resourcemolding/resource.go | 276 +++++++++---- .../resourcemolding/resource_test.go | 377 ++++++++++++------ 38 files changed, 1090 insertions(+), 1267 deletions(-) delete mode 100644 api/v1alpha1/infrastructure/resource_kind.go rename api/v1alpha1/{infrastructure => }/storage_class.go (99%) rename api/v1alpha1/{infrastructure => }/storage_class_test.go (98%) create mode 100644 api/v1alpha1/subnet_type.go create mode 100644 api/v1alpha1/subnet_type_test.go create mode 100644 internal/convention/key.go create mode 100644 internal/convention/key_test.go create mode 100644 internal/convention/node_group.go create mode 100644 internal/convention/node_group_test.go delete mode 100644 internal/convention/resource.go delete mode 100644 internal/convention/resource_test.go delete mode 100644 internal/convention/resource_type.go delete mode 100644 internal/convention/resource_type_test.go delete mode 100644 internal/convention/role.go delete mode 100644 internal/convention/role_test.go delete mode 100644 internal/convention/visibility.go delete mode 100644 internal/convention/visibility_test.go delete mode 100644 internal/convention/zone.go delete mode 100644 internal/convention/zone_test.go diff --git a/api/v1alpha1/infrastructure/casting.go b/api/v1alpha1/infrastructure/casting.go index a56a9f9e..17d56b8b 100644 --- a/api/v1alpha1/infrastructure/casting.go +++ b/api/v1alpha1/infrastructure/casting.go @@ -65,6 +65,5 @@ func (c *Casting) TrackableProperties() domain.Properties { Set("platform", c.Spec.Deployment.Platform.String()). Set("mode", c.Spec.Deployment.Mode.String()). Set("flavor", c.Spec.Deployment.Flavor.String()). - Set("resource_kind", c.Spec.Resource.Kind.String()). Set("patches_count", len(c.Spec.Patches)) } diff --git a/api/v1alpha1/infrastructure/casting.schema.json b/api/v1alpha1/infrastructure/casting.schema.json index 157c9efa..197e60f0 100644 --- a/api/v1alpha1/infrastructure/casting.schema.json +++ b/api/v1alpha1/infrastructure/casting.schema.json @@ -8,18 +8,8 @@ "additionalProperties": false, "definitions": { "InfrastructureResource": { - "required": [ - "kind" - ], "additionalProperties": false, "properties": { - "kind": { - "$ref": "#/definitions/InfrastructureResourceKind", - "description": "Kind of the resource this infrastructure serves", - "examples": [ - "Installation" - ] - }, "spec": { "$ref": "#/definitions/V1Alpha1MoldingSpec" }, @@ -30,20 +20,9 @@ }, "type": "object" }, - "InfrastructureResourceKind": { - "enum": [ - "Installation", - "CollectionAgent" - ], - "type": "string" - }, "InfrastructureResourceStatus": { "additionalProperties": false, "properties": { - "addresses": { - "$ref": "#/definitions/InfrastructureResourceStatusAddresses", - "description": "Addresses the resource admits at the substrate's edge" - }, "config": { "$ref": "#/definitions/V1Alpha1TypeConfig", "description": "Configuration for the molding" @@ -65,32 +44,6 @@ }, "type": "object" }, - "InfrastructureResourceStatusAddresses": { - "additionalProperties": false, - "properties": { - "apiserver": { - "description": "API server addresses", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - }, - "otlp": { - "description": "OTLP addresses", - "items": { - "type": "string" - }, - "type": [ - "array", - "null" - ] - } - }, - "type": "object" - }, "InfrastructureSpec": { "required": [ "deployment", diff --git a/api/v1alpha1/infrastructure/resource.go b/api/v1alpha1/infrastructure/resource.go index d6dcf981..5a936227 100644 --- a/api/v1alpha1/infrastructure/resource.go +++ b/api/v1alpha1/infrastructure/resource.go @@ -2,13 +2,9 @@ package infrastructure import "github.com/signoz/foundry/api/v1alpha1" -// Resource declares the kind of resource this infrastructure is shaped for. -// It is a declaration, not a reference: the consumer owns the binding and -// declares it on its own casting. +// Resource is the resource molding's slot on an Infrastructure casting: what a +// substrate must provide, and what foundry derived from it. type Resource struct { - // Kind of the resource this infrastructure serves. - Kind ResourceKind `json:"kind,omitzero" yaml:"kind,omitempty" required:"true" description:"Kind of the resource this infrastructure serves" examples:"[\"Installation\"]"` - // Specification for the resource. Spec v1alpha1.MoldingSpec `json:"spec" yaml:"spec" jsonschema:"description=Specification for the resource"` @@ -18,23 +14,9 @@ type Resource struct { _ struct{} `additionalProperties:"false"` } -// ResourceStatus carries the requirement set a substrate shaped for the -// resource kind must satisfy. +// ResourceStatus carries the settled requirement document. type ResourceStatus struct { v1alpha1.MoldingStatus `json:",inline" yaml:",inline"` - // Addresses the resource admits at the substrate's edge. - Addresses ResourceStatusAddresses `json:"addresses" yaml:"addresses,omitempty" description:"Addresses the resource admits at the substrate's edge"` - - _ struct{} `additionalProperties:"false"` -} - -type ResourceStatusAddresses struct { - // OTLP addresses. - OTLP []string `json:"otlp" yaml:"otlp,omitempty" description:"OTLP addresses"` - - // API server addresses. - APIServer []string `json:"apiserver" yaml:"apiserver,omitempty" description:"API server addresses"` - _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/infrastructure/resource_config.go b/api/v1alpha1/infrastructure/resource_config.go index c35bf792..e23b40c2 100644 --- a/api/v1alpha1/infrastructure/resource_config.go +++ b/api/v1alpha1/infrastructure/resource_config.go @@ -1,41 +1,207 @@ package infrastructure -// ResourceConfig is the resource requirement document, written as resource.yaml: -// what a substrate shaped for the resource kind must provide. +import "github.com/signoz/foundry/api/v1alpha1" + +// ResourceConfig is the requirement document, written as resource.yaml. +// +// Everything above Resources is declared: a molding baseline, a casting's +// contribution, then the operator's spec, which wins. Resources is derived once +// that settles. type ResourceConfig struct { - // NodeGroups is keyed by storage class, a group's only identity: a consuming - // casting selects nodes by class and cannot name a group, so a second group - // of the same class would be unreachable. - NodeGroups map[StorageClass]ResourceConfigNodeGroup `json:"nodeGroups" description:"Node groups the resource requires from the substrate, keyed by storage class"` + Networking ResourceConfigNetworking `json:"networking,omitzero" description:"The network the substrate runs in"` + + IAM ResourceConfigIAM `json:"iam,omitzero" description:"Identity the substrate's workloads assume"` + + CloudLabels map[string]string `json:"cloudLabels,omitempty" description:"Tags applied to every resource the substrate provisions"` + + InstanceGroups map[string]ResourceConfigInstanceGroup `json:"instanceGroups,omitempty" description:"Pools of nodes the resource requires, keyed by a reference of your choosing"` + + // Stating this is an error, not an override: a name that disagrees with the + // tag derived beside it matches nothing. + Resources *ResourceConfigResources `json:"resources,omitempty" description:"Derived names and tags for everything the substrate provisions; written by foundry"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigNetworking follows kOps' NetworkingSpec. +type ResourceConfigNetworking struct { + // A network is adopted whole: every subnet then states its own id. + NetworkID string `json:"networkID,omitempty" description:"Provider ID of an existing network to adopt; empty creates one" example:"vpc-0a1b2c3d"` + + NetworkCIDR string `json:"networkCIDR,omitempty" description:"CIDR block for the network" example:"10.0.0.0/16"` + + Subnets map[string]ResourceConfigSubnet `json:"subnets,omitempty" description:"Subnets carved out of the network, keyed by a reference of your choosing"` _ struct{} `additionalProperties:"false"` } -// ResourceConfigNodeGroup sizes a pool of nodes in the vocabulary every node-pool -// abstraction already uses -- kOps', GKE's and eksctl's terms -- narrowed to what -// foundry has to understand. Capacity is criteria so the document stays portable, -// with machineType as the escape hatch when a concrete type is wanted. -type ResourceConfigNodeGroup struct { +// ResourceConfigSubnet follows kOps' ClusterSubnetSpec. Zone has no default: +// letters are not contiguous within a region and the mapping is per-account. +type ResourceConfigSubnet struct { + Type v1alpha1.SubnetType `json:"type,omitzero" description:"Whether the subnet routes to an internet gateway"` + + Zone string `json:"zone,omitempty" description:"Availability zone the subnet lives in" example:"us-east-1a"` + + CIDR string `json:"cidr,omitempty" description:"CIDR block for the subnet, carved out of the network" example:"10.0.0.0/19"` + + // Private subnets only; empty creates a gateway in a public subnet of the + // same zone. + Egress string `json:"egress,omitempty" description:"Provider ID of an existing NAT gateway this private subnet routes through; empty creates one" example:"nat-0a1b2c3d"` + + ID string `json:"id,omitempty" description:"Provider ID of an existing subnet to adopt; empty creates one" example:"subnet-0a1b2c3d"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigIAM constrains the roles the substrate creates; which roles +// exist is the platform's, and their names are derived. +type ResourceConfigIAM struct { + PermissionsBoundary string `json:"permissionsBoundary,omitempty" description:"Policy ARN attached as the permissions boundary of every role the substrate creates"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigInstanceGroup follows kOps' InstanceGroupSpec, narrowed to what +// foundry has to understand. +type ResourceConfigInstanceGroup struct { + Storage v1alpha1.StorageClass `json:"storage,omitzero" description:"Durability of the group's storage, and the only fact about it a consuming casting can select on"` + + MachineType string `json:"machineType,omitempty" description:"Provider machine type for each node in the group" example:"m5.large"` + // MinSize and MaxSize are equal on a pinned group. MinSize *int `json:"minSize,omitempty" minimum:"0" description:"Minimum number of nodes in the group"` MaxSize *int `json:"maxSize,omitempty" minimum:"0" description:"Maximum number of nodes in the group"` - MachineType string `json:"machineType,omitempty" description:"Provider machine type; empty resolves one from cpu and memory" example:"m5.large"` - - CPU *int `json:"cpu,omitempty" minimum:"1" description:"CPUs per node, used when machineType is not stated"` + // References into networking.subnets; a pinned group's nodes are laid out + // across them in order. + Subnets []string `json:"subnets,omitempty" description:"Subnet references the group's nodes are placed in"` - Memory *int `json:"memory,omitempty" minimum:"1" description:"Memory per node in GB, used when machineType is not stated"` - - RootVolume ResourceConfigVolume `json:"rootVolume,omitzero" description:"The disk each node boots from"` + RootVolume ResourceConfigVolume `json:"rootVolume,omitzero" description:"The disk each node boots from, which dies with it"` DataVolume *ResourceConfigVolume `json:"dataVolume,omitempty" description:"Volume attached to each node that outlives it; persistent storage class only"` _ struct{} `additionalProperties:"false"` } +// ResourceConfigVolume follows kOps' VolumeSpec. type ResourceConfigVolume struct { Size *int `json:"size,omitempty" minimum:"1" description:"Size of the volume in GB"` + Type string `json:"type,omitempty" description:"Provider volume type" example:"gp3"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResources is every name and tag derived from the declaration. +// Templates interpolate these rather than assembling their own. +type ResourceConfigResources struct { + Cluster ResourceConfigResource `json:"cluster,omitzero"` + + VPC ResourceConfigResource `json:"vpc,omitzero"` + + InternetGateway ResourceConfigResource `json:"internetGateway,omitzero"` + + SecurityGroup ResourceConfigResource `json:"securityGroup,omitzero"` + + SecurityGroupRules map[string]ResourceConfigResource `json:"securityGroupRules,omitempty"` + + Roles map[string]ResourceConfigResource `json:"roles,omitempty"` + + InstanceProfile ResourceConfigResource `json:"instanceProfile,omitzero"` + + // Keyed by the subnet reference they serve. Not parallel: a public subnet + // has no NAT gateway, and neither has an adopted one. + Subnets map[string]ResourceConfigResourceSubnet `json:"subnets,omitempty"` + + RouteTables map[string]ResourceConfigResource `json:"routeTables,omitempty"` + + NATGateways map[string]ResourceConfigResourceNATGateway `json:"natGateways,omitempty"` + + InstanceGroups map[string]ResourceConfigResourceGroup `json:"instanceGroups,omitempty"` + + // Stamped after provisioning by whatever claims a resource; reconciling + // them reverts a live claim on every apply. + IgnoredTags []string `json:"ignoredTags,omitempty"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResource is one derived thing: what to call it and what to +// stamp on it. +type ResourceConfigResource struct { + Name string `json:"name,omitempty"` + + Tags map[string]string `json:"tags,omitempty"` + + // Set when adopted rather than created; the casting then stamps nothing. + ID string `json:"id,omitempty"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResourceSubnet resolves a declared subnet. +type ResourceConfigResourceSubnet struct { + Name string `json:"name,omitempty"` + + Tags map[string]string `json:"tags,omitempty"` + + ID string `json:"id,omitempty"` + + Public bool `json:"public"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResourceNATGateway is the egress path of one private subnet. +type ResourceConfigResourceNATGateway struct { + Name string `json:"name,omitempty"` + + Tags map[string]string `json:"tags,omitempty"` + + ID string `json:"id,omitempty"` + + // The public subnet it sits in, in the same zone as the one it serves. + Subnet string `json:"subnet,omitempty"` + + Address *ResourceConfigResource `json:"address,omitempty"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResourceGroup is what a declared instance group resolves to. A +// pinned group has Nodes and no autoscaling group; a scaling one is the +// reverse, which is how a casting tells them apart. +type ResourceConfigResourceGroup struct { + Storage v1alpha1.StorageClass `json:"storage,omitzero"` + + // The tag match that finds this group's nodes. The substrate advertises it + // wherever the platform does placement. + Selector map[string]string `json:"selector,omitempty"` + + Subnets []string `json:"subnets,omitempty"` + + LaunchTemplate *ResourceConfigResource `json:"launchTemplate,omitempty"` + + AutoscalingGroup *ResourceConfigResource `json:"autoscalingGroup,omitempty"` + + Nodes []ResourceConfigResourceNode `json:"nodes,omitempty"` + + _ struct{} `additionalProperties:"false"` +} + +// ResourceConfigResourceNode is one node of a pinned group. Its volume is +// stated inside it so the two cannot land in different zones. +type ResourceConfigResourceNode struct { + Name string `json:"name,omitempty"` + + Tags map[string]string `json:"tags,omitempty"` + + Ordinal int `json:"ordinal"` + + Subnet string `json:"subnet,omitempty"` + + Volume *ResourceConfigResource `json:"volume,omitempty"` + _ struct{} `additionalProperties:"false"` } diff --git a/api/v1alpha1/infrastructure/resource_kind.go b/api/v1alpha1/infrastructure/resource_kind.go deleted file mode 100644 index a08b7cf8..00000000 --- a/api/v1alpha1/infrastructure/resource_kind.go +++ /dev/null @@ -1,83 +0,0 @@ -package infrastructure - -import ( - "encoding/json" - "errors" - "fmt" - - "github.com/signoz/foundry/api/v1alpha1" - "github.com/swaggest/jsonschema-go" - "go.yaml.in/yaml/v3" -) - -var _ yaml.Marshaler = (*ResourceKind)(nil) -var _ yaml.Unmarshaler = (*ResourceKind)(nil) -var _ json.Marshaler = (*ResourceKind)(nil) -var _ json.Unmarshaler = (*ResourceKind)(nil) -var _ fmt.Stringer = (*ResourceKind)(nil) -var _ jsonschema.Enum = (*ResourceKind)(nil) - -var ( - ResourceKindInstallation ResourceKind = ResourceKind{s: v1alpha1.KindInstallation.String()} - ResourceKindCollectionAgent ResourceKind = ResourceKind{s: v1alpha1.KindCollectionAgent.String()} -) - -type ResourceKind struct { - s string -} - -func (kind ResourceKind) String() string { - return kind.s -} - -func ResourceKinds() []ResourceKind { - return []ResourceKind{ResourceKindInstallation, ResourceKindCollectionAgent} -} - -func (kind ResourceKind) MarshalJSON() ([]byte, error) { - return json.Marshal(kind.String()) -} - -func (kind *ResourceKind) UnmarshalJSON(text []byte) error { - var str string - if err := json.Unmarshal(text, &str); err != nil { - return err - } - - return kind.UnmarshalText([]byte(str)) -} - -func (kind *ResourceKind) UnmarshalText(text []byte) error { - for _, availableKind := range ResourceKinds() { - if availableKind.String() == string(text) { - *kind = availableKind - return nil - } - } - if text == nil { - *kind = ResourceKind{s: ""} - return nil - } - return errors.New("invalid resource kind: " + string(text)) -} - -func (kind ResourceKind) MarshalText() ([]byte, error) { - return []byte(kind.String()), nil -} - -func (kind *ResourceKind) UnmarshalYAML(node *yaml.Node) error { - return kind.UnmarshalText([]byte(node.Value)) -} - -func (kind ResourceKind) MarshalYAML() (any, error) { - return kind.String(), nil -} - -func (kind ResourceKind) Enum() []any { - kinds := []any{} - for _, kind := range ResourceKinds() { - kinds = append(kinds, kind.String()) - } - - return kinds -} diff --git a/api/v1alpha1/infrastructure/schema_test.go b/api/v1alpha1/infrastructure/schema_test.go index a9113758..fea071a4 100644 --- a/api/v1alpha1/infrastructure/schema_test.go +++ b/api/v1alpha1/infrastructure/schema_test.go @@ -4,6 +4,8 @@ import ( "encoding/json" "testing" + "github.com/signoz/foundry/api/v1alpha1" + "github.com/stretchr/testify/assert" ) @@ -18,23 +20,27 @@ func TestSchemaValidate(t *testing.T) { pass bool }{ { - name: "InstallationResource_Valid", + name: "Deployment_Valid", mutate: func(casting *Casting) { - casting.Spec.Resource.Kind = ResourceKindInstallation + casting.Spec.Deployment = v1alpha1.TypeDeployment{ + Platform: v1alpha1.PlatformECS, + Mode: v1alpha1.ModeEC2, + Flavor: v1alpha1.FlavorTerraform, + } }, pass: true, }, { - name: "CollectionAgentResource_Valid", + name: "NameMissing_Invalid", mutate: func(casting *Casting) { - casting.Spec.Resource.Kind = ResourceKindCollectionAgent + casting.Spec.Deployment = v1alpha1.TypeDeployment{ + Platform: v1alpha1.PlatformECS, + Mode: v1alpha1.ModeEC2, + Flavor: v1alpha1.FlavorTerraform, + } + casting.Metadata.Name = "" }, - pass: true, - }, - { - name: "ResourceKindMissing_Invalid", - mutate: func(casting *Casting) {}, - pass: false, + pass: false, }, } diff --git a/api/v1alpha1/infrastructure/storage_class.go b/api/v1alpha1/storage_class.go similarity index 99% rename from api/v1alpha1/infrastructure/storage_class.go rename to api/v1alpha1/storage_class.go index ddd7f165..8e33a9a7 100644 --- a/api/v1alpha1/infrastructure/storage_class.go +++ b/api/v1alpha1/storage_class.go @@ -1,4 +1,4 @@ -package infrastructure +package v1alpha1 import ( "encoding/json" diff --git a/api/v1alpha1/infrastructure/storage_class_test.go b/api/v1alpha1/storage_class_test.go similarity index 98% rename from api/v1alpha1/infrastructure/storage_class_test.go rename to api/v1alpha1/storage_class_test.go index 5b251bff..4f4e3d5c 100644 --- a/api/v1alpha1/infrastructure/storage_class_test.go +++ b/api/v1alpha1/storage_class_test.go @@ -1,4 +1,4 @@ -package infrastructure +package v1alpha1 import ( "testing" diff --git a/api/v1alpha1/subnet_type.go b/api/v1alpha1/subnet_type.go new file mode 100644 index 00000000..8a916d2f --- /dev/null +++ b/api/v1alpha1/subnet_type.go @@ -0,0 +1,101 @@ +package v1alpha1 + +import ( + "encoding/json" + "errors" + "fmt" + + "github.com/swaggest/jsonschema-go" + "go.yaml.in/yaml/v3" +) + +var _ yaml.Marshaler = (*SubnetType)(nil) +var _ yaml.Unmarshaler = (*SubnetType)(nil) +var _ json.Marshaler = (*SubnetType)(nil) +var _ json.Unmarshaler = (*SubnetType)(nil) +var _ fmt.Stringer = (*SubnetType)(nil) +var _ jsonschema.Enum = (*SubnetType)(nil) + +var ( + // SubnetTypePrivate subnets have no route to an internet gateway. Workloads + // go here; egress, where a subnet needs it, is a NAT gateway's job. + SubnetTypePrivate = SubnetType{s: "private"} + + // SubnetTypePublic subnets route to an internet gateway and are where the + // NAT gateways serving the private ones live. + SubnetTypePublic = SubnetType{s: "public", public: true} +) + +// SubnetType is whether a subnet faces the internet. A consuming casting +// filters on it: every workload it places needs a subnet, and this is the only +// fact about a subnet that both castings can predict independently. +type SubnetType struct { + s string + public bool +} + +func (subnetType SubnetType) String() string { + return subnetType.s +} + +// IsPublic reports whether the subnet routes to an internet gateway, and so +// whether a NAT gateway may be placed in it. +func (subnetType SubnetType) IsPublic() bool { + return subnetType.public +} + +func SubnetTypes() []SubnetType { + return []SubnetType{SubnetTypePrivate, SubnetTypePublic} +} + +func (subnetType SubnetType) MarshalJSON() ([]byte, error) { + return json.Marshal(subnetType.String()) +} + +func (subnetType *SubnetType) UnmarshalJSON(text []byte) error { + var str string + if err := json.Unmarshal(text, &str); err != nil { + return err + } + + return subnetType.UnmarshalText([]byte(str)) +} + +func (subnetType *SubnetType) UnmarshalText(text []byte) error { + for _, available := range SubnetTypes() { + if available.String() == string(text) { + *subnetType = available + return nil + } + } + + // A nil slice is an absent value, which leaves the zero subnetType; an + // empty string is a stated value that names none, and falls through. + if text == nil { + *subnetType = SubnetType{} + return nil + } + + return errors.New("invalid subnetType: " + string(text)) +} + +func (subnetType SubnetType) MarshalText() ([]byte, error) { + return []byte(subnetType.String()), nil +} + +func (subnetType *SubnetType) UnmarshalYAML(node *yaml.Node) error { + return subnetType.UnmarshalText([]byte(node.Value)) +} + +func (subnetType SubnetType) MarshalYAML() (any, error) { + return subnetType.String(), nil +} + +func (subnetType SubnetType) Enum() []any { + subnetTypes := []any{} + for _, subnetType := range SubnetTypes() { + subnetTypes = append(subnetTypes, subnetType.String()) + } + + return subnetTypes +} diff --git a/api/v1alpha1/subnet_type_test.go b/api/v1alpha1/subnet_type_test.go new file mode 100644 index 00000000..881d460f --- /dev/null +++ b/api/v1alpha1/subnet_type_test.go @@ -0,0 +1,65 @@ +package v1alpha1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSubnetTypeUnmarshalText(t *testing.T) { + tests := []struct { + name string + input string + pass bool + expected SubnetType + }{ + {name: "Private_Valid", input: "private", pass: true, expected: SubnetTypePrivate}, + {name: "Public_Valid", input: "public", pass: true, expected: SubnetTypePublic}, + // An absent key never reaches the unmarshaler; an explicitly empty one + // is a stated value that names no subnetType. + {name: "Empty_Invalid", input: "", pass: false}, + {name: "Unknown_Invalid", input: "internal", pass: false}, + {name: "Capitalised_Invalid", input: "Private", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + subnetType := SubnetType{} + err := subnetType.UnmarshalText([]byte(tt.input)) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expected, subnetType) + + // Round-trip: what the subnetType renders unmarshals back to itself. + roundTripped := SubnetType{} + assert.NoError(t, roundTripped.UnmarshalText([]byte(subnetType.String()))) + assert.Equal(t, subnetType, roundTripped) + }) + } +} + +func TestSubnetTypeImplications(t *testing.T) { + tests := []struct { + name string + subnetType SubnetType + expectedPublic bool + }{ + {name: "Public_RoutesToTheInternet", subnetType: SubnetTypePublic, expectedPublic: true}, + {name: "Private_DoesNot", subnetType: SubnetTypePrivate, expectedPublic: false}, + {name: "Unset_DoesNot", subnetType: SubnetType{}, expectedPublic: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedPublic, tt.subnetType.IsPublic()) + }) + } +} + +func TestSubnetTypeEnum(t *testing.T) { + assert.Equal(t, []any{"private", "public"}, SubnetTypePrivate.Enum()) +} diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 8257615d..7c0ff9d5 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -48,8 +48,13 @@ func NewPlanner(ctx context.Context, c *infrastructure.Casting, logger *slog.Log return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") } + deriver, err := registry.Deriver(c.Spec.Deployment) + if err != nil { + return nil, err + } + moldings := []infrastructuremolding.Molding{ - resourcemolding.New(logger), + resourcemolding.New(logger, deriver), } return &Planner{ diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 110c1b53..1bade53d 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -5,12 +5,16 @@ import ( "github.com/signoz/foundry/api/v1alpha1" foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" "github.com/signoz/foundry/internal/tooler" ) type CastingItem struct { Casting Casting Toolers []tooler.Tooler + + // Deriver walks this platform's topology. Same choice as the casting. + Deriver infrastructuremolding.Deriver } type Registry struct { @@ -45,3 +49,12 @@ func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler. } return item.Toolers, nil } + +func (registry *Registry) Deriver(deployment v1alpha1.TypeDeployment) (infrastructuremolding.Deriver, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "infrastructure deployment '%+v' is not supported", deployment) + } + + return item.Deriver, nil +} diff --git a/internal/config/yamlconfig/config_test.go b/internal/config/yamlconfig/config_test.go index 3e18cc52..30cb864b 100644 --- a/internal/config/yamlconfig/config_test.go +++ b/internal/config/yamlconfig/config_test.go @@ -374,13 +374,12 @@ func TestGetV1Alpha1Merge(t *testing.T) { func TestGetV1Alpha1Infrastructure(t *testing.T) { tests := []struct { - name string - input string - expectedResource infrastructure.ResourceKind - pass bool + name string + input string + pass bool }{ { - name: "InstallationResource_Valid", + name: "Deployment_Valid", input: ` apiVersion: v1alpha1 kind: Infrastructure @@ -391,19 +390,15 @@ spec: platform: ecs mode: ec2 flavor: terraform - resource: - kind: Installation `, - expectedResource: infrastructure.ResourceKindInstallation, - pass: true, + pass: true, }, { - name: "ResourceMissing_Invalid", + name: "NameMissing_Invalid", input: ` apiVersion: v1alpha1 kind: Infrastructure -metadata: - name: signoz +metadata: {} spec: deployment: platform: ecs @@ -413,7 +408,7 @@ spec: pass: false, }, { - name: "SelfReference_Invalid", + name: "UnknownPlatform_Invalid", input: ` apiVersion: v1alpha1 kind: Infrastructure @@ -421,27 +416,9 @@ metadata: name: signoz spec: deployment: - platform: ecs - mode: ec2 - flavor: terraform - resource: - kind: Infrastructure -`, - pass: false, - }, - { - name: "ResourceKindMissing_Invalid", - input: ` -apiVersion: v1alpha1 -kind: Infrastructure -metadata: - name: signoz -spec: - deployment: - platform: ecs + platform: nowhere mode: ec2 flavor: terraform - resource: {} `, pass: false, }, @@ -466,7 +443,6 @@ spec: return } assert.Equal(t, v1alpha1.KindInfrastructure, casting.Kind()) - assert.Equal(t, tt.expectedResource, casting.Spec.Resource.Kind) }) } } diff --git a/internal/convention/identity.go b/internal/convention/identity.go index d08160c5..5f3c8e3c 100644 --- a/internal/convention/identity.go +++ b/internal/convention/identity.go @@ -11,8 +11,8 @@ import ( const identitySeparator = "," // Identity is a stateful seat that claims a volume and keeps its data across an -// instance replacement: a component and its ordinals, "telemetrystore-0-0". It -// carries no substrate prefix, because deployed claims are spelled this way. +// instance replacement: a component and its ordinals, "telemetrystore-0-0". +// Deployed claims carry no substrate prefix. type Identity struct { s string } @@ -22,8 +22,7 @@ func NewIdentity(component string, ordinals ...int) (Identity, error) { return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity: component is empty") } - // The separator carries the encoding, so one inside a component would split - // into two identities on the way back. + // A separator inside a component would split into two on the way back. if strings.Contains(component, identitySeparator) { return Identity{}, errors.Newf(errors.TypeInvalidInput, "failed to create identity from %q: component contains %q", component, identitySeparator) } @@ -81,13 +80,13 @@ func (identity Identity) String() string { return identity.s } -// Identities is the claim record one volume carries, encoded as a single tag -// value: sorted and separator-joined, matching Terraform's join and split. -// Sorting keeps the value stable so an unchanged claim set produces no diff. +// Identities is the claim record one volume carries, sorted and +// separator-joined to match Terraform's join and split. Sorting keeps an +// unchanged claim set from producing a diff. // -// Only a platform with no stateful identity primitive of its own needs a claim -// record; Kubernetes, compose, swarm and systemd each bind an identity to its -// disk themselves. Empty is the norm and stamps no tag. +// Only platforms without a stateful identity primitive need this. Kubernetes, +// compose, swarm and systemd bind an identity to its disk themselves. The comma +// encoding is legal on AWS and Azure, illegal on GCP. type Identities []Identity func (identities Identities) String() string { @@ -99,8 +98,8 @@ func (identities Identities) String() string { return strings.Join(parts, identitySeparator) } -// ParseIdentities is the counterpart of String, validating through ParseIdentity -// so there is one path into the type. +// ParseIdentities is the counterpart of String. It validates through +// ParseIdentity, keeping one path into the type. func ParseIdentities(value string) (Identities, error) { if strings.TrimSpace(value) == "" { return nil, nil diff --git a/internal/convention/key.go b/internal/convention/key.go new file mode 100644 index 00000000..511a58a5 --- /dev/null +++ b/internal/convention/key.go @@ -0,0 +1,37 @@ +package convention + +import ( + "github.com/signoz/foundry/internal/errors" +) + +// Key is the reference an operator chooses for one declared thing, a subnet or +// an instance group. Identity lives in the key. A partial override restates the +// key and nothing else. +type Key struct { + s string +} + +func NewKey(key string) (Key, error) { + if key == "" { + return Key{}, errors.Newf(errors.TypeInvalidInput, "failed to create key from %q: key is empty", key) + } + + if !namePattern.MatchString(key) { + return Key{}, errors.Newf(errors.TypeInvalidInput, "failed to create key from %q: key is not lowercase alphanumeric with interior hyphens", key) + } + + return Key{s: key}, nil +} + +func MustNewKey(key string) Key { + parsed, err := NewKey(key) + if err != nil { + panic(err) + } + + return parsed +} + +func (key Key) String() string { + return key.s +} diff --git a/internal/convention/key_test.go b/internal/convention/key_test.go new file mode 100644 index 00000000..45fb737c --- /dev/null +++ b/internal/convention/key_test.go @@ -0,0 +1,50 @@ +package convention + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewKey(t *testing.T) { + tests := []struct { + name string + input string + pass bool + expectedKey string + }{ + {name: "Word_Valid", input: "persistent", pass: true, expectedKey: "persistent"}, + {name: "InteriorHyphens_Valid", input: "private-us-east-1a", pass: true, expectedKey: "private-us-east-1a"}, + {name: "Digits_Valid", input: "pool2", pass: true, expectedKey: "pool2"}, + {name: "Empty_Invalid", input: "", pass: false}, + {name: "Uppercase_Invalid", input: "Private", pass: false}, + {name: "LeadingHyphen_Invalid", input: "-private", pass: false}, + {name: "TrailingHyphen_Invalid", input: "private-", pass: false}, + {name: "Underscore_Invalid", input: "private_a", pass: false}, + {name: "Dot_Invalid", input: "private.a", pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + key, err := NewKey(tt.input) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expectedKey, key.String()) + }) + } +} + +// A key reaches a derived name unchanged, so what it accepts is what a name +// segment may contain. +func TestKeyAcceptsWhatASubstrateNameDoes(t *testing.T) { + for _, name := range []string{"foundry", "signoz-prod-eu", "signoz2"} { + key, err := NewKey(name) + + assert.NoError(t, err) + assert.Equal(t, name, key.String()) + } +} diff --git a/internal/convention/node_group.go b/internal/convention/node_group.go new file mode 100644 index 00000000..9e8712ee --- /dev/null +++ b/internal/convention/node_group.go @@ -0,0 +1,24 @@ +package convention + +import ( + "github.com/signoz/foundry/api/v1alpha1" +) + +// NodeGroup is a pool of interchangeable nodes. The key names it. The storage +// class selects it, being the only fact about it a consumer can predict. +type NodeGroup struct { + key Key + storage v1alpha1.StorageClass +} + +func NewNodeGroup(key Key, storage v1alpha1.StorageClass) NodeGroup { + return NodeGroup{key: key, storage: storage} +} + +func (group NodeGroup) Key() Key { + return group.key +} + +func (group NodeGroup) Storage() v1alpha1.StorageClass { + return group.storage +} diff --git a/internal/convention/node_group_test.go b/internal/convention/node_group_test.go new file mode 100644 index 00000000..7e3ba128 --- /dev/null +++ b/internal/convention/node_group_test.go @@ -0,0 +1,25 @@ +package convention + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/stretchr/testify/assert" +) + +// The key names the group and the class selects it: two groups may share a +// class, and a consumer that filters on the class reaches both. +func TestNodeGroup(t *testing.T) { + hot := NewNodeGroup(MustNewKey("hot"), v1alpha1.StorageClassPersistent) + cold := NewNodeGroup(MustNewKey("cold"), v1alpha1.StorageClassPersistent) + + assert.Equal(t, "hot", hot.Key().String()) + assert.Equal(t, "cold", cold.Key().String()) + assert.Equal(t, hot.Storage(), cold.Storage()) + + filter := MustNewSubstrate("foundry").Select().WithStorage(v1alpha1.StorageClassPersistent).Match() + assert.Equal(t, map[TagKey]string{ + TagKeyName: "foundry", + TagKeyStorage: "persistent", + }, filter) +} diff --git a/internal/convention/ownership.go b/internal/convention/ownership.go index c3334a2a..2a842495 100644 --- a/internal/convention/ownership.go +++ b/internal/convention/ownership.go @@ -12,7 +12,7 @@ var ( OwnershipShared = Ownership{s: "shared", shared: true} ) -// String resolves the zero value to owned, which is what saying nothing means. +// String resolves the zero value to owned. func (ownership Ownership) String() string { if ownership.s == "" { return OwnershipOwned.s diff --git a/internal/convention/resource.go b/internal/convention/resource.go deleted file mode 100644 index 60dc43f8..00000000 --- a/internal/convention/resource.go +++ /dev/null @@ -1,152 +0,0 @@ -package convention - -import ( - "strings" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" -) - -// Resource is one thing a substrate provisions, described once. Its name, its -// tags, and the selection that finds it are all derived from that description, so -// a fact stated once cannot render two ways. -// -// Callers use the constructor for what they are provisioning; it supplies the -// resource type. -type Resource struct { - substrate Substrate - resourceType resourceType - - visibility Visibility - storage infrastructure.StorageClass - zone Zone - role Role - ordinal int - - ownership Ownership - kind infrastructure.ResourceKind - identities Identities -} - -func (s Substrate) Cluster() Resource { - return Resource{substrate: s, resourceType: typeCluster} -} - -func (s Substrate) VPC() Resource { - return Resource{substrate: s, resourceType: typeVPC} -} - -func (s Substrate) InternetGateway() Resource { - return Resource{substrate: s, resourceType: typeInternetGateway} -} - -func (s Substrate) Subnet(visibility Visibility, zone Zone) Resource { - return Resource{substrate: s, resourceType: typeSubnet, visibility: visibility, zone: zone} -} - -// RouteTable is a table shared across zones, which one internet gateway serves. -func (s Substrate) RouteTable(visibility Visibility) Resource { - return Resource{substrate: s, resourceType: typeRouteTable, visibility: visibility} -} - -// RouteTableInZone is a table per zone, each routing to that zone's NAT gateway. -func (s Substrate) RouteTableInZone(visibility Visibility, zone Zone) Resource { - return Resource{substrate: s, resourceType: typeRouteTable, visibility: visibility, zone: zone} -} - -func (s Substrate) NATGateway(zone Zone) Resource { - return Resource{substrate: s, resourceType: typeNATGateway, zone: zone} -} - -func (s Substrate) SecurityGroup(role Role) Resource { - return Resource{substrate: s, resourceType: typeSecurityGroup, role: role} -} - -func (s Substrate) Role(role Role) Resource { - return Resource{substrate: s, resourceType: typeRole, role: role} -} - -// Node and Volume take the storage class, which is a node group's whole -// identity: a consuming casting selects by class and cannot name a group. -func (s Substrate) Node(storage infrastructure.StorageClass, ordinal int) Resource { - return Resource{substrate: s, resourceType: typeNode, storage: storage, ordinal: ordinal} -} - -func (s Substrate) Volume(storage infrastructure.StorageClass, ordinal int) Resource { - return Resource{substrate: s, resourceType: typeVolume, storage: storage, ordinal: ordinal} -} - -// WithOwnership marks a resource adopted rather than created. It changes no name. -func (r Resource) WithOwnership(ownership Ownership) Resource { - r.ownership = ownership - - return r -} - -// WithKind records the Kind the substrate is provisioned for. -func (r Resource) WithKind(kind infrastructure.ResourceKind) Resource { - r.kind = kind - - return r -} - -// WithClaims records the identities holding a volume. See Identities. -func (r Resource) WithClaims(identities Identities) Resource { - r.identities = identities - - return r -} - -// Name is -[-...], broad to narrow so a substrate's -// resources share a prefix and sort together. It fills a provider's name argument -// where one exists, and the display tag always -- an instance or a volume has no -// name of its own. -func (r Resource) Name() string { - parts := make([]string, 0, len(r.resourceType.qualifiers)+2) - parts = append(parts, r.substrate.name, r.resourceType.String()) - - for _, qualifier := range r.resourceType.qualifiers { - if segment := qualifier.of(r); segment != "" { - parts = append(parts, segment) - } - } - - return strings.Join(parts, "-") -} - -// Selection is the set that finds exactly this resource. -func (r Resource) Selection() Selection { - return Selection{substrate: r.substrate, storage: r.storage, identities: r.identities} -} - -// stamp is the selection's tags plus the provenance nothing reads back. -func (r Resource) stamp() Tags { - tags := r.Selection().match() - - tags = append(tags, Tag{Key: TagKeyOwner, Value: r.ownership.String()}) - - // An adopted resource keeps the name it already had. - if !r.ownership.IsShared() { - tags = append(tags, Tag{Key: TagKeyDisplayName, Value: r.Name()}) - } - - if r.kind != (infrastructure.ResourceKind{}) { - tags = append(tags, Tag{Key: TagKeyResourceKind, Value: r.kind.String()}) - } - - if r.visibility != (Visibility{}) { - tags = append(tags, Tag{Key: TagKeyVisibility, Value: r.visibility.String()}) - } - - return tags -} - -// Tags is every tag this resource carries. Ownership labels are a separate -// family: a casting merges CastingMeta.Labels() in alongside these. -func (r Resource) Tags() map[string]string { - return r.stamp().Map() -} - -// Filter is the tag match that finds this resource. -func (r Resource) Filter() map[string]string { - return r.Selection().Filter() -} diff --git a/internal/convention/resource_test.go b/internal/convention/resource_test.go deleted file mode 100644 index 57f8142f..00000000 --- a/internal/convention/resource_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package convention - -import ( - "testing" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - "github.com/stretchr/testify/assert" -) - -func TestResourceName(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - persistent := infrastructure.StorageClassPersistent - ephemeral := infrastructure.StorageClassEphemeral - - tests := []struct { - name string - resource Resource - expectedName string - }{ - {name: "Cluster_Unqualified", resource: substrate.Cluster(), expectedName: "foundry-cls"}, - {name: "VPC_Unqualified", resource: substrate.VPC(), expectedName: "foundry-vpc"}, - {name: "InternetGateway_Unqualified", resource: substrate.InternetGateway(), expectedName: "foundry-igw"}, - {name: "PrivateSubnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPrivate, zone), expectedName: "foundry-sub-prv-east1a"}, - {name: "PublicSubnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPublic, zone), expectedName: "foundry-sub-pub-east1a"}, - {name: "PrivateRouteTable_PerZone", resource: substrate.RouteTableInZone(VisibilityPrivate, zone), expectedName: "foundry-rt-prv-east1a"}, - {name: "PublicRouteTable_ZoneShared", resource: substrate.RouteTable(VisibilityPublic), expectedName: "foundry-rt-pub"}, - {name: "NATGateway_PerZone", resource: substrate.NATGateway(zone), expectedName: "foundry-nat-east1a"}, - {name: "TaskSecurityGroup_Role", resource: substrate.SecurityGroup(RoleTask), expectedName: "foundry-sg-task"}, - {name: "ExecRole_Role", resource: substrate.Role(RoleExec), expectedName: "foundry-iam-exec"}, - {name: "Node_ClassAndOrdinal", resource: substrate.Node(persistent, 0), expectedName: "foundry-node-persistent-0"}, - {name: "Volume_ClassAndOrdinal", resource: substrate.Volume(persistent, 2), expectedName: "foundry-vol-persistent-2"}, - {name: "EphemeralNode_ClassAndOrdinal", resource: substrate.Node(ephemeral, 1), expectedName: "foundry-node-ephemeral-1"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expectedName, tt.resource.Name()) - }) - } -} - -// A role name is the longest suffix a caller has to budget for against its -// provider's cap, which this package does not know. -func TestRoleNameOverheadIsBounded(t *testing.T) { - const maxRoleSuffix = len("-iam-exec") - - for _, name := range []string{"a", "foundry", "signoz-prod-eu-central"} { - substrate := MustNewSubstrate(name) - assert.LessOrEqual(t, len(substrate.Role(RoleExec).Name())-len(name), maxRoleSuffix) - } -} - -// Adopting a resource must not rename it: the name belongs to whoever created it. -func TestSharedResourceKeepsItsName(t *testing.T) { - substrate := MustNewSubstrate("foundry") - shared := substrate.VPC().WithOwnership(OwnershipShared) - - assert.Equal(t, substrate.VPC().Name(), shared.Name()) - assert.NotContains(t, shared.Tags(), TagKeyDisplayName.String()) - assert.Equal(t, "shared", shared.Tags()[TagKeyOwner.String()]) -} - -func TestResourceTags(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - persistent := infrastructure.StorageClassPersistent - - tests := []struct { - name string - resource Resource - expectedPresent map[string]string - expectedAbsent []TagKey - }{ - { - name: "Cluster_CarriesIdentityAndOwner", - resource: substrate.Cluster(), - expectedPresent: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyOwner.String(): "owned", - TagKeyDisplayName.String(): "foundry-cls", - }, - expectedAbsent: []TagKey{TagKeyVisibility, TagKeyStorage, TagKeyIdentities, TagKeyResourceKind}, - }, - { - name: "PrivateSubnet_CarriesVisibilitySpelledOut", - resource: substrate.Subnet(VisibilityPrivate, zone).WithKind(infrastructure.ResourceKindInstallation), - expectedPresent: map[string]string{ - TagKeyDisplayName.String(): "foundry-sub-prv-east1a", - TagKeyVisibility.String(): "private", - TagKeyResourceKind.String(): "Installation", - }, - expectedAbsent: []TagKey{TagKeyStorage}, - }, - { - name: "PersistentNode_CarriesStorageFromItsGroup", - resource: substrate.Node(persistent, 0), - expectedPresent: map[string]string{ - TagKeyDisplayName.String(): "foundry-node-persistent-0", - TagKeyStorage.String(): "persistent", - }, - expectedAbsent: []TagKey{TagKeyVisibility, TagKeyIdentities}, - }, - { - name: "ClaimedVolume_CarriesIdentities", - resource: substrate.Volume(persistent, 0).WithClaims(Identities{ - MustNewIdentity("telemetrystore", 0, 0), - MustNewIdentity("metastore", 0), - }), - expectedPresent: map[string]string{ - TagKeyDisplayName.String(): "foundry-vol-persistent-0", - TagKeyStorage.String(): "persistent", - TagKeyIdentities.String(): "metastore-0,telemetrystore-0-0", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tags := tt.resource.Tags() - - for key, expected := range tt.expectedPresent { - assert.Equal(t, expected, tags[key], "tag %s", key) - } - - for _, key := range tt.expectedAbsent { - assert.NotContains(t, tags, key.String()) - } - }) - } -} - -func TestResourceFilter(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - persistent := infrastructure.StorageClassPersistent - - tests := []struct { - name string - resource Resource - expectedFilter map[string]string - }{ - { - name: "VPC_SelectsIdentityOnly", - resource: substrate.VPC(), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - }, - }, - { - name: "ProvenanceOnly_IsNotSelectedOn", - resource: substrate.Subnet(VisibilityPrivate, zone).WithKind(infrastructure.ResourceKindInstallation), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - }, - }, - { - name: "PersistentNode_SelectsIdentityAndStorage", - resource: substrate.Node(persistent, 0), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyStorage.String(): "persistent", - }, - }, - { - name: "ClaimedVolume_SelectsTheClaim", - resource: substrate.Volume(persistent, 0).WithClaims(Identities{MustNewIdentity("signoz", 0)}), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyStorage.String(): "persistent", - TagKeyIdentities.String(): "signoz-0", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expectedFilter, tt.resource.Filter()) - }) - } -} - -// A fact stated once must render the same way in the name and in the tag. -func TestNameAndTagsAgreeOnTheSameFact(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - - for _, visibility := range []Visibility{VisibilityPrivate, VisibilityPublic} { - subnet := substrate.Subnet(visibility, zone) - - assert.Contains(t, subnet.Name(), visibility.Short()) - assert.Equal(t, visibility.String(), subnet.Tags()[TagKeyVisibility.String()]) - } - - for _, storage := range []infrastructure.StorageClass{infrastructure.StorageClassPersistent, infrastructure.StorageClassEphemeral} { - node := substrate.Node(storage, 0) - - assert.Contains(t, node.Name(), storage.String()) - assert.Equal(t, storage.String(), node.Tags()[TagKeyStorage.String()]) - } -} diff --git a/internal/convention/resource_type.go b/internal/convention/resource_type.go deleted file mode 100644 index f7e003a6..00000000 --- a/internal/convention/resource_type.go +++ /dev/null @@ -1,45 +0,0 @@ -package convention - -import ( - "strconv" -) - -// resourceType is what a derived name says the thing is, and the ordered -// qualifiers that narrow it. Adding a resource is one var entry below. -type resourceType struct { - short string - qualifiers []qualifier -} - -var ( - typeCluster = resourceType{short: "cls"} - typeVPC = resourceType{short: "vpc"} - typeInternetGateway = resourceType{short: "igw"} - typeSubnet = resourceType{short: "sub", qualifiers: []qualifier{qualifierVisibility, qualifierZone}} - typeRouteTable = resourceType{short: "rt", qualifiers: []qualifier{qualifierVisibility, qualifierZone}} - typeNATGateway = resourceType{short: "nat", qualifiers: []qualifier{qualifierZone}} - typeSecurityGroup = resourceType{short: "sg", qualifiers: []qualifier{qualifierRole}} - typeRole = resourceType{short: "iam", qualifiers: []qualifier{qualifierRole}} - typeNode = resourceType{short: "node", qualifiers: []qualifier{qualifierStorage, qualifierOrdinal}} - typeVolume = resourceType{short: "vol", qualifiers: []qualifier{qualifierStorage, qualifierOrdinal}} -) - -func (resource resourceType) String() string { - return resource.short -} - -// qualifier renders one axis into a name segment. An empty string drops the -// segment, so one route table declaration serves both the zonal and shared forms. -type qualifier struct { - of func(Resource) string -} - -var ( - qualifierVisibility = qualifier{of: func(r Resource) string { return r.visibility.Short() }} - qualifierZone = qualifier{of: func(r Resource) string { return r.zone.Short() }} - qualifierRole = qualifier{of: func(r Resource) string { return r.role.String() }} - qualifierStorage = qualifier{of: func(r Resource) string { return r.storage.String() }} - - // Only types that have an ordinal declare it, so zero renders as "0". - qualifierOrdinal = qualifier{of: func(r Resource) string { return strconv.Itoa(r.ordinal) }} -) diff --git a/internal/convention/resource_type_test.go b/internal/convention/resource_type_test.go deleted file mode 100644 index bc4ad504..00000000 --- a/internal/convention/resource_type_test.go +++ /dev/null @@ -1,75 +0,0 @@ -package convention - -import ( - "testing" - - "github.com/signoz/foundry/api/v1alpha1/infrastructure" - "github.com/stretchr/testify/assert" -) - -// Two types sharing a short form would derive the same name shape. -func TestResourceTypeShortFormsAreDistinct(t *testing.T) { - resourceTypes := []resourceType{ - typeCluster, typeVPC, typeInternetGateway, typeSubnet, typeRouteTable, - typeNATGateway, typeSecurityGroup, typeRole, typeNode, typeVolume, - } - - seen := make(map[string]struct{}, len(resourceTypes)) - for _, resource := range resourceTypes { - assert.NotContains(t, seen, resource.String()) - seen[resource.String()] = struct{}{} - } -} - -// An empty qualifier drops its segment, so one route table declaration serves -// both the zonal and the zone-shared form. -func TestUnsetQualifierDropsFromTheName(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - - assert.Equal(t, "foundry-rt-pub", substrate.RouteTable(VisibilityPublic).Name()) - assert.Equal(t, "foundry-rt-prv-east1a", substrate.RouteTableInZone(VisibilityPrivate, zone).Name()) -} - -// A declared qualifier that renders nothing would silently drop a segment meant -// to distinguish the name. -func TestEveryDeclaredQualifierContributes(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - persistent := infrastructure.StorageClassPersistent - - tests := []struct { - name string - resource Resource - expectedSegments int - }{ - {name: "VPC_NoQualifier", resource: substrate.VPC(), expectedSegments: 0}, - {name: "Subnet_VisibilityAndZone", resource: substrate.Subnet(VisibilityPrivate, zone), expectedSegments: 2}, - {name: "NATGateway_Zone", resource: substrate.NATGateway(zone), expectedSegments: 1}, - {name: "IAMRole_Role", resource: substrate.Role(RoleExec), expectedSegments: 1}, - {name: "Node_ClassAndOrdinal", resource: substrate.Node(persistent, 0), expectedSegments: 2}, - {name: "Volume_ClassAndOrdinal", resource: substrate.Volume(persistent, 0), expectedSegments: 2}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - rendered := 0 - for _, qualifier := range tt.resource.resourceType.qualifiers { - if qualifier.of(tt.resource) != "" { - rendered++ - } - } - - assert.Equal(t, tt.expectedSegments, rendered) - }) - } -} - -// Ordinal zero is a real ordinal, so only types that have one declare it. -func TestOrdinalZeroRenders(t *testing.T) { - substrate := MustNewSubstrate("foundry") - persistent := infrastructure.StorageClassPersistent - - assert.Equal(t, "foundry-node-persistent-0", substrate.Node(persistent, 0).Name()) - assert.Equal(t, "foundry-vpc", substrate.VPC().Name()) -} diff --git a/internal/convention/role.go b/internal/convention/role.go deleted file mode 100644 index 1e0d65e4..00000000 --- a/internal/convention/role.go +++ /dev/null @@ -1,17 +0,0 @@ -package convention - -// Role is what a security group or an IAM role is attached to. IAM roles use all -// three; a security group uses node and task. -type Role struct { - s string -} - -var ( - RoleNode = Role{s: "node"} - RoleTask = Role{s: "task"} - RoleExec = Role{s: "exec"} -) - -func (role Role) String() string { - return role.s -} diff --git a/internal/convention/role_test.go b/internal/convention/role_test.go deleted file mode 100644 index 8371348d..00000000 --- a/internal/convention/role_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package convention - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestRole(t *testing.T) { - tests := []struct { - name string - role Role - expectedWord string - }{ - {name: "Node_Rendered", role: RoleNode, expectedWord: "node"}, - {name: "Task_Rendered", role: RoleTask, expectedWord: "task"}, - {name: "Exec_Rendered", role: RoleExec, expectedWord: "exec"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expectedWord, tt.role.String()) - }) - } -} - -// Two roles sharing a rendering would collide in a derived name. -func TestRolesAreDistinct(t *testing.T) { - roles := []Role{RoleNode, RoleTask, RoleExec} - - seen := make(map[string]struct{}, len(roles)) - for _, role := range roles { - assert.NotContains(t, seen, role.String()) - seen[role.String()] = struct{}{} - } -} diff --git a/internal/convention/selection.go b/internal/convention/selection.go index 4598c67f..a862d329 100644 --- a/internal/convention/selection.go +++ b/internal/convention/selection.go @@ -1,16 +1,15 @@ package convention import ( - "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/api/v1alpha1" ) // Selection is what a consuming casting looks for: a substrate, narrowed by the -// facts it knows. Neither a name nor a resource type appears here because -// neither reaches a tag -- a filter for instances and one for volumes are the -// same tags, and the data source decides which it returns. +// facts it knows. type Selection struct { substrate Substrate - storage infrastructure.StorageClass + subnetType v1alpha1.SubnetType + storage v1alpha1.StorageClass identities Identities } @@ -18,39 +17,44 @@ func (s Substrate) Select() Selection { return Selection{substrate: s} } -func (selection Selection) WithStorage(storage infrastructure.StorageClass) Selection { +// WithSubnetType narrows to the subnets a workload may be placed in. +func (selection Selection) WithSubnetType(subnetType v1alpha1.SubnetType) Selection { + selection.subnetType = subnetType + + return selection +} + +func (selection Selection) WithStorage(storage v1alpha1.StorageClass) Selection { selection.storage = storage return selection } -// WithClaims narrows to the resource holding these identities. See Identities: -// only a platform with no stateful identity primitive of its own needs this. +// WithClaims narrows to the resource holding these identities. See Identities. func (selection Selection) WithClaims(identities Identities) Selection { selection.identities = identities return selection } -// match is the only place the tags a consumer depends on are decided. Resource -// stamps these plus provenance, so the two cannot disagree. -func (selection Selection) match() Tags { - tags := Tags{ - {Key: TagKeyName, Value: selection.substrate.name}, +// Match is the only place the facts a consumer depends on are decided. A +// resource stamps these plus provenance. +func (selection Selection) Match() map[TagKey]string { + tags := map[TagKey]string{ + TagKeyName: selection.substrate.name, } - if selection.storage != (infrastructure.StorageClass{}) { - tags = append(tags, Tag{Key: TagKeyStorage, Value: selection.storage.String()}) + if selection.subnetType != (v1alpha1.SubnetType{}) { + tags[TagKeySubnetType] = selection.subnetType.String() + } + + if selection.storage != (v1alpha1.StorageClass{}) { + tags[TagKeyStorage] = selection.storage.String() } if len(selection.identities) > 0 { - tags = append(tags, Tag{Key: TagKeyIdentities, Value: selection.identities.String()}) + tags[TagKeyIdentities] = selection.identities.String() } return tags } - -// Filter is the tag match a consuming casting writes into a data source. -func (selection Selection) Filter() map[string]string { - return selection.match().Map() -} diff --git a/internal/convention/selection_test.go b/internal/convention/selection_test.go index 35ef831b..df6088b8 100644 --- a/internal/convention/selection_test.go +++ b/internal/convention/selection_test.go @@ -3,7 +3,7 @@ package convention import ( "testing" - "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/api/v1alpha1" "github.com/stretchr/testify/assert" ) @@ -11,122 +11,57 @@ func TestSelectionFilter(t *testing.T) { substrate := MustNewSubstrate("foundry") tests := []struct { - name string - selection Selection - expectedFilter map[string]string + name string + selection Selection + expectedMatch map[TagKey]string }{ { name: "Substrate_MatchesEverythingItOwns", selection: substrate.Select(), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", + expectedMatch: map[TagKey]string{ + TagKeyName: "foundry", + }, + }, + { + name: "PrivateSubnet_MatchesTheType", + selection: substrate.Select().WithSubnetType(v1alpha1.SubnetTypePrivate), + expectedMatch: map[TagKey]string{ + TagKeyName: "foundry", + TagKeySubnetType: "private", }, }, { name: "PersistentClass_MatchesTheClass", - selection: substrate.Select().WithStorage(infrastructure.StorageClassPersistent), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyStorage.String(): "persistent", + selection: substrate.Select().WithStorage(v1alpha1.StorageClassPersistent), + expectedMatch: map[TagKey]string{ + TagKeyName: "foundry", + TagKeyStorage: "persistent", }, }, { name: "EphemeralClass_MatchesTheClass", - selection: substrate.Select().WithStorage(infrastructure.StorageClassEphemeral), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyStorage.String(): "ephemeral", + selection: substrate.Select().WithStorage(v1alpha1.StorageClassEphemeral), + expectedMatch: map[TagKey]string{ + TagKeyName: "foundry", + TagKeyStorage: "ephemeral", }, }, { name: "Claim_MatchesTheHolder", selection: substrate.Select(). - WithStorage(infrastructure.StorageClassPersistent). + WithStorage(v1alpha1.StorageClassPersistent). WithClaims(Identities{MustNewIdentity("telemetrystore", 0, 0)}), - expectedFilter: map[string]string{ - TagKeyName.String(): "foundry", - TagKeyStorage.String(): "persistent", - TagKeyIdentities.String(): "telemetrystore-0-0", + expectedMatch: map[TagKey]string{ + TagKeyName: "foundry", + TagKeyStorage: "persistent", + TagKeyIdentities: "telemetrystore-0-0", }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expectedFilter, tt.selection.Filter()) + assert.Equal(t, tt.expectedMatch, tt.selection.Match()) }) } } - -// A consumer states a class; the producer stamps it per node. The filter has to -// match at any ordinal. -func TestClassFilterMatchesWhatTheProducerStamped(t *testing.T) { - substrate := MustNewSubstrate("foundry") - consumer := substrate.Select().WithStorage(infrastructure.StorageClassPersistent).Filter() - - for ordinal := range 3 { - stamped := substrate.Node(infrastructure.StorageClassPersistent, ordinal).Tags() - - for key, value := range consumer { - assert.Equal(t, value, stamped[key], "node %d does not match the class filter on %s", ordinal, key) - } - } -} - -// A platform that tracks the identity-to-disk binding itself stamps no claim tag -// and filters on none. -func TestClaimsAreOptional(t *testing.T) { - substrate := MustNewSubstrate("foundry") - persistent := infrastructure.StorageClassPersistent - - volume := substrate.Volume(persistent, 0) - assert.NotContains(t, volume.Tags(), TagKeyIdentities.String()) - assert.NotContains(t, volume.Filter(), TagKeyIdentities.String()) - - selection := substrate.Select().WithStorage(infrastructure.StorageClassPersistent) - assert.NotContains(t, selection.Filter(), TagKeyIdentities.String()) - - // And an empty claim set is the same as never mentioning them. - assert.Equal(t, volume.Tags(), substrate.Volume(persistent, 0).WithClaims(Identities{}).Tags()) -} - -// A resource's filter is a subset of its tags, not a parallel list that has to -// agree with them. -func TestResourceContractTagsAreItsSelection(t *testing.T) { - substrate := MustNewSubstrate("foundry") - zone := MustParseZone("us-east-1a") - persistent := infrastructure.StorageClassPersistent - - resources := []Resource{ - substrate.Cluster(), - substrate.VPC().WithKind(infrastructure.ResourceKindCollectionAgent), - substrate.Subnet(VisibilityPrivate, zone), - substrate.NATGateway(zone), - substrate.Role(RoleExec), - substrate.Node(persistent, 0), - substrate.Volume(persistent, 1).WithClaims(Identities{MustNewIdentity("signoz", 0)}), - } - - for _, resource := range resources { - tags := resource.Tags() - - for key, value := range resource.Selection().Filter() { - assert.Equal(t, value, tags[key], "the selection filters on %s differently to how it is stamped", key) - } - } -} - -// A filter's keys are the only ones whose spelling live infrastructure depends -// on: renaming one leaves it unmatched, with no checkpoint to catch it. -func TestFilterKeysMatchDeployedSpelling(t *testing.T) { - filter := MustNewSubstrate("foundry").Select(). - WithStorage(infrastructure.StorageClassPersistent). - WithClaims(Identities{MustNewIdentity("signoz", 0)}). - Filter() - - assert.Equal(t, map[string]string{ - "foundry.signoz.io/name": "foundry", - "foundry.signoz.io/storage": "persistent", - "foundry.signoz.io/identities": "signoz-0", - }, filter) -} diff --git a/internal/convention/substrate.go b/internal/convention/substrate.go index cb3bb9ac..9e730f62 100644 --- a/internal/convention/substrate.go +++ b/internal/convention/substrate.go @@ -1,13 +1,9 @@ // Package convention derives the names and tags a provisioned substrate is -// identified by. +// identified by. Foundry never reads back from a platform, so a producing and a +// consuming casting must work them out the same way. Both are derived here. // -// Foundry generates rather than reconciles, so it can never ask a platform what -// it created. A consuming casting finds a producing casting's resources only by -// deriving the same names and filtering the same tags. Both sides are derived -// here so neither can drift. -// -// Provider limits are absent: a length cap belongs to the platform enforcing it, -// so a casting measures what it derives against its own provider's limits. +// Provider limits are absent. A casting measures what it derives against its +// own provider's caps. package convention import ( @@ -16,16 +12,15 @@ import ( "github.com/signoz/foundry/internal/errors" ) -// namePattern is what the strictest provider accepts as a name segment, and is -// shared by every name a caller supplies. +// namePattern is what the strictest provider accepts as a name segment. var namePattern = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`) // maxNameLength matches the metadata.name cap in the casting schema. const maxNameLength = 63 // Substrate is the infrastructure an installation runs on, known by the -// provisioning casting's metadata.name. Every resource is named and tagged from -// it, and it is the only fact a consumer needs to find them. +// provisioning casting's metadata.name. A consumer needs no other fact to find +// every resource. type Substrate struct { name string } diff --git a/internal/convention/tag.go b/internal/convention/tag.go index 36246f2a..8e83ca5e 100644 --- a/internal/convention/tag.go +++ b/internal/convention/tag.go @@ -1,44 +1,20 @@ package convention -import ( - "github.com/signoz/foundry/internal/domain" -) - -// TagKey is a tag's key. Which of these a consumer filters on is decided by -// Selection, not declared here. +// TagKey names one fact a substrate records, unqualified. Spelling is the +// provider's grammar and is rendered where the resource is created: a GCP label +// key rejects the dot and the slash, an Azure tag name rejects the slash. type TagKey struct { - key string + s string } var ( - TagKeyName = TagKey{key: domain.MetadataPrefix + "name"} - TagKeyStorage = TagKey{key: domain.MetadataPrefix + "storage"} - TagKeyIdentities = TagKey{key: domain.MetadataPrefix + "identities"} - TagKeyResourceKind = TagKey{key: domain.MetadataPrefix + "resource-kind"} - TagKeyOwner = TagKey{key: domain.MetadataPrefix + "owner"} - TagKeyVisibility = TagKey{key: domain.MetadataPrefix + "visibility"} - - // TagKeyDisplayName is unprefixed: "Name" is the provider's own convention - // for what a console shows. - TagKeyDisplayName = TagKey{key: "Name"} + TagKeyName = TagKey{s: "name"} + TagKeyStorage = TagKey{s: "storage"} + TagKeyIdentities = TagKey{s: "identities"} + TagKeyOwner = TagKey{s: "owner"} + TagKeySubnetType = TagKey{s: "subnet-type"} ) func (tagKey TagKey) String() string { - return tagKey.key -} - -type Tag struct { - Key TagKey - Value string -} - -type Tags []Tag - -func (tags Tags) Map() map[string]string { - out := make(map[string]string, len(tags)) - for _, tag := range tags { - out[tag.Key.String()] = tag.Value - } - - return out + return tagKey.s } diff --git a/internal/convention/tag_test.go b/internal/convention/tag_test.go index 0f20dd0b..fe22bef0 100644 --- a/internal/convention/tag_test.go +++ b/internal/convention/tag_test.go @@ -6,33 +6,36 @@ import ( "github.com/stretchr/testify/assert" ) +// A fact is unqualified: the provider that stamps it decides the spelling, so +// nothing here may carry a prefix a provider's grammar could reject. func TestTagKeys(t *testing.T) { tests := []struct { name string tagKey TagKey expectedKey string }{ - {name: "Name_Prefixed", tagKey: TagKeyName, expectedKey: "foundry.signoz.io/name"}, - {name: "Storage_Prefixed", tagKey: TagKeyStorage, expectedKey: "foundry.signoz.io/storage"}, - {name: "Identities_Prefixed", tagKey: TagKeyIdentities, expectedKey: "foundry.signoz.io/identities"}, - {name: "ResourceKind_Prefixed", tagKey: TagKeyResourceKind, expectedKey: "foundry.signoz.io/resource-kind"}, - {name: "Owner_Prefixed", tagKey: TagKeyOwner, expectedKey: "foundry.signoz.io/owner"}, - {name: "Visibility_Prefixed", tagKey: TagKeyVisibility, expectedKey: "foundry.signoz.io/visibility"}, - {name: "DisplayName_ProviderNative", tagKey: TagKeyDisplayName, expectedKey: "Name"}, + {name: "Name_Unqualified", tagKey: TagKeyName, expectedKey: "name"}, + {name: "Storage_Unqualified", tagKey: TagKeyStorage, expectedKey: "storage"}, + {name: "Identities_Unqualified", tagKey: TagKeyIdentities, expectedKey: "identities"}, + {name: "Owner_Unqualified", tagKey: TagKeyOwner, expectedKey: "owner"}, + {name: "SubnetType_Unqualified", tagKey: TagKeySubnetType, expectedKey: "subnet-type"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { assert.Equal(t, tt.expectedKey, tt.tagKey.String()) + assert.NotContains(t, tt.tagKey.String(), "/") + assert.NotContains(t, tt.tagKey.String(), ".") }) } } -// Two keys sharing a string would collide on one resource. +// Two facts sharing a name would collapse into one tag whichever way a +// provider spells them. func TestTagKeysAreDistinct(t *testing.T) { tagKeys := []TagKey{ TagKeyName, TagKeyStorage, TagKeyIdentities, - TagKeyResourceKind, TagKeyOwner, TagKeyVisibility, TagKeyDisplayName, + TagKeyOwner, TagKeySubnetType, } seen := make(map[string]struct{}, len(tagKeys)) diff --git a/internal/convention/visibility.go b/internal/convention/visibility.go deleted file mode 100644 index 714acbd6..00000000 --- a/internal/convention/visibility.go +++ /dev/null @@ -1,21 +0,0 @@ -package convention - -// Visibility is whether a network resource faces the internet. String is the -// form a tag value carries; Short is the form a name carries. -type Visibility struct { - s string - short string -} - -var ( - VisibilityPrivate = Visibility{s: "private", short: "prv"} - VisibilityPublic = Visibility{s: "public", short: "pub"} -) - -func (visibility Visibility) String() string { - return visibility.s -} - -func (visibility Visibility) Short() string { - return visibility.short -} diff --git a/internal/convention/visibility_test.go b/internal/convention/visibility_test.go deleted file mode 100644 index 5c95fa94..00000000 --- a/internal/convention/visibility_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package convention - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// A name is length-constrained where a tag value is not, so the two renderings -// differ. -func TestVisibility(t *testing.T) { - tests := []struct { - name string - visibility Visibility - expectedWord string - expectedShort string - }{ - {name: "Private_BothForms", visibility: VisibilityPrivate, expectedWord: "private", expectedShort: "prv"}, - {name: "Public_BothForms", visibility: VisibilityPublic, expectedWord: "public", expectedShort: "pub"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expectedWord, tt.visibility.String()) - assert.Equal(t, tt.expectedShort, tt.visibility.Short()) - }) - } -} - -// The zero value renders nothing, so a resource with no network face carries -// neither the qualifier nor the tag. -func TestVisibilityZeroValueRendersNothing(t *testing.T) { - assert.Empty(t, Visibility{}.String()) - assert.Empty(t, Visibility{}.Short()) -} diff --git a/internal/convention/zone.go b/internal/convention/zone.go deleted file mode 100644 index ef28270b..00000000 --- a/internal/convention/zone.go +++ /dev/null @@ -1,46 +0,0 @@ -package convention - -import ( - "strings" - - "github.com/signoz/foundry/internal/errors" -) - -// Zone is an availability zone. String is the provider's identifier, kept -// verbatim; Short is the form a name carries, derived from it. -type Zone struct { - s string - short string -} - -// ParseZone drops the leading locale segment and joins the rest for the short -// form: "us-east-1a" becomes "east1a", "asia-south2-c" becomes "south2c". -func ParseZone(zone string) (Zone, error) { - if zone == "" { - return Zone{}, errors.Newf(errors.TypeInvalidInput, "failed to create zone from %q: zone is empty", zone) - } - - segments := strings.Split(zone, "-") - if len(segments) < 2 { - return Zone{}, errors.Newf(errors.TypeInvalidInput, "failed to create zone from %q: zone has no locale and suffix segments", zone) - } - - return Zone{s: zone, short: strings.Join(segments[1:], "")}, nil -} - -func MustParseZone(zone string) Zone { - parsed, err := ParseZone(zone) - if err != nil { - panic(err) - } - - return parsed -} - -func (zone Zone) String() string { - return zone.s -} - -func (zone Zone) Short() string { - return zone.short -} diff --git a/internal/convention/zone_test.go b/internal/convention/zone_test.go deleted file mode 100644 index b57867c9..00000000 --- a/internal/convention/zone_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package convention - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestParseZone(t *testing.T) { - tests := []struct { - name string - input string - pass bool - expectedZone string - expectedShort string - }{ - {name: "AWSZone_Valid", input: "us-east-1a", pass: true, expectedZone: "us-east-1a", expectedShort: "east1a"}, - {name: "GCPZone_Valid", input: "asia-south2-c", pass: true, expectedZone: "asia-south2-c", expectedShort: "south2c"}, - {name: "GCPRegionZone_Valid", input: "us-central1-b", pass: true, expectedZone: "us-central1-b", expectedShort: "central1b"}, - {name: "GovCloudZone_Valid", input: "us-gov-east-1a", pass: true, expectedZone: "us-gov-east-1a", expectedShort: "goveast1a"}, - {name: "Empty_Invalid", input: "", pass: false}, - {name: "NoSeparator_Invalid", input: "useast1a", pass: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - zone, err := ParseZone(tt.input) - if !tt.pass { - assert.Error(t, err) - return - } - - assert.NoError(t, err) - - // The provider form is kept verbatim; only the short form is derived. - assert.Equal(t, tt.expectedZone, zone.String()) - assert.Equal(t, tt.expectedShort, zone.Short()) - }) - } -} diff --git a/internal/domain/metadata.go b/internal/domain/metadata.go index 097ea203..7fa8c8ad 100644 --- a/internal/domain/metadata.go +++ b/internal/domain/metadata.go @@ -5,6 +5,7 @@ package domain // on a cloud resource. Declaring it once is what keeps the three families in one // namespace even though nothing compares them. // -// Keys are one segment deep by convention -- foundry.signoz.io/managed-by, not -// foundry.signoz.io/ecs/cluster-id -- so the namespace stays flat and greppable. +// Keys are one segment deep by convention, foundry.signoz.io/managed-by rather +// than foundry.signoz.io/ecs/cluster-id, so the namespace stays flat and +// greppable. const MetadataPrefix = "foundry.signoz.io/" diff --git a/internal/molding/infrastructure/molding.go b/internal/molding/infrastructure/molding.go index f0c4d629..e56757ef 100644 --- a/internal/molding/infrastructure/molding.go +++ b/internal/molding/infrastructure/molding.go @@ -5,6 +5,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/convention" ) type MoldingEnricher interface { @@ -15,3 +16,8 @@ type Molding interface { Kind() v1alpha1.MoldingKind MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error } + +// Deriver turns a settled declaration into every name and tag the substrate +// stamps. The topology is the provider's. The registry supplies one per +// deployment. +type Deriver func(substrate convention.Substrate, declaration *infrastructure.ResourceConfig, labels map[string]string) (*infrastructure.ResourceConfigResources, error) diff --git a/internal/molding/infrastructure/resourcemolding/resource.go b/internal/molding/infrastructure/resourcemolding/resource.go index 5c6bbcef..a6f9ef1d 100644 --- a/internal/molding/infrastructure/resourcemolding/resource.go +++ b/internal/molding/infrastructure/resourcemolding/resource.go @@ -3,90 +3,68 @@ package resourcemolding import ( "context" "log/slog" + "maps" + "net/netip" + "slices" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/convention" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" ) -// The edge conventions of the SigNoz resource kinds. -var ( - otlpGRPCAddress = domain.MustNewAddress("tcp", "0.0.0.0", 4317).String() - otlpHTTPAddress = domain.MustNewAddress("tcp", "0.0.0.0", 4318).String() - apiServerAddress = domain.MustNewAddress("tcp", "0.0.0.0", 8080).String() -) - -// ResourceConfigName is the config document carrying the requirements a -// substrate shaped for the resource kind must satisfy beyond its edge. +// ResourceConfigName is the document a substrate is described by. const ResourceConfigName = "resource.yaml" +// The groups the baseline declares. A casting keys its contribution to these. +const ( + GroupPersistent = "persistent" + GroupEphemeral = "ephemeral" +) + var _ infrastructuremolding.Molding = (*resourceMolding)(nil) type resourceMolding struct { logger *slog.Logger + derive infrastructuremolding.Deriver } -func New(logger *slog.Logger) *resourceMolding { - return &resourceMolding{logger: logger} +func New(logger *slog.Logger, derive infrastructuremolding.Deriver) *resourceMolding { + return &resourceMolding{logger: logger, derive: derive} } func (molding *resourceMolding) Kind() v1alpha1.MoldingKind { return v1alpha1.MoldingKindResource } -// MoldV1Alpha1 writes the kind-level requirement set into the resource -// status: baseline first, preserving entries an enricher has already -// contributed. +// MoldV1Alpha1 settles the document from the baseline, the casting's +// contribution and the operator's spec, then derives from what settled. func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infrastructure.Casting) error { status := &config.Spec.Resource.Status - var baseline *infrastructure.ResourceConfig - switch config.Spec.Resource.Kind { - case infrastructure.ResourceKindInstallation: - status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) - status.Addresses.APIServer = append([]string{apiServerAddress}, status.Addresses.APIServer...) - baseline = &infrastructure.ResourceConfig{ - NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ - // Three persistent nodes cover the default topology: one - // keeper, the metadata node, one store node. A scaled - // installation must state its own, because Infrastructure - // provisions for a Kind and never reads the Installation - // casting. A pinned group's bounds are equal -- there is - // nothing to autoscale when every node owns a claimed volume. - infrastructure.StorageClassPersistent: { - MinSize: v1alpha1.IntPtr(3), - MaxSize: v1alpha1.IntPtr(3), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(8), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, - }, - infrastructure.StorageClassEphemeral: { - MinSize: v1alpha1.IntPtr(1), - MaxSize: v1alpha1.IntPtr(1), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(4), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - }, + // One baseline for every substrate: three persistent nodes for the stateful + // seats and one interchangeable node for the rest. A substrate that keeps + // nothing drops the persistent group with `persistent: null`. Pinned bounds + // are equal; every node owns a claimed volume, leaving nothing to scale. + baseline := &infrastructure.ResourceConfig{ + Networking: infrastructure.ResourceConfigNetworking{NetworkCIDR: "10.0.0.0/16"}, + InstanceGroups: map[string]infrastructure.ResourceConfigInstanceGroup{ + GroupPersistent: { + Storage: v1alpha1.StorageClassPersistent, + MinSize: v1alpha1.IntPtr(3), + MaxSize: v1alpha1.IntPtr(3), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, }, - } - case infrastructure.ResourceKindCollectionAgent: - status.Addresses.OTLP = append([]string{otlpGRPCAddress, otlpHTTPAddress}, status.Addresses.OTLP...) - baseline = &infrastructure.ResourceConfig{ - NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ - infrastructure.StorageClassEphemeral: { - MinSize: v1alpha1.IntPtr(1), - MaxSize: v1alpha1.IntPtr(1), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(4), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - }, + GroupEphemeral: { + Storage: v1alpha1.StorageClassEphemeral, + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, }, - } - default: - return foundryerrors.Newf(foundryerrors.TypeUnsupported, "unsupported resource kind %q", config.Spec.Resource.Kind) + }, } baselineDoc, err := domain.MarshalYAML(baseline) @@ -96,10 +74,8 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras doc := string(baselineDoc) - // Contributions (enricher deltas) merge first so casting-specific keys - // survive, then the operator's own spec, which wins: spec beats status - // wherever they disagree. Groups are keyed by class, so an override states - // only the class and the fields it changes. + // Enricher deltas first, keeping casting-specific keys, then the operator's + // spec, which wins. for _, override := range []string{ status.Config.Data[ResourceConfigName], config.Spec.Resource.Spec.Config.Data[ResourceConfigName], @@ -114,10 +90,36 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras } } - if err := validate(doc); err != nil { + declaration := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(doc), declaration); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to unmarshal resolved resource config") + } + + substrate, err := convention.NewSubstrate(config.Metadata.Name) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to resolve the substrate being provisioned") + } + + if err := validate(declaration); err != nil { + return err + } + + resources, err := molding.derive(substrate, declaration, config.Labels()) + if err != nil { return err } + // Merged, not marshalled: keys the shared shape does not know survive. + derivedDoc, err := domain.MarshalYAML(&infrastructure.ResourceConfig{Resources: resources}) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal derived resources") + } + + doc, err = domain.MergeYAML(doc, string(derivedDoc)) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to merge derived resources") + } + if status.Config.Data == nil { status.Config.Data = make(map[string]string) } @@ -126,39 +128,153 @@ func (molding *resourceMolding) MoldV1Alpha1(ctx context.Context, config *infras return nil } -// validate checks the shared shape of the resolved document; casting-specific -// keys pass through unchecked. -func validate(doc string) error { - config := &infrastructure.ResourceConfig{} - if err := domain.UnmarshalYAML([]byte(doc), config); err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to unmarshal resolved resource config") +// validate checks the shared shape; casting-specific keys pass through. +func validate(declaration *infrastructure.ResourceConfig) error { + // A stated name would disagree with the tag derived beside it. + if declaration.Resources != nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config states resources, which foundry derives from the declaration") + } + + if err := validateNetworking(declaration.Networking); err != nil { + return err + } + + return validateInstanceGroups(declaration) +} + +func validateNetworking(networking infrastructure.ResourceConfigNetworking) error { + if networking.NetworkID == "" { + if _, err := netip.ParsePrefix(networking.NetworkCIDR); err != nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config networkCIDR %q is not a CIDR block", networking.NetworkCIDR) + } + } + + if len(networking.Subnets) == 0 { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config states no subnets: a substrate cannot place a workload without one, and a zone has no safe default") + } + + // A NAT gateway sits in a public subnet in its own zone. + publicZones := map[string]struct{}{} + private := 0 + + for _, key := range slices.Sorted(maps.Keys(networking.Subnets)) { + subnet := networking.Subnets[key] + + if _, err := convention.NewKey(key); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "resource config subnet %q is not a usable reference", key) + } + + if subnet.Type == (v1alpha1.SubnetType{}) { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q states no type", key) + } + + if subnet.Zone == "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q states no zone", key) + } + + // A network is adopted whole. Half of one leaves foundry routing subnets + // it did not create. + if networking.NetworkID != "" && subnet.ID == "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config adopts network %q, so subnet %q states its own id", networking.NetworkID, key) + } + + if networking.NetworkID == "" && subnet.ID != "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q states an id, but the network it belongs to is created by foundry", key) + } + + if subnet.ID == "" { + if _, err := netip.ParsePrefix(subnet.CIDR); err != nil { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q has cidr %q, which is not a CIDR block", key, subnet.CIDR) + } + } else if subnet.Egress != "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q is adopted, so its egress is not foundry's to state", key) + } + + if subnet.Type.IsPublic() { + if subnet.Egress != "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q is public, so it states no egress", key) + } + + if subnet.ID == "" { + publicZones[subnet.Zone] = struct{}{} + } + + continue + } + + private++ } - for storage, group := range config.NodeGroups { + if private == 0 { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config states no private subnet: workloads are never placed in a public one") + } + + for _, key := range slices.Sorted(maps.Keys(networking.Subnets)) { + subnet := networking.Subnets[key] + + // An adopted subnet carries its own routing. + if subnet.Type.IsPublic() || subnet.ID != "" || subnet.Egress != "" { + continue + } + + if _, ok := publicZones[subnet.Zone]; !ok { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config subnet %q needs egress but zone %q has no public subnet to place a gateway in", key, subnet.Zone) + } + } + + return nil +} + +func validateInstanceGroups(declaration *infrastructure.ResourceConfig) error { + if len(declaration.InstanceGroups) == 0 { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config states no instance groups") + } + + for _, key := range slices.Sorted(maps.Keys(declaration.InstanceGroups)) { + group := declaration.InstanceGroups[key] + + if _, err := convention.NewKey(key); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "resource config instance group %q is not a usable reference", key) + } + + if group.Storage == (v1alpha1.StorageClass{}) { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q states no storage class", key) + } + + if group.MachineType == "" { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q states no machineType", key) + } + if group.MinSize == nil || group.MaxSize == nil || group.RootVolume.Size == nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q in resource config is incomplete", storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q is incomplete", key) } if *group.MaxSize < *group.MinSize { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q has maxSize below minSize", storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q has maxSize below minSize", key) } - // A machine is named outright or resolved from criteria; one of the - // two has to be stated or there is nothing to launch. - if group.MachineType == "" && (group.CPU == nil || group.Memory == nil) { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q states neither machineType nor cpu and memory", storage) + if group.Storage.IsPinned() && *group.MinSize != *group.MaxSize { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q is pinned, so minSize and maxSize must be equal", key) } - if storage.RequiresDataVolume() { + if group.Storage.RequiresDataVolume() { if group.DataVolume == nil || group.DataVolume.Size == nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q must state a dataVolume size", storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q must state a dataVolume size", key) } } else if group.DataVolume != nil { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q cannot state a dataVolume", storage) + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q cannot state a dataVolume", key) } - if storage.IsPinned() && *group.MinSize != *group.MaxSize { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "node group %q is pinned, so minSize and maxSize must be equal", storage) + for _, reference := range group.Subnets { + subnet, ok := declaration.Networking.Subnets[reference] + + if !ok { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q is placed in subnet %q, which is not declared", key, reference) + } + + if subnet.Type.IsPublic() { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "resource config instance group %q is placed in subnet %q, which is public", key, reference) + } } } diff --git a/internal/molding/infrastructure/resourcemolding/resource_test.go b/internal/molding/infrastructure/resourcemolding/resource_test.go index 4e837c1b..c550dcf5 100644 --- a/internal/molding/infrastructure/resourcemolding/resource_test.go +++ b/internal/molding/infrastructure/resourcemolding/resource_test.go @@ -2,6 +2,7 @@ package resourcemolding import ( "context" + "github.com/signoz/foundry/internal/convention" "log/slog" "testing" @@ -11,179 +12,315 @@ import ( "github.com/stretchr/testify/assert" ) -func TestMoldV1Alpha1(t *testing.T) { - tests := []struct { - name string - kind infrastructure.ResourceKind - pass bool - expected infrastructure.ResourceConfig - }{ - { - name: "InstallationResource_PersistentAndEphemeralNodeGroups", - kind: infrastructure.ResourceKindInstallation, - pass: true, - expected: infrastructure.ResourceConfig{ - NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ - infrastructure.StorageClassPersistent: { - MinSize: v1alpha1.IntPtr(3), - MaxSize: v1alpha1.IntPtr(3), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(8), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, - }, - infrastructure.StorageClassEphemeral: { - MinSize: v1alpha1.IntPtr(1), - MaxSize: v1alpha1.IntPtr(1), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(4), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - }, - }, - }, - }, - { - name: "CollectionAgentResource_EphemeralNodeGroup", - kind: infrastructure.ResourceKindCollectionAgent, - pass: true, - expected: infrastructure.ResourceConfig{ - NodeGroups: map[infrastructure.StorageClass]infrastructure.ResourceConfigNodeGroup{ - infrastructure.StorageClassEphemeral: { - MinSize: v1alpha1.IntPtr(1), - MaxSize: v1alpha1.IntPtr(1), - CPU: v1alpha1.IntPtr(2), - Memory: v1alpha1.IntPtr(4), - RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, - }, - }, - }, - }, - { - name: "UnknownResourceKind_Unsupported", - kind: infrastructure.ResourceKind{}, - pass: false, - }, +// What a casting contributes and an operator states between them: the baseline +// carries no zone and no machine type, because neither is the kind's to know. +const ( + networking = `networking: + subnets: + private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19} + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} +` + machineTypes = `instanceGroups: + persistent: {machineType: m5.large} + ephemeral: {machineType: c5.large} +` + ephemeralMachineType = `instanceGroups: + ephemeral: {machineType: c5.large} +` +) + +// mold merges the given documents into one declaration, runs the molding over +// a config whose spec carries it, and returns what settled. +func mold(t *testing.T, documents ...string) (string, error) { + t.Helper() + + config := infrastructure.Default() + config.Metadata.Name = "foundry" + + declaration := "" + for _, document := range documents { + if declaration == "" { + declaration = document + continue + } + + merged, err := domain.MergeYAML(declaration, document) + if err != nil { + t.Fatal(err) + } + + declaration = merged } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - config := infrastructure.Default() - config.Spec.Resource.Kind = tt.kind + if declaration != "" { + config.Spec.Resource.Spec.Config.Set(ResourceConfigName, []byte(declaration)) + } - err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) - if !tt.pass { - assert.Error(t, err) - return - } - assert.NoError(t, err) + err := New(slog.New(slog.DiscardHandler), derive).MoldV1Alpha1(context.Background(), config) - got := infrastructure.ResourceConfig{} - assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[ResourceConfigName]), &got)) - assert.Equal(t, tt.expected, got) - }) + return config.Spec.Resource.Status.Config.Data[ResourceConfigName], err +} + +// derive stands in for a provider's topology walk. The molding is neutral: it +// settles the declaration and hands it over, and what a provider makes of it is +// tested beside that provider. +func derive(_ convention.Substrate, declaration *infrastructure.ResourceConfig, _ map[string]string) (*infrastructure.ResourceConfigResources, error) { + resources := &infrastructure.ResourceConfigResources{ + InstanceGroups: map[string]infrastructure.ResourceConfigResourceGroup{}, + } + + for key, group := range declaration.InstanceGroups { + resolved := infrastructure.ResourceConfigResourceGroup{Storage: group.Storage} + + if group.Storage.IsPinned() && group.MinSize != nil { + resolved.Nodes = make([]infrastructure.ResourceConfigResourceNode, *group.MinSize) + } + + resources.InstanceGroups[key] = resolved } + + return resources, nil } -func TestMoldV1Alpha1_AddressesBaseline(t *testing.T) { - config := infrastructure.Default() - config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation +func TestMoldV1Alpha1(t *testing.T) { + doc, err := mold(t, networking, machineTypes) + assert.NoError(t, err) + + got := infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) + + // One baseline for every substrate: the default installation shape. + assert.Equal(t, map[string]infrastructure.ResourceConfigInstanceGroup{ + GroupPersistent: { + Storage: v1alpha1.StorageClassPersistent, + MachineType: "m5.large", + MinSize: v1alpha1.IntPtr(3), + MaxSize: v1alpha1.IntPtr(3), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + DataVolume: &infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(50)}, + }, + GroupEphemeral: { + Storage: v1alpha1.StorageClassEphemeral, + MachineType: "c5.large", + MinSize: v1alpha1.IntPtr(1), + MaxSize: v1alpha1.IntPtr(1), + RootVolume: infrastructure.ResourceConfigVolume{Size: v1alpha1.IntPtr(30)}, + }, + }, got.InstanceGroups) +} - err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) +// A substrate that keeps nothing drops the persistent group; the null deletes +// the key under RFC 7386. +func TestMoldV1Alpha1_StatelessSubstrate(t *testing.T) { + doc, err := mold(t, networking+"instanceGroups:\n persistent: null\n ephemeral: {machineType: c5.large}\n") assert.NoError(t, err) - assert.Equal(t, []string{"tcp://0.0.0.0:4317", "tcp://0.0.0.0:4318"}, config.Spec.Resource.Status.Addresses.OTLP) - assert.Equal(t, []string{"tcp://0.0.0.0:8080"}, config.Spec.Resource.Status.Addresses.APIServer) + + got := infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) + assert.Len(t, got.InstanceGroups, 1) + assert.Contains(t, got.InstanceGroups, GroupEphemeral) + assert.Empty(t, got.Resources.InstanceGroups[GroupPersistent].Nodes) +} + +// A declaration with no subnets cannot be completed by anything but the +// operator, so it fails rather than guessing a zone. +func TestMoldV1Alpha1_BaselineAloneIsIncomplete(t *testing.T) { + _, err := mold(t) + + assert.Error(t, err) } func TestMoldV1Alpha1_PreservesEnricherContributions(t *testing.T) { config := infrastructure.Default() - config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation - config.Spec.Resource.Status.Addresses.OTLP = []string{"tcp://0.0.0.0:9411"} - config.Spec.Resource.Status.Config.Data = map[string]string{ - ResourceConfigName: `nodeGroups: + config.Metadata.Name = "foundry" + config.Spec.Resource.Status.Config.Set(ResourceConfigName, []byte(networking+`instanceGroups: persistent: + machineType: m5.large minSize: 4 maxSize: 4 - nodes: [{ordinal: 0}, {ordinal: 1}, {ordinal: 2}, {ordinal: 3}] + spotAllocation: lowest-price ephemeral: + machineType: c5.large minSize: 2 maxSize: 2 -`, - } +`)) - err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + err := New(slog.New(slog.DiscardHandler), derive).MoldV1Alpha1(context.Background(), config) assert.NoError(t, err) - assert.Equal(t, []string{"tcp://0.0.0.0:4317", "tcp://0.0.0.0:4318", "tcp://0.0.0.0:9411"}, config.Spec.Resource.Status.Addresses.OTLP) doc := config.Spec.Resource.Status.Config.Data[ResourceConfigName] // Casting-specific keys survive the merge untouched. - assert.Contains(t, doc, "ordinal") + assert.Contains(t, doc, "spotAllocation") got := infrastructure.ResourceConfig{} assert.NoError(t, domain.UnmarshalYAML([]byte(doc), &got)) - // Groups are keyed by class, so the contribution states only the sizes it - // changes and the baseline's other fields survive -- under plain document - // merge, with no list strategy. - assert.Len(t, got.NodeGroups, 2) + // Groups are keyed, so the contribution states only the fields it changes + // and the baseline's others survive under plain document merge, with no + // list strategy. + assert.Len(t, got.InstanceGroups, 2) - persistent := got.NodeGroups[infrastructure.StorageClassPersistent] + persistent := got.InstanceGroups[GroupPersistent] assert.Equal(t, v1alpha1.IntPtr(4), persistent.MinSize) - assert.Equal(t, v1alpha1.IntPtr(8), persistent.Memory) assert.Equal(t, v1alpha1.IntPtr(50), persistent.DataVolume.Size) + assert.Len(t, got.Resources.InstanceGroups[GroupPersistent].Nodes, 4) - ephemeral := got.NodeGroups[infrastructure.StorageClassEphemeral] + ephemeral := got.InstanceGroups[GroupEphemeral] assert.Equal(t, v1alpha1.IntPtr(2), ephemeral.MinSize) - assert.Equal(t, v1alpha1.IntPtr(4), ephemeral.Memory) + assert.Equal(t, v1alpha1.IntPtr(30), ephemeral.RootVolume.Size) } -func TestMoldV1Alpha1_ContributionValidity(t *testing.T) { +// The operator's spec beats the casting's contribution wherever the two +// disagree. +func TestMoldV1Alpha1_SpecBeatsContribution(t *testing.T) { + config := infrastructure.Default() + config.Metadata.Name = "foundry" + config.Spec.Resource.Status.Config.Set(ResourceConfigName, []byte(machineTypes)) + config.Spec.Resource.Spec.Config.Set(ResourceConfigName, []byte(networking+`instanceGroups: + persistent: {machineType: r5.xlarge} +`)) + + err := New(slog.New(slog.DiscardHandler), derive).MoldV1Alpha1(context.Background(), config) + assert.NoError(t, err) + + got := infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[ResourceConfigName]), &got)) + assert.Equal(t, "r5.xlarge", got.InstanceGroups[GroupPersistent].MachineType) + assert.Equal(t, "c5.large", got.InstanceGroups[GroupEphemeral].MachineType) +} + +func TestValidate(t *testing.T) { tests := []struct { - name string - contribution string - pass bool + name string + documents []string + pass bool }{ { - // A partial override is the point of keying by class: the baseline - // supplies everything it does not mention. - name: "PartialOverride_Valid", - contribution: "nodeGroups:\n persistent:\n minSize: 3\n", - pass: true, + // A partial override is the point of keying: the baseline supplies + // everything the declaration does not mention. + name: "PartialOverride_Valid", + documents: []string{networking, machineTypes, "instanceGroups:\n persistent: {minSize: 1, maxSize: 1}\n"}, + pass: true, + }, + { + name: "NoSubnets_Invalid", + documents: []string{machineTypes}, + pass: false, + }, + { + name: "SubnetWithoutZone_Invalid", + documents: []string{"networking:\n subnets:\n private-a: {type: private, cidr: 10.0.0.0/19}\n", machineTypes}, + pass: false, + }, + { + name: "SubnetWithoutType_Invalid", + documents: []string{"networking:\n subnets:\n private-a: {zone: us-east-1a, cidr: 10.0.0.0/19}\n", machineTypes}, + pass: false, + }, + { + name: "SubnetWithBadCIDR_Invalid", + documents: []string{"networking:\n subnets:\n private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0}\n", machineTypes}, + pass: false, + }, + { + name: "SubnetKeyNotAName_Invalid", + documents: []string{"networking:\n subnets:\n Private_A: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19}\n", machineTypes}, + pass: false, + }, + { + name: "OnlyPublicSubnets_Invalid", + documents: []string{"networking:\n subnets:\n public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22}\n", machineTypes}, + pass: false, + }, + { + // Nothing can place the gateway the private subnet needs. + name: "PrivateSubnetWithNoPublicInItsZone_Invalid", + documents: []string{"networking:\n subnets:\n private-b: {type: private, zone: us-east-1b, cidr: 10.0.32.0/19}\n public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22}\n", machineTypes}, + pass: false, }, { - // An unnameable group cannot be reached by any consumer, so an - // unknown key fails at unmarshal rather than provisioning nodes - // nothing will be placed on. - name: "UnknownClass_Invalid", - contribution: "nodeGroups:\n keeper:\n minSize: 3\n", - pass: false, + // An adopted egress path is one the operator already routes through. + name: "PrivateSubnetWithAdoptedEgress_Valid", + documents: []string{"networking:\n subnets:\n private-b: {type: private, zone: us-east-1b, cidr: 10.0.32.0/19, egress: nat-0a1b2c3d}\n", machineTypes}, + pass: true, }, { - name: "PinnedGroupWithUnequalBounds_Invalid", - contribution: "nodeGroups:\n persistent:\n minSize: 3\n maxSize: 5\n", - pass: false, + name: "PublicSubnetWithEgress_Invalid", + documents: []string{"networking:\n subnets:\n private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19}\n public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22, egress: nat-0a1b2c3d}\n", machineTypes}, + pass: false, + }, + { + name: "AdoptedNetworkWithAdoptedSubnets_Valid", + documents: []string{"networking:\n networkID: vpc-0a1b2c3d\n subnets:\n private-a: {type: private, zone: us-east-1a, id: subnet-0a1b2c3d}\n", machineTypes}, + pass: true, + }, + { + name: "AdoptedNetworkWithCreatedSubnet_Invalid", + documents: []string{"networking:\n networkID: vpc-0a1b2c3d\n subnets:\n private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19}\n", machineTypes}, + pass: false, + }, + { + name: "AdoptedSubnetInCreatedNetwork_Invalid", + documents: []string{"networking:\n subnets:\n private-a: {type: private, zone: us-east-1a, id: subnet-0a1b2c3d}\n", machineTypes}, + pass: false, + }, + { + name: "GroupWithoutMachineType_Invalid", + documents: []string{networking}, + pass: false, + }, + { + name: "PinnedGroupWithUnequalBounds_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n persistent: {minSize: 3, maxSize: 5}\n"}, + pass: false, + }, + { + name: "GroupWithMaxBelowMin_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n ephemeral: {minSize: 4, maxSize: 2}\n"}, + pass: false, }, { // A null deletes the key under RFC 7386, so this removes the - // baseline's data volume from a class that requires one. - name: "PersistentWithoutDataVolume_Invalid", - contribution: "nodeGroups:\n persistent:\n dataVolume: null\n", - pass: false, + // baseline's data volume from a class that requires one. It is one + // document because a null merged onto a document that never had + // the key is dropped, not carried forward. + name: "PersistentWithoutDataVolume_Invalid", + documents: []string{networking + "instanceGroups:\n persistent: {machineType: m5.large, dataVolume: null}\n ephemeral: {machineType: c5.large}\n"}, + pass: false, + }, + { + name: "EphemeralWithDataVolume_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n ephemeral: {dataVolume: {size: 20}}\n"}, + pass: false, + }, + { + name: "GroupKeyNotAName_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n Hot_Pool: {storage: ephemeral, machineType: c5.large, minSize: 1, maxSize: 1, rootVolume: {size: 30}}\n"}, + pass: false, }, { - name: "EphemeralWithDataVolume_Invalid", - contribution: "nodeGroups:\n ephemeral:\n dataVolume:\n size: 20\n", - pass: false, + name: "GroupPlacedInUndeclaredSubnet_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n persistent: {subnets: [private-z]}\n"}, + pass: false, + }, + { + name: "GroupPlacedInPublicSubnet_Invalid", + documents: []string{networking, machineTypes, "instanceGroups:\n persistent: {subnets: [public-a]}\n"}, + pass: false, + }, + { + // A stated name and the tag derived beside it would disagree, and + // the consuming casting would match nothing. + name: "StatedResources_Invalid", + documents: []string{networking, machineTypes, "resources:\n vpc: {name: my-vpc}\n"}, + pass: false, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - config := infrastructure.Default() - config.Spec.Resource.Kind = infrastructure.ResourceKindInstallation - config.Spec.Resource.Status.Config.Data = map[string]string{ResourceConfigName: tt.contribution} - - err := New(slog.New(slog.DiscardHandler)).MoldV1Alpha1(context.Background(), config) + _, err := mold(t, tt.documents...) if !tt.pass { assert.Error(t, err) return From 7dfe9a4fabf3f815371eac6aaf82d09619290aff Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 18:00:59 +0530 Subject: [PATCH 32/38] feat(convention): split the aws derivation out of the neutral core The neutral core states what a substrate is; the aws package states how AWS spells it. Both the infrastructure casting's deriver and the installation casting's tag filters read from here, so it sits in the base rather than in either consumer. --- internal/convention/aws/derive.go | 218 +++++++++++++++++ internal/convention/aws/derive_test.go | 220 +++++++++++++++++ internal/convention/aws/resource.go | 198 +++++++++++++++ internal/convention/aws/resource_test.go | 292 +++++++++++++++++++++++ internal/convention/aws/role.go | 17 ++ internal/convention/aws/role_test.go | 36 +++ internal/convention/aws/tag.go | 26 ++ internal/convention/aws/tag_test.go | 33 +++ 8 files changed, 1040 insertions(+) create mode 100644 internal/convention/aws/derive.go create mode 100644 internal/convention/aws/derive_test.go create mode 100644 internal/convention/aws/resource.go create mode 100644 internal/convention/aws/resource_test.go create mode 100644 internal/convention/aws/role.go create mode 100644 internal/convention/aws/role_test.go create mode 100644 internal/convention/aws/tag.go create mode 100644 internal/convention/aws/tag_test.go diff --git a/internal/convention/aws/derive.go b/internal/convention/aws/derive.go new file mode 100644 index 00000000..326335af --- /dev/null +++ b/internal/convention/aws/derive.go @@ -0,0 +1,218 @@ +package aws + +import ( + "github.com/signoz/foundry/internal/convention" + "maps" + "slices" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/errors" +) + +// What the substrate's own roles and rules exist for. Each is a name segment. +var ( + purposeIntraCluster = convention.MustNewKey("intra-cluster") + purposeAllOutbound = convention.MustNewKey("all-outbound") +) + +// Resources renders a declared substrate into every name and tag it stamps. +// Labels sit over the declaration's cloudLabels and under the derived tags. An +// operator cannot rename what a consumer matches on. +func Resources(s convention.Substrate, declaration *infrastructure.ResourceConfig, labels map[string]string) (*infrastructure.ResourceConfigResources, error) { + base := map[string]string{} + maps.Copy(base, declaration.CloudLabels) + maps.Copy(base, labels) + + named := func(resource Resource) infrastructure.ResourceConfigResource { + tags := maps.Clone(base) + maps.Copy(tags, resource.Tags()) + + return infrastructure.ResourceConfigResource{Name: resource.Name(), Tags: tags} + } + + resources := &infrastructure.ResourceConfigResources{ + Cluster: named(Cluster(s)), + VPC: named(VPC(s)), + SecurityGroup: named(SecurityGroup(s, RoleTask)), + InstanceProfile: named(InstanceProfile(s, RoleNode)), + SecurityGroupRules: map[string]infrastructure.ResourceConfigResource{ + purposeIntraCluster.String(): named(SecurityGroupRule(s, RoleTask, purposeIntraCluster)), + purposeAllOutbound.String(): named(SecurityGroupRule(s, RoleTask, purposeAllOutbound)), + }, + Roles: map[string]infrastructure.ResourceConfigResource{}, + IgnoredTags: []string{Tag(convention.TagKeyIdentities)}, + } + + // An adopted network keeps its owner's name and tags, and gets no gateway. + if id := declaration.Networking.NetworkID; id != "" { + resources.VPC = infrastructure.ResourceConfigResource{ID: id} + } + + // The node's own credential only. Without it the agent cannot register the + // instance with the cluster. Workload identity belongs to the workload. + resources.Roles[RoleNode.String()] = named(IAMRole(s, RoleNode)) + + if err := networking(s, named, declaration, resources); err != nil { + return nil, err + } + + if err := instanceGroups(s, named, declaration, resources); err != nil { + return nil, err + } + + return resources, nil +} + +func networking(s convention.Substrate, + named func(Resource) infrastructure.ResourceConfigResource, + declaration *infrastructure.ResourceConfig, + resources *infrastructure.ResourceConfigResources, +) error { + subnets := declaration.Networking.Subnets + + resources.Subnets = make(map[string]infrastructure.ResourceConfigResourceSubnet, len(subnets)) + resources.RouteTables = map[string]infrastructure.ResourceConfigResource{} + resources.NATGateways = map[string]infrastructure.ResourceConfigResourceNATGateway{} + + // A gateway serves the private subnet it is keyed by and lives in a public + // one in the same zone. Walked in key order to keep the choice stable. + publicByZone := map[string]string{} + + for _, key := range slices.Sorted(maps.Keys(subnets)) { + subnet := subnets[key] + + if !subnet.Type.IsPublic() || subnet.ID != "" { + continue + } + + if _, ok := publicByZone[subnet.Zone]; !ok { + publicByZone[subnet.Zone] = key + } + } + + for _, key := range slices.Sorted(maps.Keys(subnets)) { + subnet := subnets[key] + + reference, err := convention.NewKey(key) + if err != nil { + return errors.Wrapf(err, errors.TypeInvalidInput, "failed to derive subnet %q", key) + } + + // An adopted subnet keeps the operator's own routing. + if subnet.ID != "" { + resources.Subnets[key] = infrastructure.ResourceConfigResourceSubnet{ID: subnet.ID, Public: subnet.Type.IsPublic()} + continue + } + + resource := named(Subnet(s, reference, subnet.Type)) + + resources.Subnets[key] = infrastructure.ResourceConfigResourceSubnet{ + Name: resource.Name, + Tags: resource.Tags, + Public: subnet.Type.IsPublic(), + } + + resources.RouteTables[key] = named(RouteTable(s, reference)) + + if subnet.Type.IsPublic() { + // Derived here: the gateway exists only if a public subnet does. + resources.InternetGateway = named(InternetGateway(s)) + continue + } + + if subnet.Egress != "" { + resources.NATGateways[key] = infrastructure.ResourceConfigResourceNATGateway{ID: subnet.Egress} + continue + } + + gateway := named(NATGateway(s, reference)) + address := named(ElasticIP(s, reference)) + + resources.NATGateways[key] = infrastructure.ResourceConfigResourceNATGateway{ + Name: gateway.Name, + Tags: gateway.Tags, + Subnet: publicByZone[subnet.Zone], + Address: &address, + } + } + + return nil +} + +func instanceGroups(s convention.Substrate, + named func(Resource) infrastructure.ResourceConfigResource, + declaration *infrastructure.ResourceConfig, + resources *infrastructure.ResourceConfigResources, +) error { + // A group that names no subnet is placed across every private one. + placement := []string{} + + for _, key := range slices.Sorted(maps.Keys(declaration.Networking.Subnets)) { + if !declaration.Networking.Subnets[key].Type.IsPublic() { + placement = append(placement, key) + } + } + + resources.InstanceGroups = make(map[string]infrastructure.ResourceConfigResourceGroup, len(declaration.InstanceGroups)) + + for _, key := range slices.Sorted(maps.Keys(declaration.InstanceGroups)) { + declared := declaration.InstanceGroups[key] + + reference, err := convention.NewKey(key) + if err != nil { + return errors.Wrapf(err, errors.TypeInvalidInput, "failed to derive instance group %q", key) + } + + subnets := declared.Subnets + + if len(subnets) == 0 { + subnets = placement + } + + if len(subnets) == 0 { + return errors.Newf(errors.TypeInvalidInput, "failed to derive instance group %q: there is no private subnet to place it in", key) + } + + group := convention.NewNodeGroup(reference, declared.Storage) + resolved := infrastructure.ResourceConfigResourceGroup{ + Storage: declared.Storage, + Selector: Filter(s.Select().WithStorage(declared.Storage)), + Subnets: subnets, + } + + if !declared.Storage.IsPinned() { + launchTemplate := named(LaunchTemplate(s, group)) + autoscalingGroup := named(AutoscalingGroup(s, group)) + + resolved.LaunchTemplate = &launchTemplate + resolved.AutoscalingGroup = &autoscalingGroup + resources.InstanceGroups[key] = resolved + + continue + } + + // Each node carries its volume. The two cannot land in different zones. + nodes := 0 + + if declared.MinSize != nil { + nodes = *declared.MinSize + } + + for ordinal := range nodes { + volume := named(Volume(s, group, ordinal)) + node := named(Node(s, group, ordinal)) + + resolved.Nodes = append(resolved.Nodes, infrastructure.ResourceConfigResourceNode{ + Name: node.Name, + Tags: node.Tags, + Ordinal: ordinal, + Subnet: subnets[ordinal%len(subnets)], + Volume: &volume, + }) + } + + resources.InstanceGroups[key] = resolved + } + + return nil +} diff --git a/internal/convention/aws/derive_test.go b/internal/convention/aws/derive_test.go new file mode 100644 index 00000000..1cfb2249 --- /dev/null +++ b/internal/convention/aws/derive_test.go @@ -0,0 +1,220 @@ +package aws + +import ( + "github.com/signoz/foundry/internal/convention" + "testing" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const oneZone = `networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19} + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} +instanceGroups: + persistent: {storage: persistent, machineType: m5.large, minSize: 3, maxSize: 3} + ephemeral: {storage: ephemeral, machineType: c5.large, minSize: 1, maxSize: 1} +` + +const twoZones = `networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19} + private-b: {type: private, zone: us-east-1b, cidr: 10.0.32.0/19} + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} + public-b: {type: public, zone: us-east-1b, cidr: 10.0.100.0/22} +instanceGroups: + persistent: {storage: persistent, machineType: m5.large, minSize: 3, maxSize: 3} + ephemeral: {storage: ephemeral, machineType: c5.large, minSize: 1, maxSize: 1} +` + +// derive renders a declaration written the way the molding settles it. +func derive(t *testing.T, declaration string) (*infrastructure.ResourceConfigResources, error) { + t.Helper() + + config := &infrastructure.ResourceConfig{} + require.NoError(t, domain.UnmarshalYAML([]byte(declaration), config)) + + return Resources(convention.MustNewSubstrate("foundry"), config, nil) +} + +func mustDerive(t *testing.T, declaration string) *infrastructure.ResourceConfigResources { + t.Helper() + + derived, err := derive(t, declaration) + require.NoError(t, err) + + return derived +} + +// These names reach live infrastructure. Changing one replaces the resource it +// belongs to rather than updating it. +func TestResources(t *testing.T) { + derived := mustDerive(t, oneZone) + + tests := []struct { + name string + of func(*infrastructure.ResourceConfigResources) string + expectedName string + }{ + {name: "Cluster_Unqualified", of: func(r *infrastructure.ResourceConfigResources) string { return r.Cluster.Name }, expectedName: "foundry-cls"}, + {name: "VPC_Unqualified", of: func(r *infrastructure.ResourceConfigResources) string { return r.VPC.Name }, expectedName: "foundry-vpc"}, + {name: "InternetGateway_Unqualified", of: func(r *infrastructure.ResourceConfigResources) string { return r.InternetGateway.Name }, expectedName: "foundry-igw"}, + {name: "Subnet_Keyed", of: func(r *infrastructure.ResourceConfigResources) string { return r.Subnets["private-a"].Name }, expectedName: "foundry-sub-private-a"}, + {name: "RouteTable_Keyed", of: func(r *infrastructure.ResourceConfigResources) string { return r.RouteTables["public-a"].Name }, expectedName: "foundry-rt-public-a"}, + {name: "NATGateway_KeyedByTheSubnetItServes", of: func(r *infrastructure.ResourceConfigResources) string { return r.NATGateways["private-a"].Name }, expectedName: "foundry-nat-private-a"}, + {name: "ElasticIP_KeyedByTheSubnetItServes", of: func(r *infrastructure.ResourceConfigResources) string { return r.NATGateways["private-a"].Address.Name }, expectedName: "foundry-eip-private-a"}, + {name: "SecurityGroup_Role", of: func(r *infrastructure.ResourceConfigResources) string { return r.SecurityGroup.Name }, expectedName: "foundry-sg-task"}, + {name: "SecurityGroupRule_RoleAndPurpose", of: func(r *infrastructure.ResourceConfigResources) string { + return r.SecurityGroupRules["intra-cluster"].Name + }, expectedName: "foundry-sg-task-intra-cluster"}, + {name: "Role_NodeOnly", of: func(r *infrastructure.ResourceConfigResources) string { return r.Roles["node"].Name }, expectedName: "foundry-iam-node"}, + {name: "InstanceProfile_Role", of: func(r *infrastructure.ResourceConfigResources) string { return r.InstanceProfile.Name }, expectedName: "foundry-prf-node"}, + {name: "LaunchTemplate_GroupKey", of: func(r *infrastructure.ResourceConfigResources) string { + return r.InstanceGroups["ephemeral"].LaunchTemplate.Name + }, expectedName: "foundry-lt-ephemeral"}, + {name: "AutoscalingGroup_GroupKey", of: func(r *infrastructure.ResourceConfigResources) string { + return r.InstanceGroups["ephemeral"].AutoscalingGroup.Name + }, expectedName: "foundry-asg-ephemeral"}, + {name: "Node_GroupKeyAndOrdinal", of: func(r *infrastructure.ResourceConfigResources) string { + return r.InstanceGroups["persistent"].Nodes[0].Name + }, expectedName: "foundry-node-persistent-0"}, + {name: "Volume_GroupKeyAndOrdinal", of: func(r *infrastructure.ResourceConfigResources) string { + return r.InstanceGroups["persistent"].Nodes[0].Volume.Name + }, expectedName: "foundry-vol-persistent-0"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedName, tt.of(derived)) + }) + } +} + +// The tags a consuming casting filters on are the only ones whose spelling live +// infrastructure depends on. +func TestResourcesContractTags(t *testing.T) { + derived := mustDerive(t, oneZone) + + assert.Equal(t, "private", derived.Subnets["private-a"].Tags[Tag(convention.TagKeySubnetType)]) + assert.Equal(t, "public", derived.Subnets["public-a"].Tags[Tag(convention.TagKeySubnetType)]) + assert.Equal(t, "persistent", derived.InstanceGroups["persistent"].Nodes[0].Tags[Tag(convention.TagKeyStorage)]) + assert.Equal(t, "persistent", derived.InstanceGroups["persistent"].Nodes[0].Volume.Tags[Tag(convention.TagKeyStorage)]) + + assert.Equal(t, map[string]string{ + Tag(convention.TagKeyName): "foundry", + Tag(convention.TagKeyStorage): "persistent", + }, derived.InstanceGroups["persistent"].Selector) + + // The claim tag is stamped after provisioning, so a casting that reconciles + // has to be told to leave it alone. + assert.Equal(t, []string{Tag(convention.TagKeyIdentities)}, derived.IgnoredTags) +} + +// A group's selector has to match what its own nodes are stamped with, or the +// substrate advertises a placement nothing satisfies. +func TestResourcesSelectorMatchesItsNodes(t *testing.T) { + group := mustDerive(t, oneZone).InstanceGroups["persistent"] + + for _, node := range group.Nodes { + for key, value := range group.Selector { + assert.Equal(t, value, node.Tags[key], "node %s does not match the group selector on %s", node.Name, key) + } + } +} + +// Foundry's ownership labels and the derived tags are what a consumer matches +// on, so an operator's own tags sit underneath rather than over them. +func TestResourcesCloudLabelsDoNotOverrideTheContract(t *testing.T) { + derived := mustDerive(t, oneZone+"cloudLabels:\n team: observability\n "+Tag(convention.TagKeyName)+": not-foundry\n") + + assert.Equal(t, "observability", derived.VPC.Tags["team"]) + assert.Equal(t, "foundry", derived.VPC.Tags[Tag(convention.TagKeyName)]) +} + +// Nodes are laid out across the group's subnets in ordinal order, and a node's +// volume goes wherever the node does. +func TestResourcesPlacementCyclesThroughSubnets(t *testing.T) { + derived := mustDerive(t, twoZones) + + group := derived.InstanceGroups["persistent"] + assert.Equal(t, []string{"private-a", "private-b"}, group.Subnets) + assert.Equal(t, []string{"private-a", "private-b", "private-a"}, []string{ + group.Nodes[0].Subnet, group.Nodes[1].Subnet, group.Nodes[2].Subnet, + }) + + // Each zone's gateway sits in a public subnet of that same zone. + assert.Equal(t, "public-a", derived.NATGateways["private-a"].Subnet) + assert.Equal(t, "public-b", derived.NATGateways["private-b"].Subnet) +} + +// A group that names its own subnets is placed only there. +func TestResourcesHonourStatedPlacement(t *testing.T) { + group := mustDerive(t, twoZones+" persistent:\n subnets: [private-b]\n").InstanceGroups["persistent"] + + for _, node := range group.Nodes { + assert.Equal(t, "private-b", node.Subnet) + } +} + +// An adopted network is referenced, never described: foundry adds no gateway, +// no route table and no tags to something it does not own. +func TestResourcesAdoptedNetwork(t *testing.T) { + derived := mustDerive(t, `networking: + networkID: vpc-0a1b2c3d + subnets: + private-a: {type: private, zone: us-east-1a, id: subnet-0a1b2c3d} +instanceGroups: + persistent: {storage: persistent, machineType: m5.large, minSize: 3, maxSize: 3} +`) + + assert.Equal(t, "vpc-0a1b2c3d", derived.VPC.ID) + assert.Empty(t, derived.VPC.Name) + assert.Empty(t, derived.VPC.Tags) + assert.Empty(t, derived.InternetGateway.Name) + assert.Empty(t, derived.RouteTables) + assert.Empty(t, derived.NATGateways) + + assert.Equal(t, "subnet-0a1b2c3d", derived.Subnets["private-a"].ID) + assert.Empty(t, derived.Subnets["private-a"].Name) + + // The compute placed in it is still foundry's. + assert.Equal(t, "foundry-node-persistent-0", derived.InstanceGroups["persistent"].Nodes[0].Name) +} + +// A private subnet that already routes somewhere gets no gateway of its own, +// and the id it routes through is carried through for the casting to reference. +func TestResourcesAdoptedEgress(t *testing.T) { + derived := mustDerive(t, `networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-b: {type: private, zone: us-east-1b, cidr: 10.0.32.0/19, egress: nat-0a1b2c3d} +instanceGroups: + ephemeral: {storage: ephemeral, machineType: c5.large, minSize: 1, maxSize: 1} +`) + + gateway := derived.NATGateways["private-b"] + assert.Equal(t, "nat-0a1b2c3d", gateway.ID) + assert.Empty(t, gateway.Name) + assert.Nil(t, gateway.Address) + + // No public subnet was declared, so nothing routes to a gateway foundry owns. + assert.Empty(t, derived.InternetGateway.Name) +} + +// A group with nowhere to go would provision nodes no workload can be placed +// on, so it fails instead. +func TestResourcesWithoutAPrivateSubnet(t *testing.T) { + _, err := derive(t, `networking: + subnets: + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} +instanceGroups: + ephemeral: {storage: ephemeral, machineType: c5.large, minSize: 1, maxSize: 1} +`) + + assert.Error(t, err) +} diff --git a/internal/convention/aws/resource.go b/internal/convention/aws/resource.go new file mode 100644 index 00000000..a4690e4b --- /dev/null +++ b/internal/convention/aws/resource.go @@ -0,0 +1,198 @@ +package aws + +import ( + "github.com/signoz/foundry/internal/convention" + "strconv" + "strings" + + "github.com/signoz/foundry/api/v1alpha1" +) + +// Resource is one thing a substrate provisions. Its name, tags and selection +// all derive from one description. Use the constructor for what you are +// provisioning. +type Resource struct { + substrate convention.Substrate + resourceType resourceType + + key convention.Key + purpose convention.Key + subnetType v1alpha1.SubnetType + storage v1alpha1.StorageClass + role Role + ordinal int + + ownership convention.Ownership + identities convention.Identities +} + +func Cluster(s convention.Substrate) Resource { + return Resource{substrate: s, resourceType: typeCluster} +} + +func VPC(s convention.Substrate) Resource { + return Resource{substrate: s, resourceType: typeVPC} +} + +func InternetGateway(s convention.Substrate) Resource { + return Resource{substrate: s, resourceType: typeInternetGateway} +} + +// Subnet takes its type separately from its key. The key is the operator's own +// reference and says nothing a consumer can rely on. +func Subnet(s convention.Substrate, key convention.Key, subnetType v1alpha1.SubnetType) Resource { + return Resource{substrate: s, resourceType: typeSubnet, key: key, subnetType: subnetType} +} + +func RouteTable(s convention.Substrate, key convention.Key) Resource { + return Resource{substrate: s, resourceType: typeRouteTable, key: key} +} + +func NATGateway(s convention.Substrate, key convention.Key) Resource { + return Resource{substrate: s, resourceType: typeNATGateway, key: key} +} + +func ElasticIP(s convention.Substrate, key convention.Key) Resource { + return Resource{substrate: s, resourceType: typeElasticIP, key: key} +} + +func SecurityGroup(s convention.Substrate, role Role) Resource { + return Resource{substrate: s, resourceType: typeSecurityGroup, role: role} +} + +// SecurityGroupRule is one rule of the group for the same role, distinguished +// by what it admits. +func SecurityGroupRule(s convention.Substrate, role Role, purpose convention.Key) Resource { + return Resource{substrate: s, resourceType: typeSecurityGroup, role: role, purpose: purpose} +} + +func IAMRole(s convention.Substrate, role Role) Resource { + return Resource{substrate: s, resourceType: typeRole, role: role} +} + +// RolePolicy is a policy inline on a role, distinguished by what it grants. +func IAMRolePolicy(s convention.Substrate, role Role, purpose convention.Key) Resource { + return Resource{substrate: s, resourceType: typeRole, role: role, purpose: purpose} +} + +func InstanceProfile(s convention.Substrate, role Role) Resource { + return Resource{substrate: s, resourceType: typeInstanceProfile, role: role} +} + +func LaunchTemplate(s convention.Substrate, group convention.NodeGroup) Resource { + return Resource{substrate: s, resourceType: typeLaunchTemplate, key: group.Key(), storage: group.Storage()} +} + +func AutoscalingGroup(s convention.Substrate, group convention.NodeGroup) Resource { + return Resource{substrate: s, resourceType: typeAutoscalingGroup, key: group.Key(), storage: group.Storage()} +} + +func Node(s convention.Substrate, group convention.NodeGroup, ordinal int) Resource { + return Resource{substrate: s, resourceType: typeNode, key: group.Key(), storage: group.Storage(), ordinal: ordinal} +} + +func Volume(s convention.Substrate, group convention.NodeGroup, ordinal int) Resource { + return Resource{substrate: s, resourceType: typeVolume, key: group.Key(), storage: group.Storage(), ordinal: ordinal} +} + +// WithOwnership marks a resource adopted rather than created. It changes no name. +func (r Resource) WithOwnership(ownership convention.Ownership) Resource { + r.ownership = ownership + + return r +} + +// WithClaims records the identities holding a volume. See convention.Identities. +func (r Resource) WithClaims(identities convention.Identities) Resource { + r.identities = identities + + return r +} + +// Name is -[-...], broad to narrow. It fills a +// provider's name argument where one exists and the display tag always. An +// instance has no name of its own. +func (r Resource) Name() string { + parts := make([]string, 0, len(r.resourceType.qualifiers)+2) + parts = append(parts, r.substrate.String(), r.resourceType.String()) + + for _, qualifier := range r.resourceType.qualifiers { + if segment := qualifier.of(r); segment != "" { + parts = append(parts, segment) + } + } + + return strings.Join(parts, "-") +} + +// Selection is the set that finds exactly this resource. +func (r Resource) Selection() convention.Selection { + return r.substrate.Select().WithSubnetType(r.subnetType).WithStorage(r.storage).WithClaims(r.identities) +} + +// stamp is the selection's tags plus the provenance nothing reads back. +func (r Resource) stamp() map[string]string { + tags := Filter(r.Selection()) + tags[Tag(convention.TagKeyOwner)] = r.ownership.String() + + // An adopted resource keeps the name it already had. + if !r.ownership.IsShared() { + tags[displayName] = r.Name() + } + + return tags +} + +// Tags is every tag this resource carries. A casting merges +// CastingMeta.Labels() in alongside these. +func (r Resource) Tags() map[string]string { + return r.stamp() +} + +// Filter is the tag match that finds this resource. +func (r Resource) Filter() map[string]string { + return Filter(r.Selection()) +} + +// resourceType is what a derived name says the thing is, and the ordered +// qualifiers that narrow it. Adding a resource is one var entry below. +type resourceType struct { + short string + qualifiers []qualifier +} + +var ( + typeCluster = resourceType{short: "cls"} + typeVPC = resourceType{short: "vpc"} + typeInternetGateway = resourceType{short: "igw"} + typeSubnet = resourceType{short: "sub", qualifiers: []qualifier{qualifierKey}} + typeRouteTable = resourceType{short: "rt", qualifiers: []qualifier{qualifierKey}} + typeNATGateway = resourceType{short: "nat", qualifiers: []qualifier{qualifierKey}} + typeElasticIP = resourceType{short: "eip", qualifiers: []qualifier{qualifierKey}} + typeSecurityGroup = resourceType{short: "sg", qualifiers: []qualifier{qualifierRole, qualifierPurpose}} + typeRole = resourceType{short: "iam", qualifiers: []qualifier{qualifierRole, qualifierPurpose}} + typeInstanceProfile = resourceType{short: "prf", qualifiers: []qualifier{qualifierRole}} + typeLaunchTemplate = resourceType{short: "lt", qualifiers: []qualifier{qualifierKey}} + typeAutoscalingGroup = resourceType{short: "asg", qualifiers: []qualifier{qualifierKey}} + typeNode = resourceType{short: "node", qualifiers: []qualifier{qualifierKey, qualifierOrdinal}} + typeVolume = resourceType{short: "vol", qualifiers: []qualifier{qualifierKey, qualifierOrdinal}} +) + +func (resource resourceType) String() string { + return resource.short +} + +// qualifier renders one axis into a name segment. An empty string drops the +// segment: one declaration serves a security group and its rules. +type qualifier struct { + of func(Resource) string +} + +var ( + qualifierKey = qualifier{of: func(r Resource) string { return r.key.String() }} + qualifierRole = qualifier{of: func(r Resource) string { return r.role.String() }} + qualifierPurpose = qualifier{of: func(r Resource) string { return r.purpose.String() }} + + // Only types that have an ordinal declare it. Zero renders as "0". + qualifierOrdinal = qualifier{of: func(r Resource) string { return strconv.Itoa(r.ordinal) }} +) diff --git a/internal/convention/aws/resource_test.go b/internal/convention/aws/resource_test.go new file mode 100644 index 00000000..7a3fdd22 --- /dev/null +++ b/internal/convention/aws/resource_test.go @@ -0,0 +1,292 @@ +package aws + +import ( + "github.com/signoz/foundry/internal/convention" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/stretchr/testify/assert" +) + +func TestResourceName(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + privateA := convention.MustNewKey("private-a") + publicA := convention.MustNewKey("public-a") + persistent := convention.NewNodeGroup(convention.MustNewKey("persistent"), v1alpha1.StorageClassPersistent) + ephemeral := convention.NewNodeGroup(convention.MustNewKey("ephemeral"), v1alpha1.StorageClassEphemeral) + + tests := []struct { + name string + resource Resource + expectedName string + }{ + {name: "Cluster_Unqualified", resource: Cluster(substrate), expectedName: "foundry-cls"}, + {name: "VPC_Unqualified", resource: VPC(substrate), expectedName: "foundry-vpc"}, + {name: "InternetGateway_Unqualified", resource: InternetGateway(substrate), expectedName: "foundry-igw"}, + {name: "PrivateSubnet_Key", resource: Subnet(substrate, privateA, v1alpha1.SubnetTypePrivate), expectedName: "foundry-sub-private-a"}, + {name: "PublicSubnet_Key", resource: Subnet(substrate, publicA, v1alpha1.SubnetTypePublic), expectedName: "foundry-sub-public-a"}, + {name: "RouteTable_Key", resource: RouteTable(substrate, privateA), expectedName: "foundry-rt-private-a"}, + {name: "NATGateway_Key", resource: NATGateway(substrate, publicA), expectedName: "foundry-nat-public-a"}, + {name: "ElasticIP_Key", resource: ElasticIP(substrate, publicA), expectedName: "foundry-eip-public-a"}, + {name: "SecurityGroup_Role", resource: SecurityGroup(substrate, RoleTask), expectedName: "foundry-sg-task"}, + {name: "SecurityGroupRule_RoleAndPurpose", resource: SecurityGroupRule(substrate, RoleTask, convention.MustNewKey("intra-cluster")), expectedName: "foundry-sg-task-intra-cluster"}, + {name: "Role_Role", resource: IAMRole(substrate, RoleExec), expectedName: "foundry-iam-exec"}, + {name: "RolePolicy_RoleAndPurpose", resource: IAMRolePolicy(substrate, RoleTask, convention.MustNewKey("appconfig-read")), expectedName: "foundry-iam-task-appconfig-read"}, + {name: "InstanceProfile_Role", resource: InstanceProfile(substrate, RoleNode), expectedName: "foundry-prf-node"}, + {name: "LaunchTemplate_GroupKey", resource: LaunchTemplate(substrate, ephemeral), expectedName: "foundry-lt-ephemeral"}, + {name: "AutoscalingGroup_GroupKey", resource: AutoscalingGroup(substrate, ephemeral), expectedName: "foundry-asg-ephemeral"}, + {name: "Node_GroupKeyAndOrdinal", resource: Node(substrate, persistent, 0), expectedName: "foundry-node-persistent-0"}, + {name: "Volume_GroupKeyAndOrdinal", resource: Volume(substrate, persistent, 2), expectedName: "foundry-vol-persistent-2"}, + {name: "EphemeralNode_GroupKeyAndOrdinal", resource: Node(substrate, ephemeral, 1), expectedName: "foundry-node-ephemeral-1"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedName, tt.resource.Name()) + }) + } +} + +// A role name is the longest suffix a caller has to budget for against its +// provider's cap, which this package does not know. +func TestRoleNameOverheadIsBounded(t *testing.T) { + purpose := convention.MustNewKey("appconfig-read") + maxRoleSuffix := len("-iam-exec-" + purpose.String()) + + for _, name := range []string{"a", "foundry", "signoz-prod-eu-central"} { + substrate := convention.MustNewSubstrate(name) + assert.LessOrEqual(t, len(IAMRolePolicy(substrate, RoleExec, purpose).Name())-len(name), maxRoleSuffix) + } +} + +// Adopting a resource must not rename it: the name belongs to whoever created it. +func TestSharedResourceKeepsItsName(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + shared := VPC(substrate).WithOwnership(convention.OwnershipShared) + + assert.Equal(t, VPC(substrate).Name(), shared.Name()) + assert.NotContains(t, shared.Tags(), displayName) + assert.Equal(t, "shared", shared.Tags()[Tag(convention.TagKeyOwner)]) +} + +func TestResourceTags(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + privateA := convention.MustNewKey("private-a") + persistent := convention.NewNodeGroup(convention.MustNewKey("persistent"), v1alpha1.StorageClassPersistent) + + tests := []struct { + name string + resource Resource + expectedPresent map[string]string + expectedAbsent []convention.TagKey + }{ + { + name: "Cluster_CarriesIdentityAndOwner", + resource: Cluster(substrate), + expectedPresent: map[string]string{ + Tag(convention.TagKeyName): "foundry", + Tag(convention.TagKeyOwner): "owned", + displayName: "foundry-cls", + }, + expectedAbsent: []convention.TagKey{convention.TagKeySubnetType, convention.TagKeyStorage, convention.TagKeyIdentities}, + }, + { + name: "PrivateSubnet_CarriesItsTypeSpelledOut", + resource: Subnet(substrate, privateA, v1alpha1.SubnetTypePrivate), + expectedPresent: map[string]string{ + displayName: "foundry-sub-private-a", + Tag(convention.TagKeySubnetType): "private", + }, + expectedAbsent: []convention.TagKey{convention.TagKeyStorage}, + }, + { + name: "PersistentNode_CarriesStorageFromItsGroup", + resource: Node(substrate, persistent, 0), + expectedPresent: map[string]string{ + displayName: "foundry-node-persistent-0", + Tag(convention.TagKeyStorage): "persistent", + }, + expectedAbsent: []convention.TagKey{convention.TagKeySubnetType, convention.TagKeyIdentities}, + }, + { + name: "ClaimedVolume_CarriesIdentities", + resource: Volume(substrate, persistent, 0).WithClaims(convention.Identities{ + convention.MustNewIdentity("telemetrystore", 0, 0), + convention.MustNewIdentity("metastore", 0), + }), + expectedPresent: map[string]string{ + displayName: "foundry-vol-persistent-0", + Tag(convention.TagKeyStorage): "persistent", + Tag(convention.TagKeyIdentities): "metastore-0,telemetrystore-0-0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tags := tt.resource.Tags() + + for key, expected := range tt.expectedPresent { + assert.Equal(t, expected, tags[key], "tag %s", key) + } + + for _, key := range tt.expectedAbsent { + assert.NotContains(t, tags, Tag(key)) + } + }) + } +} + +func TestResourceFilter(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + privateA := convention.MustNewKey("private-a") + persistent := convention.NewNodeGroup(convention.MustNewKey("persistent"), v1alpha1.StorageClassPersistent) + + tests := []struct { + name string + resource Resource + expectedFilter map[string]string + }{ + { + name: "VPC_SelectsIdentityOnly", + resource: VPC(substrate), + expectedFilter: map[string]string{ + Tag(convention.TagKeyName): "foundry", + }, + }, + { + name: "ProvenanceOnly_IsNotSelectedOn", + resource: Cluster(substrate), + expectedFilter: map[string]string{ + Tag(convention.TagKeyName): "foundry", + }, + }, + { + name: "PrivateSubnet_SelectsIdentityAndType", + resource: Subnet(substrate, privateA, v1alpha1.SubnetTypePrivate), + expectedFilter: map[string]string{ + Tag(convention.TagKeyName): "foundry", + Tag(convention.TagKeySubnetType): "private", + }, + }, + { + name: "PersistentNode_SelectsIdentityAndStorage", + resource: Node(substrate, persistent, 0), + expectedFilter: map[string]string{ + Tag(convention.TagKeyName): "foundry", + Tag(convention.TagKeyStorage): "persistent", + }, + }, + { + name: "ClaimedVolume_SelectsTheClaim", + resource: Volume(substrate, persistent, 0).WithClaims(convention.Identities{convention.MustNewIdentity("signoz", 0)}), + expectedFilter: map[string]string{ + Tag(convention.TagKeyName): "foundry", + Tag(convention.TagKeyStorage): "persistent", + Tag(convention.TagKeyIdentities): "signoz-0", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedFilter, tt.resource.Filter()) + }) + } +} + +// A fact stated once must render the same way wherever it is read back. The +// operator's key reaches the name; the closed enums reach the tags, which is +// all a consuming casting can predict. +func TestNameAndTagsAgreeOnTheSameFact(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + + for _, subnetType := range []v1alpha1.SubnetType{v1alpha1.SubnetTypePrivate, v1alpha1.SubnetTypePublic} { + key := convention.MustNewKey(subnetType.String() + "-a") + subnet := Subnet(substrate, key, subnetType) + + assert.Contains(t, subnet.Name(), key.String()) + assert.Equal(t, subnetType.String(), subnet.Tags()[Tag(convention.TagKeySubnetType)]) + } + + for _, storage := range []v1alpha1.StorageClass{v1alpha1.StorageClassPersistent, v1alpha1.StorageClassEphemeral} { + group := convention.NewNodeGroup(convention.MustNewKey(storage.String()), storage) + node := Node(substrate, group, 0) + + assert.Contains(t, node.Name(), group.Key().String()) + assert.Equal(t, storage.String(), node.Tags()[Tag(convention.TagKeyStorage)]) + } +} + +// Two types sharing a short form would derive the same name shape. A security +// group and a role each cover two constructors, distinguished by the trailing +// purpose rather than by a second short form. +func TestResourceTypeShortFormsAreDistinct(t *testing.T) { + resourceTypes := []resourceType{ + typeCluster, typeVPC, typeInternetGateway, typeSubnet, typeRouteTable, + typeNATGateway, typeElasticIP, typeSecurityGroup, typeRole, + typeInstanceProfile, typeLaunchTemplate, typeAutoscalingGroup, + typeNode, typeVolume, + } + + seen := make(map[string]struct{}, len(resourceTypes)) + for _, resource := range resourceTypes { + assert.NotContains(t, seen, resource.String()) + seen[resource.String()] = struct{}{} + } +} + +// An empty qualifier drops its segment, so one declaration serves both a +// security group and the rules attached to it. +func TestUnsetQualifierDropsFromTheName(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + + assert.Equal(t, "foundry-sg-task", SecurityGroup(substrate, RoleTask).Name()) + assert.Equal(t, "foundry-sg-task-intra-cluster", SecurityGroupRule(substrate, RoleTask, convention.MustNewKey("intra-cluster")).Name()) +} + +// A declared qualifier that renders nothing would silently drop a segment meant +// to distinguish the name. +func TestEveryDeclaredQualifierContributes(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + privateA := convention.MustNewKey("private-a") + persistent := convention.NewNodeGroup(convention.MustNewKey("persistent"), v1alpha1.StorageClassPersistent) + + tests := []struct { + name string + resource Resource + expectedSegments int + }{ + {name: "VPC_NoQualifier", resource: VPC(substrate), expectedSegments: 0}, + {name: "Subnet_Key", resource: Subnet(substrate, privateA, v1alpha1.SubnetTypePrivate), expectedSegments: 1}, + {name: "NATGateway_Key", resource: NATGateway(substrate, privateA), expectedSegments: 1}, + {name: "Role_Role", resource: IAMRole(substrate, RoleExec), expectedSegments: 1}, + {name: "RolePolicy_RoleAndPurpose", resource: IAMRolePolicy(substrate, RoleExec, convention.MustNewKey("ssm-session")), expectedSegments: 2}, + {name: "InstanceProfile_Role", resource: InstanceProfile(substrate, RoleNode), expectedSegments: 1}, + {name: "LaunchTemplate_GroupKey", resource: LaunchTemplate(substrate, persistent), expectedSegments: 1}, + {name: "Node_GroupKeyAndOrdinal", resource: Node(substrate, persistent, 0), expectedSegments: 2}, + {name: "Volume_GroupKeyAndOrdinal", resource: Volume(substrate, persistent, 0), expectedSegments: 2}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rendered := 0 + for _, qualifier := range tt.resource.resourceType.qualifiers { + if qualifier.of(tt.resource) != "" { + rendered++ + } + } + + assert.Equal(t, tt.expectedSegments, rendered) + }) + } +} + +// Ordinal zero is a real ordinal, so only types that have one declare it. +func TestOrdinalZeroRenders(t *testing.T) { + substrate := convention.MustNewSubstrate("foundry") + persistent := convention.NewNodeGroup(convention.MustNewKey("persistent"), v1alpha1.StorageClassPersistent) + + assert.Equal(t, "foundry-node-persistent-0", Node(substrate, persistent, 0).Name()) + assert.Equal(t, "foundry-vpc", VPC(substrate).Name()) +} diff --git a/internal/convention/aws/role.go b/internal/convention/aws/role.go new file mode 100644 index 00000000..e22765a1 --- /dev/null +++ b/internal/convention/aws/role.go @@ -0,0 +1,17 @@ +package aws + +// Role is what a security group or an IAM role is attached to. IAM roles use all +// three; a security group uses node and task. +type Role struct { + s string +} + +var ( + RoleNode = Role{s: "node"} + RoleTask = Role{s: "task"} + RoleExec = Role{s: "exec"} +) + +func (role Role) String() string { + return role.s +} diff --git a/internal/convention/aws/role_test.go b/internal/convention/aws/role_test.go new file mode 100644 index 00000000..c76dd36a --- /dev/null +++ b/internal/convention/aws/role_test.go @@ -0,0 +1,36 @@ +package aws + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestRole(t *testing.T) { + tests := []struct { + name string + role Role + expectedWord string + }{ + {name: "Node_Rendered", role: RoleNode, expectedWord: "node"}, + {name: "Task_Rendered", role: RoleTask, expectedWord: "task"}, + {name: "Exec_Rendered", role: RoleExec, expectedWord: "exec"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedWord, tt.role.String()) + }) + } +} + +// Two roles sharing a rendering would collide in a derived name. +func TestRolesAreDistinct(t *testing.T) { + roles := []Role{RoleNode, RoleTask, RoleExec} + + seen := make(map[string]struct{}, len(roles)) + for _, role := range roles { + assert.NotContains(t, seen, role.String()) + seen[role.String()] = struct{}{} + } +} diff --git a/internal/convention/aws/tag.go b/internal/convention/aws/tag.go new file mode 100644 index 00000000..83cac34a --- /dev/null +++ b/internal/convention/aws/tag.go @@ -0,0 +1,26 @@ +package aws + +import ( + "github.com/signoz/foundry/internal/convention" + "github.com/signoz/foundry/internal/domain" +) + +// displayName is unprefixed. "Name" is what an AWS console shows. +const displayName = "Name" + +// Tag renders a fact as an AWS tag key. Only AWS accepts the full prefix. +func Tag(key convention.TagKey) string { + return domain.MetadataPrefix + key.String() +} + +// Filter renders a selection as the tag match a data source is keyed by. +func Filter(selection convention.Selection) map[string]string { + match := selection.Match() + + tags := make(map[string]string, len(match)) + for key, value := range match { + tags[Tag(key)] = value + } + + return tags +} diff --git a/internal/convention/aws/tag_test.go b/internal/convention/aws/tag_test.go new file mode 100644 index 00000000..7f418e06 --- /dev/null +++ b/internal/convention/aws/tag_test.go @@ -0,0 +1,33 @@ +package aws + +import ( + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/convention" + "github.com/stretchr/testify/assert" +) + +// A filter's keys are the only ones whose spelling live infrastructure depends +// on: renaming one leaves it unmatched, with no checkpoint to catch it. AWS is +// the only provider whose tag keys accept this prefix, which is why the +// spelling is asserted here and not beside the facts. +func TestFilterKeysMatchDeployedSpelling(t *testing.T) { + filter := Filter(convention.MustNewSubstrate("foundry").Select(). + WithSubnetType(v1alpha1.SubnetTypePrivate). + WithStorage(v1alpha1.StorageClassPersistent). + WithClaims(convention.Identities{convention.MustNewIdentity("signoz", 0)})) + + assert.Equal(t, map[string]string{ + "foundry.signoz.io/name": "foundry", + "foundry.signoz.io/subnet-type": "private", + "foundry.signoz.io/storage": "persistent", + "foundry.signoz.io/identities": "signoz-0", + }, filter) +} + +// The display tag is the provider's own, not foundry's, so it carries no prefix. +func TestDisplayNameIsProviderNative(t *testing.T) { + assert.Equal(t, "Name", displayName) + assert.Equal(t, "foundry.signoz.io/owner", Tag(convention.TagKeyOwner)) +} From 876e7d08255918285cf5d1585f011c090f7e4035 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 18:00:59 +0530 Subject: [PATCH 33/38] docs(infrastructure): describe the canonical resource document --- docs/concepts/infrastructure.md | 262 ++++++++++++++++++++++---------- 1 file changed, 184 insertions(+), 78 deletions(-) diff --git a/docs/concepts/infrastructure.md b/docs/concepts/infrastructure.md index 0925b4cb..44666059 100644 --- a/docs/concepts/infrastructure.md +++ b/docs/concepts/infrastructure.md @@ -6,7 +6,7 @@ They are separate kinds, forged separately and applied separately. Infrastructur ## Declaring a substrate -An Infrastructure casting says which Kind it is provisioning for, and nothing about that Kind's internals: +An Infrastructure casting says where to provision, and nothing about what will run there: ```yaml apiVersion: v1alpha1 @@ -15,71 +15,115 @@ metadata: name: signoz spec: deployment: - platform: aws + platform: ecs mode: ec2 flavor: terraform - resource: - kind: Installation ``` -`spec.resource.kind` is either `Installation` or `CollectionAgent`. That single field is the whole input: Foundry knows what a default SigNoz installation needs, so it can size a substrate for one without being told anything about your components. +Foundry knows what a default SigNoz installation needs and sizes a substrate for one. It cannot know where to put them, so a casting states its subnets. See [The document](#the-document). + +## The document -## Requirements +Forging settles one document, `resource.yaml`, written to `casting.yaml.lock` under `spec.resource.status`. It has two halves. -Forging turns that declaration into a requirement document, written to `casting.yaml.lock` under `spec.resource.status`. For an `Installation`: +The top half is a **declaration**, layered: foundry's baseline, then what the platform decides, then whatever you put in `spec.resource.spec.config.data`, which wins. The vocabulary follows [kOps](https://kops.sigs.k8s.io/). ```yaml -nodeGroups: - ephemeral: - cpu: 2 - maxSize: 1 - memory: 4 - minSize: 1 +networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 +instanceGroups: + persistent: + storage: persistent + machineType: m5.large + minSize: 3 + maxSize: 3 rootVolume: size: 30 - persistent: - cpu: 2 + type: gp3 dataVolume: size: 50 - maxSize: 3 - memory: 8 - minSize: 3 + type: gp3 + ephemeral: + storage: ephemeral + machineType: c5.large + minSize: 1 + maxSize: 1 rootVolume: size: 30 + type: gp3 +``` + +The bottom half, `resources`, is **derived** from the settled declaration, and holds every name and tag the substrate stamps. One entry looks like this, and the rest follow the same shape: + +```yaml +resources: + vpc: + name: signoz-vpc + tags: + Name: signoz-vpc + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz + foundry.signoz.io/owner: owned ``` -Plus the ports the substrate has to admit at its edge: `4317` and `4318` for OTLP, and `8080` for the API server. A `CollectionAgent` gets the ephemeral group and the OTLP ports only, because it stores nothing. +The casting's templates interpolate `resources` and assemble no name or tag of their own. The Installation works the same tags out from the substrate name alone, and a string spelled twice is a filter that matches nothing. Stating `resources` yourself is an error, not an override. + +There is one baseline: the shape a default SigNoz installation needs. A substrate that keeps nothing drops the persistent group with `persistent: null`. The merge reads that as a deletion. + +### Subnets -Capacity is stated as CPU and memory rather than as `m5.large`, so the same document works on any provider. The casting resolves it to a real machine type at plan time. Set `machineType` yourself when you want an exact one. +Collections are keyed by a reference you choose. The key names the subnet's resources and is what an instance group points at. `private-a` above becomes `signoz-sub-private-a`. -### Node group fields +| Field | Meaning | +|---|---| +| `type` | `private` or `public`. Workloads go in private subnets | +| `zone` | The provider's availability zone, verbatim | +| `cidr` | The block carved out of `networkCIDR` | +| `egress` | ID of a NAT gateway this private subnet already routes through. Empty creates one | +| `id` | ID of an existing subnet to adopt. Empty creates one | + +**`zone` has no default and no fallback.** Zone letters are not contiguous within a region, and the mapping from letter to physical zone differs per account. Only you can state one. A casting with no subnets fails to forge. + +A private subnet with no `egress` needs a public subnet in the same zone to hold its NAT gateway. + +### Instance groups | Field | Meaning | |---|---| +| `storage` | `persistent` or `ephemeral`. See below | +| `machineType` | Provider machine type for each node | | `minSize` | Smallest the group may be | | `maxSize` | Largest the group may grow to | -| `cpu` | CPUs per node | -| `memory` | Memory per node, in GB | -| `machineType` | Provider machine type; when set, `cpu` and `memory` are ignored | -| `rootVolume.size` | Boot disk per node, in GB | -| `dataVolume.size` | Disk that outlives the node, in GB. Persistent groups only | +| `subnets` | Subnet references to place nodes in. Empty means every private subnet | +| `rootVolume` | Boot disk per node: `size` in GB, and `type` | +| `dataVolume` | Disk that outlives the node. Persistent groups only | -### Storage classes +Nodes in a pinned group are laid out across the group's subnets in ordinal order. Each node's data volume goes wherever the node does. A disk attaches only to a machine in its own zone. -Node groups are keyed by storage class, and the class decides how the group behaves: +### Storage classes | Class | Data | Size | Used by | |---|---|---|---| | `persistent` | Each node carries a disk that outlives it | Fixed: `minSize` and `maxSize` must match | ClickHouse, Keeper, PostgreSQL | | `ephemeral` | Keeps nothing | Scales between the bounds | Collector, MCP, UI | -A persistent node cannot be swapped for another, because a component's data is on the disk attached to it. That is why its bounds are pinned: there is nothing to autoscale when every node owns a claimed disk. +A persistent node cannot be swapped for another: a component's data is on the disk attached to it. Its bounds are pinned, since every node owns a claimed disk. -There is one group per class. Two persistent groups would be indistinguishable to the Installation, which selects nodes by class, so anything scheduled would land on either one at random. +The class is the only thing about a group the Installation can select on. Two groups may share a class, and the Installation reaches both. ### Overriding the defaults -Put your own values under `spec.resource.spec.config.data`, keyed by class. You only state what you are changing; everything else comes from the defaults above. +Put your own values under `spec.resource.spec.config.data`, keyed the same way. You only state what you are changing; everything else comes from the layers underneath. ```yaml apiVersion: v1alpha1 @@ -88,16 +132,33 @@ metadata: name: signoz spec: deployment: - platform: aws + platform: ecs mode: ec2 flavor: terraform resource: - kind: Installation spec: config: data: resource.yaml: | - nodeGroups: + networking: + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + private-b: + type: private + zone: us-east-1b + cidr: 10.0.32.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 + public-b: + type: public + zone: us-east-1b + cidr: 10.0.100.0/22 + instanceGroups: persistent: minSize: 6 maxSize: 6 @@ -107,7 +168,7 @@ spec: maxSize: 4 ``` -**If you scale SigNoz, you have to scale the persistent group yourself.** The default is three persistent nodes, which covers one Keeper, the metadata store, and one ClickHouse node. Three Keeper replicas and two ClickHouse shards need more, and Infrastructure cannot work that out for you because it never reads your Installation. +**If you scale SigNoz, you have to scale the persistent group yourself.** The default is three persistent nodes, which covers one Keeper, the metadata store, and one ClickHouse node. Three Keeper replicas and two ClickHouse shards need more. Infrastructure never reads your Installation and cannot work it out. ## Order of operations @@ -127,7 +188,7 @@ spec: stamps tags ---------------------> reads them back ``` -Infrastructure first. The Installation's lookups return nothing until the substrate exists, so applying it early produces a plan that places nothing. +Infrastructure first. The Installation's lookups return nothing until the substrate exists, and an early apply produces a plan that places nothing. The two runs keep separate Terraform state. Neither reads the other's, and Foundry passes nothing between them. @@ -142,16 +203,32 @@ metadata: name: signoz spec: deployment: - platform: aws + platform: ecs mode: ec2 flavor: terraform infrastructure: name: signoz ``` -That is the whole binding. Everything else travels as tags on the resources themselves: the Installation searches for `foundry.signoz.io/name` and `foundry.signoz.io/storage` to find machines and disks, and reads `foundry.signoz.io/identities` off a disk to learn which component owns it. +That is the whole binding, and it is required. Without it the Installation has no name to derive a filter from, and forging fails. + +Everything else the Installation needs, it looks up from that one name: + +| What it needs | How it finds it | +|---|---| +| The cluster | the derived name, `signoz-cls` | +| Subnets to place tasks in | the tags `foundry.signoz.io/name` and `foundry.signoz.io/subnet-type: private` | +| The security group | the derived name, `signoz-sg-task` | +| The VPC for its Cloud Map namespace | the tag `foundry.signoz.io/name` | +| The task and execution roles | the derived names, `signoz-iam-task` and `signoz-iam-exec` | +| Machines to pin stateful tasks to | the tags `foundry.signoz.io/name` and `foundry.signoz.io/storage` | +| Which component owns a disk | the tag `foundry.signoz.io/identities` on the disk | + +Each arrives in Terraform as a variable defaulted to what was derived. A one-off change needs no edit to a generated file. Any of them can be stated on the Installation instead, one at a time, for a cluster Foundry did not provision; what is stated becomes the variable's own value and nothing is looked up. + +No outputs are wired between the two, and no state file is shared. Foundry generates files and exits. It never calls a cloud API and cannot ask the provider what it created a moment ago. Both sides work the names and tags out the same way, which is why they are derived. -No outputs are wired between the two, and no state file is shared. Foundry generates files and exits; it never calls a cloud API, so it cannot ask the provider what it created a moment ago. Both sides have to work the names and tags out the same way, which is why they are derived rather than configured. +A lookup that matches nothing fails the plan. That is deliberate: the alternative is a plan that succeeds and places tasks nowhere. ## Conventions @@ -163,57 +240,69 @@ Foundry derives every name and every tag from the substrate's name and a closed -[-...] ``` -Broad to narrow, so everything belonging to one deployment shares a prefix and sorts together. A qualifier that does not apply is left out rather than padded, which is why a zone-shared route table has no zone in its name. +Broad to narrow: everything belonging to one deployment shares a prefix and sorts together. A qualifier that does not apply is left out, letting a security group and its rules share one form. | Resource | Type | Qualifiers | Example | |---|---|---|---| | Cluster | `cls` | | `signoz-cls` | | VPC | `vpc` | | `signoz-vpc` | | Internet gateway | `igw` | | `signoz-igw` | -| Subnet | `sub` | visibility, zone | `signoz-sub-prv-east1a` | -| Route table, per zone | `rt` | visibility, zone | `signoz-rt-prv-east1a` | -| Route table, zone-shared | `rt` | visibility | `signoz-rt-pub` | -| NAT gateway | `nat` | zone | `signoz-nat-east1a` | +| Subnet | `sub` | subnet key | `signoz-sub-private-a` | +| Route table | `rt` | subnet key | `signoz-rt-private-a` | +| NAT gateway | `nat` | subnet key | `signoz-nat-private-a` | +| Elastic IP | `eip` | subnet key | `signoz-eip-private-a` | | Security group | `sg` | role | `signoz-sg-task` | +| Security group rule | `sg` | role, purpose | `signoz-sg-task-intra-cluster` | | IAM role | `iam` | role | `signoz-iam-exec` | -| Node | `node` | storage class, ordinal | `signoz-node-persistent-0` | -| Volume | `vol` | storage class, ordinal | `signoz-vol-persistent-0` | +| IAM role policy | `iam` | role, purpose | `signoz-iam-task-appconfig-read` | +| Instance profile | `prf` | role | `signoz-prf-node` | +| Launch template | `lt` | group key | `signoz-lt-ephemeral` | +| Autoscaling group | `asg` | group key | `signoz-asg-ephemeral` | +| Node | `node` | group key, ordinal | `signoz-node-persistent-0` | +| Volume | `vol` | group key, ordinal | `signoz-vol-persistent-0` | + +A NAT gateway and its address are keyed by the **private** subnet they serve, not the public one they sit in. Route tables are per subnet: a private subnet's default route is its own zone's gateway. ### Values -| Axis | Values | In a name | In a tag | +| Axis | Values | Where it comes from | In a tag | |---|---|---|---| -| Visibility | private, public | `prv`, `pub` | `private`, `public` | -| Storage class | persistent, ephemeral | `persistent`, `ephemeral` | same | -| Role | node, task, exec | `node`, `task`, `exec` | not tagged | -| Zone | the provider's zone | locale dropped: `us-east-1a` becomes `east1a` | provider's own form | -| Ordinal | position in a group | zero-based: `0`, `1`, `2` | not tagged | +| Subnet key | yours | `networking.subnets` | not tagged | +| Group key | yours | `instanceGroups` | not tagged | +| Subnet type | private, public | a subnet's `type` | `private`, `public` | +| Storage class | persistent, ephemeral | a group's `storage` | `persistent`, `ephemeral` | +| Role | node, task, exec | the platform | not tagged | +| Purpose | what a rule admits or a policy grants | the platform | not tagged | +| Ordinal | position in a group | derived, zero-based | not tagged | -A value the Installation matches on is never abbreviated, because the string has to be identical on both sides. A value only a person reads can be short where space is tight, which is why visibility has two forms and the storage class has one. +A key you chose distinguishes one name from another and says nothing the Installation can predict. Everything it filters on is a closed enum, spelled out in full. The string has to be identical on both sides, so nothing reaching a tag is abbreviated. -Length caps belong to the platform. IAM role names cap at 64 characters on AWS, which is why roles are the shortest derivation above; the substrate name is capped at 63. +Length caps belong to the platform. IAM role names cap at 64 characters on AWS, keeping roles among the shortest derivations above. The substrate name is capped at 63. ### Tags -Every tag lives under `foundry.signoz.io/` and is one segment deep, so a single filter finds everything Foundry touched in an account. +Every tag lives under `foundry.signoz.io/` and is one segment deep. A single filter finds everything Foundry touched in an account. | Tag | Value | Read by | |---|---|---| | `foundry.signoz.io/name` | The substrate's name | The Installation, to find this substrate's resources | +| `foundry.signoz.io/subnet-type` | `private` or `public` | The Installation, to pick which subnets to place a workload in | | `foundry.signoz.io/storage` | `persistent` or `ephemeral` | The Installation, to pick which nodes a component runs on | | `foundry.signoz.io/identities` | Which components claim a disk | The Installation, to keep a component on its own data | -| `foundry.signoz.io/resource-kind` | The Kind the substrate serves | People | | `foundry.signoz.io/owner` | `owned` or `shared` | People, to tell what Foundry may delete | -| `foundry.signoz.io/visibility` | `private` or `public` | People | +| `foundry.signoz.io/managed-by` | `foundry` | People | +| `foundry.signoz.io/kind` | The casting Kind that stamped it | People | | `Name` | The derived name | Cloud consoles, which show this tag by convention | -The first three are how the Installation finds anything, so they are fixed. The rest describe a resource and are free to change. +The first four are how the Installation finds anything and are fixed. The rest describe a resource and are free to change. + +Your own tags go in `cloudLabels` and are applied to every resource the substrate provisions. They sit underneath the derived ones. A `cloudLabels` entry cannot rename a tag the Installation matches on. ### Identities -A component that keeps data has an identity, written `--` with zero-based ordinals: `telemetrystore-0-0`, `telemetrykeeper-1`, `metastore-0`. An identity claims a disk and stays with it, so a component keeps its data when the machine under it is replaced. +A component that keeps data has an identity, written `--` with zero-based ordinals: `telemetrystore-0-0`, `telemetrykeeper-1`, `metastore-0`. An identity claims a disk and stays with it. A component keeps its data when the machine under it is replaced. -Claims are recorded on the disk in `foundry.signoz.io/identities`, comma-joined and sorted, so the value is stable between runs. +Claims are recorded on the disk in `foundry.signoz.io/identities`, comma-joined and sorted for a stable value between runs. ## Dependencies @@ -222,30 +311,31 @@ Claims are recorded on the disk in `foundry.signoz.io/identities`, comma-joined ``` vpc | - +-- internet gateway + +-- internet gateway (only if a public subnet is declared) | - +-- subnet (one per zone, private and public) + +-- subnet (one per declared key) | | - | +-- nat gateway (public subnet, one per zone) - | +-- route table (private: per zone, via nat) - | (public: shared, via igw) + | +-- route table (private: via its own nat) + | | (public: via igw) + | +-- nat gateway + address (private only, in a public subnet + | of the same zone) +-- security group - iam role -- instance profile + iam role -> instance profile - per persistent ordinal: - instance --> subnet in zone N, instance profile, security group - volume --> zone N + per node of a pinned group: + instance --> the group's next subnet, instance profile, security group + volume --> that subnet's zone attachment --> instance + volume - per ephemeral group: - launch template --> subnets, security group, instance profile + per scaling group: + launch template --> the group's subnets, security group, instance profile autoscaling group --> launch template ``` -A persistent node and its disk are placed in the same zone, because a disk can only attach to a machine in its own zone. Ordinal 0 goes in the first zone, 1 in the second, and so on. +A node and its disk take their zone from the same subnet; a disk attaches only to a machine in its own zone. Ordinal 0 goes in the group's first subnet, 1 in the second, wrapping around. -Persistent nodes are individual machines rather than an autoscaling group. An autoscaler replacing a machine would move a disk out from under whatever component owns it. +Nodes in a pinned group are individual machines, not an autoscaling group. An autoscaler replacing a machine would move a disk out from under whatever component owns it. ### How a component reaches its data @@ -253,7 +343,7 @@ Persistent nodes are individual machines rather than an autoscaling group. An au task --pinned to--> instance --currently holds--> volume --claimed by--> identity ``` -Read it right to left. An identity such as `telemetrystore-0-0` claims a disk. The disk is attached to some machine. The task is pinned to that machine, so it starts on top of its own data. +Read it right to left. An identity such as `telemetrystore-0-0` claims a disk. The disk is attached to some machine. The task is pinned to that machine and starts on top of its own data. The claim is recorded on the **disk**, not the machine. Machines get replaced routinely, by a resize, an image update, or a failure. A claim written on the machine would be lost every time one was replaced. Written on the disk it survives, and each plan works out which machine currently holds it. @@ -269,14 +359,30 @@ The claim is recorded on the **disk**, not the machine. Machines get replaced ro ## Adopting resources you already have -A resource tagged `foundry.signoz.io/owner: shared` keeps the name it already had and is never deleted. This is how an existing VPC gets used rather than replaced. +Set `networking.networkID` to a network you already run, and give every subnet its own `id`: + +```yaml +networking: + networkID: vpc-0a1b2c3d + subnets: + private-a: + type: private + zone: us-east-1a + id: subnet-0a1b2c3d +``` + +Foundry then references the network instead of describing it. It creates no VPC, no internet gateway, no route tables and no NAT gateways, stamps no tags on anything it did not create, and provisions only the compute placed inside. Routing an adopted subnet would replace whatever you attached to it. + +A network is adopted whole. Half of one would leave Foundry routing subnets it did not create, or carving subnets out of address space it cannot see. + +A private subnet that already has a way out can keep it without adopting the whole network: set `egress` to the NAT gateway it routes through, and Foundry creates none. -You can point an Installation at a network, subnets and a cluster you already run. Persistent components are the exception: they need disks discovered by tag, so those have to come from an Infrastructure casting. +Persistent components are the exception. They need disks discovered by tag, which have to come from an Infrastructure casting. ## Limits -**One node group per storage class.** There is no way to put Keeper on cheaper machines than ClickHouse, because the Installation selects nodes by class and has no vocabulary for naming a group. +**The Installation cannot name an instance group.** You can declare two persistent groups, but the Installation selects by storage class and anything scheduled lands on either one. There is no way to put Keeper on cheaper machines than ClickHouse. **More stateful components than persistent nodes.** Two identities end up on one disk. Both components run, both write to the same volume, and it looks like replication without being replication. Count one persistent node per identity: one per Keeper replica, one per ClickHouse node, one for the metadata store. -**A claimed disk attached to nothing.** The task stays pending rather than starting empty somewhere else, which is deliberate: starting empty would look like it worked. \ No newline at end of file +**A claimed disk attached to nothing.** The task stays pending rather than starting empty somewhere else, which is deliberate: starting empty would look like it worked. From 1d5e46ad4062749d7ecbcf5666a4b0e9dcbc7c9d Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 18:08:26 +0530 Subject: [PATCH 34/38] feat(infrastructure): add the ecs ec2 terraform casting Provisions the substrate an ECS/EC2 installation lands on: a VPC with public and private subnets, an ECS cluster, node groups per storage class, and the IAM the nodes need. The enricher contributes the declaration; the molding derives every name and tag through the aws convention, so the templates interpolate the resources map rather than assembling identifiers inline. Registers the casting with terraform as its tooler and awsconvention.Resources as its deriver. --- .../infrastructure/casting.yaml.lock | 293 +++++++++++++++++ .../ec2/terraform/infrastructure/infra.yaml | 41 +++ .../infrastructure/cloud-init/ephemeral.yaml | 6 + .../infrastructure/cloud-init/persistent.yaml | 29 ++ .../pours/infrastructure/main.tf.json | 310 ++++++++++++++++++ .../pours/infrastructure/outputs.tf.json | 44 +++ .../pours/infrastructure/providers.tf.json | 10 + .../pours/infrastructure/variables.tf.json | 1 + .../pours/infrastructure/versions.tf.json | 11 + .../ecsec2terraformcasting/casting.go | 97 ++++++ .../ecsec2terraformcasting/casting_test.go | 134 ++++++++ .../ecsec2terraformcasting/embed.go | 19 ++ .../ecsec2terraformcasting/embed_test.go | 48 +++ .../ecsec2terraformcasting/enricher.go | 58 ++++ .../ecsec2terraformcasting/enricher_test.go | 66 ++++ .../templates/cloudinit.yaml.gotmpl | 33 ++ .../templates/main.tf.json.gotmpl | 297 +++++++++++++++++ .../templates/outputs.tf.json.gotmpl | 54 +++ .../templates/providers.tf.json.gotmpl | 13 + .../templates/variables.tf.json.gotmpl | 117 +++++++ .../templates/versions.tf.json.gotmpl | 11 + internal/casting/infrastructure/registry.go | 15 +- 22 files changed, 1706 insertions(+), 1 deletion(-) create mode 100644 docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock create mode 100644 docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/ephemeral.yaml create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/persistent.yaml create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/main.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/outputs.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/providers.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/variables.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/infrastructure/versions.tf.json create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/casting.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/casting_test.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/embed.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/enricher.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/enricher_test.go create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl create mode 100644 internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl diff --git a/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock b/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock new file mode 100644 index 00000000..9e971916 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock @@ -0,0 +1,293 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: foundry +spec: + deployment: + flavor: terraform + mode: ec2 + platform: ecs + patches: + - operations: + - op: replace + path: /variable/aws_region/default + value: us-east-1 + target: infrastructure/variables.tf.json + type: jsonpatch + resource: + spec: + cluster: {} + config: + data: + resource.yaml: | + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 + instanceGroups: + persistent: + minSize: 3 + maxSize: 3 + machineType: t3.medium + dataVolume: + size: 5 + ephemeral: + machineType: t3.small + status: + config: + data: + resource.yaml: | + instanceGroups: + ephemeral: + machineType: t3.small + maxSize: 1 + minSize: 1 + rootVolume: + size: 30 + type: gp3 + storage: ephemeral + persistent: + dataVolume: + size: 5 + type: gp3 + machineType: t3.medium + maxSize: 3 + minSize: 3 + rootVolume: + size: 30 + type: gp3 + storage: persistent + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + cidr: 10.0.0.0/19 + type: private + zone: us-east-1a + public-a: + cidr: 10.0.96.0/22 + type: public + zone: us-east-1a + resources: + cluster: + name: foundry-cls + tags: + Name: foundry-cls + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + ignoredTags: + - foundry.signoz.io/identities + instanceGroups: + ephemeral: + autoscalingGroup: + name: foundry-asg-ephemeral + tags: + Name: foundry-asg-ephemeral + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: ephemeral + launchTemplate: + name: foundry-lt-ephemeral + tags: + Name: foundry-lt-ephemeral + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: ephemeral + selector: + foundry.signoz.io/name: foundry + foundry.signoz.io/storage: ephemeral + storage: ephemeral + subnets: + - private-a + persistent: + nodes: + - name: foundry-node-persistent-0 + ordinal: 0 + subnet: private-a + tags: + Name: foundry-node-persistent-0 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + volume: + name: foundry-vol-persistent-0 + tags: + Name: foundry-vol-persistent-0 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + - name: foundry-node-persistent-1 + ordinal: 1 + subnet: private-a + tags: + Name: foundry-node-persistent-1 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + volume: + name: foundry-vol-persistent-1 + tags: + Name: foundry-vol-persistent-1 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + - name: foundry-node-persistent-2 + ordinal: 2 + subnet: private-a + tags: + Name: foundry-node-persistent-2 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + volume: + name: foundry-vol-persistent-2 + tags: + Name: foundry-vol-persistent-2 + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/storage: persistent + selector: + foundry.signoz.io/name: foundry + foundry.signoz.io/storage: persistent + storage: persistent + subnets: + - private-a + instanceProfile: + name: foundry-prf-node + tags: + Name: foundry-prf-node + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + internetGateway: + name: foundry-igw + tags: + Name: foundry-igw + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + natGateways: + private-a: + address: + name: foundry-eip-private-a + tags: + Name: foundry-eip-private-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + name: foundry-nat-private-a + subnet: public-a + tags: + Name: foundry-nat-private-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + roles: + node: + name: foundry-iam-node + tags: + Name: foundry-iam-node + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + routeTables: + private-a: + name: foundry-rt-private-a + tags: + Name: foundry-rt-private-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + public-a: + name: foundry-rt-public-a + tags: + Name: foundry-rt-public-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + securityGroup: + name: foundry-sg-task + tags: + Name: foundry-sg-task + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + securityGroupRules: + all-outbound: + name: foundry-sg-task-all-outbound + tags: + Name: foundry-sg-task-all-outbound + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + intra-cluster: + name: foundry-sg-task-intra-cluster + tags: + Name: foundry-sg-task-intra-cluster + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + subnets: + private-a: + name: foundry-sub-private-a + public: false + tags: + Name: foundry-sub-private-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/subnet-type: private + public-a: + name: foundry-sub-public-a + public: true + tags: + Name: foundry-sub-public-a + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned + foundry.signoz.io/subnet-type: public + vpc: + name: foundry-vpc + tags: + Name: foundry-vpc + foundry.signoz.io/kind: Infrastructure + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: foundry + foundry.signoz.io/owner: owned diff --git a/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml b/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml new file mode 100644 index 00000000..81f6baff --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml @@ -0,0 +1,41 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: foundry +spec: + deployment: + platform: ecs + mode: ec2 + flavor: terraform + resource: + spec: + config: + data: + resource.yaml: | + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 + instanceGroups: + persistent: + minSize: 3 + maxSize: 3 + machineType: t3.medium + dataVolume: + size: 5 + ephemeral: + machineType: t3.small + patches: + - target: "infrastructure/variables.tf.json" + type: jsonpatch + operations: + - op: replace + path: /variable/aws_region/default + value: us-east-1 diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/ephemeral.yaml b/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/ephemeral.yaml new file mode 100644 index 00000000..d4f08a72 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/ephemeral.yaml @@ -0,0 +1,6 @@ +#cloud-config +write_files: + - path: /etc/ecs/ecs.config + content: | + ECS_CLUSTER=foundry-cls + ECS_INSTANCE_ATTRIBUTES={"foundry.signoz.io/name":"foundry","foundry.signoz.io/storage":"ephemeral"} diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/persistent.yaml b/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/persistent.yaml new file mode 100644 index 00000000..54871eb8 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/cloud-init/persistent.yaml @@ -0,0 +1,29 @@ +#cloud-config +write_files: + - path: /etc/ecs/ecs.config + content: | + ECS_CLUSTER=foundry-cls + ECS_INSTANCE_ATTRIBUTES={"foundry.signoz.io/name":"foundry","foundry.signoz.io/storage":"persistent"} + # The agent must never register a node whose data volume is not mounted: + # tasks would bind-mount onto the root disk and state would silently land + # on a disk that dies with the instance. + - path: /etc/systemd/system/ecs.service.d/10-foundry-data.conf + content: | + [Unit] + RequiresMountsFor=/var/lib/foundry +# Terraform attaches the data volume after boot begins, and cloud-init's +# fs_setup/mounts run once, first boot only, silently skipping devices that do +# not exist yet (canonical/cloud-init#3386). bootcmd runs every boot, before +# anything downstream of cloud-init (the agent included), so it waits for the +# attachment, formats only a device with no filesystem signature, and mounts. +bootcmd: + - 'for i in $(seq 1 120); do test -b /dev/xvdf && break; sleep 5; done' + - 'test -b /dev/xvdf || { echo "foundry: data volume /dev/xvdf never attached" >&2; exit 1; }' + - 'blkid /dev/xvdf >/dev/null || mkfs.ext4 -L foundry-data /dev/xvdf' + - 'mkdir -p /var/lib/foundry' + - 'mountpoint -q /var/lib/foundry || mount /dev/xvdf /var/lib/foundry' +# The fstab entry mounts at local-fs on every later boot and is what gives the +# RequiresMountsFor gate a real mount unit to require. The device exists by the +# time this module runs, because bootcmd waited. +mounts: + - [/dev/xvdf, /var/lib/foundry, ext4, "defaults,nofail,x-systemd.device-timeout=10min", "0", "2"] diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/main.tf.json b/docs/examples/ecs/ec2/terraform/pours/infrastructure/main.tf.json new file mode 100644 index 00000000..f64d083d --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/main.tf.json @@ -0,0 +1,310 @@ +{ + "locals": { + "vpc_id": "${aws_vpc.main.id}", + "subnet_ids": { + "private-a": "${aws_subnet.private-a.id}", + "public-a": "${aws_subnet.public-a.id}" + } + }, + "data": { + "aws_ssm_parameter": { + "ecs_ami": { + "name": "/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id" + } + } + }, + "resource": { + "aws_vpc": { + "main": { + "cidr_block": "${var.network_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": {"Name":"foundry-vpc","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_subnet": { + "private-a": { + "vpc_id": "${local.vpc_id}", + "cidr_block": "${var.subnet_private-a_cidr}", + "availability_zone": "${var.subnet_private-a_zone}", + "tags": {"Name":"foundry-sub-private-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/subnet-type":"private"} + }, + "public-a": { + "vpc_id": "${local.vpc_id}", + "cidr_block": "${var.subnet_public-a_cidr}", + "availability_zone": "${var.subnet_public-a_zone}", + "map_public_ip_on_launch": true, + "tags": {"Name":"foundry-sub-public-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/subnet-type":"public"} + } + }, + "aws_internet_gateway": { + "main": { + "vpc_id": "${local.vpc_id}", + "tags": {"Name":"foundry-igw","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_eip": { + "private-a": { + "domain": "vpc", + "tags": {"Name":"foundry-eip-private-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_nat_gateway": { + "private-a": { + "allocation_id": "${aws_eip.private-a.id}", + "subnet_id": "${local.subnet_ids[\"public-a\"]}", + "tags": {"Name":"foundry-nat-private-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"}, + "depends_on": ["aws_internet_gateway.main"] + } + }, + "aws_route_table": { + "private-a": { + "vpc_id": "${local.vpc_id}", + "tags": {"Name":"foundry-rt-private-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + }, + "public-a": { + "vpc_id": "${local.vpc_id}", + "tags": {"Name":"foundry-rt-public-a","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_route": { + "private-a": { + "route_table_id": "${aws_route_table.private-a.id}", + "destination_cidr_block": "0.0.0.0/0", + "nat_gateway_id": "${aws_nat_gateway.private-a.id}" + }, + "public-a": { + "route_table_id": "${aws_route_table.public-a.id}", + "destination_cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + } + }, + "aws_route_table_association": { + "private-a": { + "subnet_id": "${local.subnet_ids[\"private-a\"]}", + "route_table_id": "${aws_route_table.private-a.id}" + }, + "public-a": { + "subnet_id": "${local.subnet_ids[\"public-a\"]}", + "route_table_id": "${aws_route_table.public-a.id}" + } + }, + "terraform_data": { + "persistent": { + "triggers_replace": "${var.group_persistent_machine_type}" + } + }, + "aws_instance": { + "persistent-0": { + "ami": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_persistent_machine_type}", + "subnet_id": "${local.subnet_ids[\"private-a\"]}", + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "iam_instance_profile": "${aws_iam_instance_profile.node.name}", + "user_data_base64": "${filebase64(\"${path.module}/cloud-init/persistent.yaml\")}", + "user_data_replace_on_change": true, + "root_block_device": [ + { + "volume_size": "${var.group_persistent_root_volume_size}", + "volume_type": "${var.group_persistent_root_volume_type}", + "delete_on_termination": true + } + ], + "tags": {"Name":"foundry-node-persistent-0","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"}, + "lifecycle": { + "replace_triggered_by": ["terraform_data.persistent"], + "ignore_changes": ["ami"] + }, + "depends_on": ["aws_ecs_cluster.main"] + }, + "persistent-1": { + "ami": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_persistent_machine_type}", + "subnet_id": "${local.subnet_ids[\"private-a\"]}", + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "iam_instance_profile": "${aws_iam_instance_profile.node.name}", + "user_data_base64": "${filebase64(\"${path.module}/cloud-init/persistent.yaml\")}", + "user_data_replace_on_change": true, + "root_block_device": [ + { + "volume_size": "${var.group_persistent_root_volume_size}", + "volume_type": "${var.group_persistent_root_volume_type}", + "delete_on_termination": true + } + ], + "tags": {"Name":"foundry-node-persistent-1","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"}, + "lifecycle": { + "replace_triggered_by": ["terraform_data.persistent"], + "ignore_changes": ["ami"] + }, + "depends_on": ["aws_ecs_cluster.main"] + }, + "persistent-2": { + "ami": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_persistent_machine_type}", + "subnet_id": "${local.subnet_ids[\"private-a\"]}", + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "iam_instance_profile": "${aws_iam_instance_profile.node.name}", + "user_data_base64": "${filebase64(\"${path.module}/cloud-init/persistent.yaml\")}", + "user_data_replace_on_change": true, + "root_block_device": [ + { + "volume_size": "${var.group_persistent_root_volume_size}", + "volume_type": "${var.group_persistent_root_volume_type}", + "delete_on_termination": true + } + ], + "tags": {"Name":"foundry-node-persistent-2","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"}, + "lifecycle": { + "replace_triggered_by": ["terraform_data.persistent"], + "ignore_changes": ["ami"] + }, + "depends_on": ["aws_ecs_cluster.main"] + } + }, + "aws_ebs_volume": { + "persistent-0": { + "availability_zone": "${var.subnet_private-a_zone}", + "size": "${var.group_persistent_data_volume_size}", + "type": "${var.group_persistent_data_volume_type}", + "tags": {"Name":"foundry-vol-persistent-0","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"} + }, + "persistent-1": { + "availability_zone": "${var.subnet_private-a_zone}", + "size": "${var.group_persistent_data_volume_size}", + "type": "${var.group_persistent_data_volume_type}", + "tags": {"Name":"foundry-vol-persistent-1","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"} + }, + "persistent-2": { + "availability_zone": "${var.subnet_private-a_zone}", + "size": "${var.group_persistent_data_volume_size}", + "type": "${var.group_persistent_data_volume_type}", + "tags": {"Name":"foundry-vol-persistent-2","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"persistent"} + } + }, + "aws_volume_attachment": { + "persistent-0": { + "device_name": "/dev/xvdf", + "volume_id": "${aws_ebs_volume.persistent-0.id}", + "instance_id": "${aws_instance.persistent-0.id}" + }, + "persistent-1": { + "device_name": "/dev/xvdf", + "volume_id": "${aws_ebs_volume.persistent-1.id}", + "instance_id": "${aws_instance.persistent-1.id}" + }, + "persistent-2": { + "device_name": "/dev/xvdf", + "volume_id": "${aws_ebs_volume.persistent-2.id}", + "instance_id": "${aws_instance.persistent-2.id}" + } + }, + "aws_launch_template": { + "ephemeral": { + "name": "foundry-lt-ephemeral", + "image_id": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_ephemeral_machine_type}", + "iam_instance_profile": { + "arn": "${aws_iam_instance_profile.node.arn}" + }, + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "user_data": "${filebase64(\"${path.module}/cloud-init/ephemeral.yaml\")}", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": [ + { + "volume_size": "${var.group_ephemeral_root_volume_size}", + "volume_type": "${var.group_ephemeral_root_volume_type}", + "delete_on_termination": true + } + ] + } + ], + "tag_specifications": [ + { + "resource_type": "instance", + "tags": {"Name":"foundry-lt-ephemeral","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"ephemeral"} + } + ], + "tags": {"Name":"foundry-lt-ephemeral","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned","foundry.signoz.io/storage":"ephemeral"} + } + }, + "aws_autoscaling_group": { + "ephemeral": { + "name": "foundry-asg-ephemeral", + "desired_capacity": "${var.group_ephemeral_min_size}", + "min_size": "${var.group_ephemeral_min_size}", + "max_size": "${var.group_ephemeral_max_size}", + "vpc_zone_identifier": ["${local.subnet_ids[\"private-a\"]}"], + "launch_template": [ + { + "id": "${aws_launch_template.ephemeral.id}", + "version": "$Latest" + } + ], + "tag": [ + {"key": "Name", "value": "foundry-asg-ephemeral", "propagate_at_launch": true}, + {"key": "foundry.signoz.io/kind", "value": "Infrastructure", "propagate_at_launch": true}, + {"key": "foundry.signoz.io/managed-by", "value": "foundry", "propagate_at_launch": true}, + {"key": "foundry.signoz.io/name", "value": "foundry", "propagate_at_launch": true}, + {"key": "foundry.signoz.io/owner", "value": "owned", "propagate_at_launch": true}, + {"key": "foundry.signoz.io/storage", "value": "ephemeral", "propagate_at_launch": true} + ], + "depends_on": ["aws_ecs_cluster.main"] + } + }, + "aws_security_group": { + "tasks": { + "name": "foundry-sg-task", + "description": "SigNoz ECS tasks and container instances", + "vpc_id": "${local.vpc_id}", + "tags": {"Name":"foundry-sg-task","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_vpc_security_group_ingress_rule": { + "intra_cluster": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "intra-cluster traffic", + "ip_protocol": "-1", + "referenced_security_group_id": "${aws_security_group.tasks.id}", + "tags": {"Name":"foundry-sg-task-intra-cluster","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_vpc_security_group_egress_rule": { + "all_outbound": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "all outbound", + "ip_protocol": "-1", + "cidr_ipv4": "0.0.0.0/0", + "tags": {"Name":"foundry-sg-task-all-outbound","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_iam_role": { + "node": { + "name": "foundry-iam-node", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + "tags": {"Name":"foundry-iam-node","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_iam_instance_profile": { + "node": { + "name": "foundry-prf-node", + "role": "${aws_iam_role.node.name}", + "tags": {"Name":"foundry-prf-node","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + }, + "aws_iam_role_policy_attachment": { + "node": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role", + "role": "${aws_iam_role.node.name}" + } + }, + "aws_ecs_cluster": { + "main": { + "name": "foundry-cls", + "tags": {"Name":"foundry-cls","foundry.signoz.io/kind":"Infrastructure","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry","foundry.signoz.io/owner":"owned"} + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/outputs.tf.json b/docs/examples/ecs/ec2/terraform/pours/infrastructure/outputs.tf.json new file mode 100644 index 00000000..5198e18e --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/outputs.tf.json @@ -0,0 +1,44 @@ +{ + "output": { + "cluster_name": { + "description": "Name of the ECS cluster", + "value": "${aws_ecs_cluster.main.name}" + }, + "cluster_arn": { + "description": "ARN of the ECS cluster", + "value": "${aws_ecs_cluster.main.arn}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${local.vpc_id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets, which is where workloads are placed", + "value": ["${local.subnet_ids[\"private-a\"]}"] + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": ["${local.subnet_ids[\"public-a\"]}"] + }, + "security_group_ids": { + "description": "IDs of the tasks security group", + "value": ["${aws_security_group.tasks.id}"] + }, + "node_role_arn": { + "description": "ARN of the ECS container-instance role", + "value": "${aws_iam_role.node.arn}" + }, + "instance_group_ephemeral_asg_name": { + "description": "Name of the ephemeral autoscaling group", + "value": "${aws_autoscaling_group.ephemeral.name}" + }, + "instance_group_persistent_instance_ids": { + "description": "IDs of the pinned persistent instances, by ordinal", + "value": ["${aws_instance.persistent-0.id}", "${aws_instance.persistent-1.id}", "${aws_instance.persistent-2.id}"] + }, + "instance_group_persistent_volume_ids": { + "description": "IDs of the persistent data volumes, by ordinal", + "value": ["${aws_ebs_volume.persistent-0.id}", "${aws_ebs_volume.persistent-1.id}", "${aws_ebs_volume.persistent-2.id}"] + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/providers.tf.json b/docs/examples/ecs/ec2/terraform/pours/infrastructure/providers.tf.json new file mode 100644 index 00000000..b498bc22 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/providers.tf.json @@ -0,0 +1,10 @@ +{ + "provider": { + "aws": { + "region": "${var.aws_region}", + "ignore_tags": { + "keys": ["foundry.signoz.io/identities"] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/variables.tf.json b/docs/examples/ecs/ec2/terraform/pours/infrastructure/variables.tf.json new file mode 100644 index 00000000..9b429bf3 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/variables.tf.json @@ -0,0 +1 @@ +{"variable":{"aws_region":{"description":"AWS region to deploy resources; it must be the region the declared zones belong to","type":"string","default":"us-east-1","nullable":false,"validation":{"condition":"${can(regex(\"^[a-z]{2}(-gov)?-[a-z]+-[0-9]$\", var.aws_region))}","error_message":"aws_region must be a region identifier such as us-east-1."}},"network_cidr":{"description":"CIDR block for the network","type":"string","default":"10.0.0.0/16","nullable":false,"validation":{"condition":"${can(cidrhost(var.network_cidr, 0))}","error_message":"network_cidr must be a valid IPv4 CIDR."}},"subnet_private-a_zone":{"description":"Availability zone the private-a subnet lives in; a volume can only attach to a machine in its own zone","type":"string","default":"us-east-1a","nullable":false},"subnet_private-a_cidr":{"description":"CIDR block for the private-a subnet, carved out of the network","type":"string","default":"10.0.0.0/19","nullable":false,"validation":{"condition":"${can(cidrhost(var.subnet_private-a_cidr, 0))}","error_message":"subnet_private-a_cidr must be a valid IPv4 CIDR."}},"subnet_public-a_zone":{"description":"Availability zone the public-a subnet lives in; a volume can only attach to a machine in its own zone","type":"string","default":"us-east-1a","nullable":false},"subnet_public-a_cidr":{"description":"CIDR block for the public-a subnet, carved out of the network","type":"string","default":"10.0.96.0/22","nullable":false,"validation":{"condition":"${can(cidrhost(var.subnet_public-a_cidr, 0))}","error_message":"subnet_public-a_cidr must be a valid IPv4 CIDR."}},"group_ephemeral_machine_type":{"description":"Provider machine type for each node in the ephemeral group","type":"string","default":"t3.small","nullable":false},"group_ephemeral_root_volume_size":{"description":"Root volume size (GB) for each ephemeral node; at least the ECS-optimized AMI snapshot size (30)","type":"number","default":30,"nullable":false,"validation":{"condition":"${var.group_ephemeral_root_volume_size \u003e= 30}","error_message":"group_ephemeral_root_volume_size must be at least 30 GB, the ECS-optimized AMI snapshot size."}},"group_ephemeral_root_volume_type":{"description":"Root volume type for each ephemeral node","type":"string","default":"gp3","nullable":false},"group_ephemeral_min_size":{"description":"Smallest the ephemeral group may be","type":"number","default":1,"nullable":false,"validation":{"condition":"${var.group_ephemeral_min_size \u003e= 0}","error_message":"group_ephemeral_min_size cannot be negative."}},"group_ephemeral_max_size":{"description":"Largest the ephemeral group may grow to","type":"number","default":1,"nullable":false,"validation":{"condition":"${var.group_ephemeral_max_size \u003e= var.group_ephemeral_min_size}","error_message":"group_ephemeral_max_size cannot be below group_ephemeral_min_size."}},"group_persistent_machine_type":{"description":"Provider machine type for each node in the persistent group","type":"string","default":"t3.medium","nullable":false},"group_persistent_root_volume_size":{"description":"Root volume size (GB) for each persistent node; at least the ECS-optimized AMI snapshot size (30)","type":"number","default":30,"nullable":false,"validation":{"condition":"${var.group_persistent_root_volume_size \u003e= 30}","error_message":"group_persistent_root_volume_size must be at least 30 GB, the ECS-optimized AMI snapshot size."}},"group_persistent_root_volume_type":{"description":"Root volume type for each persistent node","type":"string","default":"gp3","nullable":false},"group_persistent_data_volume_size":{"description":"Data volume size (GB) attached to each persistent node; it outlives the node","type":"number","default":5,"nullable":false,"validation":{"condition":"${var.group_persistent_data_volume_size \u003e= 1}","error_message":"group_persistent_data_volume_size must be at least 1 GB."}},"group_persistent_data_volume_type":{"description":"Data volume type for each persistent node","type":"string","default":"gp3","nullable":false}}} \ No newline at end of file diff --git a/docs/examples/ecs/ec2/terraform/pours/infrastructure/versions.tf.json b/docs/examples/ecs/ec2/terraform/pours/infrastructure/versions.tf.json new file mode 100644 index 00000000..a9eded70 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/infrastructure/versions.tf.json @@ -0,0 +1,11 @@ +{ + "terraform": { + "required_version": ">= 1.4.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/casting.go b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go new file mode 100644 index 00000000..666c38e7 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go @@ -0,0 +1,97 @@ +package ecsec2terraformcasting + +import ( + "bytes" + "context" + "log/slog" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/signoz/foundry/internal/pourer" +) + +// cloudInitData is what a node needs to know at first boot: which cluster to +// join, what to advertise itself as, and whether it has a disk to mount. +type cloudInitData struct { + Cluster string + Selector map[string]string + DataVolume bool +} + +type ecsEc2TerraformCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *ecsEc2TerraformCasting { + return &ecsEc2TerraformCasting{logger: logger} +} + +func (c *ecsEc2TerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return newEcsEc2TerraformMoldingEnricher(), nil +} + +func (c *ecsEc2TerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, p *pourer.Pourer) error { + doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] + + if doc == "" { + return foundryerrors.Newf(foundryerrors.TypeInternal, "resource config %q is missing from the resource status", resourcemolding.ResourceConfigName) + } + + resource := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(doc), resource); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config") + } + + // The molding derives every name and tag the templates interpolate. + if resource.Resources == nil { + return foundryerrors.Newf(foundryerrors.TypeInternal, "resource config %q carries no derived resources", resourcemolding.ResourceConfigName) + } + + items := []struct { + template *domain.Template + path string + }{ + {versionsTFTemplate, "versions.tf.json"}, + {providersTFTemplate, "providers.tf.json"}, + {mainTFTemplate, "main.tf.json"}, + {variablesTFTemplate, "variables.tf.json"}, + {outputsTFTemplate, "outputs.tf.json"}, + } + + for _, item := range items { + buf := bytes.NewBuffer(nil) + if err := item.template.Execute(buf, config); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute %s template", item.path) + } + + p.AddJSON(buf.Bytes(), item.path) + } + + // One cloud-init config per instance group, referenced with filebase64. + // Blobs stay byte-exact, preserving the #cloud-config header. + for key, group := range resource.Resources.InstanceGroups { + buf := bytes.NewBuffer(nil) + data := cloudInitData{ + Cluster: resource.Resources.Cluster.Name, + Selector: group.Selector, + DataVolume: group.Storage.RequiresDataVolume(), + } + + if err := cloudInitTemplate.Execute(buf, data); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute cloud-init template") + } + + p.AddBlob(buf.Bytes(), "cloud-init", key+".yaml") + } + + return nil +} + +func (c *ecsEc2TerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer) error { + c.logger.WarnContext(ctx, "casting the infrastructure is not implemented yet, run terraform init and apply from the pours directory", slog.String("path", filepath.Join(outputPath, p.Dir()))) + return nil +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/casting_test.go b/internal/casting/infrastructure/ecsec2terraformcasting/casting_test.go new file mode 100644 index 00000000..4c7f2efc --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/casting_test.go @@ -0,0 +1,134 @@ +package ecsec2terraformcasting + +import ( + "context" + "encoding/json" + awsconvention "github.com/signoz/foundry/internal/convention/aws" + "log/slog" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/signoz/foundry/internal/pourer" + "github.com/stretchr/testify/assert" +) + +// A zone is per-account and per-region, so the substrate cannot be described +// without the operator stating one. +const subnets = `networking: + subnets: + private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19} + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} +` + +// moldedCasting runs the enricher and the molding, which is what puts the +// derived names and tags the templates interpolate into the status. +func moldedCasting(t *testing.T) *infrastructure.Casting { + t.Helper() + + config := infrastructure.Default() + config.Spec.Resource.Spec.Config.Set(resourcemolding.ResourceConfigName, []byte(subnets)) + + logger := slog.New(slog.DiscardHandler) + assert.NoError(t, newEcsEc2TerraformMoldingEnricher().EnrichStatus(context.Background(), v1alpha1.MoldingKindResource, config)) + assert.NoError(t, resourcemolding.New(logger, awsconvention.Resources).MoldV1Alpha1(context.Background(), config)) + + return config +} + +func TestForge(t *testing.T) { + config := moldedCasting(t) + + pr := pourer.New("infrastructure") + assert.NoError(t, New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *config, pr)) + + materials, err := pr.Pour() + assert.NoError(t, err) + + paths := map[string]domain.Material{} + for _, material := range materials { + paths[material.Path()] = material + } + + for _, expected := range []string{ + "infrastructure/providers.tf.json", + "infrastructure/main.tf.json", + "infrastructure/variables.tf.json", + "infrastructure/outputs.tf.json", + "infrastructure/cloud-init/persistent.yaml", + "infrastructure/cloud-init/ephemeral.yaml", + } { + assert.Contains(t, paths, expected) + } + + pinned := string(paths["infrastructure/cloud-init/persistent.yaml"].FmtContents()) + assert.Contains(t, pinned, "ECS_CLUSTER=signoz-cls") + assert.Contains(t, pinned, `"foundry.signoz.io/storage":"persistent"`) + assert.Contains(t, pinned, "/var/lib/foundry") + + pool := string(paths["infrastructure/cloud-init/ephemeral.yaml"].FmtContents()) + assert.Contains(t, pool, `"foundry.signoz.io/storage":"ephemeral"`) + assert.NotContains(t, pool, "fs_setup") +} + +// The templates interpolate the derived document and assemble no name or tag +// of their own, so nothing in them can drift from what a consumer filters on. +func TestForge_TemplatesSpellNoContractOfTheirOwn(t *testing.T) { + config := moldedCasting(t) + + pr := pourer.New("infrastructure") + assert.NoError(t, New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *config, pr)) + + materials, err := pr.Pour() + assert.NoError(t, err) + + resource := &infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName]), resource)) + + main := map[string]any{} + for _, material := range materials { + if material.Path() == "infrastructure/main.tf.json" { + assert.NoError(t, json.Unmarshal(material.FmtContents(), &main)) + } + } + + resources, _ := main["resource"].(map[string]any) + subnet, _ := resources["aws_subnet"].(map[string]any) + private, _ := subnet["private-a"].(map[string]any) + + assert.Equal(t, toAny(resource.Resources.Subnets["private-a"].Tags), private["tags"]) + + cluster, _ := resources["aws_ecs_cluster"].(map[string]any) + main0, _ := cluster["main"].(map[string]any) + + assert.Equal(t, resource.Resources.Cluster.Name, main0["name"]) + assert.Equal(t, toAny(resource.Resources.Cluster.Tags), main0["tags"]) +} + +func toAny(tags map[string]string) map[string]any { + out := map[string]any{} + for key, value := range tags { + out[key] = value + } + + return out +} + +func TestForge_MissingResourceConfigErrors(t *testing.T) { + config := infrastructure.Default() + + err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *config, pourer.New("infrastructure")) + assert.Error(t, err) +} + +// Only the molding derives, so a status carrying a declaration and nothing +// else has no names for the templates to interpolate. +func TestForge_UnderivedResourceConfigErrors(t *testing.T) { + config := infrastructure.Default() + config.Spec.Resource.Status.Config.Set(resourcemolding.ResourceConfigName, []byte(subnets)) + + err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *config, pourer.New("infrastructure")) + assert.Error(t, err) +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/embed.go b/internal/casting/infrastructure/ecsec2terraformcasting/embed.go new file mode 100644 index 00000000..862e858b --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/embed.go @@ -0,0 +1,19 @@ +package ecsec2terraformcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + versionsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/versions.tf.json.gotmpl", domain.FormatJSON) + providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) + mainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) + variablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) + outputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/outputs.tf.json.gotmpl", domain.FormatJSON) + cloudInitTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/cloudinit.yaml.gotmpl", domain.FormatText) +) diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go b/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go new file mode 100644 index 00000000..8215839b --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go @@ -0,0 +1,48 @@ +package ecsec2terraformcasting + +import ( + "testing" + + "github.com/signoz/foundry/internal/domain" + "github.com/stretchr/testify/assert" +) + +func TestTemplates_RenderValidJSON(t *testing.T) { + config := moldedCasting(t) + + tests := []struct { + name string + template *domain.Template + }{ + {name: "ProvidersTemplate_RendersValidJSON", template: providersTFTemplate}, + {name: "MainTemplate_RendersValidJSON", template: mainTFTemplate}, + {name: "VariablesTemplate_RendersValidJSON", template: variablesTFTemplate}, + {name: "OutputsTemplate_RendersValidJSON", template: outputsTFTemplate}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material, err := tt.template.Render(*config, "out.tf.json") + assert.NoError(t, err) + assert.NotEmpty(t, material.FmtContents()) + }) + } +} + +func TestMainTemplate_PinsPersistentAndPoolsEphemeral(t *testing.T) { + config := moldedCasting(t) + + material, err := mainTFTemplate.Render(*config, "out.tf.json") + assert.NoError(t, err) + + contents := string(material.FmtContents()) + assert.Contains(t, contents, `"persistent-0"`) + assert.Contains(t, contents, `"persistent-2"`) + assert.Contains(t, contents, `"aws_ebs_volume"`) + assert.Contains(t, contents, `"aws_volume_attachment"`) + assert.Contains(t, contents, `"aws_autoscaling_group"`) + assert.Contains(t, contents, "cloud-init/persistent.yaml") + assert.Contains(t, contents, "cloud-init/ephemeral.yaml") + assert.NotContains(t, contents, "aws_launch_template.persistent") + assert.NotContains(t, contents, "aws_instance.ephemeral") +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go new file mode 100644 index 00000000..7aaafd05 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go @@ -0,0 +1,58 @@ +package ecsec2terraformcasting + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" +) + +// What the platform decides about an instance group. Sizes come from the +// molding's baseline. Neither machine type is burstable; a store that throttles +// under sustained ingest reads as an outage. +const ( + machineTypePersistent = "m5.large" + machineTypeEphemeral = "c5.large" + volumeType = "gp3" +) + +var _ infrastructuremolding.MoldingEnricher = (*ecsEc2TerraformMoldingEnricher)(nil) + +type ecsEc2TerraformMoldingEnricher struct{} + +func newEcsEc2TerraformMoldingEnricher() *ecsEc2TerraformMoldingEnricher { + return &ecsEc2TerraformMoldingEnricher{} +} + +// EnrichStatus contributes what this platform decides. Subnets are absent on +// purpose. An availability zone is per-account; only the operator can state one. +func (e *ecsEc2TerraformMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { + if kind != v1alpha1.MoldingKindResource { + return nil + } + + groups := map[string]infrastructure.ResourceConfigInstanceGroup{ + resourcemolding.GroupPersistent: { + MachineType: machineTypePersistent, + RootVolume: infrastructure.ResourceConfigVolume{Type: volumeType}, + DataVolume: &infrastructure.ResourceConfigVolume{Type: volumeType}, + }, + resourcemolding.GroupEphemeral: { + MachineType: machineTypeEphemeral, + RootVolume: infrastructure.ResourceConfigVolume{Type: volumeType}, + }, + } + + contribution, err := domain.MarshalYAML(&infrastructure.ResourceConfig{InstanceGroups: groups}) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal resource config contribution") + } + + config.Spec.Resource.Status.Config.Set(resourcemolding.ResourceConfigName, contribution) + + return nil +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/enricher_test.go b/internal/casting/infrastructure/ecsec2terraformcasting/enricher_test.go new file mode 100644 index 00000000..2da3e27c --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/enricher_test.go @@ -0,0 +1,66 @@ +package ecsec2terraformcasting + +import ( + "context" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/stretchr/testify/assert" +) + +func TestEnrichStatus(t *testing.T) { + tests := []struct { + name string + kind v1alpha1.MoldingKind + assert func(t *testing.T, config *infrastructure.Casting) + }{ + { + name: "ResourceMolding_ContributesBothGroups", + kind: v1alpha1.MoldingKindResource, + assert: func(t *testing.T, config *infrastructure.Casting) { + doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] + assert.NotEmpty(t, doc) + + resolved := &infrastructure.ResourceConfig{} + assert.NoError(t, domain.UnmarshalYAML([]byte(doc), resolved)) + assert.Len(t, resolved.InstanceGroups, 2) + assert.Equal(t, machineTypePersistent, resolved.InstanceGroups[resourcemolding.GroupPersistent].MachineType) + assert.Equal(t, machineTypeEphemeral, resolved.InstanceGroups[resourcemolding.GroupEphemeral].MachineType) + assert.Equal(t, volumeType, resolved.InstanceGroups[resourcemolding.GroupPersistent].DataVolume.Type) + }, + }, + { + // The contribution is a delta: sizes belong to the molding's + // baseline, so stating one here would override it. + name: "ResourceMolding_StatesNoSizes", + kind: v1alpha1.MoldingKindResource, + assert: func(t *testing.T, config *infrastructure.Casting) { + doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] + + assert.NotContains(t, doc, "minSize") + assert.NotContains(t, doc, "size") + assert.NotContains(t, doc, "subnets") + }, + }, + { + name: "OtherMolding_NoOp", + kind: v1alpha1.MoldingKindTelemetryStore, + assert: func(t *testing.T, config *infrastructure.Casting) { + assert.Empty(t, config.Spec.Resource.Status.Config.Data) + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + config := infrastructure.Default() + + err := newEcsEc2TerraformMoldingEnricher().EnrichStatus(context.Background(), tt.kind, config) + assert.NoError(t, err) + tt.assert(t, config) + }) + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl new file mode 100644 index 00000000..f5bc7d37 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl @@ -0,0 +1,33 @@ +#cloud-config +write_files: + - path: /etc/ecs/ecs.config + content: | + ECS_CLUSTER={{ .Cluster }} + {{- /* A container instance advertises the group's own selector. A + consuming casting's placement constraint filters on these. */}} + ECS_INSTANCE_ATTRIBUTES={{ toJson .Selector }} +{{- if .DataVolume }} + # The agent must never register a node whose data volume is not mounted: + # tasks would bind-mount onto the root disk and state would silently land + # on a disk that dies with the instance. + - path: /etc/systemd/system/ecs.service.d/10-foundry-data.conf + content: | + [Unit] + RequiresMountsFor=/var/lib/foundry +# Terraform attaches the data volume after boot begins, and cloud-init's +# fs_setup/mounts run once, first boot only, silently skipping devices that do +# not exist yet (canonical/cloud-init#3386). bootcmd runs every boot, before +# anything downstream of cloud-init (the agent included), so it waits for the +# attachment, formats only a device with no filesystem signature, and mounts. +bootcmd: + - 'for i in $(seq 1 120); do test -b /dev/xvdf && break; sleep 5; done' + - 'test -b /dev/xvdf || { echo "foundry: data volume /dev/xvdf never attached" >&2; exit 1; }' + - 'blkid /dev/xvdf >/dev/null || mkfs.ext4 -L foundry-data /dev/xvdf' + - 'mkdir -p /var/lib/foundry' + - 'mountpoint -q /var/lib/foundry || mount /dev/xvdf /var/lib/foundry' +# The fstab entry mounts at local-fs on every later boot and is what gives the +# RequiresMountsFor gate a real mount unit to require. The device exists by the +# time this module runs, because bootcmd waited. +mounts: + - [/dev/xvdf, /var/lib/foundry, ext4, "defaults,nofail,x-systemd.device-timeout=10min", "0", "2"] +{{- end }} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl new file mode 100644 index 00000000..7114a4f6 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl @@ -0,0 +1,297 @@ +{{- /* Names and tags are interpolated from resource.yaml, never assembled + here. A literal spelled twice is a filter that matches nothing. */}} +{{- $decl := fromYaml (index $.Spec.Resource.Status.Config.Data "resource.yaml") -}} +{{- $r := $decl.resources -}} +{{- $subnets := dict -}} +{{- range $key, $subnet := $r.subnets }}{{ if not $subnet.id }}{{ $_ := set $subnets $key $subnet }}{{ end }}{{ end -}} +{{- $gateways := dict -}} +{{- range $key, $gateway := $r.natGateways }}{{ if $gateway.name }}{{ $_ := set $gateways $key $gateway }}{{ end }}{{ end -}} +{{- $pinned := dict -}} +{{- $pools := dict -}} +{{- range $key, $group := $r.instanceGroups }}{{ if $group.nodes }}{{ $_ := set $pinned $key $group }}{{ else if $group.autoscalingGroup }}{{ $_ := set $pools $key $group }}{{ end }}{{ end -}} +{ + "locals": { + "vpc_id": "{{ if $r.vpc.id }}{{ $r.vpc.id }}{{ else }}${aws_vpc.main.id}{{ end }}", + "subnet_ids": { + {{- $first := true }}{{ range $key, $subnet := $r.subnets }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": "{{ if $subnet.id }}{{ $subnet.id }}{{ else }}${aws_subnet.{{ $key }}.id}{{ end }}" + {{- end }} + } + }, + "data": { + "aws_ssm_parameter": { + "ecs_ami": { + "name": "/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id" + } + } + }, + "resource": { + {{- if not $r.vpc.id }} + "aws_vpc": { + "main": { + "cidr_block": "${var.network_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": {{ toJson $r.vpc.tags }} + } + }, + {{- end }} + {{- if $subnets }} + "aws_subnet": { + {{- $first := true }}{{ range $key, $subnet := $subnets }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "vpc_id": "${local.vpc_id}", + "cidr_block": "${var.subnet_{{ $key }}_cidr}", + "availability_zone": "${var.subnet_{{ $key }}_zone}", + {{- if $subnet.public }} + "map_public_ip_on_launch": true, + {{- end }} + "tags": {{ toJson $subnet.tags }} + } + {{- end }} + }, + {{- end }} + {{- if $r.internetGateway.name }} + "aws_internet_gateway": { + "main": { + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $r.internetGateway.tags }} + } + }, + {{- end }} + {{- if $gateways }} + "aws_eip": { + {{- $first := true }}{{ range $key, $gateway := $gateways }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "domain": "vpc", + "tags": {{ toJson $gateway.address.tags }} + } + {{- end }} + }, + "aws_nat_gateway": { + {{- $first := true }}{{ range $key, $gateway := $gateways }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "allocation_id": "${aws_eip.{{ $key }}.id}", + "subnet_id": "${local.subnet_ids[\"{{ $gateway.subnet }}\"]}", + "tags": {{ toJson $gateway.tags }}, + "depends_on": ["aws_internet_gateway.main"] + } + {{- end }} + }, + {{- end }} + {{- if $r.routeTables }} + {{- /* A table per subnet. A private subnet's default route is its own + zone's gateway. */}} + "aws_route_table": { + {{- $first := true }}{{ range $key, $table := $r.routeTables }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $table.tags }} + } + {{- end }} + }, + "aws_route": { + {{- $first := true }}{{ range $key, $table := $r.routeTables }} + {{- $subnet := index $r.subnets $key }} + {{- if $subnet.public }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "route_table_id": "${aws_route_table.{{ $key }}.id}", + "destination_cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + } + {{- else }}{{ $gateway := index $r.natGateways $key }}{{ if $gateway }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "route_table_id": "${aws_route_table.{{ $key }}.id}", + "destination_cidr_block": "0.0.0.0/0", + "nat_gateway_id": "{{ if $gateway.id }}{{ $gateway.id }}{{ else }}${aws_nat_gateway.{{ $key }}.id}{{ end }}" + } + {{- end }}{{ end }}{{ end }} + }, + "aws_route_table_association": { + {{- $first := true }}{{ range $key, $table := $r.routeTables }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "subnet_id": "${local.subnet_ids[\"{{ $key }}\"]}", + "route_table_id": "${aws_route_table.{{ $key }}.id}" + } + {{- end }} + }, + {{- end }} + {{- if $pinned }} + "terraform_data": { + {{- $first := true }}{{ range $key, $group := $pinned }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "triggers_replace": "${var.group_{{ $key }}_machine_type}" + } + {{- end }} + }, + "aws_instance": { + {{- $first := true }}{{ range $key, $group := $pinned }}{{ range $node := $group.nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.ordinal }}": { + "ami": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_{{ $key }}_machine_type}", + "subnet_id": "${local.subnet_ids[\"{{ $node.subnet }}\"]}", + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "iam_instance_profile": "${aws_iam_instance_profile.node.name}", + "user_data_base64": "${filebase64(\"${path.module}/cloud-init/{{ $key }}.yaml\")}", + {{- /* Boot config runs on first boot only. A change means a new node. */}} + "user_data_replace_on_change": true, + "root_block_device": [ + { + "volume_size": "${var.group_{{ $key }}_root_volume_size}", + "volume_type": "${var.group_{{ $key }}_root_volume_type}", + "delete_on_termination": true + } + ], + "tags": {{ toJson $node.tags }}, + {{- /* ECS refuses to re-register an instance whose type changed, and + the agent then exits terminally. A type change must REPLACE the + node. The AMI is pinned so SSM rotation does not replace the + fleet as a side effect. */}} + "lifecycle": { + "replace_triggered_by": ["terraform_data.{{ $key }}"], + "ignore_changes": ["ami"] + }, + "depends_on": ["aws_ecs_cluster.main"] + } + {{- end }}{{ end }} + }, + "aws_ebs_volume": { + {{- $first := true }}{{ range $key, $group := $pinned }}{{ range $node := $group.nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.ordinal }}": { + "availability_zone": "${var.subnet_{{ $node.subnet }}_zone}", + "size": "${var.group_{{ $key }}_data_volume_size}", + "type": "${var.group_{{ $key }}_data_volume_type}", + "tags": {{ toJson $node.volume.tags }} + } + {{- end }}{{ end }} + }, + "aws_volume_attachment": { + {{- $first := true }}{{ range $key, $group := $pinned }}{{ range $node := $group.nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.ordinal }}": { + "device_name": "/dev/xvdf", + "volume_id": "${aws_ebs_volume.{{ $key }}-{{ $node.ordinal }}.id}", + "instance_id": "${aws_instance.{{ $key }}-{{ $node.ordinal }}.id}" + } + {{- end }}{{ end }} + }, + {{- end }} + {{- if $pools }} + "aws_launch_template": { + {{- $first := true }}{{ range $key, $group := $pools }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "name": "{{ $group.launchTemplate.name }}", + "image_id": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_{{ $key }}_machine_type}", + "iam_instance_profile": { + "arn": "${aws_iam_instance_profile.node.arn}" + }, + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "user_data": "${filebase64(\"${path.module}/cloud-init/{{ $key }}.yaml\")}", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": [ + { + "volume_size": "${var.group_{{ $key }}_root_volume_size}", + "volume_type": "${var.group_{{ $key }}_root_volume_type}", + "delete_on_termination": true + } + ] + } + ], + "tag_specifications": [ + { + "resource_type": "instance", + "tags": {{ toJson $group.launchTemplate.tags }} + } + ], + "tags": {{ toJson $group.launchTemplate.tags }} + } + {{- end }} + }, + "aws_autoscaling_group": { + {{- $first := true }}{{ range $key, $group := $pools }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "name": "{{ $group.autoscalingGroup.name }}", + "desired_capacity": "${var.group_{{ $key }}_min_size}", + "min_size": "${var.group_{{ $key }}_min_size}", + "max_size": "${var.group_{{ $key }}_max_size}", + "vpc_zone_identifier": [{{ range $i, $subnet := $group.subnets }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $subnet }}\"]}"{{ end }}], + "launch_template": [ + { + "id": "${aws_launch_template.{{ $key }}.id}", + "version": "$Latest" + } + ], + "tag": [ + {{- $first := true }}{{ range $tag, $value := $group.autoscalingGroup.tags }}{{ if not $first }},{{ end }}{{ $first = false }} + {"key": "{{ $tag }}", "value": "{{ $value }}", "propagate_at_launch": true} + {{- end }} + ], + "depends_on": ["aws_ecs_cluster.main"] + } + {{- end }} + }, + {{- end }} + "aws_security_group": { + "tasks": { + "name": "{{ $r.securityGroup.name }}", + "description": "SigNoz ECS tasks and container instances", + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $r.securityGroup.tags }} + } + }, + "aws_vpc_security_group_ingress_rule": { + "intra_cluster": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "intra-cluster traffic", + "ip_protocol": "-1", + "referenced_security_group_id": "${aws_security_group.tasks.id}", + "tags": {{ toJson (index $r.securityGroupRules "intra-cluster").tags }} + } + }, + {{- /* Egress stays open. Image pulls and OS packages have no stable CIDR. + Narrow it with VPC endpoints. */}} + "aws_vpc_security_group_egress_rule": { + "all_outbound": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "all outbound", + "ip_protocol": "-1", + "cidr_ipv4": "0.0.0.0/0", + "tags": {{ toJson (index $r.securityGroupRules "all-outbound").tags }} + } + }, + {{- /* The node's own credential only. Without the managed policy the ECS + agent cannot register the instance. Task and execution roles belong + to the casting that runs the tasks. */}} + "aws_iam_role": { + "node": { + "name": "{{ $r.roles.node.name }}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + {{- if $decl.iam.permissionsBoundary }} + "permissions_boundary": "{{ $decl.iam.permissionsBoundary }}", + {{- end }} + "tags": {{ toJson $r.roles.node.tags }} + } + }, + "aws_iam_instance_profile": { + "node": { + "name": "{{ $r.instanceProfile.name }}", + "role": "${aws_iam_role.node.name}", + "tags": {{ toJson $r.instanceProfile.tags }} + } + }, + "aws_iam_role_policy_attachment": { + "node": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role", + "role": "${aws_iam_role.node.name}" + } + }, + "aws_ecs_cluster": { + "main": { + "name": "{{ $r.cluster.name }}", + "tags": {{ toJson $r.cluster.tags }} + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl new file mode 100644 index 00000000..2d1a1dc8 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl @@ -0,0 +1,54 @@ +{{- $decl := fromYaml (index $.Spec.Resource.Status.Config.Data "resource.yaml") -}} +{{- $r := $decl.resources -}} +{{- $private := list -}} +{{- $public := list -}} +{{- range $key, $subnet := $r.subnets }}{{ if $subnet.public }}{{ $public = append $public $key }}{{ else }}{{ $private = append $private $key }}{{ end }}{{ end -}} +{ + "output": { + "cluster_name": { + "description": "Name of the ECS cluster", + "value": "${aws_ecs_cluster.main.name}" + }, + "cluster_arn": { + "description": "ARN of the ECS cluster", + "value": "${aws_ecs_cluster.main.arn}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${local.vpc_id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets, which is where workloads are placed", + "value": [{{ range $i, $key := $private }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $key }}\"]}"{{ end }}] + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": [{{ range $i, $key := $public }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $key }}\"]}"{{ end }}] + }, + "security_group_ids": { + "description": "IDs of the tasks security group", + "value": ["${aws_security_group.tasks.id}"] + }, + "node_role_arn": { + "description": "ARN of the ECS container-instance role", + "value": "${aws_iam_role.node.arn}" + } + {{- range $key, $group := $r.instanceGroups }} + {{- if $group.nodes }}, + "instance_group_{{ $key }}_instance_ids": { + "description": "IDs of the pinned {{ $key }} instances, by ordinal", + "value": [{{ range $i, $node := $group.nodes }}{{ if $i }}, {{ end }}"${aws_instance.{{ $key }}-{{ $node.ordinal }}.id}"{{ end }}] + }, + "instance_group_{{ $key }}_volume_ids": { + "description": "IDs of the {{ $key }} data volumes, by ordinal", + "value": [{{ range $i, $node := $group.nodes }}{{ if $i }}, {{ end }}"${aws_ebs_volume.{{ $key }}-{{ $node.ordinal }}.id}"{{ end }}] + } + {{- else if $group.autoscalingGroup }}, + "instance_group_{{ $key }}_asg_name": { + "description": "Name of the {{ $key }} autoscaling group", + "value": "${aws_autoscaling_group.{{ $key }}.name}" + } + {{- end }} + {{- end }} + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl new file mode 100644 index 00000000..9c07878c --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl @@ -0,0 +1,13 @@ +{{- /* A volume's claim is stamped by whoever claims it, long after this + applies. Reconciling it reverts a live claim on every apply. */}} +{{- $decl := fromYaml (index $.Spec.Resource.Status.Config.Data "resource.yaml") -}} +{ + "provider": { + "aws": { + "region": "${var.aws_region}", + "ignore_tags": { + "keys": {{ toJson $decl.resources.ignoredTags }} + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl new file mode 100644 index 00000000..2e64d19f --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl @@ -0,0 +1,117 @@ +{{- /* Every declared knob arrives as a variable defaulted to what was + declared. Names and tags are not knobs; a consumer matches on them. */}} +{{- $decl := fromYaml (index $.Spec.Resource.Status.Config.Data "resource.yaml") -}} +{{- $r := $decl.resources -}} +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources; it must be the region the declared zones belong to", + "type": "string", + "default": "us-east-1", + "nullable": false, + "validation": { + "condition": "${can(regex(\"^[a-z]{2}(-gov)?-[a-z]+-[0-9]$\", var.aws_region))}", + "error_message": "aws_region must be a region identifier such as us-east-1." + } + } + {{- if not $r.vpc.id }}, + "network_cidr": { + "description": "CIDR block for the network", + "type": "string", + "default": "{{ $decl.networking.networkCIDR }}", + "nullable": false, + "validation": { + "condition": "${can(cidrhost(var.network_cidr, 0))}", + "error_message": "network_cidr must be a valid IPv4 CIDR." + } + } + {{- end }} + {{- range $key, $subnet := $r.subnets }}, + "subnet_{{ $key }}_zone": { + "description": "Availability zone the {{ $key }} subnet lives in; a volume can only attach to a machine in its own zone", + "type": "string", + "default": "{{ (index $decl.networking.subnets $key).zone }}", + "nullable": false + } + {{- if not $subnet.id }}, + "subnet_{{ $key }}_cidr": { + "description": "CIDR block for the {{ $key }} subnet, carved out of the network", + "type": "string", + "default": "{{ (index $decl.networking.subnets $key).cidr }}", + "nullable": false, + "validation": { + "condition": "${can(cidrhost(var.subnet_{{ $key }}_cidr, 0))}", + "error_message": "subnet_{{ $key }}_cidr must be a valid IPv4 CIDR." + } + } + {{- end }} + {{- end }} + {{- range $key, $group := $r.instanceGroups }}{{ $declared := index $decl.instanceGroups $key }}, + "group_{{ $key }}_machine_type": { + "description": "Provider machine type for each node in the {{ $key }} group", + "type": "string", + "default": "{{ $declared.machineType }}", + "nullable": false + }, + {{- /* The root disk carries the AMI. Its floor is the snapshot size. */}} + "group_{{ $key }}_root_volume_size": { + "description": "Root volume size (GB) for each {{ $key }} node; at least the ECS-optimized AMI snapshot size (30)", + "type": "number", + "default": {{ $declared.rootVolume.size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ $key }}_root_volume_size >= 30}", + "error_message": "group_{{ $key }}_root_volume_size must be at least 30 GB, the ECS-optimized AMI snapshot size." + } + }, + "group_{{ $key }}_root_volume_type": { + "description": "Root volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.rootVolume.type }}", + "nullable": false + } + {{- if $declared.dataVolume }}, + "group_{{ $key }}_data_volume_size": { + "description": "Data volume size (GB) attached to each {{ $key }} node; it outlives the node", + "type": "number", + "default": {{ $declared.dataVolume.size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ $key }}_data_volume_size >= 1}", + "error_message": "group_{{ $key }}_data_volume_size must be at least 1 GB." + } + }, + "group_{{ $key }}_data_volume_type": { + "description": "Data volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.dataVolume.type }}", + "nullable": false + } + {{- end }} + {{- /* A pinned group has no size variable. Each node is its own resource, + and the count changes the plan's shape. */}} + {{- if $group.autoscalingGroup }}, + "group_{{ $key }}_min_size": { + "description": "Smallest the {{ $key }} group may be", + "type": "number", + "default": {{ $declared.minSize }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ $key }}_min_size >= 0}", + "error_message": "group_{{ $key }}_min_size cannot be negative." + } + }, + "group_{{ $key }}_max_size": { + "description": "Largest the {{ $key }} group may grow to", + "type": "number", + "default": {{ $declared.maxSize }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ $key }}_max_size >= var.group_{{ $key }}_min_size}", + "error_message": "group_{{ $key }}_max_size cannot be below group_{{ $key }}_min_size." + } + } + {{- end }} + {{- end }} + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl new file mode 100644 index 00000000..a9eded70 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl @@ -0,0 +1,11 @@ +{ + "terraform": { + "required_version": ">= 1.4.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + } +} diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 1bade53d..940745fe 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,9 +4,12 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting/infrastructure/ecsec2terraformcasting" + awsconvention "github.com/signoz/foundry/internal/convention/aws" foundryerrors "github.com/signoz/foundry/internal/errors" infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" ) type CastingItem struct { @@ -23,7 +26,17 @@ type Registry struct { func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ - castings: map[v1alpha1.TypeDeployment]CastingItem{}, + castings: map[v1alpha1.TypeDeployment]CastingItem{ + { + Platform: v1alpha1.PlatformECS, + Mode: v1alpha1.ModeEC2, + Flavor: v1alpha1.FlavorTerraform, + }: { + Casting: ecsec2terraformcasting.New(logger), + Toolers: []tooler.Tooler{terraformtooler.New()}, + Deriver: awsconvention.Resources, + }, + }, } } From 557801a3054489993a77821b66e838c31137aa0b Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 18:23:05 +0530 Subject: [PATCH 35/38] feat(casting/ecs): run multi-node clusters and find the substrate by tag Emits one ECS service per ClickHouse Keeper and per shard replica, each pinned to the substrate node holding its data, so ClickHouse and Keeper can run as clusters rather than single tasks. Every identifier the stack needs arrives as a variable defaulted to what the casting resolved. Left alone, the stack finds the cluster, subnets, VPC, security group and roles by their foundry.signoz.io tags, which is what the infrastructure casting stamps. Stating the matching annotation replaces that lookup with the value itself and emits no data source, so an existing cluster still works. Flattens the templates out of module/ so the deployment forges as one stack. --- api/v1alpha1/installation/annotations.go | 49 +++ docs/examples/ecs/ec2/terraform/README.md | 149 ++++++--- docs/examples/ecs/ec2/terraform/casting.yaml | 14 +- .../ecs/ec2/terraform/casting.yaml.lock | 62 ++-- .../pours/deployment/ingester.tf.json | 185 +++++++++++ .../{module => }/ingester/ingester.yaml | 10 +- .../pours/deployment/ingester/opamp.yaml | 1 + .../terraform/pours/deployment/main.tf.json | 153 +++++++-- .../deployment/{module => }/metastore.tf.json | 55 +-- .../pours/deployment/module/ingester.tf.json | 125 ------- .../deployment/module/ingester/opamp.yaml | 1 - .../pours/deployment/module/main.tf.json | 10 - .../pours/deployment/module/signoz.tf.json | 79 ----- .../deployment/module/telemetrykeeper.tf.json | 130 -------- .../deployment/module/telemetrystore.tf.json | 163 --------- .../pours/deployment/module/variables.tf.json | 40 --- .../deployment/{module => }/outputs.tf.json | 22 +- .../pours/deployment/providers.tf.json | 7 + .../terraform/pours/deployment/signoz.tf.json | 79 +++++ .../pours/deployment/telemetrykeeper.tf.json | 158 +++++++++ .../clickhousekeeper/keeper-0.yaml | 2 +- .../pours/deployment/telemetrystore.tf.json | 213 ++++++++++++ .../telemetrystore/clickhouse/config-0-0.yaml | 4 +- .../telemetrystore/clickhouse/functions.yaml | 0 .../telemetrystore_migrator.tf.json | 26 +- .../pours/deployment/terraform.tfvars.json | 10 +- .../pours/deployment/variables.tf.json | 87 +++-- .../pours/deployment/versions.tf.json | 11 + .../casting/ecsterraformcasting/casting.go | 242 ++++++++------ internal/casting/ecsterraformcasting/embed.go | 29 +- .../casting/ecsterraformcasting/embed_test.go | 241 +++++++++++--- .../casting/ecsterraformcasting/enricher.go | 106 ++++-- .../templates/ingester.tf.json.gotmpl | 205 ++++++++++++ .../templates/main.tf.json.gotmpl | 217 ++++++++++-- .../templates/mcp.tf.json.gotmpl | 78 +++++ .../{module => }/metastore.tf.json.gotmpl | 49 +-- .../templates/module/ingester.tf.json.gotmpl | 133 -------- .../templates/module/main.tf.json.gotmpl | 11 - .../templates/module/outputs.tf.json.gotmpl | 62 ---- .../module/telemetrykeeper.tf.json.gotmpl | 236 ------------- .../module/telemetrystore.tf.json.gotmpl | 168 ---------- .../templates/module/variables.tf.json.gotmpl | 40 --- .../templates/outputs.tf.json.gotmpl | 73 ++++ .../templates/providers.tf.json.gotmpl | 7 + .../{module => }/signoz.tf.json.gotmpl | 58 ++-- .../templates/telemetrykeeper.tf.json.gotmpl | 312 ++++++++++++++++++ .../templates/telemetrystore.tf.json.gotmpl | 252 ++++++++++++++ .../telemetrystore_migrator.tf.json.gotmpl | 24 +- .../templates/terraform.tfvars.json.gotmpl | 20 +- .../templates/variables.tf.json.gotmpl | 129 ++++++-- .../templates/versions.tf.json.gotmpl | 11 + 51 files changed, 2859 insertions(+), 1689 deletions(-) create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/ingester.tf.json rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/ingester/ingester.yaml (86%) create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/ingester/opamp.yaml rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/metastore.tf.json (50%) delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json delete mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/outputs.tf.json (70%) create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/providers.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/signoz.tf.json create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper.tf.json rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/telemetrykeeper/clickhousekeeper/keeper-0.yaml (89%) create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/telemetrystore/clickhouse/config-0-0.yaml (95%) rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/telemetrystore/clickhouse/functions.yaml (100%) rename docs/examples/ecs/ec2/terraform/pours/deployment/{module => }/telemetrystore_migrator.tf.json (56%) create mode 100644 docs/examples/ecs/ec2/terraform/pours/deployment/versions.tf.json create mode 100644 internal/casting/ecsterraformcasting/templates/ingester.tf.json.gotmpl create mode 100644 internal/casting/ecsterraformcasting/templates/mcp.tf.json.gotmpl rename internal/casting/ecsterraformcasting/templates/{module => }/metastore.tf.json.gotmpl (63%) delete mode 100644 internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl delete mode 100644 internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl delete mode 100644 internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl delete mode 100644 internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl delete mode 100644 internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl delete mode 100644 internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl create mode 100644 internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl create mode 100644 internal/casting/ecsterraformcasting/templates/providers.tf.json.gotmpl rename internal/casting/ecsterraformcasting/templates/{module => }/signoz.tf.json.gotmpl (52%) create mode 100644 internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl create mode 100644 internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl rename internal/casting/ecsterraformcasting/templates/{module => }/telemetrystore_migrator.tf.json.gotmpl (68%) create mode 100644 internal/casting/ecsterraformcasting/templates/versions.tf.json.gotmpl diff --git a/api/v1alpha1/installation/annotations.go b/api/v1alpha1/installation/annotations.go index 9ae5e2f4..d98f0cb5 100644 --- a/api/v1alpha1/installation/annotations.go +++ b/api/v1alpha1/installation/annotations.go @@ -51,6 +51,48 @@ var ( } ) +// Cluster annotations for the ECS/EC2 deployment of the Installation Kind. The +// casting places tasks onto a cluster it does not provision, so each of these +// names an existing AWS object it must be handed. They have no defaults: an +// absent value is a missing cluster, not a fallback. +var ( + ECSRegion = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-region", + Mode: v1alpha1.ModeEC2, + Description: "AWS region holding the cluster.", + } + ECSClusterARN = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-cluster-arn", + Mode: v1alpha1.ModeEC2, + Description: "ARN of the ECS cluster to deploy services into.", + } + ECSSubnetIDs = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-subnet-ids", + Mode: v1alpha1.ModeEC2, + Description: "Comma-separated subnet IDs for task networking (awsvpc).", + } + ECSSecurityGroupIDs = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-security-group-ids", + Mode: v1alpha1.ModeEC2, + Description: "Comma-separated security group IDs for task networking (awsvpc); must permit intra-cluster traffic.", + } + ECSVPCID = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-vpc-id", + Mode: v1alpha1.ModeEC2, + Description: "VPC ID the Cloud Map private DNS namespace is created in.", + } + ECSTaskRoleARN = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-task-role-arn", + Mode: v1alpha1.ModeEC2, + Description: "IAM role ARN assumed by the tasks; needs read access to AWS AppConfig.", + } + ECSTaskExecutionRoleARN = v1alpha1.Annotation{ + Key: "foundry.signoz.io/ecs-task-execution-role-arn", + Mode: v1alpha1.ModeEC2, + Description: "IAM role ARN the ECS agent assumes to pull images and start tasks.", + } +) + // Annotations returns the Installation annotation catalog. func Annotations() []v1alpha1.Annotation { return []v1alpha1.Annotation{ @@ -61,5 +103,12 @@ func Annotations() []v1alpha1.Annotation { TelemetryKeeperClickHouseKeeperBinaryPath, TelemetryKeeperZookeeperBinaryPath, MCPBinaryPath, + ECSRegion, + ECSClusterARN, + ECSSubnetIDs, + ECSSecurityGroupIDs, + ECSVPCID, + ECSTaskRoleARN, + ECSTaskExecutionRoleARN, } } diff --git a/docs/examples/ecs/ec2/terraform/README.md b/docs/examples/ecs/ec2/terraform/README.md index 8b1a3f1d..5df9a6b7 100644 --- a/docs/examples/ecs/ec2/terraform/README.md +++ b/docs/examples/ecs/ec2/terraform/README.md @@ -20,35 +20,102 @@ Components: ## Prerequisites -- An existing ECS cluster with an EC2 capacity provider +- An ECS cluster tagged the way foundry names things, either from the Infrastructure casting or your own (see [Infrastructure contract](#infrastructure-contract-you-provide-the-cluster)) +- Registered EC2 container instances advertising the substrate's attributes - A VPC with private subnets -- An S3 bucket for storing component configs - IAM roles for ECS task and task execution - [Terraform](https://developer.hashicorp.com/terraform/install) >= 1.0 ## Configuration +`spec.infrastructure.name` names the substrate this installation runs on. That +one field is the whole binding: the cluster, its subnets, its security group, +its VPC and its two IAM roles are all found by the names and tags both castings +derive from it, and each arrives in Terraform as a variable defaulted to what +was derived. The region is the only thing left to state; neither the casting +nor a tag can carry it. + ```yaml apiVersion: v1alpha1 +kind: Installation metadata: name: signoz annotations: - foundry.signoz.io/ecs/region: us-east-1 - foundry.signoz.io/ecs/cluster-id: arn:aws:ecs:us-east-1:123456789012:cluster/my-cluster - foundry.signoz.io/ecs/subnet-ids: subnet-abc123,subnet-def456 - foundry.signoz.io/ecs/security-group-ids: sg-abc123 - foundry.signoz.io/ecs/vpc-id: vpc-abc123 - foundry.signoz.io/ecs/config-bucket: my-signoz-configs - foundry.signoz.io/ecs/task-role-arn: arn:aws:iam::123456789012:role/ecs-task-role - foundry.signoz.io/ecs/task-execution-role-arn: arn:aws:iam::123456789012:role/ecs-execution-role - foundry.signoz.io/ecs/capacity-provider: my-capacity-provider + foundry.signoz.io/ecs-region: us-east-1 spec: deployment: platform: ecs mode: ec2 flavor: terraform + infrastructure: + name: signoz +``` + +### Bringing your own cluster + +Each identifier can be stated instead of discovered, one at a time. What you +state is used verbatim and no lookup is emitted for it; what you leave out is +still discovered. You can adopt a substrate and pin one piece of it. + +```yaml +metadata: + annotations: + foundry.signoz.io/ecs-region: us-east-1 + foundry.signoz.io/ecs-cluster-arn: arn:aws:ecs:us-east-1:123456789012:cluster/my-cluster + foundry.signoz.io/ecs-subnet-ids: subnet-abc123,subnet-def456 + foundry.signoz.io/ecs-security-group-ids: sg-abc123 + foundry.signoz.io/ecs-vpc-id: vpc-abc123 + foundry.signoz.io/ecs-task-role-arn: arn:aws:iam::123456789012:role/ecs-task-role + foundry.signoz.io/ecs-task-execution-role-arn: arn:aws:iam::123456789012:role/ecs-execution-role ``` +`spec.infrastructure.name` is still required: the container instances and their +volumes are always found by tag, whoever provisioned them. + +## Multi-node clusters + +ClickHouse (telemetry store) and the keeper can run as a multi-node cluster. Set the cluster sizes in the spec: + +```yaml +spec: + telemetrykeeper: + spec: + cluster: + replicas: 3 # keeper nodes; use an odd number for raft quorum + telemetrystore: + spec: + cluster: + shards: 2 # number of shards + replicas: 1 # replicas in addition to the primary, so 2 nodes per shard +``` + +This generates one ECS service, one Cloud Map service, and one task definition **per node**: ClickHouse nodes are `shards x (replicas+1)` (named `telemetrystore-clickhouse--`) and keeper nodes are `replicas` (named `telemetrykeeper--`). Each ClickHouse node fetches its own `config--.yaml`; each keeper node fetches its own `keeper-.yaml` (ClickHouse Keeper) or is clustered via `ZOO_SERVER_ID` / `ZOO_SERVERS` env (ZooKeeper). + +Each stateful task places onto the persistent storage class and bind-mounts its data under `/var/lib/foundry//...`; stateless tasks place onto the ephemeral pool. Tasks use `launch_type = EC2`, so ECS places them onto your registered container instances: + +| Component | Placed on instances advertising | +| --- | --- | +| ClickHouse, Keeper, Postgres | `foundry.signoz.io/name == ` and `foundry.signoz.io/storage == persistent` | +| SigNoz with `sqlite` | `foundry.signoz.io/name == ` and `foundry.signoz.io/storage == persistent` | +| Ingester, SigNoz with `postgres` | `foundry.signoz.io/name == ` and `foundry.signoz.io/storage == ephemeral` | + +Node identity is claimed, not computed. At plan time, Terraform reads the `foundry.signoz.io/identities` tag off the persistent instances: identities already claimed stay exactly where they are, new identities take unclaimed instances first and then wrap round-robin onto the fleet. At apply, the claim is written back as the tag and each stateful service is pinned with `ec2InstanceId == ''` (plus the storage attribute as a bootstrap check). The claim record lives on the instances themselves: neither foundry nor its lock ever holds a binding, re-forging never moves data, and an operator can move an identity by editing the tag with plain AWS knowledge. + +Placement is best effort by design: with at least two persistent instances, replicas of one shard land on distinct machines (identities assign round-robin in shard-major order); with fewer instances than identities, they share machines and the ECS scheduler's own limits (task ENIs, memory reservations) decide what actually fits, leaving the rest PENDING. A replaced instance loses its tag, so its identities re-claim automatically on the next apply and start on a fresh disk (replicated components resync; unreplicated start empty). + +### Infrastructure contract (you provide the cluster) + +The Installation kind **deploys onto** an ECS cluster; it does not provision compute or storage. Whether you use the Infrastructure kind or bring your own cluster, it must satisfy the same convention: + +- **Registered EC2 container instances**, enough for the topology: one persistent instance per stateful node. `shards: 2 / replicas: 1` + `3` keepers + Postgres is `4 + 3 + 1 = 8` persistent instances; SigNoz and Ingester run on `ephemeral` capacity. +- **Instances advertising `foundry.signoz.io/name` and `foundry.signoz.io/storage`** (`persistent` or `ephemeral`). Set them via the agent's `ECS_INSTANCE_ATTRIBUTES={"foundry.signoz.io/name":"signoz","foundry.signoz.io/storage":"persistent"}` or `aws ecs put-attributes`. The name is what keeps one installation off another's nodes in a shared cluster. A task with no matching instance stays PENDING (fail-loud, never silent data loss). +- **Instances and volumes tagged the same way**. The claim controller finds them by tag, not from a list it is handed. +- **A durable volume (e.g. EBS) mounted at `/var/lib/foundry`** on each `persistent` instance, so bind-mounted data survives restarts. Keep the volume's lifecycle independent of the instance (a standalone EBS volume re-attached to a replacement) and the node's data survives termination too. + +Data survives task restarts on the same instance. A task rescheduled onto a different persistent instance starts empty and re-replicates from its peers (or, for an unreplicated component, starts fresh). Counts, sizing, and autoscaling are yours to manage; foundry only wires the deployment. + +Leaving the cluster blocks unset deploys a single node of each component (the default). + ## Deploy Run the full pipeline (generate Terraform files and apply): @@ -106,9 +173,9 @@ pours/deployment/ Verify the ECS services are running: ```bash -aws ecs list-services --cluster my-cluster --region us-east-1 +aws ecs list-services --cluster signoz-cls --region us-east-1 aws ecs describe-services \ - --cluster my-cluster \ + --cluster signoz-cls \ --services signoz-signoz signoz-ingester signoz-telemetrystore-clickhouse \ --region us-east-1 ``` @@ -167,19 +234,18 @@ Run `foundryctl forge` to see the generated files and identify the JSON paths yo ## Annotations -Annotations populate `terraform.tfvars.json` so Foundry can generate a ready-to-apply Terraform configuration. +Only the region is required. The rest override one discovered identifier each, +and are for a cluster foundry did not provision. -| Annotation | Maps to tfvar | Description | +| Annotation | Replaces the lookup variable | With | | --- | --- | --- | -| `foundry.signoz.io/ecs/region` | `region` | AWS region | -| `foundry.signoz.io/ecs/cluster-id` | `ecs_cluster_id` | ECS cluster ARN or ID | -| `foundry.signoz.io/ecs/subnet-ids` | `subnet_ids` | Comma-separated subnet IDs | -| `foundry.signoz.io/ecs/security-group-ids` | `security_group_ids` | Comma-separated security group IDs | -| `foundry.signoz.io/ecs/vpc-id` | `vpc_id` | VPC ID for Cloud Map namespace | -| `foundry.signoz.io/ecs/config-bucket` | `config_bucket` | S3 bucket for component configs | -| `foundry.signoz.io/ecs/task-role-arn` | `task_role_arn` | IAM role ARN for ECS tasks | -| `foundry.signoz.io/ecs/task-execution-role-arn` | `task_execution_role_arn` | IAM role ARN for task execution | -| `foundry.signoz.io/ecs/capacity-provider` | `capacity_provider` | ECS capacity provider name | +| `foundry.signoz.io/ecs-region` | | required; nothing else carries it | +| `foundry.signoz.io/ecs-cluster-arn` | `cluster_name` | `cluster_arn` | +| `foundry.signoz.io/ecs-subnet-ids` | `subnet_tags` | `subnet_ids` | +| `foundry.signoz.io/ecs-security-group-ids` | `security_group_name` | `security_group_ids` | +| `foundry.signoz.io/ecs-vpc-id` | `vpc_tags` | `vpc_id` | +| `foundry.signoz.io/ecs-task-role-arn` | `task_role_name` | `task_role_arn` | +| `foundry.signoz.io/ecs-task-execution-role-arn` | `execution_role_name` | `execution_role_arn` | ## Platform details @@ -204,17 +270,26 @@ The module creates the following AWS resources: ### Variables -| Variable | Type | Description | +Everything the casting resolved about the substrate is a variable whose default +is what it resolved. A one-off change needs no edit to a generated file. +Change `casting.yaml` and the default moves with it; pass `-var` to override a +single apply. + +| Variable | Type | Default | | --- | --- | --- | -| `region` | `string` | AWS region | -| `ecs_cluster_id` | `string` | ID of the existing ECS cluster | -| `subnet_ids` | `list(string)` | Subnet IDs for ECS service networking (awsvpc) | -| `security_group_ids` | `list(string)` | Security group IDs for ECS service networking | -| `vpc_id` | `string` | VPC ID for the Cloud Map private DNS namespace | -| `config_bucket` | `string` | S3 bucket for storing component config files | -| `task_role_arn` | `string` | IAM role ARN for ECS tasks | -| `task_execution_role_arn` | `string` | IAM role ARN for ECS task execution (pull images, write logs) | -| `capacity_provider` | `string` | Name of the ECS capacity provider | +| `aws_region` | `string` | from the region annotation | +| `cluster_name` | `string` | `-cls` | +| `subnet_tags` | `map(string)` | `name=`, `visibility=private` | +| `security_group_name` | `string` | `-sg-task` | +| `vpc_tags` | `map(string)` | `name=` | +| `task_role_name` | `string` | `-iam-task` | +| `execution_role_name` | `string` | `-iam-exec` | +| `node_tags` | `map(string)` | `name=`, `storage=persistent` | +| `claim_tag` | `string` | `foundry.signoz.io/identities` | + +Stating an identifier on the casting swaps its lookup variable for the value +itself: `cluster_name` becomes `cluster_arn`, `subnet_tags` becomes +`subnet_ids`, and no data source is emitted for it. ### Outputs @@ -244,12 +319,12 @@ Components communicate via Cloud Map DNS within the `{name}.local` namespace: ### IAM requirements -The **task execution role** (`task_execution_role_arn`) needs: +The **task execution role** (`execution_role_name`) needs: - `ecr:GetAuthorizationToken`, `ecr:BatchGetImage`, `ecr:GetDownloadUrlForLayer` (pull images) - `logs:CreateLogStream`, `logs:PutLogEvents` (CloudWatch logs) -The **task role** (`task_role_arn`) needs: -- `s3:GetObject` on the config bucket (config-fetcher sidecar reads configs from S3) +The **task role** (`task_role_name`) needs: +- `appconfig:StartConfigurationSession` and `appconfig:GetLatestConfiguration` (the config sidecar reads each component's config from AWS AppConfig) ### Security groups diff --git a/docs/examples/ecs/ec2/terraform/casting.yaml b/docs/examples/ecs/ec2/terraform/casting.yaml index 14264602..0d0d3c44 100644 --- a/docs/examples/ecs/ec2/terraform/casting.yaml +++ b/docs/examples/ecs/ec2/terraform/casting.yaml @@ -1,9 +1,17 @@ +# yaml-language-server: $schema=../../../../../api/v1alpha1/casting.schema.json apiVersion: v1alpha1 kind: Installation metadata: - name: signoz + name: foundry + annotations: + # The region is the one thing neither the casting nor a tag can carry. + # Everything else about the cluster is found by the tags the Infrastructure + # casting named below stamped on it. + foundry.signoz.io/ecs-region: us-east-1 spec: deployment: - flavor: terraform - mode: ec2 platform: ecs + mode: ec2 + flavor: terraform + infrastructure: + name: foundry \ No newline at end of file diff --git a/docs/examples/ecs/ec2/terraform/casting.yaml.lock b/docs/examples/ecs/ec2/terraform/casting.yaml.lock index edfa7419..61df2462 100644 --- a/docs/examples/ecs/ec2/terraform/casting.yaml.lock +++ b/docs/examples/ecs/ec2/terraform/casting.yaml.lock @@ -1,12 +1,16 @@ apiVersion: v1alpha1 kind: Installation metadata: - name: signoz + annotations: + foundry.signoz.io/ecs-region: us-east-1 + name: foundry spec: deployment: flavor: terraform mode: ec2 platform: ecs + infrastructure: + name: foundry ingester: spec: cluster: @@ -70,31 +74,31 @@ spec: - name: service.version exporters: clickhousetraces: - datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + datasource: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_traces low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} use_new_schema: true timeout: 45s sending_queue: enabled: false signozclickhousemetrics: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metrics timeout: 45s sending_queue: enabled: false clickhouselogsexporter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_logs use_new_schema: true timeout: 45s sending_queue: enabled: false signozclickhousemeter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_meter timeout: 45s sending_queue: enabled: false metadataexporter: enabled: true - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metadata timeout: 45s cache: provider: in_memory @@ -147,7 +151,7 @@ spec: exporters: - signozclickhousemeter opamp.yaml: | - server_endpoint: ws://signoz.signoz.local:4320/v1/opamp + server_endpoint: ws://signoz.foundry.local:4320/v1/opamp enabled: true env: SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m @@ -215,31 +219,31 @@ spec: - name: service.version exporters: clickhousetraces: - datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + datasource: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_traces low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} use_new_schema: true timeout: 45s sending_queue: enabled: false signozclickhousemetrics: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metrics timeout: 45s sending_queue: enabled: false clickhouselogsexporter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_logs use_new_schema: true timeout: 45s sending_queue: enabled: false signozclickhousemeter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_meter timeout: 45s sending_queue: enabled: false metadataexporter: enabled: true - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metadata timeout: 45s cache: provider: in_memory @@ -292,7 +296,7 @@ spec: exporters: - signozclickhousemeter opamp.yaml: | - server_endpoint: ws://signoz.signoz.local:4320/v1/opamp + server_endpoint: ws://signoz.foundry.local:4320/v1/opamp env: SIGNOZ_OTEL_COLLECTOR_TIMEOUT: 10m mcp: @@ -323,7 +327,7 @@ spec: status: addresses: dsn: - - tcp://metastore-postgres.signoz.local:5432 + - tcp://metastore-postgres-0.foundry.local:5432 config: {} env: POSTGRES_DB: signoz @@ -336,23 +340,23 @@ spec: config: {} enabled: true env: - SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres-0.foundry.local:5432/signoz?sslmode=disable SIGNOZ_SQLSTORE_PROVIDER: postgres - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse.signoz.local:9000 + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000 SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse image: signoz/signoz:latest version: latest status: addresses: apiserver: - - tcp://signoz.signoz.local:8080 + - tcp://signoz.foundry.local:8080 opamp: - - ws://signoz.signoz.local:4320 + - ws://signoz.foundry.local:4320 config: {} env: - SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable + SIGNOZ_SQLSTORE_POSTGRES_DSN: postgres://signoz:signoz@metastore-postgres-0.foundry.local:5432/signoz?sslmode=disable SIGNOZ_SQLSTORE_PROVIDER: postgres - SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse.signoz.local:9000 + SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000 SIGNOZ_TELEMETRYSTORE_PROVIDER: clickhouse telemetrykeeper: kind: clickhousekeeper @@ -378,7 +382,7 @@ spec: log_storage_path: /var/lib/clickhouse-keeper/coordination/log raft_configuration: server: - - hostname: telemetrykeeper-clickhousekeeper.signoz.local + - hostname: telemetrykeeper-clickhousekeeper-0.foundry.local port: 9234 id: 0 server_id: 0 @@ -390,9 +394,9 @@ spec: status: addresses: client: - - tcp://telemetrykeeper-clickhousekeeper.signoz.local:9181 + - tcp://telemetrykeeper-clickhousekeeper-0.foundry.local:9181 raft: - - tcp://telemetrykeeper-clickhousekeeper.signoz.local:9234 + - tcp://telemetrykeeper-clickhousekeeper-0.foundry.local:9234 config: data: keeper-0.yaml: | @@ -412,7 +416,7 @@ spec: log_storage_path: /var/lib/clickhouse-keeper/coordination/log raft_configuration: server: - - hostname: telemetrykeeper-clickhousekeeper.signoz.local + - hostname: telemetrykeeper-clickhousekeeper-0.foundry.local port: 9234 id: 0 server_id: 0 @@ -469,11 +473,11 @@ spec: cluster: shard: - replica: - - host: telemetrystore-clickhouse.signoz.local + - host: telemetrystore-clickhouse-0-0.foundry.local port: 9000 zookeeper: node: - - host: telemetrykeeper-clickhousekeeper.signoz.local + - host: telemetrykeeper-clickhousekeeper-0.foundry.local port: 9181 query_log: flush_interval_milliseconds: 30000 @@ -543,7 +547,7 @@ spec: status: addresses: tcp: - - tcp://telemetrystore-clickhouse.signoz.local:9000 + - tcp://telemetrystore-clickhouse-0-0.foundry.local:9000 config: data: config-0-0.yaml: | @@ -589,11 +593,11 @@ spec: cluster: shard: - replica: - - host: telemetrystore-clickhouse.signoz.local + - host: telemetrystore-clickhouse-0-0.foundry.local port: 9000 zookeeper: node: - - host: telemetrykeeper-clickhousekeeper.signoz.local + - host: telemetrykeeper-clickhousekeeper-0.foundry.local port: 9181 query_log: flush_interval_milliseconds: 30000 diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/ingester.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester.tf.json new file mode 100644 index 00000000..da1c2890 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester.tf.json @@ -0,0 +1,185 @@ +{ + "locals": { + "containers_ingester": [ + { + "name": "foundry-ingester-config-init", + "image": "signoz/signoz-otel-collector:latest", + "essential": false, + "user": "0", + "entryPoint": ["/bin/sh", "-c"], + "command": ["chown 10001:0 /conf"], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "memoryReservation": 102 + }, + { + "name": "foundry-ingester-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + "user": "10001:0", + "dependsOn": [ + {"containerName": "foundry-ingester-config-init", "condition": "SUCCESS"} + ], + "environment": [ + {"name": "PREFETCH_LIST", "value": "foundry:default:ingester,foundry:default:ingester-opamp"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"foundry:default:ingester\":{\"writeTo\":{\"path\":\"/conf/ingester.yaml\"}},\"foundry:default:ingester-opamp\":{\"writeTo\":{\"path\":\"/conf/opamp.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "test -s /conf/ingester.yaml && test -s /conf/opamp.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "foundry-ingester", + "image": "signoz/signoz-otel-collector:latest", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], + "environment": [{"name":"FOUNDRY_CONFIG_DIGEST","value":"449e8d4a848e1880ecf8d9e30df69d0a7da71bb4b38c56b7e8b3945549a482b7"},{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse-0-0.foundry.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], + "portMappings": [ + {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, + {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "dependsOn": [ + {"containerName": "foundry-ingester-config-init", "condition": "SUCCESS"}, + {"containerName": "foundry-ingester-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 512, + "memoryReservation": 512 + } + ] + }, + "resource": { + "aws_appconfig_configuration_profile": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "ingester", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "ingester-opamp", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_appconfig_hosted_configuration_version": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/ingester/ingester.yaml\")}" + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester_opamp.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/ingester/opamp.yaml\")}" + } + }, + "aws_appconfig_deployment": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.ingester.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester_opamp.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.ingester_opamp.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_ecs_task_definition": { + "ingester": { + "family": "foundry-ingester", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_ingester)}", + "volume": [ + { + "name": "ingester-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_appconfig_deployment.ingester", "aws_appconfig_deployment.ingester_opamp"] + } + }, + "aws_service_discovery_service": { + "ingester": { + "name": "ingester", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "ingester": { + "name": "foundry-ingester", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.ingester.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.ingester.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "attribute:foundry.signoz.io/name == foundry and attribute:foundry.signoz.io/storage == ephemeral" + } + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester/ingester.yaml similarity index 86% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml rename to docs/examples/ecs/ec2/terraform/pours/deployment/ingester/ingester.yaml index 0e5b7c83..0a430fd1 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/ingester.yaml +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester/ingester.yaml @@ -7,13 +7,13 @@ connectors: metrics_flush_interval: 1h exporters: clickhouselogsexporter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_logs + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_logs sending_queue: enabled: false timeout: 45s use_new_schema: true clickhousetraces: - datasource: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_traces + datasource: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_traces low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING} sending_queue: enabled: false @@ -22,16 +22,16 @@ exporters: metadataexporter: cache: provider: in_memory - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metadata + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metadata enabled: true timeout: 45s signozclickhousemeter: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_meter + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_meter sending_queue: enabled: false timeout: 45s signozclickhousemetrics: - dsn: tcp://telemetrystore-clickhouse.signoz.local:9000/signoz_metrics + dsn: tcp://telemetrystore-clickhouse-0-0.foundry.local:9000/signoz_metrics sending_queue: enabled: false timeout: 45s diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/ingester/opamp.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester/opamp.yaml new file mode 100644 index 00000000..6388b351 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/ingester/opamp.yaml @@ -0,0 +1 @@ +server_endpoint: ws://signoz.foundry.local:4320/v1/opamp diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json index 7cdd6a42..71646edc 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/main.tf.json @@ -1,30 +1,139 @@ { - "terraform": { - "required_version": ">= 1.0", - "required_providers": { - "aws": { - "source": "hashicorp/aws", - "version": ">= 5.0" + "data": { + "aws_ecs_cluster": { + "main": { + "cluster_name": "${var.cluster_name}" + } + }, + "aws_subnets": { + "private": { + "tags": "${var.subnet_tags}" + } + }, + "aws_security_group": { + "tasks": { + "name": "${var.security_group_name}" + } + }, + "aws_vpc": { + "main": { + "tags": "${var.vpc_tags}" + } + }, + "aws_instances": { + "persistent": { + "instance_tags": "${var.node_tags}", + "instance_state_names": ["running", "pending"] + } + }, + "aws_instance": { + "persistent": { + "for_each": "${toset(data.aws_instances.persistent.ids)}", + "instance_id": "${each.value}" + } + }, + "aws_ebs_volumes": { + "persistent": { + "tags": "${var.node_tags}" + } + }, + "aws_ebs_volume": { + "persistent": { + "for_each": "${toset(data.aws_ebs_volumes.persistent.ids)}", + "filter": [ + { + "name": "volume-id", + "values": ["${each.value}"] + } + ] } } }, - "provider": { - "aws": { - "region": "${var.region}" - } + "locals": { + "cluster_arn": "${data.aws_ecs_cluster.main.arn}", + "subnet_ids": "${data.aws_subnets.private.ids}", + "security_group_ids": "${[data.aws_security_group.tasks.id]}", + "vpc_id": "${data.aws_vpc.main.id}", + "task_role_arn": "${aws_iam_role.task.arn}", + "execution_role_arn": "${aws_iam_role.exec.arn}", + "identities": ["telemetrykeeper-0", "metastore-0", "telemetrystore-0-0"], + "volume_instance": "${ merge([ for id, instance in data.aws_instance.persistent : { for device in instance.ebs_block_device : device.volume_id => id } ]...) }", + "instance_volume": "${ { for volume, instance in local.volume_instance : instance => volume } }", + "volume_claims": "${ merge([ for id, volume in data.aws_ebs_volume.persistent : { for identity in split(\",\", volume.tags[var.claim_tag]) : identity => id } if contains(keys(volume.tags), var.claim_tag) ]...) }", + "instance_claims": "${ merge([ for id, instance in data.aws_instance.persistent : { for identity in split(\",\", instance.tags[var.claim_tag]) : identity => id } if contains(keys(instance.tags), var.claim_tag) ]...) }", + "inherited_claims": "${ { for identity, instance in local.instance_claims : identity => local.instance_volume[instance] if contains(keys(local.instance_volume), instance) } }", + "claims": "${ merge(local.inherited_claims, local.volume_claims) }", + "unclaimed_volume_ids": "${ sort([ for id, volume in data.aws_ebs_volume.persistent : id if !contains(values(local.claims), id) ]) }", + "claimed_volume_ids": "${ sort([ for id, volume in data.aws_ebs_volume.persistent : id if contains(values(local.claims), id) ]) }", + "assignment_pool": "${ concat(local.unclaimed_volume_ids, local.claimed_volume_ids) }", + "new_identities": "${ [ for identity in local.identities : identity if !contains(keys(local.claims), identity) ] }", + "volumes": "${ { for identity in local.identities : identity => contains(keys(local.claims), identity) ? local.claims[identity] : element(local.assignment_pool, index(local.new_identities, identity)) } }", + "seats": "${ { for identity, volume in local.volumes : identity => lookup(local.volume_instance, volume, \"unattached\") } }" }, - "module": { - "signoz": { - "source": "./module", - "region": "${var.region}", - "ecs_cluster_id": "${var.ecs_cluster_id}", - "subnet_ids": "${var.subnet_ids}", - "security_group_ids": "${var.security_group_ids}", - "vpc_id": "${var.vpc_id}", - "config_bucket": "${var.config_bucket}", - "task_role_arn": "${var.task_role_arn}", - "task_execution_role_arn": "${var.task_execution_role_arn}", - "capacity_provider": "${var.capacity_provider}" + "resource": { + "aws_iam_role": { + "task": { + "name": "${var.task_role_name}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ecs-tasks.amazonaws.com\"}}]})}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + }, + "exec": { + "name": "${var.execution_role_name}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ecs-tasks.amazonaws.com\"}}]})}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_iam_role_policy_attachment": { + "exec": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "role": "${aws_iam_role.exec.name}" + } + }, + "aws_iam_role_policy": { + "task_appconfig_read": { + "name": "${var.task_role_name}-appconfig-read", + "role": "${aws_iam_role.task.id}", + "policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Effect\" = \"Allow\", \"Action\" = [\"appconfig:StartConfigurationSession\"], \"Resource\" = [aws_appconfig_application.main.arn]}, {\"Effect\" = \"Allow\", \"Action\" = [\"appconfig:GetLatestConfiguration\"], \"Resource\" = \"*\"}]})}" + } + }, + "aws_service_discovery_private_dns_namespace": { + "main": { + "name": "foundry.local", + "vpc": "${local.vpc_id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_ec2_tag": { + "claims": { + "for_each": "${ toset(values(local.volumes)) }", + "resource_id": "${each.value}", + "key": "${var.claim_tag}", + "value": "${ join(\",\", sort([ for identity, volume in local.volumes : identity if volume == each.value ])) }" + } + }, + "aws_appconfig_application": { + "main": { + "name": "foundry", + "description": "SigNoz component configuration", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_appconfig_environment": { + "main": { + "name": "default", + "application_id": "${aws_appconfig_application.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_appconfig_deployment_strategy": { + "main": { + "name": "foundry-config", + "deployment_duration_in_minutes": 0, + "final_bake_time_in_minutes": 0, + "growth_factor": 100, + "replicate_to": "NONE", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } } } } diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/metastore.tf.json similarity index 50% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json rename to docs/examples/ecs/ec2/terraform/pours/deployment/metastore.tf.json index df1ccf27..05f9de08 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/metastore.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/metastore.tf.json @@ -1,8 +1,8 @@ { "locals": { - "containers": [ + "containers_metastore": [ { - "name": "signoz-metastore-postgres-0", + "name": "foundry-metastore-postgres-0", "image": "postgres:16", "essential": true, "environment": [{"name":"POSTGRES_DB","value":"signoz"},{"name":"POSTGRES_PASSWORD","value":"signoz"},{"name":"POSTGRES_USER","value":"signoz"}], @@ -11,15 +11,14 @@ ], "mountPoints": [ { - "sourceVolume": "postgres-data", + "sourceVolume": "metastore-data", "containerPath": "/var/lib/postgresql/data" } ], "cpu": 256, - "memory": 256, "memoryReservation": 256, "healthCheck": { - "command": ["CMD-SHELL", "pg_isready -U postgres || exit 1"], + "command": ["CMD-SHELL", "pg_isready -U signoz -d signoz || exit 1"], "interval": 30, "timeout": 5, "retries": 3, @@ -31,26 +30,25 @@ "resource": { "aws_ecs_task_definition": { "metastore": { - "family": "signoz-metastore-postgres-0", + "family": "foundry-metastore-postgres-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, "network_mode": "awsvpc", "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_metastore)}", "volume": [ { - "name": "postgres-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } + "name": "metastore-data", + "host_path": "/var/lib/foundry/foundry/metastore/0" } ] } }, "aws_service_discovery_service": { "metastore": { - "name": "metastore-postgres", + "name": "metastore-postgres-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", "dns_records": [ @@ -65,24 +63,27 @@ }, "aws_ecs_service": { "metastore": { - "name": "signoz-metastore-postgres-0", - "cluster": "${var.ecs_cluster_id}", + "name": "foundry-metastore-postgres-0", + "cluster": "${local.cluster_arn}", "task_definition": "${aws_ecs_task_definition.metastore.arn}", "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" }, "service_registries": { "registry_arn": "${aws_service_discovery_service.metastore.arn}" - } + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"metastore-0\"]}' and attribute:foundry.signoz.io/name == foundry and attribute:foundry.signoz.io/storage == persistent" + } + ] } } } diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json deleted file mode 100644 index 40e043d4..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester.tf.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "locals": { - "containers": [ - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/signoz/ingester/ /configs/ingester/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "ingester-config", - "containerPath": "/configs/ingester" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "ingester", - "image": "signoz/signoz-otel-collector:latest", - "essential": true, - "entryPoint": ["/bin/sh", "-c"], - "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], - "environment": [{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], - "portMappings": [ - {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, - {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} - ], - "mountPoints": [ - { - "sourceVolume": "ingester-config", - "containerPath": "/conf" - } - ], - "dependsOn": [ - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 512, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q localhost:13133 || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "ingester_configs": { - "for_each": "${fileset(\"${path.module}/ingester\", \"*.yaml\")}", - "bucket": "${var.config_bucket}", - "key": "signoz/ingester/${each.value}", - "source": "${path.module}/ingester/${each.value}", - "etag": "${filemd5(\"${path.module}/ingester/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "ingester": { - "family": "signoz-ingester", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "ingester-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.ingester_configs"] - } - }, - "aws_service_discovery_service": { - "ingester": { - "name": "ingester", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "ingester": { - "name": "signoz-ingester", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.ingester.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.ingester.arn}" - }, -"depends_on": [ - "aws_ecs_service.signoz", - "aws_ecs_service.telemetrystore" - ] - } - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml deleted file mode 100644 index 46c46c21..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/ingester/opamp.yaml +++ /dev/null @@ -1 +0,0 @@ -server_endpoint: ws://signoz.signoz.local:4320/v1/opamp diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json deleted file mode 100644 index 9868aa99..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/main.tf.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "resource": { - "aws_service_discovery_private_dns_namespace": { - "main": { - "name": "signoz.local", - "vpc": "${var.vpc_id}" - } - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json deleted file mode 100644 index e3f57a53..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/signoz.tf.json +++ /dev/null @@ -1,79 +0,0 @@ -{ - "locals": { - "containers": [ - { - "name": "signoz", - "image": "signoz/signoz:latest", - "essential": true, - "environment": [{"name":"SIGNOZ_SQLSTORE_POSTGRES_DSN","value":"postgres://signoz:signoz@metastore-postgres.signoz.local:5432/signoz?sslmode=disable"},{"name":"SIGNOZ_SQLSTORE_PROVIDER","value":"postgres"},{"name":"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_TELEMETRYSTORE_PROVIDER","value":"clickhouse"}], - "portMappings": [ - {"name": "http", "containerPort": 8080, "protocol": "tcp", "appProtocol": "http"}, - {"name": "opamp", "containerPort": 4320, "protocol": "tcp"} - ], - "cpu": 512, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q localhost:8080/api/v1/health || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_ecs_task_definition": { - "signoz": { - "family": "signoz-signoz", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}" - } - }, - "aws_service_discovery_service": { - "signoz": { - "name": "signoz", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "signoz": { - "name": "signoz-signoz", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.signoz.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.signoz.arn}" - }, -"depends_on": [ - "aws_ecs_service.metastore", - "aws_ecs_service.telemetrystore" - ] - } - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json deleted file mode 100644 index e5d524b1..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper.tf.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "locals": { - "containers": [ - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/signoz/telemetrykeeper/clickhousekeeper/ /configs/telemetrykeeper/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "keeper-config", - "containerPath": "/configs/telemetrykeeper" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "signoz-telemetrykeeper-clickhousekeeper-0", - "image": "clickhouse/clickhouse-keeper:25.12.5", - "essential": true, - "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], - "portMappings": [ - {"name": "client", "containerPort": 9181, "protocol": "tcp"}, - {"name": "raft", "containerPort": 9234, "protocol": "tcp"} - ], - "mountPoints": [ - { - "sourceVolume": "keeper-data", - "containerPath": "/var/lib/clickhouse-keeper" - }, - { - "sourceVolume": "keeper-config", - "containerPath": "/etc/clickhouse-keeper" - } - ], - "dependsOn": [ - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 256, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "echo ruok | nc localhost 9181 || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "telemetrykeeper_configs": { - "for_each": "${fileset(\"${path.module}/telemetrykeeper/clickhousekeeper\", \"**\")}", - "bucket": "${var.config_bucket}", - "key": "signoz/telemetrykeeper/clickhousekeeper/${each.value}", - "source": "${path.module}/telemetrykeeper/clickhousekeeper/${each.value}", - "etag": "${filemd5(\"${path.module}/telemetrykeeper/clickhousekeeper/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "telemetrykeeper": { - "family": "signoz-telemetrykeeper-clickhousekeeper-0", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "keeper-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "keeper-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.telemetrykeeper_configs"] - } - }, - "aws_service_discovery_service": { - "telemetrykeeper": { - "name": "telemetrykeeper-clickhousekeeper", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "telemetrykeeper": { - "name": "signoz-telemetrykeeper-clickhousekeeper-0", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.telemetrykeeper.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.telemetrykeeper.arn}" - } - } - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json deleted file mode 100644 index 124fc1dc..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore.tf.json +++ /dev/null @@ -1,163 +0,0 @@ -{ - "locals": { - "containers": [ - { - "name": "init-clickhouse", - "image": "alpine:3.18.2", - "essential": false, - "command": ["/bin/sh", "-c", "cd /tmp && wget -O histogram-quantile.tar.gz 'https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_linux_amd64.tar.gz' && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile"], - "mountPoints": [ - { - "sourceVolume": "shared-binary-volume", - "containerPath": "/var/lib/clickhouse/user_scripts" - } - ], - "cpu": 256, - "memory": 256, - "memoryReservation": 256 - }, - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/signoz/telemetrystore/clickhouse/ /configs/telemetrystore/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "telemetrystore-clickhouse-config", - "containerPath": "/configs/telemetrystore" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "signoz-telemetrystore-clickhouse-0-0", - "image": "clickhouse/clickhouse-server:25.12.5", - "essential": true, - "environment": [{"name":"CLICKHOUSE_SKIP_USER_SETUP","value":"1"}], - "entryPoint": ["/bin/sh", "-c"], - "command": ["ln -sf /etc/clickhouse-server/config.d/config-0-0.yaml /etc/clickhouse-server/config-0-0.yaml && exec /entrypoint.sh"], - "portMappings": [ - {"name": "native", "containerPort": 9000, "protocol": "tcp"}, - {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, - {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} - ], - "mountPoints": [ - { - "sourceVolume": "shared-binary-volume", - "containerPath": "/var/lib/clickhouse/user_scripts" - }, - { - "sourceVolume": "telemetrystore-clickhouse-config", - "containerPath": "/etc/clickhouse-server/config.d" - }, - { - "sourceVolume": "clickhouse-data", - "containerPath": "/var/lib/clickhouse" - } - ], - "dependsOn": [ - {"containerName": "init-clickhouse", "condition": "SUCCESS"}, - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 1024, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q 0.0.0.0:8123/ping || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "telemetrystore_configs": { - "for_each": "${fileset(\"${path.module}/telemetrystore/clickhouse\", \"**\")}", - "bucket": "${var.config_bucket}", - "key": "signoz/telemetrystore/clickhouse/${each.value}", - "source": "${path.module}/telemetrystore/clickhouse/${each.value}", - "etag": "${filemd5(\"${path.module}/telemetrystore/clickhouse/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "telemetrystore": { - "family": "signoz-telemetrystore-clickhouse-0-0", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "shared-binary-volume", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "telemetrystore-clickhouse-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "clickhouse-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.telemetrystore_configs"] - } - }, - "aws_service_discovery_service": { - "telemetrystore": { - "name": "telemetrystore-clickhouse", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "telemetrystore": { - "name": "signoz-telemetrystore-clickhouse-0-0", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.telemetrystore.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.telemetrystore.arn}" - }, -"depends_on": [ - "aws_ecs_service.telemetrykeeper" - ] - } - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json deleted file mode 100644 index fd7e081e..00000000 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/variables.tf.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "variable": { - "region": { - "description": "AWS region", - "type": "string" - }, - "ecs_cluster_id": { - "description": "ID of the existing ECS cluster to deploy services into", - "type": "string" - }, - "subnet_ids": { - "description": "List of subnet IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "security_group_ids": { - "description": "List of security group IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "vpc_id": { - "description": "VPC ID for the private DNS namespace", - "type": "string" - }, - "config_bucket": { - "description": "S3 bucket name for storing config files", - "type": "string" - }, - "task_role_arn": { - "description": "IAM role ARN for ECS tasks", - "type": "string" - }, - "task_execution_role_arn": { - "description": "IAM role ARN for ECS task execution (pull images, write logs)", - "type": "string" - }, - "capacity_provider": { - "description": "Name of the ECS capacity provider", - "type": "string" - } - } -} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/outputs.tf.json similarity index 70% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json rename to docs/examples/ecs/ec2/terraform/pours/deployment/outputs.tf.json index d2799c07..3b17c839 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/outputs.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/outputs.tf.json @@ -1,8 +1,8 @@ { "output": { - "ecs_cluster_id": { - "description": "ECS cluster ID", - "value": "${var.ecs_cluster_id}" + "cluster_arn": { + "description": "ARN of the ECS cluster", + "value": "${local.cluster_arn}" }, "namespace_id": { "description": "Cloud Map private DNS namespace ID", @@ -14,11 +14,11 @@ }, "subnet_ids": { "description": "Subnet IDs used by ECS services", - "value": "${var.subnet_ids}" + "value": "${local.subnet_ids}" }, "security_group_ids": { "description": "Security group IDs used by ECS services", - "value": "${var.security_group_ids}" + "value": "${local.security_group_ids}" }, "signoz_service_arn": { "description": "SigNoz ECS service ARN (target for ALB on port 8080)", @@ -36,13 +36,13 @@ "description": "Ingester ECS service name", "value": "${aws_ecs_service.ingester.name}" }, - "telemetrystore_service_name": { - "description": "TelemetryStore ECS service name", - "value": "${aws_ecs_service.telemetrystore.name}" + "telemetrystore_service_names": { + "description": "TelemetryStore ECS service names (one per node)", + "value": ["${aws_ecs_service.telemetrystore_0_0.name}"] }, - "telemetrykeeper_service_name": { - "description": "TelemetryKeeper ECS service name", - "value": "${aws_ecs_service.telemetrykeeper.name}" + "telemetrykeeper_service_names": { + "description": "TelemetryKeeper ECS service names (one per node)", + "value": ["${aws_ecs_service.telemetrykeeper_0.name}"] }, "metastore_service_name": { "description": "MetaStore ECS service name", diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/providers.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/providers.tf.json new file mode 100644 index 00000000..fcaa4a46 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/providers.tf.json @@ -0,0 +1,7 @@ +{ + "provider": { + "aws": { + "region": "${var.aws_region}" + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/signoz.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/signoz.tf.json new file mode 100644 index 00000000..f9b32d5a --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/signoz.tf.json @@ -0,0 +1,79 @@ +{ + "locals": { + "containers_signoz": [ + { + "name": "foundry-signoz", + "image": "signoz/signoz:latest", + "essential": true, + "environment": [{"name":"SIGNOZ_SQLSTORE_POSTGRES_DSN","value":"postgres://signoz:signoz@metastore-postgres-0.foundry.local:5432/signoz?sslmode=disable"},{"name":"SIGNOZ_SQLSTORE_PROVIDER","value":"postgres"},{"name":"SIGNOZ_TELEMETRYSTORE_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse-0-0.foundry.local:9000"},{"name":"SIGNOZ_TELEMETRYSTORE_PROVIDER","value":"clickhouse"}], + "portMappings": [ + {"name": "http", "containerPort": 8080, "protocol": "tcp", "appProtocol": "http"}, + {"name": "opamp", "containerPort": 4320, "protocol": "tcp"} + ], + "cpu": 512, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q http://localhost:8080/api/v1/health || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "signoz": { + "family": "foundry-signoz", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_signoz)}" + } + }, + "aws_service_discovery_service": { + "signoz": { + "name": "signoz", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "signoz": { + "name": "foundry-signoz", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.signoz.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.signoz.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "attribute:foundry.signoz.io/name == foundry and attribute:foundry.signoz.io/storage == ephemeral" + } + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper.tf.json new file mode 100644 index 00000000..1601fc48 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper.tf.json @@ -0,0 +1,158 @@ +{ + "locals": { + "containers_telemetrykeeper_0": [ + { + "name": "foundry-telemetrykeeper-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + "environment": [ + {"name": "PREFETCH_LIST", "value": "foundry:default:telemetrykeeper-clickhousekeeper-0"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"foundry:default:telemetrykeeper-clickhousekeeper-0\":{\"writeTo\":{\"path\":\"/etc/clickhouse-keeper/keeper.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrykeeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "test -s /etc/clickhouse-keeper/keeper.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "foundry-telemetrykeeper-clickhousekeeper-0", + "image": "clickhouse/clickhouse-keeper:25.12.5", + "essential": true, + "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], + "environment": [], + "portMappings": [ + {"name": "client", "containerPort": 9181, "protocol": "tcp"}, + {"name": "raft", "containerPort": 9234, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrykeeper-data", + "containerPath": "/var/lib/clickhouse-keeper" + }, + { + "sourceVolume": "telemetrykeeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + "dependsOn": [ + {"containerName": "foundry-telemetrykeeper-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 256, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "clickhouse-keeper-client -h localhost -p 9181 -q ls || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_appconfig_configuration_profile": { + "telemetrykeeper_0": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrykeeper-clickhousekeeper-0", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_appconfig_hosted_configuration_version": { + "telemetrykeeper_0": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrykeeper_0.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrykeeper/clickhousekeeper/keeper-0.yaml\")}" + } + }, + "aws_appconfig_deployment": { + "telemetrykeeper_0": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrykeeper_0.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrykeeper_0.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_ecs_task_definition": { + "telemetrykeeper_0": { + "family": "foundry-telemetrykeeper-clickhousekeeper-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrykeeper_0)}", + "volume": [ + { + "name": "telemetrykeeper-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrykeeper-data", + "host_path": "/var/lib/foundry/foundry/telemetrykeeper/0" + } + ], + "depends_on": ["aws_appconfig_deployment.telemetrykeeper_0"] + } + }, + "aws_service_discovery_service": { + "telemetrykeeper_0": { + "name": "telemetrykeeper-clickhousekeeper-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrykeeper_0": { + "name": "foundry-telemetrykeeper-clickhousekeeper-0", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.telemetrykeeper_0.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrykeeper_0.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"telemetrykeeper-0\"]}' and attribute:foundry.signoz.io/name == foundry and attribute:foundry.signoz.io/storage == persistent" + } + ] + } + } + } +} \ No newline at end of file diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml similarity index 89% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml rename to docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml index 53f4bb57..eb9ec30a 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrykeeper/clickhousekeeper/keeper-0.yaml +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrykeeper/clickhousekeeper/keeper-0.yaml @@ -10,7 +10,7 @@ keeper_server: log_storage_path: /var/lib/clickhouse-keeper/coordination/log raft_configuration: server: - - hostname: telemetrykeeper-clickhousekeeper.signoz.local + - hostname: telemetrykeeper-clickhousekeeper-0.foundry.local id: 0 port: 9234 server_id: 0 diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json new file mode 100644 index 00000000..ec6faa45 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json @@ -0,0 +1,213 @@ +{ + "locals": { + "containers_telemetrystore_0_0": [ + { + "name": "foundry-telemetrystore-user-scripts", + "image": "clickhouse/clickhouse-server:25.12.5", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["node_os=$(uname -s | tr '[:upper:]' '[:lower:]') && node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && cd /tmp && wget -O histogram-quantile.tar.gz https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_$${node_os}_$${node_arch}.tar.gz && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && chown clickhouse:clickhouse /etc/clickhouse-server/config.d"], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-user-scripts", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + } + ], + "memoryReservation": 256 + }, + { + "name": "foundry-telemetrystore-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + "user": "101:101", + "dependsOn": [ + {"containerName": "foundry-telemetrystore-user-scripts", "condition": "SUCCESS"} + ], + "environment": [ + {"name": "PREFETCH_LIST", "value": "foundry:default:telemetrystore-clickhouse-0-0,foundry:default:telemetrystore-clickhouse-functions"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"foundry:default:telemetrystore-clickhouse-0-0\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-0-0.yaml\"}},\"foundry:default:telemetrystore-clickhouse-functions\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-0-0.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "foundry-telemetrystore-clickhouse-0-0", + "image": "clickhouse/clickhouse-server:25.12.5", + "essential": true, + "environment": [{"name":"CLICKHOUSE_SKIP_USER_SETUP","value":"1"},{"name":"CLICKHOUSE_CONFIG","value":"/etc/clickhouse-server/config.d/config-0-0.yaml"}], + "portMappings": [ + {"name": "native", "containerPort": 9000, "protocol": "tcp"}, + {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, + {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-user-scripts", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + }, + { + "sourceVolume": "telemetrystore-data", + "containerPath": "/var/lib/clickhouse" + } + ], + "dependsOn": [ + {"containerName": "foundry-telemetrystore-user-scripts", "condition": "SUCCESS"}, + {"containerName": "foundry-telemetrystore-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 1024, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + }, + "resource": { + "aws_appconfig_configuration_profile": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrystore-clickhouse-functions", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + }, + "telemetrystore_0_0": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrystore-clickhouse-0-0", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_appconfig_hosted_configuration_version": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_functions.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrystore/clickhouse/functions.yaml\")}" + }, + "telemetrystore_0_0": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_0_0.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrystore/clickhouse/config-0-0.yaml\")}" + } + }, + "aws_appconfig_deployment": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_functions.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrystore_functions.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + }, + "telemetrystore_0_0": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_0_0.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrystore_0_0.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"} + } + }, + "aws_ecs_task_definition": { + "telemetrystore_0_0": { + "family": "foundry-telemetrystore-clickhouse-0-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrystore_0_0)}", + "volume": [ + { + "name": "telemetrystore-user-scripts", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-data", + "host_path": "/var/lib/foundry/foundry/telemetrystore/0-0" + } + ], + "depends_on": ["aws_appconfig_deployment.telemetrystore_0_0", "aws_appconfig_deployment.telemetrystore_functions"] + } + }, + "aws_service_discovery_service": { + "telemetrystore_0_0": { + "name": "telemetrystore-clickhouse-0-0", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "telemetrystore_0_0": { + "name": "foundry-telemetrystore-clickhouse-0-0", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.telemetrystore_0_0.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrystore_0_0.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"telemetrystore-0-0\"]}' and attribute:foundry.signoz.io/name == foundry and attribute:foundry.signoz.io/storage == persistent" + } + ] + } + } + } +} diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml similarity index 95% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml rename to docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml index e24cf5d7..9e912dec 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/config-0-0.yaml +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore/clickhouse/config-0-0.yaml @@ -59,7 +59,7 @@ remote_servers: cluster: shard: - replica: - - host: telemetrystore-clickhouse.signoz.local + - host: telemetrystore-clickhouse-0-0.foundry.local port: 9000 session_log: ttl: event_date + INTERVAL 1 DAY DELETE @@ -90,7 +90,7 @@ users: show_named_collection_secrets: 1 zookeeper: node: - - host: telemetrykeeper-clickhousekeeper.signoz.local + - host: telemetrykeeper-clickhousekeeper-0.foundry.local port: 9181 zookeeper_log: ttl: event_date + INTERVAL 1 DAY DELETE diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/functions.yaml b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore/clickhouse/functions.yaml similarity index 100% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore/clickhouse/functions.yaml rename to docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore/clickhouse/functions.yaml diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore_migrator.tf.json similarity index 56% rename from docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json rename to docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore_migrator.tf.json index 9019493f..3162724b 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/module/telemetrystore_migrator.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore_migrator.tf.json @@ -1,13 +1,13 @@ { "locals": { - "containers": [ + "containers_telemetrystore_migrator": [ { - "name": "telemetrystore-migrator", + "name": "foundry-telemetrystore-migrator", "image": "signoz/signoz-otel-collector:latest", "essential": true, "entryPoint": ["/bin/sh", "-c"], "command": ["/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up"], - "environment": [{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse.signoz.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], + "environment": [{"name":"SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN","value":"tcp://telemetrystore-clickhouse-0-0.foundry.local:9000"},{"name":"SIGNOZ_OTEL_COLLECTOR_TIMEOUT","value":"10m"}], "cpu": 256, "memory": 512 } @@ -16,32 +16,30 @@ "resource": { "aws_ecs_task_definition": { "telemetrystore_migrator": { - "family": "signoz-telemetrystore-migrator", + "family": "foundry-telemetrystore-migrator", + "tags": {"foundry.signoz.io/kind":"Installation","foundry.signoz.io/managed-by":"foundry","foundry.signoz.io/name":"foundry"}, "network_mode": "awsvpc", "requires_compatibilities": ["FARGATE"], "cpu": 256, "memory": 512, - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}" + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrystore_migrator)}" } } }, "data": { "aws_ecs_task_execution": { "telemetrystore_migrator": { - "cluster": "${var.ecs_cluster_id}", + "cluster": "${local.cluster_arn}", "task_definition": "${aws_ecs_task_definition.telemetrystore_migrator.arn}", "desired_count": 1, "launch_type": "FARGATE", "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}", + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}", "assign_public_ip": false - }, - "depends_on": [ - "aws_ecs_service.telemetrystore" - ] + } } } } diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json b/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json index cceaa206..860b4336 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/terraform.tfvars.json @@ -1,11 +1,3 @@ { - "region": "", - "ecs_cluster_id": "", - "subnet_ids": [""], - "security_group_ids": [""], - "vpc_id": "", - "config_bucket": "", - "task_role_arn": "", - "task_execution_role_arn": "", - "capacity_provider": "" + "aws_region": "us-east-1" } diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json index fd7e081e..2ab11538 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/variables.tf.json @@ -1,40 +1,61 @@ { "variable": { - "region": { - "description": "AWS region", + "aws_region": { + "nullable": false, + "validation": { + "condition": "${can(regex(\"^[a-z]{2}(-gov)?-[a-z]+-[0-9]$\", var.aws_region))}", + "error_message": "aws_region must be a region identifier such as us-east-1." + }, + "description": "AWS region holding the cluster", "type": "string" }, - "ecs_cluster_id": { - "description": "ID of the existing ECS cluster to deploy services into", - "type": "string" - }, - "subnet_ids": { - "description": "List of subnet IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "security_group_ids": { - "description": "List of security group IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "vpc_id": { - "description": "VPC ID for the private DNS namespace", - "type": "string" - }, - "config_bucket": { - "description": "S3 bucket name for storing config files", - "type": "string" - }, - "task_role_arn": { - "description": "IAM role ARN for ECS tasks", - "type": "string" - }, - "task_execution_role_arn": { - "description": "IAM role ARN for ECS task execution (pull images, write logs)", - "type": "string" - }, - "capacity_provider": { - "description": "Name of the ECS capacity provider", - "type": "string" + "cluster_name": { + "nullable": false, + "description": "Name of the ECS cluster to deploy into", + "type": "string", + "default": "foundry-cls" + }, + "subnet_tags": { + "nullable": false, + "description": "Tags that find the subnets tasks are placed in", + "type": "map(string)", + "default": {"foundry.signoz.io/name":"foundry","foundry.signoz.io/subnet-type":"private"} + }, + "security_group_name": { + "nullable": false, + "description": "Name of the security group tasks join", + "type": "string", + "default": "foundry-sg-task" + }, + "vpc_tags": { + "nullable": false, + "description": "Tags that find the VPC the Cloud Map namespace is created in", + "type": "map(string)", + "default": {"foundry.signoz.io/name":"foundry"} + }, + "task_role_name": { + "nullable": false, + "description": "Name of the IAM role this stack creates for its tasks", + "type": "string", + "default": "foundry-installation-task" + }, + "execution_role_name": { + "nullable": false, + "description": "Name of the IAM role this stack creates for the ECS agent to pull images and write logs", + "type": "string", + "default": "foundry-installation-exec" + }, + "node_tags": { + "nullable": false, + "description": "Tags that find the persistent instances and the volumes attached to them", + "type": "map(string)", + "default": {"foundry.signoz.io/name":"foundry","foundry.signoz.io/storage":"persistent"} + }, + "claim_tag": { + "nullable": false, + "description": "Tag key recording which identities hold a volume", + "type": "string", + "default": "foundry.signoz.io/identities" } } } diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/versions.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/versions.tf.json new file mode 100644 index 00000000..a9eded70 --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/versions.tf.json @@ -0,0 +1,11 @@ +{ + "terraform": { + "required_version": ">= 1.4.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/casting.go b/internal/casting/ecsterraformcasting/casting.go index f47205d8..0f511944 100644 --- a/internal/casting/ecsterraformcasting/casting.go +++ b/internal/casting/ecsterraformcasting/casting.go @@ -3,14 +3,19 @@ package ecsterraformcasting import ( "context" "log/slog" + "maps" "os" "os/exec" "path/filepath" + "slices" "strings" "time" + "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/installation" rootcasting "github.com/signoz/foundry/internal/casting" + "github.com/signoz/foundry/internal/convention" + awsconvention "github.com/signoz/foundry/internal/convention/aws" "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" @@ -29,125 +34,76 @@ func New(logger *slog.Logger) *ecsCasting { } func (c *ecsCasting) Enricher(ctx context.Context, config *installation.Casting) (molding.MoldingEnricher, error) { - return newEcsMoldingEnricher(config) + data, err := c.templateData(*config) + if err != nil { + return nil, err + } + + return newEcsMoldingEnricher(data) } func (c *ecsCasting) Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) { var materials []domain.Material - deployDir := rootcasting.DeploymentDir - moduleDir := filepath.Join(deployDir, "module") + dir := rootcasting.DeploymentDir + + data, err := c.templateData(config) + if err != nil { + return nil, err + } - // Root Terraform files - rootTemplates := map[string]*domain.Template{ + for filename, tmpl := range map[string]*domain.Template{ + "versions.tf.json": versionsTF, + "providers.tf.json": providersTF, "main.tf.json": mainTF, "variables.tf.json": variablesTF, + "outputs.tf.json": outputsTF, "terraform.tfvars.json": tfarsTF, - } - for filename, tmpl := range rootTemplates { - m, err := tmpl.Render(config, filepath.Join(deployDir, filename)) + } { + m, err := tmpl.Render(data, filepath.Join(dir, filename)) if err != nil { return nil, err } - materials = append(materials, m) - } - // Module shared files - moduleTemplates := map[string]*domain.Template{ - "main.tf.json": moduleMainTF, - "variables.tf.json": moduleVariablesTF, - "outputs.tf.json": moduleOutputsTF, - } - for filename, tmpl := range moduleTemplates { - m, err := tmpl.Render(config, filepath.Join(moduleDir, filename)) - if err != nil { - return nil, err - } materials = append(materials, m) } - // TelemetryKeeper - if config.Spec.TelemetryKeeper.Spec.IsEnabled() { - m, err := moduleTelemetryKeeperTF.Render(config, filepath.Join(moduleDir, "telemetrykeeper.tf.json")) - if err != nil { - return nil, err - } - materials = append(materials, m) - - for filename, content := range config.Spec.TelemetryKeeper.Spec.Config.Data { - material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String(), filename)) - if err != nil { - return nil, err - } - materials = append(materials, material) - } + // One file per component, beside the molding config that component fetches + // at task start. A component configured only through env has no configDir. + components := []struct { + enabled bool + filename string + template *domain.Template + configDir string + config map[string]string + }{ + {config.Spec.TelemetryKeeper.Spec.IsEnabled(), "telemetrykeeper.tf.json", telemetryKeeperTF, filepath.Join("telemetrykeeper", config.Spec.TelemetryKeeper.Kind.String()), config.Spec.TelemetryKeeper.Spec.Config.Data}, + {config.Spec.TelemetryStore.Spec.IsEnabled(), "telemetrystore.tf.json", telemetryStoreTF, filepath.Join("telemetrystore", config.Spec.TelemetryStore.Kind.String()), config.Spec.TelemetryStore.Spec.Config.Data}, + {config.Spec.TelemetryStore.Spec.IsEnabled(), "telemetrystore_migrator.tf.json", migratorTF, "", nil}, + {config.Spec.MetaStore.Spec.IsEnabled(), "metastore.tf.json", metaStoreTF, filepath.Join("metastore", config.Spec.MetaStore.Kind.String()), config.Spec.MetaStore.Spec.Config.Data}, + {config.Spec.Signoz.Spec.IsEnabled(), "signoz.tf.json", signozTF, "", nil}, + {config.Spec.Ingester.Spec.IsEnabled(), "ingester.tf.json", ingesterTF, "ingester", config.Spec.Ingester.Spec.Config.Data}, + {config.Spec.MCP.Spec.IsEnabled(), "mcp.tf.json", mcpTF, "", nil}, } - // TelemetryStore - if config.Spec.TelemetryStore.Spec.IsEnabled() { - m, err := moduleTelemetryStoreTF.Render(config, filepath.Join(moduleDir, "telemetrystore.tf.json")) - if err != nil { - return nil, err - } - materials = append(materials, m) - - for filename, content := range config.Spec.TelemetryStore.Spec.Config.Data { - material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "telemetrystore", config.Spec.TelemetryStore.Kind.String(), filename)) - if err != nil { - return nil, err - } - materials = append(materials, material) + for _, component := range components { + if !component.enabled { + continue } - } - // TelemetryStore migrator - if config.Spec.TelemetryStore.Spec.IsEnabled() { - m, err := moduleMigratorTF.Render(config, filepath.Join(moduleDir, "telemetrystore_migrator.tf.json")) + m, err := component.template.Render(data, filepath.Join(dir, component.filename)) if err != nil { return nil, err } - materials = append(materials, m) - } - // MetaStore - if config.Spec.MetaStore.Spec.IsEnabled() { - m, err := moduleMetaStoreTF.Render(config, filepath.Join(moduleDir, "metastore.tf.json")) - if err != nil { - return nil, err - } materials = append(materials, m) - for filename, content := range config.Spec.MetaStore.Spec.Config.Data { - material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "metastore", config.Spec.MetaStore.Kind.String(), filename)) + for filename, content := range component.config { + material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(dir, component.configDir, filename)) if err != nil { return nil, err } - materials = append(materials, material) - } - } - // Signoz - if config.Spec.Signoz.Spec.IsEnabled() { - m, err := moduleSignozTF.Render(config, filepath.Join(moduleDir, "signoz.tf.json")) - if err != nil { - return nil, err - } - materials = append(materials, m) - } - - // Ingester - if config.Spec.Ingester.Spec.IsEnabled() { - m, err := moduleIngesterTF.Render(config, filepath.Join(moduleDir, "ingester.tf.json")) - if err != nil { - return nil, err - } - materials = append(materials, m) - - for filename, content := range config.Spec.Ingester.Spec.Config.Data { - material, err := domain.NewYAMLMaterial([]byte(content), filepath.Join(moduleDir, "ingester", filename)) - if err != nil { - return nil, err - } materials = append(materials, material) } } @@ -196,21 +152,109 @@ func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outp return nil } -// getMaterials renders all module templates and returns them as JSONMaterials. -func getMaterials(config *installation.Casting) ([]domain.StructuredMaterial, error) { +// templateData binds the casting to the substrate it runs on. Forge and the +// enricher each render from their own config; both come through here. +func (c *ecsCasting) templateData(config installation.Casting) (templateData, error) { + name := config.Spec.Infrastructure.Name + + if name == "" { + return templateData{}, errors.Newf(errors.TypeInvalidInput, "spec.infrastructure.name is not set: this casting finds its cluster, its subnets and its nodes by the substrate's own tags, so it has to be told which substrate it runs on") + } + + substrate, err := convention.NewSubstrate(name) + if err != nil { + return templateData{}, errors.Wrapf(err, errors.TypeInvalidInput, "failed to resolve the substrate this installation runs on") + } + + persistent := substrate.Select().WithStorage(v1alpha1.StorageClassPersistent) + ephemeral := substrate.Select().WithStorage(v1alpha1.StorageClassEphemeral) + + // Every service uses awsvpc. A task needs a subnet, and never a public one. + private := substrate.Select().WithSubnetType(v1alpha1.SubnetTypePrivate) + + return templateData{ + Casting: config, + + ClusterName: awsconvention.Cluster(substrate).Name(), + SecurityGroupName: awsconvention.SecurityGroup(substrate, awsconvention.RoleTask).Name(), + + // Named after the workload. Several workloads share one substrate, and a + // substrate-derived name collides on the second apply. + TaskRoleName: config.Metadata.Name + "-" + strings.ToLower(config.Kind().String()) + "-task", + ExecutionRoleName: config.Metadata.Name + "-" + strings.ToLower(config.Kind().String()) + "-exec", + + VPCTags: awsconvention.Filter(substrate.Select()), + SubnetTags: awsconvention.Filter(private), + NodeTags: awsconvention.Filter(persistent), + + ClaimTag: awsconvention.Tag(convention.TagKeyIdentities), + PersistentPlacement: placement(persistent), + EphemeralPlacement: placement(ephemeral), + }, nil +} + +// templateData is what every template renders against. The casting is embedded, +// leaving `.Spec` and `.Metadata` as they were. The rest is derived from the +// substrate this installation is bound to. +type templateData struct { + installation.Casting + + ClusterName string + SecurityGroupName string + + // TaskRoleName and ExecutionRoleName are this workload's own identity, + // created and destroyed with this stack. + TaskRoleName string + ExecutionRoleName string + + VPCTags map[string]string + SubnetTags map[string]string + + // NodeTags finds the substrate's persistent instances and the volumes + // attached to them. The claim controller reads both. + NodeTags map[string]string + + // ClaimTag records which identities hold a volume. It is written after + // provisioning, and the Infrastructure casting does not reconcile it. + ClaimTag string + + // Placement expressions in ECS' own syntax, matching the attributes a + // container instance advertises. + PersistentPlacement string + EphemeralPlacement string +} + +// placement renders a selection as an ECS placement constraint. A container +// instance advertises exactly these attributes. +func placement(selection convention.Selection) string { + filter := awsconvention.Filter(selection) + + parts := make([]string, 0, len(filter)) + + for _, key := range slices.Sorted(maps.Keys(filter)) { + parts = append(parts, "attribute:"+key+" == "+filter[key]) + } + + return strings.Join(parts, " and ") +} + +// getMaterials renders the component templates for the enricher to read service +// names back from. The template is the single source of node names. +func getMaterials(data templateData) ([]domain.StructuredMaterial, error) { var materials []domain.StructuredMaterial for _, tmpl := range []*domain.Template{ - moduleMainTF, - moduleTelemetryStoreTF, - moduleTelemetryKeeperTF, - moduleMetaStoreTF, - moduleSignozTF, - moduleIngesterTF, + mainTF, + telemetryStoreTF, + telemetryKeeperTF, + metaStoreTF, + signozTF, + ingesterTF, + mcpTF, } { - m, err := tmpl.Render(*config, tmpl.Path()) + m, err := tmpl.Render(data, tmpl.Path()) if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to create material") + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to render material") } sm, ok := m.(domain.StructuredMaterial) if !ok { diff --git a/internal/casting/ecsterraformcasting/embed.go b/internal/casting/ecsterraformcasting/embed.go index 3a0d80ee..a36fb9d3 100644 --- a/internal/casting/ecsterraformcasting/embed.go +++ b/internal/casting/ecsterraformcasting/embed.go @@ -6,26 +6,29 @@ import ( "github.com/signoz/foundry/internal/domain" ) -//go:embed templates/*.gotmpl templates/module/*.gotmpl +//go:embed templates/*.gotmpl var templates embed.FS -// Root Terraform templates. +// The pour is a composition: one root Terraform module holding concrete +// values and its own state. Components are files within it, not child +// modules, because a module with a single generated caller is indirection +// without reuse. var ( + versionsTF = domain.MustNewTemplateFromFS(templates, "templates/versions.tf.json.gotmpl", domain.FormatJSON) + providersTF = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) mainTF = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) variablesTF = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) + outputsTF = domain.MustNewTemplateFromFS(templates, "templates/outputs.tf.json.gotmpl", domain.FormatJSON) tfarsTF = domain.MustNewTemplateFromFS(templates, "templates/terraform.tfvars.json.gotmpl", domain.FormatJSON) ) -// Module Terraform templates. +// One file per component. var ( - moduleMainTF = domain.MustNewTemplateFromFS(templates, "templates/module/main.tf.json.gotmpl", domain.FormatJSON) - moduleVariablesTF = domain.MustNewTemplateFromFS(templates, "templates/module/variables.tf.json.gotmpl", domain.FormatJSON) - moduleOutputsTF = domain.MustNewTemplateFromFS(templates, "templates/module/outputs.tf.json.gotmpl", domain.FormatJSON) - - moduleTelemetryKeeperTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrykeeper.tf.json.gotmpl", domain.FormatJSON) - moduleTelemetryStoreTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrystore.tf.json.gotmpl", domain.FormatJSON) - moduleMigratorTF = domain.MustNewTemplateFromFS(templates, "templates/module/telemetrystore_migrator.tf.json.gotmpl", domain.FormatJSON) - moduleMetaStoreTF = domain.MustNewTemplateFromFS(templates, "templates/module/metastore.tf.json.gotmpl", domain.FormatJSON) - moduleSignozTF = domain.MustNewTemplateFromFS(templates, "templates/module/signoz.tf.json.gotmpl", domain.FormatJSON) - moduleIngesterTF = domain.MustNewTemplateFromFS(templates, "templates/module/ingester.tf.json.gotmpl", domain.FormatJSON) + telemetryKeeperTF = domain.MustNewTemplateFromFS(templates, "templates/telemetrykeeper.tf.json.gotmpl", domain.FormatJSON) + telemetryStoreTF = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore.tf.json.gotmpl", domain.FormatJSON) + migratorTF = domain.MustNewTemplateFromFS(templates, "templates/telemetrystore_migrator.tf.json.gotmpl", domain.FormatJSON) + metaStoreTF = domain.MustNewTemplateFromFS(templates, "templates/metastore.tf.json.gotmpl", domain.FormatJSON) + signozTF = domain.MustNewTemplateFromFS(templates, "templates/signoz.tf.json.gotmpl", domain.FormatJSON) + ingesterTF = domain.MustNewTemplateFromFS(templates, "templates/ingester.tf.json.gotmpl", domain.FormatJSON) + mcpTF = domain.MustNewTemplateFromFS(templates, "templates/mcp.tf.json.gotmpl", domain.FormatJSON) ) diff --git a/internal/casting/ecsterraformcasting/embed_test.go b/internal/casting/ecsterraformcasting/embed_test.go index 060c348f..1d58ba10 100644 --- a/internal/casting/ecsterraformcasting/embed_test.go +++ b/internal/casting/ecsterraformcasting/embed_test.go @@ -2,27 +2,54 @@ package ecsterraformcasting import ( "bytes" + "log/slog" "testing" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/installation" "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// boundCasting returns a fully-defaulted Installation bound to a substrate. The +// binding is what lets the casting derive the tags it looks resources up by, so +// nothing renders without it. +func boundCasting(declared *installation.Casting) *installation.Casting { + c := installation.Default(declared) + c.Spec.Infrastructure.Name = "signoz" + + return c +} + +// clusteredCasting returns a bound Installation with the telemetry store and +// keeper cluster sizes overridden. +func clusteredCasting(keeperKind installation.TelemetryKeeperKind, shards, replicas int) *installation.Casting { + declared := &installation.Casting{} + declared.Spec.TelemetryKeeper.Kind = keeperKind + + c := boundCasting(declared) + c.Spec.TelemetryStore.Spec.Cluster.Shards = v1alpha1.IntPtr(shards) + c.Spec.TelemetryStore.Spec.Cluster.Replicas = v1alpha1.IntPtr(replicas) + c.Spec.TelemetryKeeper.Spec.Cluster.Replicas = v1alpha1.IntPtr(3) + + return c +} + func TestNotEmptyAndValid(t *testing.T) { templates := map[string]*domain.Template{ - "mainTF": mainTF, - "variablesTF": variablesTF, - "moduleMainTF": moduleMainTF, - "moduleVariablesTF": moduleVariablesTF, - "moduleOutputsTF": moduleOutputsTF, - "moduleTelemetryKeeperTF": moduleTelemetryKeeperTF, - "moduleTelemetryStoreTF": moduleTelemetryStoreTF, - "moduleMigratorTF": moduleMigratorTF, - "moduleMetaStoreTF": moduleMetaStoreTF, - "moduleSignozTF": moduleSignozTF, - "moduleIngesterTF": moduleIngesterTF, + "versionsTF": versionsTF, + "providersTF": providersTF, + "mainTF": mainTF, + "variablesTF": variablesTF, + "outputsTF": outputsTF, + "telemetryKeeperTF": telemetryKeeperTF, + "telemetryStoreTF": telemetryStoreTF, + "migratorTF": migratorTF, + "metaStoreTF": metaStoreTF, + "signozTF": signozTF, + "ingesterTF": ingesterTF, + "mcpTF": mcpTF, } for name, tmpl := range templates { @@ -34,33 +61,173 @@ func TestNotEmptyAndValid(t *testing.T) { } } -func TestTfvarsTemplateWithAnnotations(t *testing.T) { - assert.NotEmpty(t, tfarsTF) - - casting := &installation.Casting{ - CastingMeta: v1alpha1.CastingMeta{ - Metadata: v1alpha1.TypeMetadata{ - Name: "signoz", - Annotations: map[string]string{ - "foundry.signoz.io/ecs/region": "us-east-1", - "foundry.signoz.io/ecs/cluster-id": "arn:aws:ecs:us-east-1:123456789012:cluster/test", - "foundry.signoz.io/ecs/subnet-ids": "subnet-abc123,subnet-def456", - "foundry.signoz.io/ecs/security-group-ids": "sg-abc123", - "foundry.signoz.io/ecs/vpc-id": "vpc-abc123", - "foundry.signoz.io/ecs/config-bucket": "test-configs", - "foundry.signoz.io/ecs/task-role-arn": "arn:aws:iam::123456789012:role/task", - "foundry.signoz.io/ecs/task-execution-role-arn": "arn:aws:iam::123456789012:role/exec", - "foundry.signoz.io/ecs/capacity-provider": "test-provider", - }, - }, +// The region is the one input that is neither declared nor discoverable, so it +// is all that is left in the tfvars. +func TestTfvarsTemplateCarriesTheRegion(t *testing.T) { + casting := boundCasting(&installation.Casting{}) + casting.Metadata.Annotations = map[string]string{installation.ECSRegion.Key: "us-east-1"} + + buf := bytes.NewBuffer(nil) + require.NoError(t, tfarsTF.Execute(buf, render(t, casting))) + + assert.JSONEq(t, `{"aws_region": "us-east-1"}`, buf.String()) +} + +// Nothing the substrate provisioned is named twice: the variables carry the +// names and tags the other casting derived from the same substrate name, and +// main.tf looks the objects up by them. +func TestSubstrateIsLookedUpThroughVariables(t *testing.T) { + data := render(t, boundCasting(&installation.Casting{})) + + variables := bytes.NewBuffer(nil) + require.NoError(t, variablesTF.Execute(variables, data)) + + for _, expected := range []string{ + `"default": "signoz-cls"`, + `"default": "signoz-sg-task"`, + `"default": "signoz-installation-task"`, + `"default": "signoz-installation-exec"`, + `"foundry.signoz.io/subnet-type":"private"`, + `"foundry.signoz.io/storage":"persistent"`, + } { + assert.Contains(t, variables.String(), expected) + } + + main := bytes.NewBuffer(nil) + require.NoError(t, mainTF.Execute(main, data)) + + for _, expected := range []string{ + `"cluster_name": "${var.cluster_name}"`, + `"tags": "${var.subnet_tags}"`, + `"instance_tags": "${var.node_tags}"`, + `"cluster_arn": "${data.aws_ecs_cluster.main.arn}"`, + `"subnet_ids": "${data.aws_subnets.private.ids}"`, + `volume.tags[var.claim_tag]`, + } { + assert.Contains(t, main.String(), expected) + } +} + +// An operator who runs a cluster foundry did not provision states the +// identifiers, and no lookup is emitted for them. +func TestStatedIdentifiersReplaceTheirLookup(t *testing.T) { + casting := boundCasting(&installation.Casting{}) + casting.Metadata.Annotations = map[string]string{ + installation.ECSClusterARN.Key: "arn:aws:ecs:us-east-1:123456789012:cluster/test", + installation.ECSSubnetIDs.Key: "subnet-abc123, subnet-def456", + } + + data := render(t, casting) + + variables := bytes.NewBuffer(nil) + require.NoError(t, variablesTF.Execute(variables, data)) + + assert.Contains(t, variables.String(), `"default": "arn:aws:ecs:us-east-1:123456789012:cluster/test"`) + assert.Contains(t, variables.String(), `"default": ["subnet-abc123","subnet-def456"]`) + + main := bytes.NewBuffer(nil) + require.NoError(t, mainTF.Execute(main, data)) + + out := main.String() + assert.Contains(t, out, `"cluster_arn": "${var.cluster_arn}"`) + assert.Contains(t, out, `"subnet_ids": "${var.subnet_ids}"`) + assert.NotContains(t, out, "aws_ecs_cluster") + assert.NotContains(t, out, "aws_subnets") + + // The ones they did not state are still discovered. + assert.Contains(t, out, `"name": "${var.task_role_name}"`) + assert.Contains(t, variables.String(), `"default": "signoz-installation-task"`) +} + +func render(t *testing.T, casting *installation.Casting) templateData { + t.Helper() + + data, err := New(slog.New(slog.DiscardHandler)).templateData(*casting) + require.NoError(t, err) + + return data +} + +func TestModulePlacement(t *testing.T) { + t.Parallel() + + sqlite := boundCasting(&installation.Casting{}) + sqlite.Spec.MetaStore.Kind = installation.MetaStoreKindSQLite + + // The attribute clauses are the same match the substrate stamps on its + // nodes, so a component reaches only the nodes of the substrate it is + // bound to. + seat := func(identity string) string { + return `ec2InstanceId == '${local.seats[\"` + identity + `\"]}' and attribute:foundry.signoz.io/name == signoz and attribute:foundry.signoz.io/storage == persistent` + } + ephemeral := "attribute:foundry.signoz.io/name == signoz and attribute:foundry.signoz.io/storage == ephemeral" + + // Each stateful identity pins to the instance its claim resolved + // (ec2InstanceId), with the storage attribute kept as a bootstrap + // integrity check. Stateless services place onto the ephemeral pool. + testCases := []struct { + name string + template *domain.Template + casting *installation.Casting + expectedExpressions []string + }{ + { + name: "TelemetryKeeper_PinnedToClaimedSeats", + template: telemetryKeeperTF, + casting: clusteredCasting(installation.TelemetryKeeperKindClickhouseKeeper, 2, 1), + expectedExpressions: []string{seat("telemetrykeeper-0"), seat("telemetrykeeper-2")}, + }, + { + name: "TelemetryStore_PinnedToClaimedSeats", + template: telemetryStoreTF, + casting: clusteredCasting(installation.TelemetryKeeperKindClickhouseKeeper, 2, 1), + expectedExpressions: []string{seat("telemetrystore-0-0"), seat("telemetrystore-1-1")}, + }, + { + name: "Metastore_PinnedToClaimedSeat", + template: metaStoreTF, + casting: boundCasting(&installation.Casting{}), + expectedExpressions: []string{seat("metastore-0")}, + }, + { + name: "Ingester_Ephemeral", + template: ingesterTF, + casting: boundCasting(&installation.Casting{}), + expectedExpressions: []string{ephemeral}, + }, + { + name: "SignozPostgres_Ephemeral", + template: signozTF, + casting: boundCasting(&installation.Casting{}), + expectedExpressions: []string{ephemeral}, + }, + { + name: "SignozSqlite_PinnedToClaimedSeat", + template: signozTF, + casting: sqlite, + expectedExpressions: []string{seat("signoz-0")}, + }, + { + name: "MCP_Ephemeral", + template: mcpTF, + casting: boundCasting(&installation.Casting{}), + expectedExpressions: []string{ephemeral}, }, } - buf := bytes.NewBuffer(nil) - err := tfarsTF.Execute(buf, casting) - assert.NoError(t, err) - assert.NotEmpty(t, buf.String()) - assert.Contains(t, buf.String(), "us-east-1") - assert.Contains(t, buf.String(), "subnet-abc123") - assert.Contains(t, buf.String(), "subnet-def456") + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + buf := bytes.NewBuffer(nil) + require.NoError(t, tc.template.Execute(buf, render(t, tc.casting))) + out := buf.String() + + assert.Contains(t, out, `"launch_type": "EC2"`) + for _, expression := range tc.expectedExpressions { + assert.Contains(t, out, expression) + } + assert.NotContains(t, out, "capacity_provider") + }) + } } diff --git a/internal/casting/ecsterraformcasting/enricher.go b/internal/casting/ecsterraformcasting/enricher.go index 4b7cc85d..0727bbf2 100644 --- a/internal/casting/ecsterraformcasting/enricher.go +++ b/internal/casting/ecsterraformcasting/enricher.go @@ -2,7 +2,6 @@ package ecsterraformcasting import ( "context" - "fmt" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/installation" @@ -11,77 +10,126 @@ import ( "github.com/signoz/foundry/internal/molding" ) +var _ molding.MoldingEnricher = (*ecsMoldingEnricher)(nil) + const ( telemetryStorePort = 9000 telemetryKeeperClientPort = 9181 telemetryKeeperRaftPort = 9234 + zookeeperClientPort = 2181 + zookeeperRaftPort = 2888 metaStorePort = 5432 - signozAPIPort = 8080 + signozAPIServerPort = 8080 signozOpampPort = 4320 + mcpHTTPPort = 8000 ) -var _ molding.MoldingEnricher = (*ecsMoldingEnricher)(nil) +const ( + // sdNamesPath selects every Cloud Map service name a module renders, in the + // order the template emits them. The enricher reads names back rather than + // recomputing them, so the template stays the single source of node names + // and ordering (no drift). + sdNamesPath = "resource.aws_service_discovery_service.@values.#.name" + namespacePath = "resource.aws_service_discovery_private_dns_namespace.main.name" +) type ecsMoldingEnricher struct { materials []domain.StructuredMaterial } -func newEcsMoldingEnricher(config *installation.Casting) (*ecsMoldingEnricher, error) { - materials, err := getMaterials(config) +func newEcsMoldingEnricher(data templateData) (*ecsMoldingEnricher, error) { + materials, err := getMaterials(data) if err != nil { - return nil, errors.Wrapf(err, errors.TypeInternal, "failed to get materials") + return nil, errors.Wrapf(err, errors.TypeInternal, "failed to render materials") } return &ecsMoldingEnricher{materials: materials}, nil } -func (enricher *ecsMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { - namespaceBytes, err := enricher.materials[0].GetBytes("resource.aws_service_discovery_private_dns_namespace.main.name") +func (e *ecsMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *installation.Casting) error { + ns, err := e.materials[0].GetBytes(namespacePath) if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "failed to get namespace") + return errors.Wrapf(err, errors.TypeInternal, "failed to read service discovery namespace") } - namespace := string(namespaceBytes) switch kind { case v1alpha1.MoldingKindTelemetryStore: - sdName, err := enricher.materials[1].GetBytes("resource.aws_service_discovery_service.telemetrystore.name") + names, err := e.materials[1].GetStringSlice(sdNamesPath) if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrystore service discovery name") + return errors.Wrapf(err, errors.TypeInternal, "failed to read telemetrystore service names") + } + + addresses := make([]string, 0, len(names)) + for _, name := range names { + host := name + "." + string(ns) + addresses = append(addresses, domain.MustNewAddress("tcp", host, telemetryStorePort).String()) } - fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) - config.Spec.TelemetryStore.Status.Addresses.TCP = []string{domain.MustNewAddress("tcp", fqdn, telemetryStorePort).String()} + + config.Spec.TelemetryStore.Status.Addresses.TCP = addresses case v1alpha1.MoldingKindTelemetryKeeper: - sdName, err := enricher.materials[2].GetBytes("resource.aws_service_discovery_service.telemetrykeeper.name") + names, err := e.materials[2].GetStringSlice(sdNamesPath) if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "failed to get telemetrykeeper service discovery name") + return errors.Wrapf(err, errors.TypeInternal, "failed to read telemetrykeeper service names") } clientPort, raftPort := telemetryKeeperClientPort, telemetryKeeperRaftPort if config.Spec.TelemetryKeeper.Kind == installation.TelemetryKeeperKindZookeeper { - clientPort, raftPort = 2181, 2888 + clientPort, raftPort = zookeeperClientPort, zookeeperRaftPort + } + + clientAddresses := make([]string, 0, len(names)) + raftAddresses := make([]string, 0, len(names)) + for _, name := range names { + host := name + "." + string(ns) + clientAddresses = append(clientAddresses, domain.MustNewAddress("tcp", host, clientPort).String()) + raftAddresses = append(raftAddresses, domain.MustNewAddress("tcp", host, raftPort).String()) } - fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) - config.Spec.TelemetryKeeper.Status.Addresses.Client = []string{domain.MustNewAddress("tcp", fqdn, clientPort).String()} - config.Spec.TelemetryKeeper.Status.Addresses.Raft = []string{domain.MustNewAddress("tcp", fqdn, raftPort).String()} + config.Spec.TelemetryKeeper.Status.Addresses.Client = clientAddresses + config.Spec.TelemetryKeeper.Status.Addresses.Raft = raftAddresses case v1alpha1.MoldingKindMetaStore: - sdName, err := enricher.materials[3].GetBytes("resource.aws_service_discovery_service.metastore.name") + if config.Spec.MetaStore.Kind != installation.MetaStoreKindPostgres { + return nil + } + + names, err := e.materials[3].GetStringSlice(sdNamesPath) if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "failed to get metastore service discovery name") + return errors.Wrapf(err, errors.TypeInternal, "failed to read metastore service names") + } + + if len(names) > 0 { + host := names[0] + "." + string(ns) + config.Spec.MetaStore.Status.Addresses.DSN = []string{domain.MustNewAddress("tcp", host, metaStorePort).String()} } - fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) - config.Spec.MetaStore.Status.Addresses.DSN = []string{domain.MustNewAddress("tcp", fqdn, metaStorePort).String()} case v1alpha1.MoldingKindSignoz: - sdName, err := enricher.materials[4].GetBytes("resource.aws_service_discovery_service.signoz.name") + names, err := e.materials[4].GetStringSlice(sdNamesPath) if err != nil { - return errors.Wrapf(err, errors.TypeInternal, "failed to get signoz service discovery name") + return errors.Wrapf(err, errors.TypeInternal, "failed to read signoz service names") + } + + if len(names) > 0 { + host := names[0] + "." + string(ns) + config.Spec.Signoz.Status.Addresses.APIServer = []string{domain.MustNewAddress("tcp", host, signozAPIServerPort).String()} + config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("ws", host, signozOpampPort).String()} + } + + case v1alpha1.MoldingKindMCP: + if !config.Spec.MCP.Spec.IsEnabled() { + return nil + } + + names, err := e.materials[6].GetStringSlice(sdNamesPath) + if err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to read mcp service names") + } + + if len(names) > 0 { + host := names[0] + "." + string(ns) + config.Spec.MCP.Status.Addresses.HTTP = []string{domain.MustNewAddress("http", host, mcpHTTPPort).String()} } - fqdn := fmt.Sprintf("%s.%s", string(sdName), namespace) - config.Spec.Signoz.Status.Addresses.APIServer = []string{domain.MustNewAddress("tcp", fqdn, signozAPIPort).String()} - config.Spec.Signoz.Status.Addresses.Opamp = []string{domain.MustNewAddress("ws", fqdn, signozOpampPort).String()} } return nil diff --git a/internal/casting/ecsterraformcasting/templates/ingester.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/ingester.tf.json.gotmpl new file mode 100644 index 00000000..870b7127 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/ingester.tf.json.gotmpl @@ -0,0 +1,205 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers_ingester": [ + {{- /* The collector image runs as uid 10001 and a fresh task volume is + owned by root, so the config directory is handed to that uid + before the agent, which runs as the same uid, writes into it. */}} + { + "name": "{{ $name }}-ingester-config-init", + "image": "{{ $.Spec.Ingester.Spec.Image }}", + "essential": false, + "user": "0", + "entryPoint": ["/bin/sh", "-c"], + "command": ["chown 10001:0 /conf"], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-ingester-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + {{- /* writeTo has no mode or owner setting; the agent writes 0600 as + its own uid, so it runs as the collector's. */}} + "user": "10001:0", + "dependsOn": [ + {"containerName": "{{ $name }}-ingester-config-init", "condition": "SUCCESS"} + ], + "environment": [ + {"name": "PREFETCH_LIST", "value": "{{ $name }}:default:ingester,{{ $name }}:default:ingester-opamp"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"{{ $name }}:default:ingester\":{\"writeTo\":{\"path\":\"/conf/ingester.yaml\"}},\"{{ $name }}:default:ingester-opamp\":{\"writeTo\":{\"path\":\"/conf/opamp.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "test -s /conf/ingester.yaml && test -s /conf/opamp.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-ingester", + "image": "{{ $.Spec.Ingester.Spec.Image }}", + "essential": true, + "entryPoint": ["/bin/sh", "-c"], + "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], + {{- /* The collector reads its config once at start, so a change has to + replace the task. Keeper and telemetrystore reload in place and + carry no digest. */}} + {{- $digest := "" }} + {{- with $.Spec.Ingester.Spec.Config.Data }} + {{- $digest = sha256sum (printf "%s%s" (index . "ingester.yaml") (index . "opamp.yaml")) }} + {{- end }} + {{- $env := list (dict "name" "FOUNDRY_CONFIG_DIGEST" "value" $digest) }} + {{- if and (derefBool $.Spec.TelemetryStore.Spec.Enabled) $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + {{- $env = append $env (dict "name" "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN" "value" (index $.Spec.TelemetryStore.Status.Addresses.TCP 0)) }} + {{- end }} + {{- range $key, $value := $.Spec.Ingester.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, + {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "ingester-config", + "containerPath": "/conf" + } + ], + "dependsOn": [ + {"containerName": "{{ $name }}-ingester-config-init", "condition": "SUCCESS"}, + {"containerName": "{{ $name }}-ingester-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 512, + "memoryReservation": 512 + } + ] + }, + "resource": { + "aws_appconfig_configuration_profile": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "ingester", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {{ toJson $.Labels }} + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "ingester-opamp", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {{ toJson $.Labels }} + } + }, + "aws_appconfig_hosted_configuration_version": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/ingester/ingester.yaml\")}" + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester_opamp.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/ingester/opamp.yaml\")}" + } + }, + "aws_appconfig_deployment": { + "ingester": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.ingester.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {{ toJson $.Labels }} + }, + "ingester_opamp": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.ingester_opamp.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.ingester_opamp.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {{ toJson $.Labels }} + } + }, + "aws_ecs_task_definition": { + "ingester": { + "family": "{{ $name }}-ingester", + "tags": {{ toJson $.Labels }}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_ingester)}", + "volume": [ + { + "name": "ingester-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + } + ], + "depends_on": ["aws_appconfig_deployment.ingester", "aws_appconfig_deployment.ingester_opamp"] + } + }, + "aws_service_discovery_service": { + "ingester": { + "name": "ingester", + "tags": {{ toJson $.Labels }}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "ingester": { + "name": "{{ $name }}-ingester", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.ingester.arn}", + "desired_count": {{ derefIntDefault $.Spec.Ingester.Spec.Cluster.Replicas 1 }}, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.ingester.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "{{ $.EphemeralPlacement }}" + } + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl index 7cdd6a42..62004790 100644 --- a/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/main.tf.json.gotmpl @@ -1,30 +1,203 @@ +{{- $name := $.Metadata.Name -}} +{{- $annotations := default (dict) $.Metadata.Annotations -}} +{{- $clusterARN := index $annotations "foundry.signoz.io/ecs-cluster-arn" -}} +{{- $subnetIDs := index $annotations "foundry.signoz.io/ecs-subnet-ids" -}} +{{- $securityGroupIDs := index $annotations "foundry.signoz.io/ecs-security-group-ids" -}} +{{- $vpcID := index $annotations "foundry.signoz.io/ecs-vpc-id" -}} +{{- $taskRoleARN := index $annotations "foundry.signoz.io/ecs-task-role-arn" -}} +{{- $executionRoleARN := index $annotations "foundry.signoz.io/ecs-task-execution-role-arn" -}} +{{- /* The claim controller. An identity claims the VOLUME, not the instance. + The volume outlives an instance, and the pairing survives replacement. + The claim is a tag on the volume; plans re-read it, existing claims + resolve first, new identities take unclaimed volumes then wrap. ECS pins + only by ec2InstanceId, so each plan looks up whichever instance holds + the claimed volume. */}} +{{- $keeperReplicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $keeperReplicas 1 }}{{- $keeperReplicas = 1 -}}{{- end -}} +{{- $shards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $shards 1 }}{{- $shards = 1 -}}{{- end -}} +{{- $nodesPerShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} +{{- $identities := list -}} +{{- range $i := until $keeperReplicas -}} +{{- $identities = append $identities (printf "telemetrykeeper-%d" $i) -}} +{{- end -}} +{{- if eq $.Spec.MetaStore.Kind.String "sqlite" -}} +{{- $identities = append $identities "signoz-0" -}} +{{- else -}} +{{- $identities = append $identities "metastore-0" -}} +{{- end -}} +{{- range $s := until $shards -}} +{{- range $r := until $nodesPerShard -}} +{{- $identities = append $identities (printf "telemetrystore-%d-%d" $s $r) -}} +{{- end -}} +{{- end -}} { - "terraform": { - "required_version": ">= 1.0", - "required_providers": { - "aws": { - "source": "hashicorp/aws", - "version": ">= 5.0" + "data": { + {{- if not $clusterARN }} + "aws_ecs_cluster": { + "main": { + "cluster_name": "${var.cluster_name}" + } + }, + {{- end }} + {{- if not $subnetIDs }} + "aws_subnets": { + "private": { + "tags": "${var.subnet_tags}" + } + }, + {{- end }} + {{- if not $securityGroupIDs }} + "aws_security_group": { + "tasks": { + "name": "${var.security_group_name}" + } + }, + {{- end }} + {{- if not $vpcID }} + "aws_vpc": { + "main": { + "tags": "${var.vpc_tags}" + } + }, + {{- end }} + "aws_instances": { + "persistent": { + "instance_tags": "${var.node_tags}", + "instance_state_names": ["running", "pending"] + } + }, + "aws_instance": { + "persistent": { + "for_each": "${toset(data.aws_instances.persistent.ids)}", + "instance_id": "${each.value}" + } + }, + "aws_ebs_volumes": { + "persistent": { + "tags": "${var.node_tags}" + } + }, + "aws_ebs_volume": { + "persistent": { + "for_each": "${toset(data.aws_ebs_volumes.persistent.ids)}", + "filter": [ + { + "name": "volume-id", + "values": ["${each.value}"] + } + ] } } }, - "provider": { - "aws": { - "region": "${var.region}" - } + "locals": { + "cluster_arn": "{{ if $clusterARN }}${var.cluster_arn}{{ else }}${data.aws_ecs_cluster.main.arn}{{ end }}", + "subnet_ids": "{{ if $subnetIDs }}${var.subnet_ids}{{ else }}${data.aws_subnets.private.ids}{{ end }}", + "security_group_ids": "{{ if $securityGroupIDs }}${var.security_group_ids}{{ else }}${[data.aws_security_group.tasks.id]}{{ end }}", + "vpc_id": "{{ if $vpcID }}${var.vpc_id}{{ else }}${data.aws_vpc.main.id}{{ end }}", + "task_role_arn": "{{ if $taskRoleARN }}${var.task_role_arn}{{ else }}${aws_iam_role.task.arn}{{ end }}", + "execution_role_arn": "{{ if $executionRoleARN }}${var.execution_role_arn}{{ else }}${aws_iam_role.exec.arn}{{ end }}", + "identities": [{{ range $i, $identity := $identities }}{{ if $i }}, {{ end }}"{{ $identity }}"{{ end }}], + "volume_instance": "${ merge([ for id, instance in data.aws_instance.persistent : { for device in instance.ebs_block_device : device.volume_id => id } ]...) }", + "instance_volume": "${ { for volume, instance in local.volume_instance : instance => volume } }", + "volume_claims": "${ merge([ for id, volume in data.aws_ebs_volume.persistent : { for identity in split(\",\", volume.tags[var.claim_tag]) : identity => id } if contains(keys(volume.tags), var.claim_tag) ]...) }", + "instance_claims": "${ merge([ for id, instance in data.aws_instance.persistent : { for identity in split(\",\", instance.tags[var.claim_tag]) : identity => id } if contains(keys(instance.tags), var.claim_tag) ]...) }", + "inherited_claims": "${ { for identity, instance in local.instance_claims : identity => local.instance_volume[instance] if contains(keys(local.instance_volume), instance) } }", + "claims": "${ merge(local.inherited_claims, local.volume_claims) }", + "unclaimed_volume_ids": "${ sort([ for id, volume in data.aws_ebs_volume.persistent : id if !contains(values(local.claims), id) ]) }", + "claimed_volume_ids": "${ sort([ for id, volume in data.aws_ebs_volume.persistent : id if contains(values(local.claims), id) ]) }", + "assignment_pool": "${ concat(local.unclaimed_volume_ids, local.claimed_volume_ids) }", + "new_identities": "${ [ for identity in local.identities : identity if !contains(keys(local.claims), identity) ] }", + "volumes": "${ { for identity in local.identities : identity => contains(keys(local.claims), identity) ? local.claims[identity] : element(local.assignment_pool, index(local.new_identities, identity)) } }", + "seats": "${ { for identity, volume in local.volumes : identity => lookup(local.volume_instance, volume, \"unattached\") } }" }, - "module": { - "signoz": { - "source": "./module", - "region": "${var.region}", - "ecs_cluster_id": "${var.ecs_cluster_id}", - "subnet_ids": "${var.subnet_ids}", - "security_group_ids": "${var.security_group_ids}", - "vpc_id": "${var.vpc_id}", - "config_bucket": "${var.config_bucket}", - "task_role_arn": "${var.task_role_arn}", - "task_execution_role_arn": "${var.task_execution_role_arn}", - "capacity_provider": "${var.capacity_provider}" + "resource": { + {{- /* Workload identity is the workload's own. A role holds no data and + dies with this stack. An operator states an ARN instead and nothing + is created. */}} + {{- if or (not $taskRoleARN) (not $executionRoleARN) }} + "aws_iam_role": { + {{- $first := true }} + {{- if not $taskRoleARN }}{{ if not $first }},{{ end }}{{ $first = false }} + "task": { + "name": "${var.task_role_name}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ecs-tasks.amazonaws.com\"}}]})}", + "tags": {{ toJson $.Labels }} + } + {{- end }} + {{- if not $executionRoleARN }}{{ if not $first }},{{ end }}{{ $first = false }} + "exec": { + "name": "${var.execution_role_name}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ecs-tasks.amazonaws.com\"}}]})}", + "tags": {{ toJson $.Labels }} + } + {{- end }} + }, + {{- end }} + {{- if not $executionRoleARN }} + "aws_iam_role_policy_attachment": { + "exec": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy", + "role": "${aws_iam_role.exec.name}" + } + }, + {{- end }} + {{- if not $taskRoleARN }} + {{- /* Scoped to the application this stack creates. GetLatestConfiguration + acts on a session token and cannot be scoped. */}} + "aws_iam_role_policy": { + "task_appconfig_read": { + "name": "${var.task_role_name}-appconfig-read", + "role": "${aws_iam_role.task.id}", + "policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Effect\" = \"Allow\", \"Action\" = [\"appconfig:StartConfigurationSession\"], \"Resource\" = [aws_appconfig_application.main.arn]}, {\"Effect\" = \"Allow\", \"Action\" = [\"appconfig:GetLatestConfiguration\"], \"Resource\" = \"*\"}]})}" + } + }, + {{- end }} + "aws_service_discovery_private_dns_namespace": { + "main": { + "name": "{{ $name }}.local", + "vpc": "${local.vpc_id}", + "tags": {{ toJson $.Labels }} + } + }, + "aws_ec2_tag": { + "claims": { + "for_each": "${ toset(values(local.volumes)) }", + "resource_id": "${each.value}", + "key": "${var.claim_tag}", + "value": "${ join(\",\", sort([ for identity, volume in local.volumes : identity if volume == each.value ])) }" + } + }, + {{- /* AppConfig is the ECS analog of a ConfigMap. The sidecar rewrites the + config in place, reloading the process instead of replacing the + task. Load-bearing for quorum members. Verified against keeper + 25.12.5. */}} + "aws_appconfig_application": { + "main": { + "name": "{{ $name }}", + "description": "SigNoz component configuration", + "tags": {{ toJson $.Labels }} + } + }, + "aws_appconfig_environment": { + "main": { + "name": "default", + "application_id": "${aws_appconfig_application.main.id}", + "tags": {{ toJson $.Labels }} + } + }, + {{- /* No bake. AppConfig serializes deployments per environment, paying + bake time once per component on every apply. Add it back with + CloudWatch monitors that can trigger a rollback. */}} + "aws_appconfig_deployment_strategy": { + "main": { + "name": "{{ $name }}-config", + "deployment_duration_in_minutes": 0, + "final_bake_time_in_minutes": 0, + "growth_factor": 100, + "replicate_to": "NONE", + "tags": {{ toJson $.Labels }} + } } } } diff --git a/internal/casting/ecsterraformcasting/templates/mcp.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/mcp.tf.json.gotmpl new file mode 100644 index 00000000..8a03d2c2 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/mcp.tf.json.gotmpl @@ -0,0 +1,78 @@ +{{- $name := $.Metadata.Name -}} +{ + "locals": { + "containers_mcp": [ + { + "name": "{{ $name }}-mcp", + "image": "{{ $.Spec.MCP.Spec.Image }}", + "essential": true, + {{- $env := list }} + {{- range $key, $value := $.Spec.MCP.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + {{- if $env }} + "environment": {{ toJson $env }}, + {{- end }} + "portMappings": [ + {"name": "http", "containerPort": 8000, "protocol": "tcp", "appProtocol": "http"} + ], + "cpu": 256, + "memoryReservation": 512 + } + ] + }, + "resource": { + "aws_ecs_task_definition": { + "mcp": { + "family": "{{ $name }}-mcp", + "tags": {{ toJson $.Labels }}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_mcp)}" + } + }, + "aws_service_discovery_service": { + "mcp": { + "name": "mcp", + "tags": {{ toJson $.Labels }}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + }, + "aws_ecs_service": { + "mcp": { + "name": "{{ $name }}-mcp", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.mcp.arn}", + "desired_count": {{ derefIntDefault $.Spec.MCP.Spec.Cluster.Replicas 1 }}, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.mcp.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "{{ $.EphemeralPlacement }}" + } + ] + } + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl similarity index 63% rename from internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl rename to internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl index 24002310..664c609b 100644 --- a/internal/casting/ecsterraformcasting/templates/module/metastore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl @@ -1,7 +1,7 @@ {{- $name := $.Metadata.Name -}} { "locals": { - "containers": [ + "containers_metastore": [ { "name": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", "image": "{{ $.Spec.MetaStore.Spec.Image }}", @@ -16,15 +16,14 @@ ], "mountPoints": [ { - "sourceVolume": "postgres-data", + "sourceVolume": "metastore-data", "containerPath": "/var/lib/postgresql/data" } ], "cpu": 256, - "memory": 256, "memoryReservation": 256, "healthCheck": { - "command": ["CMD-SHELL", "pg_isready -U postgres || exit 1"], + "command": ["CMD-SHELL", "pg_isready -U signoz -d signoz || exit 1"], "interval": 30, "timeout": 5, "retries": 3, @@ -37,25 +36,24 @@ "aws_ecs_task_definition": { "metastore": { "family": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", + "tags": {{ toJson $.Labels }}, "network_mode": "awsvpc", "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_metastore)}", "volume": [ { - "name": "postgres-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } + "name": "metastore-data", + "host_path": "/var/lib/foundry/{{ $name }}/metastore/0" } ] } }, "aws_service_discovery_service": { "metastore": { - "name": "metastore-{{ $.Spec.MetaStore.Kind }}", + "name": "metastore-{{ $.Spec.MetaStore.Kind }}-0", + "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", "dns_records": [ @@ -71,23 +69,26 @@ "aws_ecs_service": { "metastore": { "name": "{{ $name }}-metastore-{{ $.Spec.MetaStore.Kind }}-0", - "cluster": "${var.ecs_cluster_id}", + "cluster": "${local.cluster_arn}", "task_definition": "${aws_ecs_task_definition.metastore.arn}", "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" }, "service_registries": { "registry_arn": "${aws_service_discovery_service.metastore.arn}" - } + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"metastore-0\"]}' and {{ $.PersistentPlacement }}" + } + ] } } } diff --git a/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl deleted file mode 100644 index cb3946cb..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/ingester.tf.json.gotmpl +++ /dev/null @@ -1,133 +0,0 @@ -{{- $name := $.Metadata.Name -}} -{ - "locals": { - "containers": [ - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/ingester/ /configs/ingester/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "ingester-config", - "containerPath": "/configs/ingester" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "ingester", - "image": "{{ $.Spec.Ingester.Spec.Image }}", - "essential": true, - "entryPoint": ["/bin/sh", "-c"], - "command": ["/signoz-otel-collector migrate sync check && /signoz-otel-collector --config=/conf/ingester.yaml --manager-config=/conf/opamp.yaml --copy-path=/var/tmp/collector-config.yaml"], - {{- $env := list }} - {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} - {{- $env = append $env (dict "name" "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN" "value" (index $.Spec.TelemetryStore.Status.Addresses.TCP 0)) }} - {{- end }} - {{- range $key, $value := $.Spec.Ingester.Spec.Env }} - {{- $env = append $env (dict "name" $key "value" $value) }} - {{- end }} - "environment": {{ toJson $env }}, - "portMappings": [ - {"name": "grpc", "containerPort": 4317, "protocol": "tcp", "appProtocol": "grpc"}, - {"name": "http", "containerPort": 4318, "protocol": "tcp", "appProtocol": "http"} - ], - "mountPoints": [ - { - "sourceVolume": "ingester-config", - "containerPath": "/conf" - } - ], - "dependsOn": [ - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 512, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q localhost:13133 || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "ingester_configs": { - "for_each": "${fileset(\"${path.module}/ingester\", \"*.yaml\")}", - "bucket": "${var.config_bucket}", - "key": "{{ $name }}/ingester/${each.value}", - "source": "${path.module}/ingester/${each.value}", - "etag": "${filemd5(\"${path.module}/ingester/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "ingester": { - "family": "{{ $name }}-ingester", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "ingester-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.ingester_configs"] - } - }, - "aws_service_discovery_service": { - "ingester": { - "name": "ingester", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "ingester": { - "name": "{{ $name }}-ingester", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.ingester.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.ingester.arn}" - }, -"depends_on": [ - "aws_ecs_service.signoz", - "aws_ecs_service.telemetrystore" - ] - } - } - } -} diff --git a/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl deleted file mode 100644 index db843606..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/main.tf.json.gotmpl +++ /dev/null @@ -1,11 +0,0 @@ -{{- $name := $.Metadata.Name -}} -{ - "resource": { - "aws_service_discovery_private_dns_namespace": { - "main": { - "name": "{{ $name }}.local", - "vpc": "${var.vpc_id}" - } - } - } -} diff --git a/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl deleted file mode 100644 index 4df0dcb1..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/outputs.tf.json.gotmpl +++ /dev/null @@ -1,62 +0,0 @@ -{ - "output": { - "ecs_cluster_id": { - "description": "ECS cluster ID", - "value": "${var.ecs_cluster_id}" - }, - "namespace_id": { - "description": "Cloud Map private DNS namespace ID", - "value": "${aws_service_discovery_private_dns_namespace.main.id}" - }, - "namespace_name": { - "description": "Cloud Map private DNS namespace name", - "value": "${aws_service_discovery_private_dns_namespace.main.name}" - }, - "subnet_ids": { - "description": "Subnet IDs used by ECS services", - "value": "${var.subnet_ids}" - }, - "security_group_ids": { - "description": "Security group IDs used by ECS services", - "value": "${var.security_group_ids}" - } - {{- if $.Spec.Signoz.Spec.Enabled }}, - "signoz_service_arn": { - "description": "SigNoz ECS service ARN (target for ALB on port 8080)", - "value": "${aws_ecs_service.signoz.id}" - }, - "signoz_service_name": { - "description": "SigNoz ECS service name", - "value": "${aws_ecs_service.signoz.name}" - } - {{- end }} - {{- if $.Spec.Ingester.Spec.Enabled }}, - "ingester_service_arn": { - "description": "Ingester ECS service ARN (target for NLB on port 4317/4318)", - "value": "${aws_ecs_service.ingester.id}" - }, - "ingester_service_name": { - "description": "Ingester ECS service name", - "value": "${aws_ecs_service.ingester.name}" - } - {{- end }} - {{- if $.Spec.TelemetryStore.Spec.Enabled }}, - "telemetrystore_service_name": { - "description": "TelemetryStore ECS service name", - "value": "${aws_ecs_service.telemetrystore.name}" - } - {{- end }} - {{- if $.Spec.TelemetryKeeper.Spec.Enabled }}, - "telemetrykeeper_service_name": { - "description": "TelemetryKeeper ECS service name", - "value": "${aws_ecs_service.telemetrykeeper.name}" - } - {{- end }} - {{- if $.Spec.MetaStore.Spec.Enabled }}, - "metastore_service_name": { - "description": "MetaStore ECS service name", - "value": "${aws_ecs_service.metastore.name}" - } - {{- end }} - } -} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl deleted file mode 100644 index 098450bf..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/telemetrykeeper.tf.json.gotmpl +++ /dev/null @@ -1,236 +0,0 @@ -{{- $name := $.Metadata.Name -}} -{{- if eq $.Spec.TelemetryKeeper.Kind.String "zookeeper" -}} -{ - "locals": { - "containers": [ - { - "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "image": "{{ $.Spec.TelemetryKeeper.Spec.Image }}", - "essential": true, - {{- $env := list (dict "name" "ZOO_SERVER_ID" "value" "1") }} - {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} - {{- $env = append $env (dict "name" $key "value" $value) }} - {{- end }} - "environment": {{ toJson $env }}, - "portMappings": [ - {"name": "client", "containerPort": 2181, "protocol": "tcp"}, - {"name": "raft", "containerPort": 2888, "protocol": "tcp"}, - {"name": "election", "containerPort": 3888, "protocol": "tcp"} - ], - "mountPoints": [ - { - "sourceVolume": "keeper-data", - "containerPath": "/bitnami/zookeeper" - } - ], - "cpu": 256, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "echo ruok | nc localhost 2181 || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_ecs_task_definition": { - "telemetrykeeper": { - "family": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "keeper-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ] - } - }, - "aws_service_discovery_service": { - "telemetrykeeper": { - "name": "telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "telemetrykeeper": { - "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.telemetrykeeper.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.telemetrykeeper.arn}" - } - } - } - } -} -{{- else -}} -{ - "locals": { - "containers": [ - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/ /configs/telemetrykeeper/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "keeper-config", - "containerPath": "/configs/telemetrykeeper" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "image": "{{ $.Spec.TelemetryKeeper.Spec.Image }}", - "essential": true, - "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], - {{- $env := list }} - {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} - {{- $env = append $env (dict "name" $key "value" $value) }} - {{- end }} - {{- if $env }} - "environment": {{ toJson $env }}, - {{- end }} - "portMappings": [ - {"name": "client", "containerPort": 9181, "protocol": "tcp"}, - {"name": "raft", "containerPort": 9234, "protocol": "tcp"} - ], - "mountPoints": [ - { - "sourceVolume": "keeper-data", - "containerPath": "/var/lib/clickhouse-keeper" - }, - { - "sourceVolume": "keeper-config", - "containerPath": "/etc/clickhouse-keeper" - } - ], - "dependsOn": [ - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 256, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "echo ruok | nc localhost 9181 || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "telemetrykeeper_configs": { - "for_each": "${fileset(\"${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}\", \"**\")}", - "bucket": "${var.config_bucket}", - "key": "{{ $name }}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}", - "source": "${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}", - "etag": "${filemd5(\"${path.module}/telemetrykeeper/{{ $.Spec.TelemetryKeeper.Kind }}/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "telemetrykeeper": { - "family": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "keeper-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "keeper-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.telemetrykeeper_configs"] - } - }, - "aws_service_discovery_service": { - "telemetrykeeper": { - "name": "telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "telemetrykeeper": { - "name": "{{ $name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-0", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.telemetrykeeper.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.telemetrykeeper.arn}" - } - } - } - } -} -{{- end }} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl deleted file mode 100644 index 9fc3e02d..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/telemetrystore.tf.json.gotmpl +++ /dev/null @@ -1,168 +0,0 @@ -{{- $name := $.Metadata.Name -}} -{ - "locals": { - "containers": [ - { - "name": "init-clickhouse", - "image": "alpine:3.18.2", - "essential": false, - "command": ["/bin/sh", "-c", "cd /tmp && wget -O histogram-quantile.tar.gz 'https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_linux_amd64.tar.gz' && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile"], - "mountPoints": [ - { - "sourceVolume": "shared-binary-volume", - "containerPath": "/var/lib/clickhouse/user_scripts" - } - ], - "cpu": 256, - "memory": 256, - "memoryReservation": 256 - }, - { - "name": "config-fetcher", - "image": "amazon/aws-cli:2.27.32", - "essential": false, - "entryPoint": ["/bin/sh", "-c"], - "command": ["aws s3 cp s3://${var.config_bucket}/{{ $name }}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/ /configs/telemetrystore/ --recursive"], - "mountPoints": [ - { - "sourceVolume": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", - "containerPath": "/configs/telemetrystore" - } - ], - "cpu": 10, - "memory": 102, - "memoryReservation": 102 - }, - { - "name": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", - "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", - "essential": true, - {{- $env := list (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} - {{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} - {{- $env = append $env (dict "name" $key "value" $value) }} - {{- end }} - "environment": {{ toJson $env }}, - "entryPoint": ["/bin/sh", "-c"], - "command": ["ln -sf /etc/clickhouse-server/config.d/config-0-0.yaml /etc/clickhouse-server/config-0-0.yaml && exec /entrypoint.sh"], - "portMappings": [ - {"name": "native", "containerPort": 9000, "protocol": "tcp"}, - {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, - {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} - ], - "mountPoints": [ - { - "sourceVolume": "shared-binary-volume", - "containerPath": "/var/lib/clickhouse/user_scripts" - }, - { - "sourceVolume": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", - "containerPath": "/etc/clickhouse-server/config.d" - }, - { - "sourceVolume": "clickhouse-data", - "containerPath": "/var/lib/clickhouse" - } - ], - "dependsOn": [ - {"containerName": "init-clickhouse", "condition": "SUCCESS"}, - {"containerName": "config-fetcher", "condition": "SUCCESS"} - ], - "cpu": 1024, - "memory": 512, - "memoryReservation": 512, - "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q 0.0.0.0:8123/ping || exit 1"], - "interval": 30, - "timeout": 5, - "retries": 3, - "startPeriod": 30 - } - } - ] - }, - "resource": { - "aws_s3_object": { - "telemetrystore_configs": { - "for_each": "${fileset(\"${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}\", \"**\")}", - "bucket": "${var.config_bucket}", - "key": "{{ $name }}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}", - "source": "${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}", - "etag": "${filemd5(\"${path.module}/telemetrystore/{{ $.Spec.TelemetryStore.Kind }}/${each.value}\")}" - } - }, - "aws_ecs_task_definition": { - "telemetrystore": { - "family": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", - "network_mode": "awsvpc", - "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}", - "volume": [ - { - "name": "shared-binary-volume", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-config", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - }, - { - "name": "clickhouse-data", - "docker_volume_configuration": { - "scope": "task", - "driver": "local" - } - } - ], - "depends_on": ["aws_s3_object.telemetrystore_configs"] - } - }, - "aws_service_discovery_service": { - "telemetrystore": { - "name": "telemetrystore-{{ $.Spec.TelemetryStore.Kind }}", - "dns_config": { - "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", - "dns_records": [ - { - "ttl": 10, - "type": "A" - } - ], - "routing_policy": "MULTIVALUE" - } - } - }, - "aws_ecs_service": { - "telemetrystore": { - "name": "{{ $name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-0-0", - "cluster": "${var.ecs_cluster_id}", - "task_definition": "${aws_ecs_task_definition.telemetrystore.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], - "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" - }, - "service_registries": { - "registry_arn": "${aws_service_discovery_service.telemetrystore.arn}" - }, -"depends_on": [ - "aws_ecs_service.telemetrykeeper" - ] - } - } - } -} diff --git a/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl deleted file mode 100644 index fd7e081e..00000000 --- a/internal/casting/ecsterraformcasting/templates/module/variables.tf.json.gotmpl +++ /dev/null @@ -1,40 +0,0 @@ -{ - "variable": { - "region": { - "description": "AWS region", - "type": "string" - }, - "ecs_cluster_id": { - "description": "ID of the existing ECS cluster to deploy services into", - "type": "string" - }, - "subnet_ids": { - "description": "List of subnet IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "security_group_ids": { - "description": "List of security group IDs for ECS service networking (awsvpc)", - "type": "list(string)" - }, - "vpc_id": { - "description": "VPC ID for the private DNS namespace", - "type": "string" - }, - "config_bucket": { - "description": "S3 bucket name for storing config files", - "type": "string" - }, - "task_role_arn": { - "description": "IAM role ARN for ECS tasks", - "type": "string" - }, - "task_execution_role_arn": { - "description": "IAM role ARN for ECS task execution (pull images, write logs)", - "type": "string" - }, - "capacity_provider": { - "description": "Name of the ECS capacity provider", - "type": "string" - } - } -} diff --git a/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl new file mode 100644 index 00000000..cbad8a19 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl @@ -0,0 +1,73 @@ +{ + "output": { + "cluster_arn": { + "description": "ARN of the ECS cluster", + "value": "${local.cluster_arn}" + }, + "namespace_id": { + "description": "Cloud Map private DNS namespace ID", + "value": "${aws_service_discovery_private_dns_namespace.main.id}" + }, + "namespace_name": { + "description": "Cloud Map private DNS namespace name", + "value": "${aws_service_discovery_private_dns_namespace.main.name}" + }, + "subnet_ids": { + "description": "Subnet IDs used by ECS services", + "value": "${local.subnet_ids}" + }, + "security_group_ids": { + "description": "Security group IDs used by ECS services", + "value": "${local.security_group_ids}" + } + {{- if derefBool $.Spec.Signoz.Spec.Enabled }}, + "signoz_service_arn": { + "description": "SigNoz ECS service ARN (target for ALB on port 8080)", + "value": "${aws_ecs_service.signoz.id}" + }, + "signoz_service_name": { + "description": "SigNoz ECS service name", + "value": "${aws_ecs_service.signoz.name}" + } + {{- end }} + {{- if derefBool $.Spec.Ingester.Spec.Enabled }}, + "ingester_service_arn": { + "description": "Ingester ECS service ARN (target for NLB on port 4317/4318)", + "value": "${aws_ecs_service.ingester.id}" + }, + "ingester_service_name": { + "description": "Ingester ECS service name", + "value": "${aws_ecs_service.ingester.name}" + } + {{- end }} + {{- if derefBool $.Spec.TelemetryStore.Spec.Enabled }}, + "telemetrystore_service_names": { + "description": "TelemetryStore ECS service names (one per node)", + {{- $tsShards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 }}{{- if lt $tsShards 1 }}{{- $tsShards = 1 }}{{- end }} + {{- $tsPerShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) }} + {{- $tsKeys := list }} + {{- range $s := until $tsShards }}{{- range $r := until $tsPerShard }}{{- $tsKeys = append $tsKeys (printf "%d_%d" $s $r) }}{{- end }}{{- end }} + "value": [{{ range $i, $k := $tsKeys }}{{ if $i }}, {{ end }}"${aws_ecs_service.telemetrystore_{{ $k }}.name}"{{ end }}] + } + {{- end }} + {{- if derefBool $.Spec.TelemetryKeeper.Spec.Enabled }}, + "telemetrykeeper_service_names": { + "description": "TelemetryKeeper ECS service names (one per node)", + {{- $kpReplicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 }}{{- if lt $kpReplicas 1 }}{{- $kpReplicas = 1 }}{{- end }} + "value": [{{ range $i := until $kpReplicas }}{{ if $i }}, {{ end }}"${aws_ecs_service.telemetrykeeper_{{ $i }}.name}"{{ end }}] + } + {{- end }} + {{- if derefBool $.Spec.MetaStore.Spec.Enabled }}, + "metastore_service_name": { + "description": "MetaStore ECS service name", + "value": "${aws_ecs_service.metastore.name}" + } + {{- end }} + {{- if derefBool $.Spec.MCP.Spec.Enabled }}, + "mcp_service_name": { + "description": "MCP ECS service name", + "value": "${aws_ecs_service.mcp.name}" + } + {{- end }} + } +} diff --git a/internal/casting/ecsterraformcasting/templates/providers.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/providers.tf.json.gotmpl new file mode 100644 index 00000000..fcaa4a46 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/providers.tf.json.gotmpl @@ -0,0 +1,7 @@ +{ + "provider": { + "aws": { + "region": "${var.aws_region}" + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl similarity index 52% rename from internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl rename to internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl index 07822a0d..89fe23f4 100644 --- a/internal/casting/ecsterraformcasting/templates/module/signoz.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl @@ -1,9 +1,9 @@ {{- $name := $.Metadata.Name -}} { "locals": { - "containers": [ + "containers_signoz": [ { - "name": "signoz", + "name": "{{ $name }}-signoz", "image": "{{ $.Spec.Signoz.Spec.Image }}", "essential": true, {{- $env := list }} @@ -17,11 +17,18 @@ {"name": "http", "containerPort": 8080, "protocol": "tcp", "appProtocol": "http"}, {"name": "opamp", "containerPort": 4320, "protocol": "tcp"} ], + {{- if eq $.Spec.MetaStore.Kind.String "sqlite" }} + "mountPoints": [ + { + "sourceVolume": "signoz-data", + "containerPath": "/var/lib/signoz" + } + ], + {{- end }} "cpu": 512, - "memory": 512, "memoryReservation": 512, "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q localhost:8080/api/v1/health || exit 1"], + "command": ["CMD-SHELL", "wget --spider -q http://localhost:8080/api/v1/health || exit 1"], "interval": 30, "timeout": 5, "retries": 3, @@ -34,16 +41,26 @@ "aws_ecs_task_definition": { "signoz": { "family": "{{ $name }}-signoz", + "tags": {{ toJson $.Labels }}, "network_mode": "awsvpc", "requires_compatibilities": ["EC2"], - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}" + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_signoz)}" + {{- if eq $.Spec.MetaStore.Kind.String "sqlite" }}, + "volume": [ + { + "name": "signoz-data", + "host_path": "/var/lib/foundry/{{ $name }}/signoz/0" + } + ] + {{- end }} } }, "aws_service_discovery_service": { "signoz": { "name": "signoz", + "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", "dns_records": [ @@ -59,26 +76,25 @@ "aws_ecs_service": { "signoz": { "name": "{{ $name }}-signoz", - "cluster": "${var.ecs_cluster_id}", + "cluster": "${local.cluster_arn}", "task_definition": "${aws_ecs_task_definition.signoz.arn}", - "desired_count": 1, - "capacity_provider_strategy": [ - { - "capacity_provider": "${var.capacity_provider}", - "weight": 1, - "base": 0 - } - ], + "desired_count": {{ if eq $.Spec.MetaStore.Kind.String "sqlite" }}1{{ else }}{{ derefIntDefault $.Spec.Signoz.Spec.Cluster.Replicas 1 }}{{ end }}, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}" + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" }, "service_registries": { "registry_arn": "${aws_service_discovery_service.signoz.arn}" }, -"depends_on": [ - "aws_ecs_service.metastore", - "aws_ecs_service.telemetrystore" + "placement_constraints": [ + { + "type": "memberOf", + "expression": "{{ if eq $.Spec.MetaStore.Kind.String "sqlite" }}ec2InstanceId == '${local.seats[\"signoz-0\"]}' and {{ $.PersistentPlacement }}{{ else }}{{ $.EphemeralPlacement }}{{ end }}" + } ] } } diff --git a/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl new file mode 100644 index 00000000..0ba5ea19 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl @@ -0,0 +1,312 @@ +{{- $name := $.Metadata.Name -}} +{{- $kind := $.Spec.TelemetryKeeper.Kind.String -}} +{{- $replicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} +{{- $namespace := printf "%s.local" $name -}} +{{- if eq $kind "zookeeper" -}} +{ + "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "containers_telemetrykeeper_{{ $i }}": [ + { + "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "image": "{{ $.Spec.TelemetryKeeper.Spec.Image }}", + "essential": true, + {{- $env := list }} + {{- $env = append $env (dict "name" "ZOO_SERVER_ID" "value" (printf "%d" (add $i 1))) }} + {{- if gt $replicas 1 }} + {{- $servers := list }} + {{- range $j := until $replicas }} + {{- if eq $j $i }} + {{- $servers = append $servers "0.0.0.0:2888:3888" }} + {{- else }} + {{- $servers = append $servers (printf "telemetrykeeper-%s-%d.%s:2888:3888" $kind $j $namespace) }} + {{- end }} + {{- end }} + {{- $env = append $env (dict "name" "ZOO_SERVERS" "value" (join "," $servers)) }} + {{- end }} + {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "client", "containerPort": 2181, "protocol": "tcp"}, + {"name": "raft", "containerPort": 2888, "protocol": "tcp"}, + {"name": "election", "containerPort": 3888, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrykeeper-data", + "containerPath": "/bitnami/zookeeper" + } + ], + "cpu": 256, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "curl -s -m 2 http://localhost:8080/commands/ruok | grep error | grep null"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + {{- end }} + }, + "resource": { + "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "tags": {{ toJson $.Labels }}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrykeeper_{{ $i }})}", + "volume": [ + { + "name": "telemetrykeeper-data", + "host_path": "/var/lib/foundry/{{ $name }}/telemetrykeeper/{{ $i }}" + } + ] + } + {{- end }} + }, + "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", + "tags": {{ toJson $.Labels }}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + {{- end }} + }, + "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.telemetrykeeper_{{ $i }}.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrykeeper_{{ $i }}.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"telemetrykeeper-{{ $i }}\"]}' and {{ $.PersistentPlacement }}" + } + ] + } + {{- end }} + } + } +} +{{- else -}} +{ + "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "containers_telemetrykeeper_{{ $i }}": [ + {{- /* The agent keeps the config file current while the process runs, so + a config change reloads keeper instead of replacing the task. That + matters more here than anywhere else: replacing the node that + holds the raft log is how an ensemble loses its data. */}} + {{- $profile := printf "telemetrykeeper-%s-%d" $kind $i }} + { + "name": "{{ $name }}-telemetrykeeper-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + "environment": [ + {"name": "PREFETCH_LIST", "value": "{{ $name }}:default:{{ $profile }}"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"{{ $name }}:default:{{ $profile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-keeper/keeper.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrykeeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + {{- /* Assert the precondition keeper actually has, rather than probing + the agent's HTTP port: the file is present and non-empty. */}} + "healthCheck": { + "command": ["CMD-SHELL", "test -s /etc/clickhouse-keeper/keeper.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "image": "{{ $.Spec.TelemetryKeeper.Spec.Image }}", + "essential": true, + "entryPoint": ["/usr/bin/clickhouse-keeper", "--config-file=/etc/clickhouse-keeper/keeper.yaml"], + {{- $env := list }} + {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "client", "containerPort": 9181, "protocol": "tcp"}, + {"name": "raft", "containerPort": 9234, "protocol": "tcp"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrykeeper-data", + "containerPath": "/var/lib/clickhouse-keeper" + }, + { + "sourceVolume": "telemetrykeeper-config", + "containerPath": "/etc/clickhouse-keeper" + } + ], + "dependsOn": [ + {"containerName": "{{ $name }}-telemetrykeeper-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 256, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "clickhouse-keeper-client -h localhost -p 9181 -q ls || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + {{- end }} + }, + "resource": { + {{- /* One profile per node. The pour's YAML stays the source of truth -- + it remains readable and spec.patches-targetable, and file() lifts + it into a hosted version at plan time. Hosted versions are + immutable, so editing the YAML mints a new version and a new + deployment, which the agent picks up on its next poll. */}} + "aws_appconfig_configuration_profile": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {{ toJson $.Labels }} + } + {{- end }} + }, + "aws_appconfig_hosted_configuration_version": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrykeeper_{{ $i }}.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrykeeper/{{ $kind }}/keeper-{{ $i }}.yaml\")}" + } + {{- end }} + }, + "aws_appconfig_deployment": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrykeeper_{{ $i }}.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrykeeper_{{ $i }}.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {{ toJson $.Labels }} + } + {{- end }} + }, + "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "tags": {{ toJson $.Labels }}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrykeeper_{{ $i }})}", + "volume": [ + { + "name": "telemetrykeeper-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrykeeper-data", + "host_path": "/var/lib/foundry/{{ $name }}/telemetrykeeper/{{ $i }}" + } + ], + "depends_on": ["aws_appconfig_deployment.telemetrykeeper_{{ $i }}"] + } + {{- end }} + }, + "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", + "tags": {{ toJson $.Labels }}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + {{- end }} + }, + "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + "telemetrykeeper_{{ $i }}": { + "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.telemetrykeeper_{{ $i }}.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrykeeper_{{ $i }}.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"telemetrykeeper-{{ $i }}\"]}' and {{ $.PersistentPlacement }}" + } + ] + } + {{- end }} + } + } +} +{{- end -}} diff --git a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl new file mode 100644 index 00000000..a240d1cd --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl @@ -0,0 +1,252 @@ +{{- $name := $.Metadata.Name -}} +{{- $kind := $.Spec.TelemetryStore.Kind.String -}} +{{- $shards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $shards 1 }}{{- $shards = 1 -}}{{- end -}} +{{- $nodesPerShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} +{{- $nodes := list -}} +{{- range $s := until $shards -}} +{{- range $r := until $nodesPerShard -}} +{{- $nodes = append $nodes (dict "key" (printf "%d_%d" $s $r) "id" (printf "%d-%d" $s $r)) -}} +{{- end -}} +{{- end -}} +{ + "locals": { + {{- range $i, $n := $nodes }}{{ if $i }},{{ end }} + "containers_telemetrystore_{{ $n.key }}": [ + { + "name": "{{ $name }}-telemetrystore-user-scripts", + "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", + "essential": false, + "entryPoint": ["/bin/sh", "-c"], + "command": ["node_os=$(uname -s | tr '[:upper:]' '[:lower:]') && node_arch=$(uname -m | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && cd /tmp && wget -O histogram-quantile.tar.gz https://github.com/SigNoz/signoz/releases/download/histogram-quantile%2Fv0.0.1/histogram-quantile_$${node_os}_$${node_arch}.tar.gz && tar -xzf histogram-quantile.tar.gz && mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile && chown clickhouse:clickhouse /etc/clickhouse-server/config.d"], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-user-scripts", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + } + ], + "memoryReservation": 256 + }, + {{- /* One agent per task, two configurations: this node's server config + and the UDF file, which is identical across nodes and so has a + single profile every node reads. */}} + {{- $cfgProfile := printf "telemetrystore-%s-%s" $kind $n.id }} + {{- $fnProfile := printf "telemetrystore-%s-functions" $kind }} + { + "name": "{{ $name }}-telemetrystore-appconfig-agent", + "image": "public.ecr.aws/aws-appconfig/aws-appconfig-agent:2.x", + "essential": true, + {{- /* writeTo has no mode or owner setting and the agent writes 0600 + as whichever uid it runs as. ClickHouse drops to uid 101, so the + agent runs as 101 and every rewrite stays readable. */}} + "user": "101:101", + "dependsOn": [ + {"containerName": "{{ $name }}-telemetrystore-user-scripts", "condition": "SUCCESS"} + ], + "environment": [ + {"name": "PREFETCH_LIST", "value": "{{ $name }}:default:{{ $cfgProfile }},{{ $name }}:default:{{ $fnProfile }}"}, + {"name": "POLL_INTERVAL", "value": "45s"}, + {"name": "MANIFEST", "value": "{\"{{ $name }}:default:{{ $cfgProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-{{ $n.id }}.yaml\"}},\"{{ $name }}:default:{{ $fnProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + } + ], + "healthCheck": { + "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-{{ $n.id }}.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], + "interval": 5, + "timeout": 3, + "retries": 10, + "startPeriod": 30 + }, + "memoryReservation": 102 + }, + { + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $n.id }}", + "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", + "essential": true, + {{- $env := list (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} + {{- $env = append $env (dict "name" "CLICKHOUSE_CONFIG" "value" (printf "/etc/clickhouse-server/config.d/config-%s.yaml" $n.id)) }} + {{- range $key, $value := $.Spec.TelemetryStore.Spec.Env }} + {{- $env = append $env (dict "name" $key "value" $value) }} + {{- end }} + "environment": {{ toJson $env }}, + "portMappings": [ + {"name": "native", "containerPort": 9000, "protocol": "tcp"}, + {"name": "http", "containerPort": 8123, "protocol": "tcp", "appProtocol": "http"}, + {"name": "prometheus", "containerPort": 9363, "protocol": "tcp", "appProtocol": "http"} + ], + "mountPoints": [ + { + "sourceVolume": "telemetrystore-user-scripts", + "containerPath": "/var/lib/clickhouse/user_scripts" + }, + { + "sourceVolume": "telemetrystore-config", + "containerPath": "/etc/clickhouse-server/config.d" + }, + { + "sourceVolume": "telemetrystore-data", + "containerPath": "/var/lib/clickhouse" + } + ], + "dependsOn": [ + {"containerName": "{{ $name }}-telemetrystore-user-scripts", "condition": "SUCCESS"}, + {"containerName": "{{ $name }}-telemetrystore-appconfig-agent", "condition": "HEALTHY"} + ], + "cpu": 1024, + "memoryReservation": 512, + "healthCheck": { + "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "interval": 30, + "timeout": 5, + "retries": 3, + "startPeriod": 30 + } + } + ] + {{- end }} + }, + "resource": { + {{- /* A profile per node plus one for the shared UDF file. */}} + "aws_appconfig_configuration_profile": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrystore-{{ $kind }}-functions", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {{ toJson $.Labels }} + } + {{- range $i, $n := $nodes }}, + "telemetrystore_{{ $n.key }}": { + "application_id": "${aws_appconfig_application.main.id}", + "name": "telemetrystore-{{ $kind }}-{{ $n.id }}", + "location_uri": "hosted", + "type": "AWS.Freeform", + "tags": {{ toJson $.Labels }} + } + {{- end }} + }, + "aws_appconfig_hosted_configuration_version": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_functions.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/functions.yaml\")}" + } + {{- range $i, $n := $nodes }}, + "telemetrystore_{{ $n.key }}": { + "application_id": "${aws_appconfig_application.main.id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_{{ $n.key }}.configuration_profile_id}", + "content_type": "application/x-yaml", + "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/config-{{ $n.id }}.yaml\")}" + } + {{- end }} + }, + "aws_appconfig_deployment": { + "telemetrystore_functions": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_functions.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrystore_functions.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {{ toJson $.Labels }} + } + {{- range $i, $n := $nodes }}, + "telemetrystore_{{ $n.key }}": { + "application_id": "${aws_appconfig_application.main.id}", + "environment_id": "${aws_appconfig_environment.main.environment_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.telemetrystore_{{ $n.key }}.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.telemetrystore_{{ $n.key }}.version_number}", + "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", + "tags": {{ toJson $.Labels }} + } + {{- end }} + }, + "aws_ecs_task_definition": { + {{- range $i, $n := $nodes }}{{ if $i }},{{ end }} + "telemetrystore_{{ $n.key }}": { + "family": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $n.id }}", + "tags": {{ toJson $.Labels }}, + "network_mode": "awsvpc", + "requires_compatibilities": ["EC2"], + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrystore_{{ $n.key }})}", + "volume": [ + { + "name": "telemetrystore-user-scripts", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-config", + "docker_volume_configuration": { + "scope": "task", + "driver": "local" + } + }, + { + "name": "telemetrystore-data", + "host_path": "/var/lib/foundry/{{ $name }}/telemetrystore/{{ $n.id }}" + } + ], + "depends_on": ["aws_appconfig_deployment.telemetrystore_{{ $n.key }}", "aws_appconfig_deployment.telemetrystore_functions"] + } + {{- end }} + }, + "aws_service_discovery_service": { + {{- range $i, $n := $nodes }}{{ if $i }},{{ end }} + "telemetrystore_{{ $n.key }}": { + "name": "telemetrystore-{{ $kind }}-{{ $n.id }}", + "tags": {{ toJson $.Labels }}, + "dns_config": { + "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", + "dns_records": [ + { + "ttl": 10, + "type": "A" + } + ], + "routing_policy": "MULTIVALUE" + } + } + {{- end }} + }, + "aws_ecs_service": { + {{- range $i, $n := $nodes }}{{ if $i }},{{ end }} + "telemetrystore_{{ $n.key }}": { + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $n.id }}", + "cluster": "${local.cluster_arn}", + "task_definition": "${aws_ecs_task_definition.telemetrystore_{{ $n.key }}.arn}", + "desired_count": 1, + "deployment_minimum_healthy_percent": 0, + "deployment_maximum_percent": 100, + "launch_type": "EC2", + "tags": {{ toJson $.Labels }}, + "network_configuration": { + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}" + }, + "service_registries": { + "registry_arn": "${aws_service_discovery_service.telemetrystore_{{ $n.key }}.arn}" + }, + "placement_constraints": [ + { + "type": "memberOf", + "expression": "ec2InstanceId == '${local.seats[\"telemetrystore-{{ $n.id }}\"]}' and {{ $.PersistentPlacement }}" + } + ] + } + {{- end }} + } + } +} diff --git a/internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrystore_migrator.tf.json.gotmpl similarity index 68% rename from internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl rename to internal/casting/ecsterraformcasting/templates/telemetrystore_migrator.tf.json.gotmpl index c1a1bd72..ce49a6f0 100644 --- a/internal/casting/ecsterraformcasting/templates/module/telemetrystore_migrator.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrystore_migrator.tf.json.gotmpl @@ -1,15 +1,15 @@ {{- $name := $.Metadata.Name -}} { "locals": { - "containers": [ + "containers_telemetrystore_migrator": [ { - "name": "telemetrystore-migrator", + "name": "{{ $name }}-telemetrystore-migrator", "image": "{{ $.Spec.Ingester.Spec.Image }}", "essential": true, "entryPoint": ["/bin/sh", "-c"], "command": ["/signoz-otel-collector migrate ready && /signoz-otel-collector migrate bootstrap && /signoz-otel-collector migrate sync up && /signoz-otel-collector migrate async up"], {{- $env := list }} - {{- if and $.Spec.TelemetryStore.Spec.Enabled $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} + {{- if and (derefBool $.Spec.TelemetryStore.Spec.Enabled) $.Spec.TelemetryStore.Status.Addresses.TCP (index $.Spec.TelemetryStore.Status.Addresses.TCP 0) }} {{- $env = append $env (dict "name" "SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN" "value" (index $.Spec.TelemetryStore.Status.Addresses.TCP 0)) }} {{- end }} {{- range $key, $value := $.Spec.Ingester.Spec.Env }} @@ -25,31 +25,29 @@ "aws_ecs_task_definition": { "telemetrystore_migrator": { "family": "{{ $name }}-telemetrystore-migrator", + "tags": {{ toJson $.Labels }}, "network_mode": "awsvpc", "requires_compatibilities": ["FARGATE"], "cpu": 256, "memory": 512, - "task_role_arn": "${var.task_role_arn}", - "execution_role_arn": "${var.task_execution_role_arn}", - "container_definitions": "${jsonencode(local.containers)}" + "task_role_arn": "${local.task_role_arn}", + "execution_role_arn": "${local.execution_role_arn}", + "container_definitions": "${jsonencode(local.containers_telemetrystore_migrator)}" } } }, "data": { "aws_ecs_task_execution": { "telemetrystore_migrator": { - "cluster": "${var.ecs_cluster_id}", + "cluster": "${local.cluster_arn}", "task_definition": "${aws_ecs_task_definition.telemetrystore_migrator.arn}", "desired_count": 1, "launch_type": "FARGATE", "network_configuration": { - "subnets": "${var.subnet_ids}", - "security_groups": "${var.security_group_ids}", + "subnets": "${local.subnet_ids}", + "security_groups": "${local.security_group_ids}", "assign_public_ip": false - }, - "depends_on": [ - "aws_ecs_service.telemetrystore" - ] + } } } } diff --git a/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl b/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl index 867159ca..3c93888c 100644 --- a/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/terraform.tfvars.json.gotmpl @@ -1,20 +1,4 @@ -{{- $region := index $.Metadata.Annotations "foundry.signoz.io/ecs/region" -}} -{{- $configBucket := index $.Metadata.Annotations "foundry.signoz.io/ecs/config-bucket" -}} -{{- $taskRoleArn := index $.Metadata.Annotations "foundry.signoz.io/ecs/task-role-arn" -}} -{{- $taskExecutionRoleArn := index $.Metadata.Annotations "foundry.signoz.io/ecs/task-execution-role-arn" -}} -{{- $clusterID := index $.Metadata.Annotations "foundry.signoz.io/ecs/cluster-id" -}} -{{- $subnetIDs := index $.Metadata.Annotations "foundry.signoz.io/ecs/subnet-ids" -}} -{{- $securityGroupIDs := index $.Metadata.Annotations "foundry.signoz.io/ecs/security-group-ids" -}} -{{- $vpcID := index $.Metadata.Annotations "foundry.signoz.io/ecs/vpc-id" -}} -{{- $capacityProvider := index $.Metadata.Annotations "foundry.signoz.io/ecs/capacity-provider" -}} +{{- $region := index $.Metadata.Annotations "foundry.signoz.io/ecs-region" -}} { - "region": "{{ $region }}", - "ecs_cluster_id": "{{ $clusterID }}", - "subnet_ids": [{{ range $i, $v := splitList "," $subnetIDs }}{{ if $i }}, {{ end }}"{{ trim $v }}"{{ end }}], - "security_group_ids": [{{ range $i, $v := splitList "," $securityGroupIDs }}{{ if $i }}, {{ end }}"{{ trim $v }}"{{ end }}], - "vpc_id": "{{ $vpcID }}", - "config_bucket": "{{ $configBucket }}", - "task_role_arn": "{{ $taskRoleArn }}", - "task_execution_role_arn": "{{ $taskExecutionRoleArn }}", - "capacity_provider": "{{ $capacityProvider }}" + "aws_region": "{{ $region }}" } diff --git a/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl index fd7e081e..8093d941 100644 --- a/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/variables.tf.json.gotmpl @@ -1,40 +1,125 @@ +{{- /* Every identifier arrives as a variable defaulted to what the casting + resolved. Stating one on the casting replaces its lookup variable with + the value itself, and main.tf emits no data source for it. */}} +{{- $annotations := default (dict) $.Metadata.Annotations -}} +{{- $clusterARN := index $annotations "foundry.signoz.io/ecs-cluster-arn" -}} +{{- $subnetIDs := index $annotations "foundry.signoz.io/ecs-subnet-ids" -}} +{{- $securityGroupIDs := index $annotations "foundry.signoz.io/ecs-security-group-ids" -}} +{{- $vpcID := index $annotations "foundry.signoz.io/ecs-vpc-id" -}} +{{- $taskRoleARN := index $annotations "foundry.signoz.io/ecs-task-role-arn" -}} +{{- $executionRoleARN := index $annotations "foundry.signoz.io/ecs-task-execution-role-arn" -}} { "variable": { - "region": { - "description": "AWS region", + "aws_region": { + "nullable": false, + "validation": { + "condition": "${can(regex(\"^[a-z]{2}(-gov)?-[a-z]+-[0-9]$\", var.aws_region))}", + "error_message": "aws_region must be a region identifier such as us-east-1." + }, + "description": "AWS region holding the cluster", "type": "string" }, - "ecs_cluster_id": { - "description": "ID of the existing ECS cluster to deploy services into", - "type": "string" + {{- if $clusterARN }} + "cluster_arn": { + "nullable": false, + "description": "ARN of the ECS cluster to deploy into", + "type": "string", + "default": "{{ $clusterARN }}" + }, + {{- else }} + "cluster_name": { + "nullable": false, + "description": "Name of the ECS cluster to deploy into", + "type": "string", + "default": "{{ $.ClusterName }}" }, + {{- end }} + {{- if $subnetIDs }} "subnet_ids": { - "description": "List of subnet IDs for ECS service networking (awsvpc)", - "type": "list(string)" + "nullable": false, + "description": "IDs of the subnets tasks are placed in", + "type": "list(string)", + "default": {{ $ids := list }}{{ range splitList "," $subnetIDs }}{{ $ids = append $ids (trim .) }}{{ end }}{{ toJson (compact $ids) }} + }, + {{- else }} + "subnet_tags": { + "nullable": false, + "description": "Tags that find the subnets tasks are placed in", + "type": "map(string)", + "default": {{ toJson $.SubnetTags }} }, + {{- end }} + {{- if $securityGroupIDs }} "security_group_ids": { - "description": "List of security group IDs for ECS service networking (awsvpc)", - "type": "list(string)" + "nullable": false, + "description": "IDs of the security groups tasks join", + "type": "list(string)", + "default": {{ $ids := list }}{{ range splitList "," $securityGroupIDs }}{{ $ids = append $ids (trim .) }}{{ end }}{{ toJson (compact $ids) }} }, + {{- else }} + "security_group_name": { + "nullable": false, + "description": "Name of the security group tasks join", + "type": "string", + "default": "{{ $.SecurityGroupName }}" + }, + {{- end }} + {{- if $vpcID }} "vpc_id": { - "description": "VPC ID for the private DNS namespace", - "type": "string" + "nullable": false, + "description": "ID of the VPC the Cloud Map namespace is created in", + "type": "string", + "default": "{{ $vpcID }}" }, - "config_bucket": { - "description": "S3 bucket name for storing config files", - "type": "string" + {{- else }} + "vpc_tags": { + "nullable": false, + "description": "Tags that find the VPC the Cloud Map namespace is created in", + "type": "map(string)", + "default": {{ toJson $.VPCTags }} }, + {{- end }} + {{- if $taskRoleARN }} "task_role_arn": { - "description": "IAM role ARN for ECS tasks", - "type": "string" + "nullable": false, + "description": "ARN of the IAM role tasks assume", + "type": "string", + "default": "{{ $taskRoleARN }}" }, - "task_execution_role_arn": { - "description": "IAM role ARN for ECS task execution (pull images, write logs)", - "type": "string" + {{- else }} + "task_role_name": { + "nullable": false, + "description": "Name of the IAM role this stack creates for its tasks", + "type": "string", + "default": "{{ $.TaskRoleName }}" }, - "capacity_provider": { - "description": "Name of the ECS capacity provider", - "type": "string" + {{- end }} + {{- if $executionRoleARN }} + "execution_role_arn": { + "nullable": false, + "description": "ARN of the IAM role the ECS agent assumes", + "type": "string", + "default": "{{ $executionRoleARN }}" + }, + {{- else }} + "execution_role_name": { + "nullable": false, + "description": "Name of the IAM role this stack creates for the ECS agent to pull images and write logs", + "type": "string", + "default": "{{ $.ExecutionRoleName }}" + }, + {{- end }} + "node_tags": { + "nullable": false, + "description": "Tags that find the persistent instances and the volumes attached to them", + "type": "map(string)", + "default": {{ toJson $.NodeTags }} + }, + "claim_tag": { + "nullable": false, + "description": "Tag key recording which identities hold a volume", + "type": "string", + "default": "{{ $.ClaimTag }}" } } } diff --git a/internal/casting/ecsterraformcasting/templates/versions.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/versions.tf.json.gotmpl new file mode 100644 index 00000000..a9eded70 --- /dev/null +++ b/internal/casting/ecsterraformcasting/templates/versions.tf.json.gotmpl @@ -0,0 +1,11 @@ +{ + "terraform": { + "required_version": ">= 1.4.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "~> 5.0" + } + } + } +} From 0bd60a6f5684cf6b5643b8aec02b664ab131ce51 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Thu, 6 Aug 2026 18:23:05 +0530 Subject: [PATCH 36/38] docs(reference): correct the ecs annotation table Drops ecs-config-bucket, which no longer exists, fixes the task execution role to map to execution_role_arn, and records that every annotation but the region is an optional override of a tag lookup. --- docs/reference/casting-file.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/reference/casting-file.md b/docs/reference/casting-file.md index 01f5a38d..6acb91c1 100644 --- a/docs/reference/casting-file.md +++ b/docs/reference/casting-file.md @@ -183,15 +183,24 @@ Required when using `platform: ecs`, `mode: ec2`, `flavor: terraform`. | Annotation | Maps to tfvar | Description | | --- | --- | --- | -| `foundry.signoz.io/ecs/region` | `region` | AWS region | -| `foundry.signoz.io/ecs/cluster-id` | `ecs_cluster_id` | ECS cluster ARN or ID | -| `foundry.signoz.io/ecs/subnet-ids` | `subnet_ids` | Comma-separated subnet IDs | -| `foundry.signoz.io/ecs/security-group-ids` | `security_group_ids` | Comma-separated security group IDs | -| `foundry.signoz.io/ecs/vpc-id` | `vpc_id` | VPC ID for Cloud Map namespace | -| `foundry.signoz.io/ecs/config-bucket` | `config_bucket` | S3 bucket for component configs | -| `foundry.signoz.io/ecs/task-role-arn` | `task_role_arn` | IAM role ARN for ECS tasks | -| `foundry.signoz.io/ecs/task-execution-role-arn` | `task_execution_role_arn` | IAM role ARN for task execution | -| `foundry.signoz.io/ecs/capacity-provider` | `capacity_provider` | ECS capacity provider name | +| `foundry.signoz.io/ecs-region` | `aws_region` | AWS region holding the cluster | + +The rest are optional. Each names an existing AWS object, for a cluster the +casting places tasks onto but does not provision. Leave one out and the stack +finds that object by its `foundry.signoz.io/*` tags instead. State it and the +lookup variable is replaced by the value itself, with no data source emitted +for it. + +| Annotation | Maps to tfvar | Replaces the lookup on | +| --- | --- | --- | +| `foundry.signoz.io/ecs-cluster-arn` | `cluster_arn` | `cluster_name` | +| `foundry.signoz.io/ecs-subnet-ids` | `subnet_ids` | `subnet_tags` | +| `foundry.signoz.io/ecs-security-group-ids` | `security_group_ids` | `security_group_name` | +| `foundry.signoz.io/ecs-vpc-id` | `vpc_id` | `vpc_tags` | +| `foundry.signoz.io/ecs-task-role-arn` | `task_role_arn` | `task_role_name` | +| `foundry.signoz.io/ecs-task-execution-role-arn` | `execution_role_arn` | `execution_role_name` | + +The ID annotations take a comma-separated list. ## Schema From 3e1db241b49d2d15879c594b2766f112386f129a Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Sun, 9 Aug 2026 17:29:39 +0530 Subject: [PATCH 37/38] feat: better handle tool invocations --- cmd/foundryctl/config.go | 11 ++ cmd/foundryctl/main.go | 1 + cmd/foundryctl/uncast.go | 53 +++++++ internal/casting/casting.go | 10 +- internal/casting/collectionagent/casting.go | 4 +- .../dockercomposecasting/casting.go | 63 ++++---- .../dockerswarmcasting/casting.go | 7 +- internal/casting/collectionagent/planner.go | 16 +- internal/casting/collectionagent/registry.go | 15 +- .../casting/collectionagent/registry_test.go | 40 +++++ internal/casting/coolifycasting/casting.go | 9 +- .../casting/dockercomposecasting/casting.go | 73 ++++----- .../casting/dockerswarmcasting/casting.go | 7 +- .../casting/ecsterraformcasting/casting.go | 7 +- internal/casting/installation/planner.go | 18 ++- internal/casting/installation/registry.go | 18 ++- .../casting/installation/registry_test.go | 40 +++++ .../casting/kuberneteshelmcasting/casting.go | 7 +- .../kuberneteskustomizecasting/casting.go | 7 +- .../casting/railwaytemplatecasting/casting.go | 8 +- internal/casting/rendercasting/casting.go | 9 +- internal/casting/systemdcasting/casting.go | 7 +- internal/domain/event.go | 3 +- internal/foundry/gauge.go | 8 + internal/foundry/uncast.go | 15 ++ internal/planner/planner.go | 7 +- internal/runner/composerunner/runner.go | 141 ++++++++++++++++++ internal/runner/composerunner/runner_test.go | 94 ++++++++++++ internal/runner/runner.go | 25 ++++ 29 files changed, 630 insertions(+), 93 deletions(-) create mode 100644 cmd/foundryctl/uncast.go create mode 100644 internal/casting/collectionagent/registry_test.go create mode 100644 internal/casting/installation/registry_test.go create mode 100644 internal/foundry/uncast.go create mode 100644 internal/runner/composerunner/runner.go create mode 100644 internal/runner/composerunner/runner_test.go create mode 100644 internal/runner/runner.go diff --git a/cmd/foundryctl/config.go b/cmd/foundryctl/config.go index a02572aa..89aae17f 100644 --- a/cmd/foundryctl/config.go +++ b/cmd/foundryctl/config.go @@ -12,6 +12,9 @@ var ( // Stores cast configuration. castCfg castConfig + // Stores uncast configuration. + uncastCfg uncastConfig + // Stores catalog configuration. catalogCfg catalogConfig ) @@ -45,6 +48,14 @@ type castConfig struct { NoForge bool } +type uncastConfig struct { + Yes bool +} + +func (c *uncastConfig) RegisterFlags(cmd *cobra.Command) { + cmd.PersistentFlags().BoolVar(&c.Yes, "yes", false, "Confirm removing the deployment.") +} + func (c *castConfig) RegisterFlags(cmd *cobra.Command) { cmd.PersistentFlags().BoolVar(&c.NoGauge, "no-gauge", false, "Do not run gauge before forge and cast.") cmd.PersistentFlags().BoolVar(&c.NoForge, "no-forge", false, "Do not run forge before cast.") diff --git a/cmd/foundryctl/main.go b/cmd/foundryctl/main.go index bcc3b560..d13a3d76 100644 --- a/cmd/foundryctl/main.go +++ b/cmd/foundryctl/main.go @@ -26,6 +26,7 @@ func main() { registerGaugeCmd(rootCmd) registerForgeCmd(rootCmd) registerCastCmd(rootCmd) + registerUncastCmd(rootCmd) registerGenCmd(rootCmd) registerCatalogCmd(rootCmd) registerVersionCmd(rootCmd) diff --git a/cmd/foundryctl/uncast.go b/cmd/foundryctl/uncast.go new file mode 100644 index 00000000..6346768d --- /dev/null +++ b/cmd/foundryctl/uncast.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "log/slog" + "path/filepath" + + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/foundry" + "github.com/spf13/cobra" +) + +func registerUncastCmd(rootCmd *cobra.Command) { + uncastCmd := &cobra.Command{ + Use: "uncast", + Short: "Remove the cast deployment. Definitions are removed; data is never touched.", + RunE: recoverRunE(domain.EventUncast, func(cmd *cobra.Command, args []string) (domain.Properties, error) { + ctx := cmd.Context() + + if !uncastCfg.Yes { + return domain.NewProperties(), errors.Newf(errors.TypeInvalidInput, "uncast removes the deployment (data and volumes always stay); re-run with --yes to confirm") + } + + return runUncast(ctx, rootLogger, poursCfg.Path, commonCfg.File) + }), + } + + rootCmd.AddCommand(uncastCmd) + uncastCfg.RegisterFlags(uncastCmd) +} + +func runUncast(ctx context.Context, logger *slog.Logger, poursPath string, configPath string) (domain.Properties, error) { + foundry, err := foundry.New(logger) + if err != nil { + return domain.NewProperties(), err + } + + poursPath, err = filepath.Abs(poursPath) + if err != nil { + return domain.NewProperties(), errors.Wrapf(err, errors.TypeInternal, "failed to resolve pours path") + } + + machinery, err := foundry.Config.GetV1Alpha1Lock(ctx, configPath) + if err != nil { + return domain.NewProperties(), err + } + + props := machinery.TrackableProperties() + + err = foundry.Uncast(ctx, machinery, poursPath) + return props, err +} diff --git a/internal/casting/casting.go b/internal/casting/casting.go index 5d0ba829..974b1998 100644 --- a/internal/casting/casting.go +++ b/internal/casting/casting.go @@ -6,6 +6,7 @@ import ( "github.com/signoz/foundry/api/v1alpha1/installation" "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) // DeploymentDir is the subdirectory within the pours directory where @@ -19,6 +20,11 @@ type Casting interface { // Generates all the files needed for casting. Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) - // Runs the forged files. - Cast(ctx context.Context, config installation.Casting, poursPath string) error + // Runs the forged files. Runners are the registry-listed tool + // interfaces for this casting; usage options are passed per call. + Cast(ctx context.Context, config installation.Casting, poursPath string, runners []runner.Runner) error + + // Removes what Cast deployed: definitions only, never data, never + // users, config stays. + Uncast(ctx context.Context, config installation.Casting, poursPath string, runners []runner.Runner) error } diff --git a/internal/casting/collectionagent/casting.go b/internal/casting/collectionagent/casting.go index 30c94126..80cb6062 100644 --- a/internal/casting/collectionagent/casting.go +++ b/internal/casting/collectionagent/casting.go @@ -6,10 +6,12 @@ import ( "github.com/signoz/foundry/api/v1alpha1/collectionagent" collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/runner" ) type Casting interface { Enricher(ctx context.Context, config *collectionagent.Casting) (collectionagentmolding.MoldingEnricher, error) Forge(ctx context.Context, config collectionagent.Casting, p *pourer.Pourer) error - Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer) error + Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error + Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error } diff --git a/internal/casting/collectionagent/dockercomposecasting/casting.go b/internal/casting/collectionagent/dockercomposecasting/casting.go index c93e4a05..ab032a10 100644 --- a/internal/casting/collectionagent/dockercomposecasting/casting.go +++ b/internal/casting/collectionagent/dockercomposecasting/casting.go @@ -7,7 +7,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/collectionagent" @@ -15,6 +14,8 @@ import ( foundryerrors "github.com/signoz/foundry/internal/errors" collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/runner" + "github.com/signoz/foundry/internal/runner/composerunner" ) type dockerComposeCasting struct { @@ -44,31 +45,49 @@ func (c *dockerComposeCasting) Forge(ctx context.Context, config collectionagent return nil } -func (c *dockerComposeCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer) error { - composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") - - if _, err := os.Stat(composeFile); os.IsNotExist(err) { - return foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) +func (c *dockerComposeCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error { + composeFile, compose, err := c.deployment(outputPath, p, runners) + if err != nil { + return err } if err := c.checkOwnership(ctx, config); err != nil { return err } - composeCmd, err := getComposeCommand(ctx) + c.logger.InfoContext(ctx, "Removing the collection agent; volumes and their data stay") + + return compose.Down(ctx, composeFile, composerunner.Options{}) +} + +func (c *dockerComposeCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error { + composeFile, compose, err := c.deployment(outputPath, p, runners) if err != nil { - return foundryerrors.Wrapf(err, foundryerrors.TypeNotFound, "docker compose not available") + return err } - args := append(composeCmd[1:], "-f", composeFile, "up", "-d") + if err := c.checkOwnership(ctx, config); err != nil { + return err + } - c.logger.DebugContext(ctx, "running command", slog.String("command", strings.Join(append([]string{composeCmd[0]}, args...), " "))) + return compose.Up(ctx, composeFile, composerunner.Options{}) +} - cmd := exec.CommandContext(ctx, composeCmd[0], args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr +// deployment resolves the compose file this casting forged and the runner +// that executes it. +func (c *dockerComposeCasting) deployment(outputPath string, p *pourer.Pourer, runners []runner.Runner) (string, *composerunner.Runner, error) { + composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") - return cmd.Run() + if _, err := os.Stat(composeFile); os.IsNotExist(err) { + return "", nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) + } + + compose, err := composerunner.From(runners) + if err != nil { + return "", nil, err + } + + return composeFile, compose, nil } // checkOwnership refuses to deploy over a compose project of the same name @@ -95,19 +114,3 @@ func (c *dockerComposeCasting) checkOwnership(ctx context.Context, config collec return nil } - -func getComposeCommand(ctx context.Context) ([]string, error) { - if _, err := exec.LookPath("docker"); err == nil { - cmd := exec.CommandContext(ctx, "docker", "compose", "version") - - if err := cmd.Run(); err == nil { - return []string{"docker", "compose"}, nil - } - } - - if _, err := exec.LookPath("docker-compose"); err == nil { - return []string{"docker-compose"}, nil - } - - return nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "neither 'docker compose' nor 'docker-compose' is available") -} diff --git a/internal/casting/collectionagent/dockerswarmcasting/casting.go b/internal/casting/collectionagent/dockerswarmcasting/casting.go index 9d08d466..c865cfe7 100644 --- a/internal/casting/collectionagent/dockerswarmcasting/casting.go +++ b/internal/casting/collectionagent/dockerswarmcasting/casting.go @@ -16,6 +16,7 @@ import ( foundryerrors "github.com/signoz/foundry/internal/errors" collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/runner" ) type dockerSwarmCasting struct { @@ -45,7 +46,11 @@ func (c *dockerSwarmCasting) Forge(ctx context.Context, config collectionagent.C return nil } -func (c *dockerSwarmCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer) error { +func (c *dockerSwarmCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, _ []runner.Runner) error { + return foundryerrors.Newf(foundryerrors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (c *dockerSwarmCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, _ []runner.Runner) error { composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") if _, err := os.Stat(composeFile); os.IsNotExist(err) { diff --git a/internal/casting/collectionagent/planner.go b/internal/casting/collectionagent/planner.go index d4d7c34a..e3459191 100644 --- a/internal/casting/collectionagent/planner.go +++ b/internal/casting/collectionagent/planner.go @@ -13,6 +13,7 @@ import ( "github.com/signoz/foundry/internal/molding/collectionagent/collectormolding" "github.com/signoz/foundry/internal/planner" "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/runner" "github.com/signoz/foundry/internal/tooler" ) @@ -26,6 +27,7 @@ type Planner struct { logger *slog.Logger casting Casting toolers []tooler.Tooler + runners []runner.Runner enricher collectionagentmolding.MoldingEnricher moldings []collectionagentmolding.Molding } @@ -43,6 +45,11 @@ func NewPlanner(ctx context.Context, c *collectionagent.Casting, logger *slog.Lo return nil, err } + runners, err := registry.Runners(c.Spec.Deployment) + if err != nil { + return nil, err + } + enricher, err := castingStrategy.Enricher(ctx, c) if err != nil { return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") @@ -57,6 +64,7 @@ func NewPlanner(ctx context.Context, c *collectionagent.Casting, logger *slog.Lo logger: logger, casting: castingStrategy, toolers: toolers, + runners: runners, enricher: enricher, moldings: moldings, }, nil @@ -99,7 +107,13 @@ func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, } func (p *Planner) Cast(ctx context.Context, poursPath string) error { - return p.casting.Cast(ctx, *p.config, poursPath, pourer.New(strings.ToLower(p.config.Kind().String()))) + return p.casting.Cast(ctx, *p.config, poursPath, pourer.New(strings.ToLower(p.config.Kind().String())), p.runners) +} + +func (p *Planner) Uncast(ctx context.Context, poursPath string) error { + return p.casting.Uncast(ctx, *p.config, poursPath, pourer.New(strings.ToLower(p.config.Kind().String())), p.runners) } func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } + +func (p *Planner) Runners() []runner.Runner { return p.runners } diff --git a/internal/casting/collectionagent/registry.go b/internal/casting/collectionagent/registry.go index 3a293f00..00208285 100644 --- a/internal/casting/collectionagent/registry.go +++ b/internal/casting/collectionagent/registry.go @@ -7,8 +7,9 @@ import ( "github.com/signoz/foundry/internal/casting/collectionagent/dockercomposecasting" "github.com/signoz/foundry/internal/casting/collectionagent/dockerswarmcasting" foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/runner" + "github.com/signoz/foundry/internal/runner/composerunner" "github.com/signoz/foundry/internal/tooler" - "github.com/signoz/foundry/internal/tooler/dockercomposetooler" "github.com/signoz/foundry/internal/tooler/dockerswarmtooler" "github.com/signoz/foundry/internal/tooler/dockertooler" ) @@ -16,6 +17,8 @@ import ( type CastingItem struct { Casting Casting Toolers []tooler.Tooler + + Runners []runner.Runner } type Registry struct { @@ -30,7 +33,7 @@ func NewRegistry(logger *slog.Logger) *Registry { Flavor: v1alpha1.FlavorCompose, }: { Casting: dockercomposecasting.New(logger), - Toolers: []tooler.Tooler{dockertooler.New(), dockercomposetooler.New()}, + Runners: []runner.Runner{composerunner.New(logger, composerunner.Config{})}, }, { Mode: v1alpha1.ModeDocker, @@ -69,3 +72,11 @@ func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler. } return item.Toolers, nil } + +func (registry *Registry) Runners(deployment v1alpha1.TypeDeployment) ([]runner.Runner, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "collectionagent deployment '%+v' is not supported", deployment) + } + return item.Runners, nil +} diff --git a/internal/casting/collectionagent/registry_test.go b/internal/casting/collectionagent/registry_test.go new file mode 100644 index 00000000..9df10562 --- /dev/null +++ b/internal/casting/collectionagent/registry_test.go @@ -0,0 +1,40 @@ +package collectionagent + +import ( + "log/slog" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/runner/composerunner" + "github.com/stretchr/testify/assert" +) + +// The compose casting resolves its runner from the registry entry at cast +// time; this pins the entry so the lookup cannot silently go empty. +func TestRegistryRunners(t *testing.T) { + registry := NewRegistry(slog.New(slog.DiscardHandler)) + + tests := []struct { + name string + deployment v1alpha1.TypeDeployment + pass bool + }{ + {name: "DockerCompose_ComposeRunner", deployment: v1alpha1.TypeDeployment{Mode: v1alpha1.ModeDocker, Flavor: v1alpha1.FlavorCompose}, pass: true}, + {name: "DockerSwarm_NoRunnerYet", deployment: v1alpha1.TypeDeployment{Mode: v1alpha1.ModeDocker, Flavor: v1alpha1.FlavorSwarm}, pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runners, err := registry.Runners(tt.deployment) + assert.NoError(t, err) + + _, err = composerunner.From(runners) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + }) + } +} diff --git a/internal/casting/coolifycasting/casting.go b/internal/casting/coolifycasting/casting.go index e7c58187..ac1a430f 100644 --- a/internal/casting/coolifycasting/casting.go +++ b/internal/casting/coolifycasting/casting.go @@ -11,6 +11,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ rootcasting.Casting = (*coolifyCasting)(nil) @@ -48,7 +49,7 @@ func (c *coolifyCasting) Forge(ctx context.Context, config installation.Casting, return []domain.Material{coolifyMaterial}, nil } -func (c *coolifyCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *coolifyCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Please run 'forge' first to generate the Coolify Casting", slog.String("pours_path", poursPath)) c.logger.InfoContext(ctx, "After forging, deploy coolify.yaml to Coolify using the stack feature", @@ -56,6 +57,12 @@ func (c *coolifyCasting) Cast(ctx context.Context, config installation.Casting, return nil } +func (c *coolifyCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + c.logger.InfoContext(ctx, "Remove the stack from Coolify directly; foundry does not manage Coolify resources", + slog.String("docs", "https://coolify.io/docs/knowledge-base/docker/compose")) + return nil +} + func getCoolifyMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := coolifyYAMLTemplate.Execute(buf, config) diff --git a/internal/casting/dockercomposecasting/casting.go b/internal/casting/dockercomposecasting/casting.go index a3af3c88..51fc47cf 100644 --- a/internal/casting/dockercomposecasting/casting.go +++ b/internal/casting/dockercomposecasting/casting.go @@ -3,18 +3,17 @@ package dockercomposecasting import ( "bytes" "context" - "errors" "log/slog" "os" - "os/exec" "path/filepath" - "strings" "github.com/signoz/foundry/api/v1alpha1/installation" rootcasting "github.com/signoz/foundry/internal/casting" "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" + "github.com/signoz/foundry/internal/runner/composerunner" ) var _ rootcasting.Casting = (*dockerComposeCasting)(nil) @@ -99,33 +98,26 @@ func (casting *dockerComposeCasting) Forge(ctx context.Context, config installat return materials, nil } -func (casting *dockerComposeCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { - casting.logger.InfoContext(ctx, "Executing commands for platform") +func (casting *dockerComposeCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { + casting.logger.InfoContext(ctx, "Removing the compose deployment; volumes and their data stay") - // Check if compose file exists - composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") - if _, err := os.Stat(composeFile); os.IsNotExist(err) { - return foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) - } - - // Get the available docker compose command - composeCmd, err := getComposeCommand(ctx) + composeFile, compose, err := casting.deployment(outputPath, runners) if err != nil { - casting.logger.ErrorContext(ctx, "Docker compose not available", slog.String("error", err.Error())) - return foundryerrors.Wrapf(err, foundryerrors.TypeNotFound, "docker compose not available") + return err } - args := append(composeCmd[1:], "-f", composeFile, "up", "-d") - - casting.logger.DebugContext(ctx, "Running command", slog.String("command", strings.Join(append([]string{composeCmd[0]}, args...), " "))) + return compose.Down(ctx, composeFile, composerunner.Options{}) +} - cmd := exec.CommandContext(ctx, composeCmd[0], args...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr +func (casting *dockerComposeCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { + casting.logger.InfoContext(ctx, "Executing commands for platform") - err = cmd.Run() + composeFile, compose, err := casting.deployment(outputPath, runners) if err != nil { - casting.logger.ErrorContext(ctx, "Command execution failed", slog.String("error", err.Error())) + return err + } + + if err := compose.Up(ctx, composeFile, composerunner.Options{}); err != nil { return err } @@ -134,6 +126,22 @@ func (casting *dockerComposeCasting) Cast(ctx context.Context, config installati return nil } +// deployment resolves the compose file this casting forged and the runner +// that executes it. +func (casting *dockerComposeCasting) deployment(outputPath string, runners []runner.Runner) (string, *composerunner.Runner, error) { + composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") + if _, err := os.Stat(composeFile); os.IsNotExist(err) { + return "", nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) + } + + compose, err := composerunner.From(runners) + if err != nil { + return "", nil, err + } + + return composeFile, compose, nil +} + func getComposeMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := composeYAMLTemplate.Execute(buf, config) @@ -143,22 +151,3 @@ func getComposeMaterial(config *installation.Casting, path string) (domain.Struc return domain.NewYAMLMaterial(buf.Bytes(), path) } - -// getComposeCommand detects the available docker compose command. -// It checks for "docker compose" (newer, preferred) first, then falls back to "docker-compose" (legacy). -func getComposeCommand(ctx context.Context) ([]string, error) { - // Check "docker compose" first (newer, preferred) - if _, err := exec.LookPath("docker"); err == nil { - cmd := exec.CommandContext(ctx, "docker", "compose", "version") - if err := cmd.Run(); err == nil { - return []string{"docker", "compose"}, nil - } - } - - // Fallback to "docker-compose" (legacy) - if _, err := exec.LookPath("docker-compose"); err == nil { - return []string{"docker-compose"}, nil - } - - return nil, errors.New("neither 'docker compose' nor 'docker-compose' is available") -} diff --git a/internal/casting/dockerswarmcasting/casting.go b/internal/casting/dockerswarmcasting/casting.go index e9c8dc80..61a626e1 100644 --- a/internal/casting/dockerswarmcasting/casting.go +++ b/internal/casting/dockerswarmcasting/casting.go @@ -15,6 +15,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ rootcasting.Casting = (*dockerSwarmCasting)(nil) @@ -95,7 +96,11 @@ func (casting *dockerSwarmCasting) Forge(ctx context.Context, config installatio return materials, nil } -func (casting *dockerSwarmCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { +func (casting *dockerSwarmCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (casting *dockerSwarmCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { casting.logger.InfoContext(ctx, "Deploying stack to Docker Swarm") composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") diff --git a/internal/casting/ecsterraformcasting/casting.go b/internal/casting/ecsterraformcasting/casting.go index 0f511944..5a701243 100644 --- a/internal/casting/ecsterraformcasting/casting.go +++ b/internal/casting/ecsterraformcasting/casting.go @@ -19,6 +19,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ rootcasting.Casting = (*ecsCasting)(nil) @@ -111,7 +112,11 @@ func (c *ecsCasting) Forge(ctx context.Context, config installation.Casting, pou return materials, nil } -func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outputPath string) error { +func (c *ecsCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Running Terraform for ECS deployment") deploymentDir := filepath.Join(outputPath, rootcasting.DeploymentDir) diff --git a/internal/casting/installation/planner.go b/internal/casting/installation/planner.go index c6cc0294..a91bda87 100644 --- a/internal/casting/installation/planner.go +++ b/internal/casting/installation/planner.go @@ -17,6 +17,7 @@ import ( "github.com/signoz/foundry/internal/molding/telemetrykeepermolding" "github.com/signoz/foundry/internal/molding/telemetrystoremolding" "github.com/signoz/foundry/internal/planner" + "github.com/signoz/foundry/internal/runner" "github.com/signoz/foundry/internal/tooler" ) @@ -30,6 +31,7 @@ type Planner struct { logger *slog.Logger casting casting.Casting toolers []tooler.Tooler + runners []runner.Runner enricher molding.MoldingEnricher moldings []molding.Molding } @@ -47,6 +49,11 @@ func NewPlanner(ctx context.Context, c *installation.Casting, logger *slog.Logge return nil, err } + runners, err := registry.Runners(c.Spec.Deployment) + if err != nil { + return nil, err + } + enricher, err := castingStrategy.Enricher(ctx, c) if err != nil { return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to get molding enricher") @@ -66,6 +73,7 @@ func NewPlanner(ctx context.Context, c *installation.Casting, logger *slog.Logge logger: logger, casting: castingStrategy, toolers: toolers, + runners: runners, enricher: enricher, moldings: moldings, }, nil @@ -104,9 +112,17 @@ func (p *Planner) Forge(ctx context.Context, target string) ([]domain.Material, } func (p *Planner) Cast(ctx context.Context, poursPath string) error { - return p.casting.Cast(ctx, *p.config, poursPath) + return p.casting.Cast(ctx, *p.config, poursPath, p.runners) +} + +func (p *Planner) Uncast(ctx context.Context, poursPath string) error { + return p.casting.Uncast(ctx, *p.config, poursPath, p.runners) } func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } + +func (p *Planner) Runners() []runner.Runner { + return p.runners +} diff --git a/internal/casting/installation/registry.go b/internal/casting/installation/registry.go index bd6661f2..739a6238 100644 --- a/internal/casting/installation/registry.go +++ b/internal/casting/installation/registry.go @@ -15,8 +15,9 @@ import ( "github.com/signoz/foundry/internal/casting/rendercasting" "github.com/signoz/foundry/internal/casting/systemdcasting" foundryerrors "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/runner" + "github.com/signoz/foundry/internal/runner/composerunner" "github.com/signoz/foundry/internal/tooler" - "github.com/signoz/foundry/internal/tooler/dockercomposetooler" "github.com/signoz/foundry/internal/tooler/dockerswarmtooler" "github.com/signoz/foundry/internal/tooler/dockertooler" "github.com/signoz/foundry/internal/tooler/helmtooler" @@ -32,6 +33,11 @@ type CastingItem struct { // The toolers for the particular casting. Toolers []tooler.Tooler + + // The runners for the particular casting. The same objects gauge + // preflights are handed to Cast and Uncast, so the tools checked and + // the tools used cannot drift. + Runners []runner.Runner } type Registry struct { @@ -47,7 +53,7 @@ func NewRegistry(logger *slog.Logger) *Registry { Flavor: v1alpha1.FlavorCompose, }: { Casting: dockercomposecasting.New(logger), - Toolers: []tooler.Tooler{dockertooler.New(), dockercomposetooler.New()}, + Runners: []runner.Runner{composerunner.New(logger, composerunner.Config{})}, }, { Mode: v1alpha1.ModeSystemd, @@ -139,3 +145,11 @@ func (registry *Registry) Toolers(deployment v1alpha1.TypeDeployment) ([]tooler. } return item.Toolers, nil } + +func (registry *Registry) Runners(deployment v1alpha1.TypeDeployment) ([]runner.Runner, error) { + item, ok := registry.lookup(deployment) + if !ok { + return nil, foundryerrors.Newf(foundryerrors.TypeUnsupported, "deployment '%+v' is not supported, raise an issue at https://github.com/signoz/foundry/issues to request support for this deployment", deployment) + } + return item.Runners, nil +} diff --git a/internal/casting/installation/registry_test.go b/internal/casting/installation/registry_test.go new file mode 100644 index 00000000..cfef973a --- /dev/null +++ b/internal/casting/installation/registry_test.go @@ -0,0 +1,40 @@ +package installation + +import ( + "log/slog" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/runner/composerunner" + "github.com/stretchr/testify/assert" +) + +// The compose casting resolves its runner from the registry entry at cast +// time; this pins the entry so the lookup cannot silently go empty. +func TestRegistryRunners(t *testing.T) { + registry := NewRegistry(slog.New(slog.DiscardHandler)) + + tests := []struct { + name string + deployment v1alpha1.TypeDeployment + pass bool + }{ + {name: "DockerCompose_ComposeRunner", deployment: v1alpha1.TypeDeployment{Mode: v1alpha1.ModeDocker, Flavor: v1alpha1.FlavorCompose}, pass: true}, + {name: "DockerSwarm_NoRunnerYet", deployment: v1alpha1.TypeDeployment{Mode: v1alpha1.ModeDocker, Flavor: v1alpha1.FlavorSwarm}, pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runners, err := registry.Runners(tt.deployment) + assert.NoError(t, err) + + _, err = composerunner.From(runners) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + }) + } +} diff --git a/internal/casting/kuberneteshelmcasting/casting.go b/internal/casting/kuberneteshelmcasting/casting.go index b9a66662..7468f219 100644 --- a/internal/casting/kuberneteshelmcasting/casting.go +++ b/internal/casting/kuberneteshelmcasting/casting.go @@ -14,6 +14,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli" @@ -69,7 +70,11 @@ func (c *helmCasting) Forge(ctx context.Context, config installation.Casting, po return []domain.Material{valuesMaterial}, nil } -func (c *helmCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *helmCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (c *helmCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { valuesFile := filepath.Join(poursPath, rootcasting.DeploymentDir, "values.yaml") if _, err := os.Stat(valuesFile); os.IsNotExist(err) { diff --git a/internal/casting/kuberneteskustomizecasting/casting.go b/internal/casting/kuberneteskustomizecasting/casting.go index 06b76619..3efd8c8b 100644 --- a/internal/casting/kuberneteskustomizecasting/casting.go +++ b/internal/casting/kuberneteskustomizecasting/casting.go @@ -15,6 +15,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ rootcasting.Casting = (*kustomizeCasting)(nil) @@ -89,7 +90,11 @@ var clickhouseCRDs = []string{ "clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml", } -func (c *kustomizeCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *kustomizeCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (c *kustomizeCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Applying kustomize manifests") kustomizeDir := filepath.Join(poursPath, rootcasting.DeploymentDir) diff --git a/internal/casting/railwaytemplatecasting/casting.go b/internal/casting/railwaytemplatecasting/casting.go index d94ce1d9..baf73838 100644 --- a/internal/casting/railwaytemplatecasting/casting.go +++ b/internal/casting/railwaytemplatecasting/casting.go @@ -11,6 +11,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ casting.Casting = (*railwayTemplateCasting)(nil) @@ -160,11 +161,16 @@ func (c *railwayTemplateCasting) Forge(ctx context.Context, config installation. return materials, nil } -func (c *railwayTemplateCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *railwayTemplateCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Please use the template.") return nil } +func (c *railwayTemplateCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + c.logger.InfoContext(ctx, "Remove the deployment from Railway directly; foundry does not manage Railway resources.") + return nil +} + func getRailwayMaterial(config *installation.Casting) ([]domain.StructuredMaterial, error) { var materials []domain.StructuredMaterial diff --git a/internal/casting/rendercasting/casting.go b/internal/casting/rendercasting/casting.go index e47ccef9..654d5902 100644 --- a/internal/casting/rendercasting/casting.go +++ b/internal/casting/rendercasting/casting.go @@ -12,6 +12,7 @@ import ( "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" ) var _ casting.Casting = (*renderCasting)(nil) @@ -111,7 +112,7 @@ func (c *renderCasting) Forge(ctx context.Context, config installation.Casting, return materials, nil } -func (c *renderCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *renderCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Please run 'forge' first to generate the Render Casting", slog.String("pours_path", poursPath)) c.logger.InfoContext(ctx, "After forging, deploy render.yaml to Render using Infrastructure as Code", @@ -119,6 +120,12 @@ func (c *renderCasting) Cast(ctx context.Context, config installation.Casting, p return nil } +func (c *renderCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + c.logger.InfoContext(ctx, "Remove the deployment from Render directly; foundry does not manage Render resources", + slog.String("Docs", "https://render.com/docs/infrastructure-as-code#setup")) + return nil +} + func getRenderMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := renderYAMLTemplate.Execute(buf, config) diff --git a/internal/casting/systemdcasting/casting.go b/internal/casting/systemdcasting/casting.go index 2f2a171b..b3f52731 100644 --- a/internal/casting/systemdcasting/casting.go +++ b/internal/casting/systemdcasting/casting.go @@ -8,6 +8,7 @@ import ( rootcasting "github.com/signoz/foundry/internal/casting" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/molding" + "github.com/signoz/foundry/internal/runner" "github.com/signoz/foundry/internal/tooler" "github.com/signoz/foundry/internal/tooler/binarytooler" @@ -62,7 +63,11 @@ func (c *systemdCasting) Forge(ctx context.Context, cfg installation.Casting, po return materials, nil } -func (c *systemdCasting) Cast(ctx context.Context, config installation.Casting, poursPath string) error { +func (c *systemdCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + +func (c *systemdCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() diff --git a/internal/domain/event.go b/internal/domain/event.go index 28f06fcb..7ed3dfee 100644 --- a/internal/domain/event.go +++ b/internal/domain/event.go @@ -20,10 +20,11 @@ var ( EventGauge = Event{name: "gauge"} EventForge = Event{name: "forge"} EventCast = Event{name: "cast"} + EventUncast = Event{name: "uncast"} EventCatalog = Event{name: "catalog"} ) -var allEvents = []Event{EventGauge, EventForge, EventCast, EventCatalog} +var allEvents = []Event{EventGauge, EventForge, EventCast, EventUncast, EventCatalog} // NewEvent accepts only the names of declared base Event values. The returned // Event has no outcome; use Succeeded or Failed to attach one. diff --git a/internal/foundry/gauge.go b/internal/foundry/gauge.go index 43698d38..cbd8e057 100644 --- a/internal/foundry/gauge.go +++ b/internal/foundry/gauge.go @@ -24,6 +24,14 @@ func (foundry *Foundry) Gauge(ctx context.Context, machinery v1alpha1.Machinery) } foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", tooler.Name())) } + for _, r := range p.Runners() { + if err := r.Preflight(ctx); err != nil { + foundry.Logger.ErrorContext(ctx, "tool is not available or cannot be detected properly", slog.String("tool.name", r.Name()), foundryerrors.LogAttr(err)) + unavailableTools = append(unavailableTools, r.Name()) + continue + } + foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", r.Name())) + } if len(unavailableTools) > 0 { return foundryerrors.Newf(foundryerrors.TypeNotFound, "tools are not available, please install them and try again: %s", strings.Join(unavailableTools, ", ")) } diff --git a/internal/foundry/uncast.go b/internal/foundry/uncast.go new file mode 100644 index 00000000..121b3c3c --- /dev/null +++ b/internal/foundry/uncast.go @@ -0,0 +1,15 @@ +package foundry + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" +) + +func (foundry *Foundry) Uncast(ctx context.Context, machinery v1alpha1.Machinery, poursPath string) error { + p, err := foundry.newPlanner(ctx, machinery) + if err != nil { + return err + } + return p.Uncast(ctx, poursPath) +} diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 5791ca91..9c21cb9c 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -5,20 +5,22 @@ import ( "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/runner" "github.com/signoz/foundry/internal/tooler" ) // Planner is the per-Kind contract Foundry iterates against. Every Kind // expresses itself in the same vocabulary: // -// - identity: Machinery, Patches, Toolers +// - identity: Machinery, Patches, Toolers, Runners // - ordering: MoldingKinds (the moldings this Kind processes, in order) // - stages: EnrichStatus, Mold, MergeStatusIntoSpec -// - lifecycle: Forge, Cast +// - lifecycle: Forge, Cast, Uncast type Planner interface { Machinery() v1alpha1.Machinery Patches() []v1alpha1.PatchEntry Toolers() []tooler.Tooler + Runners() []runner.Runner MoldingKinds() []v1alpha1.MoldingKind EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind) error @@ -27,4 +29,5 @@ type Planner interface { Forge(ctx context.Context, target string) ([]domain.Material, error) Cast(ctx context.Context, poursPath string) error + Uncast(ctx context.Context, poursPath string) error } diff --git a/internal/runner/composerunner/runner.go b/internal/runner/composerunner/runner.go new file mode 100644 index 00000000..53bd9903 --- /dev/null +++ b/internal/runner/composerunner/runner.go @@ -0,0 +1,141 @@ +package composerunner + +import ( + "context" + "io" + "log/slog" + "os" + "os/exec" + "strings" + "sync" + + "github.com/signoz/foundry/internal/errors" + "github.com/signoz/foundry/internal/runner" +) + +var _ runner.Runner = (*Runner)(nil) + +// Config is what must be in place for the runner to serve any casting. +// Everything that varies per run (the compose file, usage options) is a +// method argument instead. +type Config struct { + // Stdout and Stderr carry the tool's own output. Zero values use the + // process streams. + Stdout io.Writer + Stderr io.Writer +} + +// Runner interacts with docker compose. It holds no per-run state: the +// compose file and options are always arguments. The only mutable field is +// the memoized command probe, which is environment-derived, not run-derived. +type Runner struct { + logger *slog.Logger + config Config + + mu sync.Mutex + command []string +} + +func New(logger *slog.Logger, config Config) *Runner { + if config.Stdout == nil { + config.Stdout = os.Stdout + } + + if config.Stderr == nil { + config.Stderr = os.Stderr + } + + return &Runner{logger: logger, config: config} +} + +// From picks the compose runner out of the runners a casting receives. It +// lives here rather than in package runner so the contract package never +// imports its implementations. +func From(runners []runner.Runner) (*Runner, error) { + for _, r := range runners { + if compose, ok := r.(*Runner); ok { + return compose, nil + } + } + + return nil, errors.Newf(errors.TypeNotFound, "compose runner is not registered for this casting") +} + +// Options is the per-call usage slot. Empty today; profiles and their kin +// land here when a casting needs them. +type Options struct{} + +func (r *Runner) Name() string { + return "docker compose" +} + +// Preflight resolves how compose is invoked on this machine: the docker +// compose plugin, or the legacy docker-compose binary. This absorbs what the +// docker and docker compose toolers checked separately. +func (r *Runner) Preflight(ctx context.Context) error { + _, err := r.compose(ctx) + + return err +} + +// Up converges the deployment the compose file describes. +func (r *Runner) Up(ctx context.Context, composeFile string, _ Options) error { + if err := r.run(ctx, "-f", composeFile, "up", "-d"); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "docker compose up failed") + } + + return nil +} + +// Down removes the containers and networks the compose file created. Volumes +// stay: uncast never crosses the data line. +func (r *Runner) Down(ctx context.Context, composeFile string, _ Options) error { + if err := r.run(ctx, "-f", composeFile, "down"); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "docker compose down failed") + } + + return nil +} + +func (r *Runner) run(ctx context.Context, args ...string) error { + command, err := r.compose(ctx) + if err != nil { + return err + } + + full := append(append([]string{}, command[1:]...), args...) + r.logger.DebugContext(ctx, "running command", slog.String("command", strings.Join(append([]string{command[0]}, full...), " "))) + + cmd := exec.CommandContext(ctx, command[0], full...) + cmd.Stdout = r.config.Stdout + cmd.Stderr = r.config.Stderr + + return cmd.Run() +} + +// compose probes for the docker compose plugin, then for the legacy +// docker-compose binary, memoizing the answer. +func (r *Runner) compose(ctx context.Context) ([]string, error) { + r.mu.Lock() + defer r.mu.Unlock() + + if r.command != nil { + return r.command, nil + } + + if _, err := exec.LookPath("docker"); err == nil { + if err := exec.CommandContext(ctx, "docker", "compose", "version").Run(); err == nil { + r.command = []string{"docker", "compose"} + + return r.command, nil + } + } + + if _, err := exec.LookPath("docker-compose"); err == nil { + r.command = []string{"docker-compose"} + + return r.command, nil + } + + return nil, errors.Newf(errors.TypeNotFound, "docker compose is not available: install the docker compose plugin or docker-compose") +} diff --git a/internal/runner/composerunner/runner_test.go b/internal/runner/composerunner/runner_test.go new file mode 100644 index 00000000..62f62269 --- /dev/null +++ b/internal/runner/composerunner/runner_test.go @@ -0,0 +1,94 @@ +package composerunner + +import ( + "context" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/signoz/foundry/internal/runner" + "github.com/stretchr/testify/assert" +) + +type otherRunner struct{} + +func (otherRunner) Name() string { return "other" } +func (otherRunner) Preflight(ctx context.Context) error { return nil } + +func TestNew(t *testing.T) { + tests := []struct { + name string + config Config + }{ + {name: "ZeroValue_ProcessStreams", config: Config{}}, + {name: "Streams_Kept", config: Config{Stdout: io.Discard, Stderr: io.Discard}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := New(slog.New(slog.DiscardHandler), tt.config) + + assert.NotNil(t, r.config.Stdout) + assert.NotNil(t, r.config.Stderr) + assert.Equal(t, "docker compose", r.Name()) + }) + } +} + +func TestFrom(t *testing.T) { + compose := New(slog.New(slog.DiscardHandler), Config{}) + + tests := []struct { + name string + runners []runner.Runner + pass bool + }{ + {name: "Registered_Found", runners: []runner.Runner{compose}, pass: true}, + {name: "AmongOthers_Found", runners: []runner.Runner{otherRunner{}, compose}, pass: true}, + {name: "Empty_NotFound", runners: nil, pass: false}, + {name: "OnlyOthers_NotFound", runners: []runner.Runner{otherRunner{}}, pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + found, err := From(tt.runners) + if !tt.pass { + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, compose, found) + }) + } +} + +// Up then Down against a minimal compose file; needs a running docker engine, +// so it skips wherever one is absent. +func TestUpDown(t *testing.T) { + if testing.Short() { + t.Skip("skipping docker engine test in short mode") + } + + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker is not available") + } + + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skip("docker engine is not running") + } + + dir := t.TempDir() + composeFile := filepath.Join(dir, "compose.yaml") + contents := "name: composerunner-test\nservices:\n ok:\n image: busybox:stable\n command: [\"sleep\", \"300\"]\n" + assert.NoError(t, os.WriteFile(composeFile, []byte(contents), 0o644)) + + r := New(slog.New(slog.DiscardHandler), Config{Stdout: io.Discard, Stderr: io.Discard}) + + assert.NoError(t, r.Preflight(context.Background())) + assert.NoError(t, r.Up(context.Background(), composeFile, Options{})) + assert.NoError(t, r.Down(context.Background(), composeFile, Options{})) +} diff --git a/internal/runner/runner.go b/internal/runner/runner.go new file mode 100644 index 00000000..81c9ef26 --- /dev/null +++ b/internal/runner/runner.go @@ -0,0 +1,25 @@ +// Package runner is the contract for foundry's tool-interaction layer: one +// package per tool under internal/runner, shared by every casting whose +// flavor speaks that tool. +// +// A runner is the interface between foundry and one tool. It is casting-, +// kind- and set-blind: no casting types, no ordering, no bindings. Runners +// hold no per-run state -- the root directory and usage options are always +// method arguments, never fields -- and all deployment state lives in the +// tool's own record. +package runner + +import "context" + +// Runner is the surface generic consumers need: gauge preflights every +// runner a casting is registered with. The tool's own operations live on +// each package's concrete type, which castings receive and use directly. +type Runner interface { + Name() string + + // Preflight verifies the environment satisfies what the runner's + // configuration demands: the engine is present or can be pinned into + // place. It needs no root and no connection, which is what makes it + // runnable at gauge time. + Preflight(ctx context.Context) error +} From 87289fee202eae1bb67976ebf0b4fa9b8a505dc1 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Mon, 10 Aug 2026 02:32:30 +0530 Subject: [PATCH 38/38] fix: lift off the ownership to runners --- .../compose-mcp/pours/deployment/compose.yaml | 32 +++ .../compose/pours/deployment/compose.yaml | 28 +++ internal/casting/casting.go | 4 +- .../dockercomposecasting/casting.go | 73 ++---- .../dockerswarmcasting/casting.go | 28 ++- internal/casting/collectionagent/registry.go | 2 +- .../casting/collectionagent/registry_test.go | 2 +- internal/casting/coolifycasting/casting.go | 2 + internal/casting/coolifycasting/enricher.go | 1 - .../casting/dockercomposecasting/casting.go | 45 ++-- .../templates/compose.yaml.gotmpl | 32 +++ .../casting/dockerswarmcasting/casting.go | 9 +- .../casting/ecsterraformcasting/casting.go | 9 +- internal/casting/infrastructure/planner.go | 9 + internal/casting/installation/registry.go | 2 +- .../casting/installation/registry_test.go | 2 +- .../casting/kuberneteshelmcasting/casting.go | 9 +- .../kuberneteskustomizecasting/casting.go | 9 +- .../casting/railwaytemplatecasting/casting.go | 4 +- internal/casting/rendercasting/casting.go | 4 +- internal/casting/systemdcasting/casting.go | 9 +- internal/domain/ownership.go | 105 ++++++--- internal/domain/ownership_test.go | 151 ++++++++---- internal/foundry/gauge.go | 35 ++- internal/runner/composerunner/runner.go | 218 +++++++++++++----- internal/runner/composerunner/runner_test.go | 188 ++++++++++++--- internal/runner/runner.go | 22 +- 27 files changed, 710 insertions(+), 324 deletions(-) diff --git a/docs/examples/docker/compose-mcp/pours/deployment/compose.yaml b/docs/examples/docker/compose-mcp/pours/deployment/compose.yaml index 0c7ce940..a59c73f4 100644 --- a/docs/examples/docker/compose-mcp/pours/deployment/compose.yaml +++ b/docs/examples/docker/compose-mcp/pours/deployment/compose.yaml @@ -17,6 +17,10 @@ services: - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m image: signoz/signoz-otel-collector:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: signoz-network: aliases: @@ -36,6 +40,10 @@ services: - SIGNOZ_URL=http://signoz-signoz-0:8080 - TRANSPORT_MODE=http image: signoz/signoz-mcp-server:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: signoz-network: aliases: @@ -58,6 +66,10 @@ services: - pg_isready -U signoz -d signoz timeout: 10s image: postgres:16 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -82,6 +94,10 @@ services: - http://localhost:8080/api/v1/health timeout: 10s image: signoz/signoz:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network ports: @@ -107,6 +123,10 @@ services: - ls timeout: 10s image: clickhouse/clickhouse-keeper:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -135,6 +155,10 @@ services: - http://localhost:8123/ping timeout: 10s image: clickhouse/clickhouse-server:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -158,6 +182,10 @@ services: mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile container_name: signoz-telemetrystore-clickhouse-user-scripts image: clickhouse/clickhouse-server:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: on-failure @@ -178,6 +206,10 @@ services: - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m image: signoz/signoz-otel-collector:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: on-failure diff --git a/docs/examples/docker/compose/pours/deployment/compose.yaml b/docs/examples/docker/compose/pours/deployment/compose.yaml index c3f1acb1..8f6150d0 100644 --- a/docs/examples/docker/compose/pours/deployment/compose.yaml +++ b/docs/examples/docker/compose/pours/deployment/compose.yaml @@ -17,6 +17,10 @@ services: - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m image: signoz/signoz-otel-collector:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: signoz-network: aliases: @@ -43,6 +47,10 @@ services: - pg_isready -U signoz -d signoz timeout: 10s image: postgres:16 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -67,6 +75,10 @@ services: - http://localhost:8080/api/v1/health timeout: 10s image: signoz/signoz:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network ports: @@ -92,6 +104,10 @@ services: - ls timeout: 10s image: clickhouse/clickhouse-keeper:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -120,6 +136,10 @@ services: - http://localhost:8123/ping timeout: 10s image: clickhouse/clickhouse-server:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: unless-stopped @@ -143,6 +163,10 @@ services: mv histogram-quantile /var/lib/clickhouse/user_scripts/histogramQuantile container_name: signoz-telemetrystore-clickhouse-user-scripts image: clickhouse/clickhouse-server:25.12.5 + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: on-failure @@ -163,6 +187,10 @@ services: - SIGNOZ_OTEL_COLLECTOR_CLICKHOUSE_DSN=tcp://signoz-telemetrystore-clickhouse-0-0:9000 - SIGNOZ_OTEL_COLLECTOR_TIMEOUT=10m image: signoz/signoz-otel-collector:latest + labels: + foundry.signoz.io/kind: Installation + foundry.signoz.io/managed-by: foundry + foundry.signoz.io/name: signoz networks: - signoz-network restart: on-failure diff --git a/internal/casting/casting.go b/internal/casting/casting.go index 974b1998..813f74df 100644 --- a/internal/casting/casting.go +++ b/internal/casting/casting.go @@ -20,8 +20,8 @@ type Casting interface { // Generates all the files needed for casting. Forge(ctx context.Context, config installation.Casting, poursPath string) ([]domain.Material, error) - // Runs the forged files. Runners are the registry-listed tool - // interfaces for this casting; usage options are passed per call. + // Runs the forged files. Runners are the tool interfaces the registry + // lists for this casting. Cast(ctx context.Context, config installation.Casting, poursPath string, runners []runner.Runner) error // Removes what Cast deployed: definitions only, never data, never diff --git a/internal/casting/collectionagent/dockercomposecasting/casting.go b/internal/casting/collectionagent/dockercomposecasting/casting.go index ab032a10..37690eb5 100644 --- a/internal/casting/collectionagent/dockercomposecasting/casting.go +++ b/internal/casting/collectionagent/dockercomposecasting/casting.go @@ -4,13 +4,9 @@ import ( "bytes" "context" "log/slog" - "os" - "os/exec" "path/filepath" - "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/collectionagent" - "github.com/signoz/foundry/internal/domain" foundryerrors "github.com/signoz/foundry/internal/errors" collectionagentmolding "github.com/signoz/foundry/internal/molding/collectionagent" "github.com/signoz/foundry/internal/pourer" @@ -45,72 +41,31 @@ func (c *dockerComposeCasting) Forge(ctx context.Context, config collectionagent return nil } -func (c *dockerComposeCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error { - composeFile, compose, err := c.deployment(outputPath, p, runners) - if err != nil { - return err - } - - if err := c.checkOwnership(ctx, config); err != nil { - return err - } - - c.logger.InfoContext(ctx, "Removing the collection agent; volumes and their data stay") - - return compose.Down(ctx, composeFile, composerunner.Options{}) -} - func (c *dockerComposeCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error { - composeFile, compose, err := c.deployment(outputPath, p, runners) + compose, err := composerunner.Lookup(runners) if err != nil { return err } - if err := c.checkOwnership(ctx, config); err != nil { - return err - } - - return compose.Up(ctx, composeFile, composerunner.Options{}) + return compose.Up(ctx, c.options(config, outputPath, p)) } -// deployment resolves the compose file this casting forged and the runner -// that executes it. -func (c *dockerComposeCasting) deployment(outputPath string, p *pourer.Pourer, runners []runner.Runner) (string, *composerunner.Runner, error) { - composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") - - if _, err := os.Stat(composeFile); os.IsNotExist(err) { - return "", nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) - } - - compose, err := composerunner.From(runners) +// Uncast removes the agent's containers and networks; volumes stay. +func (c *dockerComposeCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, runners []runner.Runner) error { + compose, err := composerunner.Lookup(runners) if err != nil { - return "", nil, err + return err } - return composeFile, compose, nil + return compose.Down(ctx, c.options(config, outputPath, p)) } -// checkOwnership refuses to deploy over a compose project of the same name -// that belongs to a different foundry Kind. Unlabeled containers only warn: -// they are either a pre-label foundry deployment or a foreign project. -func (c *dockerComposeCasting) checkOwnership(ctx context.Context, config collectionagent.Casting) error { - out, err := exec.CommandContext(ctx, "docker", "ps", "-a", - "--filter", "label=com.docker.compose.project="+config.Metadata.Name, - "--format", `{{.Label "`+v1alpha1.LabelKind.Key+`"}}`).Output() - if err != nil { - c.logger.WarnContext(ctx, "skipping the ownership check: could not read labels from docker", foundryerrors.LogAttr(err)) - return nil +// options states the project this casting owns, so the runner refuses a +// project of the same name that belongs to another foundry Kind. +func (c *dockerComposeCasting) options(config collectionagent.Casting, outputPath string, p *pourer.Pourer) composerunner.Options { + return composerunner.Options{ + File: filepath.Join(outputPath, p.Dir(), "compose.yaml"), + Project: config.Metadata.Name, + Owner: config.Labels(), } - - ownership := domain.ParseOwnership(string(out)) - - if foreign, conflict := ownership.Foreign(config.Kind().String()); conflict { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "%q already belongs to a foundry %s on this host: choose a different metadata.name or remove the existing deployment", config.Metadata.Name, foreign) - } - - if ownership.HasUnlabeled() { - c.logger.WarnContext(ctx, "compose project has containers without foundry ownership labels", slog.String("project", config.Metadata.Name)) - } - - return nil } diff --git a/internal/casting/collectionagent/dockerswarmcasting/casting.go b/internal/casting/collectionagent/dockerswarmcasting/casting.go index c865cfe7..70041fe7 100644 --- a/internal/casting/collectionagent/dockerswarmcasting/casting.go +++ b/internal/casting/collectionagent/dockerswarmcasting/casting.go @@ -46,10 +46,6 @@ func (c *dockerSwarmCasting) Forge(ctx context.Context, config collectionagent.C return nil } -func (c *dockerSwarmCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, _ []runner.Runner) error { - return foundryerrors.Newf(foundryerrors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (c *dockerSwarmCasting) Cast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, _ []runner.Runner) error { composeFile := filepath.Join(outputPath, p.Dir(), "compose.yaml") @@ -75,9 +71,15 @@ func (c *dockerSwarmCasting) Cast(ctx context.Context, config collectionagent.Ca return cmd.Run() } +// Uncast is not implemented for this casting yet. +func (c *dockerSwarmCasting) Uncast(ctx context.Context, config collectionagent.Casting, outputPath string, p *pourer.Pourer, _ []runner.Runner) error { + return foundryerrors.Newf(foundryerrors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + // checkOwnership refuses to deploy over a swarm stack of the same name that -// belongs to a different foundry Kind. Unlabeled task containers only warn: -// they are either a pre-label foundry deployment or a foreign stack. +// belongs to a different foundry Kind. Task containers that record no owner +// only warn: they are either a pre-label foundry deployment or a foreign +// stack. func (c *dockerSwarmCasting) checkOwnership(ctx context.Context, config collectionagent.Casting) error { out, err := exec.CommandContext(ctx, "docker", "ps", "-a", "--filter", "label=com.docker.stack.namespace="+config.Metadata.Name, @@ -87,13 +89,19 @@ func (c *dockerSwarmCasting) checkOwnership(ctx context.Context, config collecti return nil } - ownership := domain.ParseOwnership(string(out)) + owners := []domain.Owner{} + for _, kind := range strings.Split(strings.TrimRight(string(out), "\n"), "\n") { + owners = append(owners, domain.Owner{v1alpha1.LabelKind.Key: kind}) + } + + ownership := domain.NewOwnership(owners...) + self := domain.Owner{v1alpha1.LabelKind.Key: config.Kind().String()} - if foreign, conflict := ownership.Foreign(config.Kind().String()); conflict { - return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "%q already belongs to a foundry %s on this host: choose a different metadata.name or remove the existing deployment", config.Metadata.Name, foreign) + if foreign, conflict := ownership.Foreign(self); conflict { + return foundryerrors.Newf(foundryerrors.TypeInvalidInput, "%q already belongs to a foundry %s on this host: choose a different metadata.name or remove the existing deployment", config.Metadata.Name, foreign[v1alpha1.LabelKind.Key]) } - if ownership.HasUnlabeled() { + if ownership.HasUnowned() { c.logger.WarnContext(ctx, "swarm stack has task containers without foundry ownership labels", slog.String("stack", config.Metadata.Name)) } diff --git a/internal/casting/collectionagent/registry.go b/internal/casting/collectionagent/registry.go index 00208285..7da006a9 100644 --- a/internal/casting/collectionagent/registry.go +++ b/internal/casting/collectionagent/registry.go @@ -33,7 +33,7 @@ func NewRegistry(logger *slog.Logger) *Registry { Flavor: v1alpha1.FlavorCompose, }: { Casting: dockercomposecasting.New(logger), - Runners: []runner.Runner{composerunner.New(logger, composerunner.Config{})}, + Runners: []runner.Runner{composerunner.New(logger)}, }, { Mode: v1alpha1.ModeDocker, diff --git a/internal/casting/collectionagent/registry_test.go b/internal/casting/collectionagent/registry_test.go index 9df10562..dee863c5 100644 --- a/internal/casting/collectionagent/registry_test.go +++ b/internal/casting/collectionagent/registry_test.go @@ -28,7 +28,7 @@ func TestRegistryRunners(t *testing.T) { runners, err := registry.Runners(tt.deployment) assert.NoError(t, err) - _, err = composerunner.From(runners) + _, err = composerunner.Lookup(runners) if !tt.pass { assert.Error(t, err) return diff --git a/internal/casting/coolifycasting/casting.go b/internal/casting/coolifycasting/casting.go index ac1a430f..9d6687b3 100644 --- a/internal/casting/coolifycasting/casting.go +++ b/internal/casting/coolifycasting/casting.go @@ -57,6 +57,8 @@ func (c *coolifyCasting) Cast(ctx context.Context, config installation.Casting, return nil } +// Uncast tells the operator where to remove the deployment: foundry does not +// drive Coolify. func (c *coolifyCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Remove the stack from Coolify directly; foundry does not manage Coolify resources", slog.String("docs", "https://coolify.io/docs/knowledge-base/docker/compose")) diff --git a/internal/casting/coolifycasting/enricher.go b/internal/casting/coolifycasting/enricher.go index 802faeb3..4e496a69 100644 --- a/internal/casting/coolifycasting/enricher.go +++ b/internal/casting/coolifycasting/enricher.go @@ -112,7 +112,6 @@ func (enricher *coolifyMoldingEnricher) EnrichStatus(ctx context.Context, kind v config.Spec.Ingester.Status.Addresses.OTLP = []string{ domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4318).String(), domain.MustNewAddress("tcp", config.Metadata.Name+"-ingester", 4317).String(), - } case v1alpha1.MoldingKindMCP: if !config.Spec.MCP.Spec.IsEnabled() { diff --git a/internal/casting/dockercomposecasting/casting.go b/internal/casting/dockercomposecasting/casting.go index 51fc47cf..5715bbbb 100644 --- a/internal/casting/dockercomposecasting/casting.go +++ b/internal/casting/dockercomposecasting/casting.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "log/slog" - "os" "path/filepath" "github.com/signoz/foundry/api/v1alpha1/installation" @@ -98,48 +97,34 @@ func (casting *dockerComposeCasting) Forge(ctx context.Context, config installat return materials, nil } -func (casting *dockerComposeCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { - casting.logger.InfoContext(ctx, "Removing the compose deployment; volumes and their data stay") - - composeFile, compose, err := casting.deployment(outputPath, runners) +func (casting *dockerComposeCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { + compose, err := composerunner.Lookup(runners) if err != nil { return err } - return compose.Down(ctx, composeFile, composerunner.Options{}) + return compose.Up(ctx, casting.options(config, outputPath)) } -func (casting *dockerComposeCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { - casting.logger.InfoContext(ctx, "Executing commands for platform") - - composeFile, compose, err := casting.deployment(outputPath, runners) +// Uncast removes the containers and networks; the volumes holding component +// data stay. +func (casting *dockerComposeCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, runners []runner.Runner) error { + compose, err := composerunner.Lookup(runners) if err != nil { return err } - if err := compose.Up(ctx, composeFile, composerunner.Options{}); err != nil { - return err - } - - casting.logger.InfoContext(ctx, "Command executed successfully") - - return nil + return compose.Down(ctx, casting.options(config, outputPath)) } -// deployment resolves the compose file this casting forged and the runner -// that executes it. -func (casting *dockerComposeCasting) deployment(outputPath string, runners []runner.Runner) (string, *composerunner.Runner, error) { - composeFile := filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml") - if _, err := os.Stat(composeFile); os.IsNotExist(err) { - return "", nil, foundryerrors.Newf(foundryerrors.TypeNotFound, "compose file does not exist at path: %s", composeFile) +// options states the project this casting owns, so the runner refuses a +// project of the same name that belongs to another foundry Kind. +func (casting *dockerComposeCasting) options(config installation.Casting, outputPath string) composerunner.Options { + return composerunner.Options{ + File: filepath.Join(outputPath, rootcasting.DeploymentDir, "compose.yaml"), + Project: config.Metadata.Name, + Owner: config.Labels(), } - - compose, err := composerunner.From(runners) - if err != nil { - return "", nil, err - } - - return composeFile, compose, nil } func getComposeMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { diff --git a/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl b/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl index 2032ffb2..7f271539 100644 --- a/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl +++ b/internal/casting/dockercomposecasting/templates/compose.yaml.gotmpl @@ -5,6 +5,10 @@ services: container_name: {{ $.Metadata.Name }}-telemetrykeeper-{{ $.Spec.TelemetryKeeper.Kind }}-{{ $replicaIdx }} image: {{ $.Spec.TelemetryKeeper.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network {{- if eq $.Spec.TelemetryKeeper.Kind.String "zookeeper" }} @@ -68,6 +72,10 @@ services: {{- end }} image: {{ $.Spec.TelemetryStore.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network environment: @@ -100,6 +108,10 @@ services: container_name: {{ $.Metadata.Name }}-telemetrystore-{{ $.Spec.TelemetryStore.Kind }}-user-scripts image: {{ $.Spec.TelemetryStore.Spec.Image }} restart: on-failure + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network volumes: @@ -122,6 +134,10 @@ services: container_name: {{ $.Metadata.Name }}-metastore-{{ $.Spec.MetaStore.Kind }}-{{ $replicaIdx }} image: {{ $.Spec.MetaStore.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network {{- if $.Spec.MetaStore.Spec.Env }} @@ -146,6 +162,10 @@ services: ingester: image: {{ $.Spec.Ingester.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: {{ $.Metadata.Name }}-network: aliases: @@ -185,6 +205,10 @@ services: container_name: {{ $.Metadata.Name }}-signoz-{{ $replicaIdx }} image: {{ $.Spec.Signoz.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network {{- if $.Spec.Signoz.Spec.Env }} @@ -216,6 +240,10 @@ services: mcp: image: {{ $.Spec.MCP.Spec.Image }} restart: unless-stopped + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: {{ $.Metadata.Name }}-network: aliases: @@ -240,6 +268,10 @@ services: container_name: {{ $.Metadata.Name }}-telemetrystore-migrator image: {{ $.Spec.Ingester.Spec.Image }} restart: on-failure + labels: + {{- range $key, $value := $.Labels }} + {{ $key }}: {{ $value | quote }} + {{- end }} networks: - {{ $.Metadata.Name }}-network entrypoint: diff --git a/internal/casting/dockerswarmcasting/casting.go b/internal/casting/dockerswarmcasting/casting.go index 61a626e1..95b1b7f1 100644 --- a/internal/casting/dockerswarmcasting/casting.go +++ b/internal/casting/dockerswarmcasting/casting.go @@ -96,10 +96,6 @@ func (casting *dockerSwarmCasting) Forge(ctx context.Context, config installatio return materials, nil } -func (casting *dockerSwarmCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { - return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (casting *dockerSwarmCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { casting.logger.InfoContext(ctx, "Deploying stack to Docker Swarm") @@ -132,6 +128,11 @@ func (casting *dockerSwarmCasting) Cast(ctx context.Context, config installation return nil } +// Uncast is not implemented for this casting yet. +func (casting *dockerSwarmCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + func getComposeMaterial(config *installation.Casting, path string) (domain.StructuredMaterial, error) { buf := bytes.NewBuffer(nil) err := composeYAMLTemplate.Execute(buf, config) diff --git a/internal/casting/ecsterraformcasting/casting.go b/internal/casting/ecsterraformcasting/casting.go index 5a701243..c39b966f 100644 --- a/internal/casting/ecsterraformcasting/casting.go +++ b/internal/casting/ecsterraformcasting/casting.go @@ -112,10 +112,6 @@ func (c *ecsCasting) Forge(ctx context.Context, config installation.Casting, pou return materials, nil } -func (c *ecsCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { - return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Running Terraform for ECS deployment") @@ -157,6 +153,11 @@ func (c *ecsCasting) Cast(ctx context.Context, config installation.Casting, outp return nil } +// Uncast is not implemented for this casting yet. +func (c *ecsCasting) Uncast(ctx context.Context, config installation.Casting, outputPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + // templateData binds the casting to the substrate it runs on. Forge and the // enricher each render from their own config; both come through here. func (c *ecsCasting) templateData(config installation.Casting) (templateData, error) { diff --git a/internal/casting/infrastructure/planner.go b/internal/casting/infrastructure/planner.go index 7c0ff9d5..6dd1c6c8 100644 --- a/internal/casting/infrastructure/planner.go +++ b/internal/casting/infrastructure/planner.go @@ -13,6 +13,7 @@ import ( "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" "github.com/signoz/foundry/internal/planner" "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/runner" "github.com/signoz/foundry/internal/tooler" ) @@ -107,4 +108,12 @@ func (p *Planner) Cast(ctx context.Context, poursPath string) error { return p.casting.Cast(ctx, *p.config, poursPath, pourer.New(strings.ToLower(p.config.Kind().String()))) } +func (p *Planner) Uncast(ctx context.Context, poursPath string) error { + return foundryerrors.Newf(foundryerrors.TypeUnsupported, "uncast is not implemented for the infrastructure kind yet") +} + func (p *Planner) Toolers() []tooler.Tooler { return p.toolers } + +// Runners is empty: terraform is still invoked by the casting itself, and the +// terraform tooler is what gauge checks. +func (p *Planner) Runners() []runner.Runner { return nil } diff --git a/internal/casting/installation/registry.go b/internal/casting/installation/registry.go index 739a6238..0cf8cb78 100644 --- a/internal/casting/installation/registry.go +++ b/internal/casting/installation/registry.go @@ -53,7 +53,7 @@ func NewRegistry(logger *slog.Logger) *Registry { Flavor: v1alpha1.FlavorCompose, }: { Casting: dockercomposecasting.New(logger), - Runners: []runner.Runner{composerunner.New(logger, composerunner.Config{})}, + Runners: []runner.Runner{composerunner.New(logger)}, }, { Mode: v1alpha1.ModeSystemd, diff --git a/internal/casting/installation/registry_test.go b/internal/casting/installation/registry_test.go index cfef973a..83caa3b2 100644 --- a/internal/casting/installation/registry_test.go +++ b/internal/casting/installation/registry_test.go @@ -28,7 +28,7 @@ func TestRegistryRunners(t *testing.T) { runners, err := registry.Runners(tt.deployment) assert.NoError(t, err) - _, err = composerunner.From(runners) + _, err = composerunner.Lookup(runners) if !tt.pass { assert.Error(t, err) return diff --git a/internal/casting/kuberneteshelmcasting/casting.go b/internal/casting/kuberneteshelmcasting/casting.go index 7468f219..e0474059 100644 --- a/internal/casting/kuberneteshelmcasting/casting.go +++ b/internal/casting/kuberneteshelmcasting/casting.go @@ -70,10 +70,6 @@ func (c *helmCasting) Forge(ctx context.Context, config installation.Casting, po return []domain.Material{valuesMaterial}, nil } -func (c *helmCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { - return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (c *helmCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { valuesFile := filepath.Join(poursPath, rootcasting.DeploymentDir, "values.yaml") @@ -195,6 +191,11 @@ func (c *helmCasting) Cast(ctx context.Context, config installation.Casting, pou return nil } +// Uncast is not implemented for this casting yet. +func (c *helmCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + func (c *helmCasting) shouldForgeChart(config *installation.Casting) bool { if config.Metadata.Annotations == nil { return false diff --git a/internal/casting/kuberneteskustomizecasting/casting.go b/internal/casting/kuberneteskustomizecasting/casting.go index 3efd8c8b..6cca8ac5 100644 --- a/internal/casting/kuberneteskustomizecasting/casting.go +++ b/internal/casting/kuberneteskustomizecasting/casting.go @@ -90,10 +90,6 @@ var clickhouseCRDs = []string{ "clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml", } -func (c *kustomizeCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { - return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (c *kustomizeCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { c.logger.InfoContext(ctx, "Applying kustomize manifests") @@ -125,6 +121,11 @@ func (c *kustomizeCasting) Cast(ctx context.Context, config installation.Casting return nil } +// Uncast is not implemented for this casting yet. +func (c *kustomizeCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + func (c *kustomizeCasting) applyCRDs(ctx context.Context) error { c.logger.InfoContext(ctx, "Applying ClickHouse CRDs", slog.String("version", clickhouseOperatorVersion)) diff --git a/internal/casting/railwaytemplatecasting/casting.go b/internal/casting/railwaytemplatecasting/casting.go index baf73838..52abaf38 100644 --- a/internal/casting/railwaytemplatecasting/casting.go +++ b/internal/casting/railwaytemplatecasting/casting.go @@ -166,8 +166,10 @@ func (c *railwayTemplateCasting) Cast(ctx context.Context, config installation.C return nil } +// Uncast tells the operator where to remove the deployment: foundry does not +// drive Railway. func (c *railwayTemplateCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { - c.logger.InfoContext(ctx, "Remove the deployment from Railway directly; foundry does not manage Railway resources.") + c.logger.InfoContext(ctx, "Remove the services from Railway directly; foundry does not manage Railway resources.") return nil } diff --git a/internal/casting/rendercasting/casting.go b/internal/casting/rendercasting/casting.go index 654d5902..0aabf83e 100644 --- a/internal/casting/rendercasting/casting.go +++ b/internal/casting/rendercasting/casting.go @@ -120,8 +120,10 @@ func (c *renderCasting) Cast(ctx context.Context, config installation.Casting, p return nil } +// Uncast tells the operator where to remove the deployment: foundry does not +// drive Render. func (c *renderCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { - c.logger.InfoContext(ctx, "Remove the deployment from Render directly; foundry does not manage Render resources", + c.logger.InfoContext(ctx, "Remove the services from Render directly; foundry does not manage Render resources", slog.String("Docs", "https://render.com/docs/infrastructure-as-code#setup")) return nil } diff --git a/internal/casting/systemdcasting/casting.go b/internal/casting/systemdcasting/casting.go index b3f52731..5d3315cb 100644 --- a/internal/casting/systemdcasting/casting.go +++ b/internal/casting/systemdcasting/casting.go @@ -63,10 +63,6 @@ func (c *systemdCasting) Forge(ctx context.Context, cfg installation.Casting, po return materials, nil } -func (c *systemdCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { - return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") -} - func (c *systemdCasting) Cast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) defer cancel() @@ -100,6 +96,11 @@ func (c *systemdCasting) Cast(ctx context.Context, config installation.Casting, return nil } +// Uncast is not implemented for this casting yet. +func (c *systemdCasting) Uncast(ctx context.Context, config installation.Casting, poursPath string, _ []runner.Runner) error { + return errors.Newf(errors.TypeUnsupported, "uncast is not implemented for this casting yet") +} + func (c *systemdCasting) forgeTelemetryKeeper(cfg *installation.Casting) ([]domain.Material, error) { if !cfg.Spec.TelemetryKeeper.Spec.IsEnabled() { return nil, nil diff --git a/internal/domain/ownership.go b/internal/domain/ownership.go index 976e78b6..afbedea7 100644 --- a/internal/domain/ownership.go +++ b/internal/domain/ownership.go @@ -1,58 +1,101 @@ package domain -import "strings" +import ( + "maps" + "slices" + "strings" +) -// Ownership captures which foundry Kinds own a group of platform workloads, -// derived from the foundry.signoz.io/kind label carried by each workload. -type Ownership struct { - kinds []string - unlabeled bool +// Owner is whoever a workload belongs to, as the attributes a platform records +// on it: labels on a container, tags on a cloud resource. Two workloads share +// an owner when every attribute they were asked for matches, so an owner is +// compared as a whole and never by one attribute at a time. +type Owner map[string]string + +// IsZero reports an owner that recorded nothing. A workload carrying none of +// the attributes asked for belongs to no one foundry can name, which is not +// the same as belonging to an owner whose every attribute is empty. +func (owner Owner) IsZero() bool { + for _, value := range owner { + if value != "" { + return false + } + } + + return true } -// ParseOwnership derives ownership from label values, one workload per line; -// an empty line is a workload without the label. -func ParseOwnership(labels string) Ownership { - lines := strings.Split(labels, "\n") +// Equal treats an absent attribute and an empty one as the same, so an owner +// asked for fewer attributes still compares against one asked for more. +func (owner Owner) Equal(other Owner) bool { + for key, value := range owner { + if other[key] != value { + return false + } + } - if n := len(lines); n > 0 && lines[n-1] == "" { - lines = lines[:n-1] + for key, value := range other { + if owner[key] != value { + return false + } } - ownership := Ownership{} - seen := map[string]bool{} + return true +} + +// String renders the attributes in key order, so the same owner always reads +// the same way in a message. +func (owner Owner) String() string { + pairs := make([]string, 0, len(owner)) + for _, key := range slices.Sorted(maps.Keys(owner)) { + pairs = append(pairs, key+"="+owner[key]) + } + + return strings.Join(pairs, ",") +} - for _, line := range lines { - kind := strings.TrimSpace(line) +// Ownership is the owners a group of workloads reports, one owner per +// workload, deduplicated. +type Ownership struct { + owners []Owner + unowned bool +} + +// NewOwnership records what each workload reported. A workload that recorded +// nothing marks the group as partly unowned rather than becoming an owner in +// its own right. +func NewOwnership(owners ...Owner) Ownership { + ownership := Ownership{} - if kind == "" { - ownership.unlabeled = true + for _, owner := range owners { + if owner.IsZero() { + ownership.unowned = true continue } - if seen[kind] { + if slices.ContainsFunc(ownership.owners, owner.Equal) { continue } - seen[kind] = true - ownership.kinds = append(ownership.kinds, kind) + ownership.owners = append(ownership.owners, owner) } return ownership } -// Foreign returns the owning Kind that is not self, when one exists. -func (ownership Ownership) Foreign(self string) (string, bool) { - for _, kind := range ownership.kinds { - if kind != self { - return kind, true +// Foreign returns an owner that is not self, when one exists. +func (ownership Ownership) Foreign(self Owner) (Owner, bool) { + for _, owner := range ownership.owners { + if !owner.Equal(self) { + return owner, true } } - return "", false + return nil, false } -// HasUnlabeled reports workloads carrying no ownership label: either a -// pre-label foundry deployment or a foreign project sharing the name. -func (ownership Ownership) HasUnlabeled() bool { - return ownership.unlabeled +// HasUnowned reports workloads that recorded no owner: either a deployment +// made before foundry stamped them, or a foreign one sharing the same name. +func (ownership Ownership) HasUnowned() bool { + return ownership.unowned } diff --git a/internal/domain/ownership_test.go b/internal/domain/ownership_test.go index c1e08213..15aa2117 100644 --- a/internal/domain/ownership_test.go +++ b/internal/domain/ownership_test.go @@ -6,69 +6,140 @@ import ( "github.com/stretchr/testify/assert" ) -func TestParseOwnership(t *testing.T) { +func TestOwnerIsZero(t *testing.T) { tests := []struct { - name string - out string - self string - expectedForeign string - expectedConflict bool - expectedUnlabeled bool + name string + owner Owner + expectedZero bool }{ + {name: "Nil_Zero", owner: nil, expectedZero: true}, + {name: "Empty_Zero", owner: Owner{}, expectedZero: true}, + {name: "AllValuesEmpty_Zero", owner: Owner{"kind": "", "name": ""}, expectedZero: true}, + {name: "OneValueSet_NotZero", owner: Owner{"kind": "", "name": "signoz"}, expectedZero: false}, + {name: "AllValuesSet_NotZero", owner: Owner{"kind": "Installation", "name": "signoz"}, expectedZero: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedZero, tt.owner.IsZero()) + }) + } +} + +func TestOwnerEqual(t *testing.T) { + tests := []struct { + name string + owner Owner + other Owner + expectedEqual bool + }{ + { + name: "SameAttributes_Equal", + owner: Owner{"kind": "Installation", "name": "signoz"}, + other: Owner{"kind": "Installation", "name": "signoz"}, + expectedEqual: true, + }, + { + // One attribute of the set differing is a different owner: an + // owner is compared as a whole. + name: "OneAttributeDiffers_NotEqual", + owner: Owner{"kind": "Installation", "name": "signoz"}, + other: Owner{"kind": "CollectionAgent", "name": "signoz"}, + expectedEqual: false, + }, + { + name: "AbsentMatchesEmpty_Equal", + owner: Owner{"kind": "Installation", "name": ""}, + other: Owner{"kind": "Installation"}, + expectedEqual: true, + }, { - name: "Empty_NoOwnership", - out: "", - self: "CollectionAgent", + name: "ExtraAttribute_NotEqual", + owner: Owner{"kind": "Installation"}, + other: Owner{"kind": "Installation", "name": "signoz"}, + expectedEqual: false, }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedEqual, tt.owner.Equal(tt.other)) + assert.Equal(t, tt.expectedEqual, tt.other.Equal(tt.owner)) + }) + } +} + +func TestOwnerString(t *testing.T) { + owner := Owner{"name": "signoz", "kind": "Installation", "managed-by": "foundry"} + + assert.Equal(t, "kind=Installation,managed-by=foundry,name=signoz", owner.String()) + assert.Empty(t, Owner{}.String()) +} + +func TestOwnership(t *testing.T) { + installation := Owner{"kind": "Installation", "name": "signoz"} + agent := Owner{"kind": "CollectionAgent", "name": "signoz"} + + tests := []struct { + name string + owners []Owner + self Owner + expectedForeign Owner + expectedUnowned bool + }{ { - name: "SelfKind_NoConflict", - out: "CollectionAgent\n", - self: "CollectionAgent", + name: "NoWorkloads_NothingOwned", + owners: nil, + self: installation, }, { - name: "ForeignKind_Conflicts", - out: "Installation\n", - self: "CollectionAgent", - expectedForeign: "Installation", - expectedConflict: true, + name: "OnlySelf_NoConflict", + owners: []Owner{installation, installation}, + self: installation, }, { - name: "UnlabeledOnly_UnlabeledWithoutConflict", - out: "\n\n", - self: "CollectionAgent", - expectedUnlabeled: true, + name: "OtherOwner_Conflict", + owners: []Owner{agent}, + self: installation, + expectedForeign: agent, }, { - name: "UnlabeledAndForeign_Conflicts", - out: "\nInstallation\n", - self: "CollectionAgent", - expectedForeign: "Installation", - expectedConflict: true, - expectedUnlabeled: true, + name: "SelfBesideOther_Conflict", + owners: []Owner{installation, agent}, + self: installation, + expectedForeign: agent, }, { - name: "DuplicateForeign_SingleForeign", - out: "Installation\nInstallation\n", - self: "CollectionAgent", - expectedForeign: "Installation", - expectedConflict: true, + name: "NothingRecorded_Unowned", + owners: []Owner{{"kind": "", "name": ""}}, + self: installation, + expectedUnowned: true, }, { - name: "SelfAmongUnlabeled_NoConflict", - out: "CollectionAgent\n\nCollectionAgent\n", - self: "CollectionAgent", - expectedUnlabeled: true, + name: "SelfBesideUnrecorded_UnownedNoConflict", + owners: []Owner{installation, {"kind": "", "name": ""}}, + self: installation, + expectedUnowned: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ownership := ParseOwnership(tt.out) + ownership := NewOwnership(tt.owners...) foreign, conflict := ownership.Foreign(tt.self) + assert.Equal(t, tt.expectedForeign != nil, conflict) assert.Equal(t, tt.expectedForeign, foreign) - assert.Equal(t, tt.expectedConflict, conflict) - assert.Equal(t, tt.expectedUnlabeled, ownership.HasUnlabeled()) + assert.Equal(t, tt.expectedUnowned, ownership.HasUnowned()) }) } } + +// The same owner reported by many workloads is one owner. +func TestOwnershipDeduplicates(t *testing.T) { + installation := Owner{"kind": "Installation", "name": "signoz"} + + ownership := NewOwnership(installation, installation, installation) + + assert.Len(t, ownership.owners, 1) +} diff --git a/internal/foundry/gauge.go b/internal/foundry/gauge.go index cbd8e057..b896728b 100644 --- a/internal/foundry/gauge.go +++ b/internal/foundry/gauge.go @@ -9,31 +9,42 @@ import ( foundryerrors "github.com/signoz/foundry/internal/errors" ) +// gaugeable is what toolers and runners both expose. Runners replace toolers +// one tool at a time, so gauge checks whichever a casting is registered with. +type gaugeable interface { + Name() string + Gauge(ctx context.Context) error +} + func (foundry *Foundry) Gauge(ctx context.Context, machinery v1alpha1.Machinery) error { p, err := foundry.newPlanner(ctx, machinery) if err != nil { return err } - unavailableTools := []string{} - for _, tooler := range p.Toolers() { - if err := tooler.Gauge(ctx); err != nil { - foundry.Logger.ErrorContext(ctx, "tool is not available or cannot be detected properly", slog.String("tool.name", tooler.Name()), foundryerrors.LogAttr(err)) - unavailableTools = append(unavailableTools, tooler.Name()) - continue - } - foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", tooler.Name())) + tools := make([]gaugeable, 0, len(p.Toolers())+len(p.Runners())) + for _, t := range p.Toolers() { + tools = append(tools, t) } + for _, r := range p.Runners() { - if err := r.Preflight(ctx); err != nil { - foundry.Logger.ErrorContext(ctx, "tool is not available or cannot be detected properly", slog.String("tool.name", r.Name()), foundryerrors.LogAttr(err)) - unavailableTools = append(unavailableTools, r.Name()) + tools = append(tools, r) + } + + unavailableTools := []string{} + for _, tool := range tools { + if err := tool.Gauge(ctx); err != nil { + foundry.Logger.ErrorContext(ctx, "tool is not available or cannot be detected properly", slog.String("tool.name", tool.Name()), foundryerrors.LogAttr(err)) + unavailableTools = append(unavailableTools, tool.Name()) continue } - foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", r.Name())) + + foundry.Logger.InfoContext(ctx, "tool is available", slog.String("tool.name", tool.Name())) } + if len(unavailableTools) > 0 { return foundryerrors.Newf(foundryerrors.TypeNotFound, "tools are not available, please install them and try again: %s", strings.Join(unavailableTools, ", ")) } + return nil } diff --git a/internal/runner/composerunner/runner.go b/internal/runner/composerunner/runner.go index 53bd9903..8dd02461 100644 --- a/internal/runner/composerunner/runner.go +++ b/internal/runner/composerunner/runner.go @@ -4,138 +4,228 @@ import ( "context" "io" "log/slog" + "maps" "os" "os/exec" + "slices" "strings" - "sync" + "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/runner" ) var _ runner.Runner = (*Runner)(nil) -// Config is what must be in place for the runner to serve any casting. -// Everything that varies per run (the compose file, usage options) is a -// method argument instead. -type Config struct { - // Stdout and Stderr carry the tool's own output. Zero values use the - // process streams. +// ownerSeparator separates the label values docker returns for one container. +// No label foundry stamps carries it. +const ownerSeparator = "|" + +// Options is what a casting states about one call. The runner declares the +// tool's vocabulary; each casting fills in what it needs, so one casting's +// knobs never reach another's calls and the runner carries no casting state. +type Options struct { + // File is the compose file to run against. + File string + + // Project names the compose project. Stating it together with Owner + // guards the call. + Project string + + // Owner guards the call: the runner refuses a project whose containers + // report a different owner. What a casting stamps is what it claims. + Owner domain.Owner + + // Stdout and Stderr carry the tool's own output. Nil uses the process + // streams. Stdout io.Writer Stderr io.Writer } -// Runner interacts with docker compose. It holds no per-run state: the -// compose file and options are always arguments. The only mutable field is -// the memoized command probe, which is environment-derived, not run-derived. +// command is how compose is invoked on this machine: the binary, plus the +// subcommand that precedes every call when the plugin is what is installed. +type command struct { + name string + args []string +} + +// Runner interacts with docker compose. Everything about a call arrives with +// it, so the runner holds nothing but how to reach the tool. type Runner struct { logger *slog.Logger - config Config - mu sync.Mutex - command []string + // command is probed once: it is a property of the machine, not of a call. + command command } -func New(logger *slog.Logger, config Config) *Runner { - if config.Stdout == nil { - config.Stdout = os.Stdout - } - - if config.Stderr == nil { - config.Stderr = os.Stderr - } - - return &Runner{logger: logger, config: config} +func New(logger *slog.Logger) *Runner { + return &Runner{logger: logger} } -// From picks the compose runner out of the runners a casting receives. It -// lives here rather than in package runner so the contract package never -// imports its implementations. -func From(runners []runner.Runner) (*Runner, error) { +// Lookup picks the compose runner out of the runners a casting receives. It +// lives here so the contract package never imports its implementations. +func Lookup(runners []runner.Runner) (*Runner, error) { for _, r := range runners { if compose, ok := r.(*Runner); ok { return compose, nil } } - return nil, errors.Newf(errors.TypeNotFound, "compose runner is not registered for this casting") + return nil, errors.Newf(errors.TypeNotFound, "failed to look up the compose runner: it is not registered for this casting") } -// Options is the per-call usage slot. Empty today; profiles and their kin -// land here when a casting needs them. -type Options struct{} - func (r *Runner) Name() string { return "docker compose" } -// Preflight resolves how compose is invoked on this machine: the docker -// compose plugin, or the legacy docker-compose binary. This absorbs what the -// docker and docker compose toolers checked separately. -func (r *Runner) Preflight(ctx context.Context) error { +// Gauge resolves how compose is invoked on this machine: the docker compose +// plugin, or the legacy docker-compose binary. +func (r *Runner) Gauge(ctx context.Context) error { _, err := r.compose(ctx) return err } -// Up converges the deployment the compose file describes. -func (r *Runner) Up(ctx context.Context, composeFile string, _ Options) error { - if err := r.run(ctx, "-f", composeFile, "up", "-d"); err != nil { - return errors.Wrapf(err, errors.TypeInternal, "docker compose up failed") - } - - return nil +// Up converges the deployment and returns once the containers are started, +// not once they are healthy. +func (r *Runner) Up(ctx context.Context, options Options) error { + return r.run(ctx, options, "up", "-d") } // Down removes the containers and networks the compose file created. Volumes // stay: uncast never crosses the data line. -func (r *Runner) Down(ctx context.Context, composeFile string, _ Options) error { - if err := r.run(ctx, "-f", composeFile, "down"); err != nil { - return errors.Wrapf(err, errors.TypeInternal, "docker compose down failed") +func (r *Runner) Down(ctx context.Context, options Options) error { + return r.run(ctx, options, "down") +} + +func (r *Runner) run(ctx context.Context, options Options, verb string, args ...string) error { + if options.File == "" { + return errors.Newf(errors.TypeInvalidInput, "failed to run docker compose: no compose file is stated") } - return nil -} + if _, err := os.Stat(options.File); os.IsNotExist(err) { + return errors.Newf(errors.TypeNotFound, "failed to run docker compose: no compose file at %q", options.File) + } + + if err := r.checkOwner(ctx, options); err != nil { + return err + } -func (r *Runner) run(ctx context.Context, args ...string) error { command, err := r.compose(ctx) if err != nil { return err } - full := append(append([]string{}, command[1:]...), args...) - r.logger.DebugContext(ctx, "running command", slog.String("command", strings.Join(append([]string{command[0]}, full...), " "))) + full := append(append([]string{}, command.args...), "-f", options.File, verb) + full = append(full, args...) + + r.logger.DebugContext(ctx, "running command", slog.String("command", strings.Join(append([]string{command.name}, full...), " "))) + + cmd := exec.CommandContext(ctx, command.name, full...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if options.Stdout != nil { + cmd.Stdout = options.Stdout + } + + if options.Stderr != nil { + cmd.Stderr = options.Stderr + } + + if err := cmd.Run(); err != nil { + return errors.Wrapf(err, errors.TypeInternal, "failed to run docker compose %s", verb) + } + + return nil +} - cmd := exec.CommandContext(ctx, command[0], full...) - cmd.Stdout = r.config.Stdout - cmd.Stderr = r.config.Stderr +// checkOwner refuses a project another owner already holds. Workloads that +// record no owner only warn: they are either a pre-label deployment or a +// foreign project. An unreadable engine skips the check rather than blocking +// the call, and a caller that states no owner is not guarded at all. +func (r *Runner) checkOwner(ctx context.Context, options Options) error { + if options.Project == "" || len(options.Owner) == 0 { + return nil + } + + keys := slices.Sorted(maps.Keys(options.Owner)) + + directives := make([]string, 0, len(keys)) + for _, key := range keys { + directives = append(directives, `{{.Label "`+key+`"}}`) + } + + out, err := exec.CommandContext(ctx, "docker", "ps", "-a", + "--filter", "label=com.docker.compose.project="+options.Project, + "--format", strings.Join(directives, ownerSeparator)).Output() + if err != nil { + r.logger.WarnContext(ctx, "skipping the ownership check: could not read labels from docker", errors.LogAttr(err)) + + return nil + } + + ownership := domain.NewOwnership(owners(keys, string(out))...) + + if foreign, conflict := ownership.Foreign(options.Owner); conflict { + return errors.Newf(errors.TypeInvalidInput, "failed to run docker compose: project %q already belongs to [%s] on this host, not [%s]: remove that deployment, or give this one a different name", options.Project, foreign, options.Owner) + } - return cmd.Run() + if ownership.HasUnowned() { + r.logger.WarnContext(ctx, "compose project has containers without ownership labels", slog.String("project", options.Project)) + } + + return nil } -// compose probes for the docker compose plugin, then for the legacy -// docker-compose binary, memoizing the answer. -func (r *Runner) compose(ctx context.Context) ([]string, error) { - r.mu.Lock() - defer r.mu.Unlock() +// owners reads back what the format asked for: one container per line, its +// label values in the order the keys were asked. +func owners(keys []string, out string) []domain.Owner { + lines := strings.Split(strings.TrimRight(out, "\n"), "\n") + + parsed := make([]domain.Owner, 0, len(lines)) + for _, line := range lines { + if line == "" { + continue + } + + values := strings.Split(line, ownerSeparator) - if r.command != nil { + owner := domain.Owner{} + for i, key := range keys { + if i < len(values) { + owner[key] = values[i] + } + } + + parsed = append(parsed, owner) + } + + return parsed +} + +// compose probes for the docker compose plugin, then for the legacy +// docker-compose binary, remembering the answer for later calls. The memo is +// unguarded on purpose: foundry runs single-threaded, so a lock here would +// claim a concurrency contract runners do not have. +func (r *Runner) compose(ctx context.Context) (command, error) { + if r.command.name != "" { return r.command, nil } if _, err := exec.LookPath("docker"); err == nil { if err := exec.CommandContext(ctx, "docker", "compose", "version").Run(); err == nil { - r.command = []string{"docker", "compose"} + r.command = command{name: "docker", args: []string{"compose"}} return r.command, nil } } if _, err := exec.LookPath("docker-compose"); err == nil { - r.command = []string{"docker-compose"} + r.command = command{name: "docker-compose"} return r.command, nil } - return nil, errors.Newf(errors.TypeNotFound, "docker compose is not available: install the docker compose plugin or docker-compose") + return command{}, errors.Newf(errors.TypeNotFound, "failed to find docker compose: install the docker compose plugin or docker-compose") } diff --git a/internal/runner/composerunner/runner_test.go b/internal/runner/composerunner/runner_test.go index 62f62269..012681cc 100644 --- a/internal/runner/composerunner/runner_test.go +++ b/internal/runner/composerunner/runner_test.go @@ -4,42 +4,47 @@ import ( "context" "io" "log/slog" + "maps" "os" "os/exec" "path/filepath" "testing" + "github.com/signoz/foundry/internal/domain" "github.com/signoz/foundry/internal/runner" "github.com/stretchr/testify/assert" ) -type otherRunner struct{} - -func (otherRunner) Name() string { return "other" } -func (otherRunner) Preflight(ctx context.Context) error { return nil } +// requireEngine skips a test that drives a real docker engine. +func requireEngine(t *testing.T) { + t.Helper() -func TestNew(t *testing.T) { - tests := []struct { - name string - config Config - }{ - {name: "ZeroValue_ProcessStreams", config: Config{}}, - {name: "Streams_Kept", config: Config{Stdout: io.Discard, Stderr: io.Discard}}, + if testing.Short() { + t.Skip("skipping docker engine test in short mode") } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - r := New(slog.New(slog.DiscardHandler), tt.config) + if _, err := exec.LookPath("docker"); err != nil { + t.Skip("docker is not available") + } - assert.NotNil(t, r.config.Stdout) - assert.NotNil(t, r.config.Stderr) - assert.Equal(t, "docker compose", r.Name()) - }) + if err := exec.Command("docker", "info").Run(); err != nil { + t.Skip("docker engine is not running") } } -func TestFrom(t *testing.T) { - compose := New(slog.New(slog.DiscardHandler), Config{}) +type otherRunner struct{} + +func (otherRunner) Name() string { return "other" } +func (otherRunner) Gauge(ctx context.Context) error { return nil } + +func TestNew(t *testing.T) { + r := New(slog.New(slog.DiscardHandler)) + + assert.Equal(t, "docker compose", r.Name()) +} + +func TestLookup(t *testing.T) { + compose := New(slog.New(slog.DiscardHandler)) tests := []struct { name string @@ -48,13 +53,13 @@ func TestFrom(t *testing.T) { }{ {name: "Registered_Found", runners: []runner.Runner{compose}, pass: true}, {name: "AmongOthers_Found", runners: []runner.Runner{otherRunner{}, compose}, pass: true}, - {name: "Empty_NotFound", runners: nil, pass: false}, - {name: "OnlyOthers_NotFound", runners: []runner.Runner{otherRunner{}}, pass: false}, + {name: "Empty_Invalid", runners: nil, pass: false}, + {name: "OnlyOthers_Invalid", runners: []runner.Runner{otherRunner{}}, pass: false}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - found, err := From(tt.runners) + found, err := Lookup(tt.runners) if !tt.pass { assert.Error(t, err) return @@ -66,29 +71,138 @@ func TestFrom(t *testing.T) { } } +// The verbs validate what they execute against, so a casting that forgot to +// state the file, or forged nothing, fails before the tool is spawned. +func TestRunValidatesFile(t *testing.T) { + tests := []struct { + name string + file string + pass bool + }{ + {name: "Unset_Invalid", file: "", pass: false}, + {name: "Missing_Invalid", file: filepath.Join(t.TempDir(), "absent", "compose.yaml"), pass: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := New(slog.New(slog.DiscardHandler)) + + if !tt.pass { + assert.Error(t, r.Up(context.Background(), Options{File: tt.file})) + assert.Error(t, r.Down(context.Background(), Options{File: tt.file})) + return + } + + assert.NoError(t, r.Up(context.Background(), Options{File: tt.file})) + }) + } +} + +// owners reads docker's flat output back into the owners domain compares. +// The encoding is the runner's: a container reporting nothing must read as +// unowned, not as an owner whose every value is empty. +func TestOwners(t *testing.T) { + keys := []string{"foundry.signoz.io/kind", "foundry.signoz.io/managed-by", "foundry.signoz.io/name"} + + tests := []struct { + name string + out string + expectedOwners []domain.Owner + }{ + {name: "NoContainers_None", out: "", expectedOwners: []domain.Owner{}}, + { + name: "OneContainer_OneOwner", + out: "Installation|foundry|signoz\n", + expectedOwners: []domain.Owner{{ + "foundry.signoz.io/kind": "Installation", + "foundry.signoz.io/managed-by": "foundry", + "foundry.signoz.io/name": "signoz", + }}, + }, + { + name: "NoLabels_ZeroOwner", + out: "||\n", + expectedOwners: []domain.Owner{{ + "foundry.signoz.io/kind": "", + "foundry.signoz.io/managed-by": "", + "foundry.signoz.io/name": "", + }}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expectedOwners, owners(keys, tt.out)) + }) + } + + // A container reporting nothing marks the group unowned rather than + // conflicting with the caller. + ownership := domain.NewOwnership(owners(keys, "||\n")...) + _, conflict := ownership.Foreign(domain.Owner{"foundry.signoz.io/kind": "Installation"}) + + assert.False(t, conflict) + assert.True(t, ownership.HasUnowned()) +} + // Up then Down against a minimal compose file; needs a running docker engine, // so it skips wherever one is absent. func TestUpDown(t *testing.T) { - if testing.Short() { - t.Skip("skipping docker engine test in short mode") - } + requireEngine(t) - if _, err := exec.LookPath("docker"); err != nil { - t.Skip("docker is not available") + composeFile := filepath.Join(t.TempDir(), "compose.yaml") + contents := "name: composerunner-test\nservices:\n ok:\n image: busybox:stable\n command: [\"sleep\", \"300\"]\n" + assert.NoError(t, os.WriteFile(composeFile, []byte(contents), 0o644)) + + r := New(slog.New(slog.DiscardHandler)) + + assert.NoError(t, r.Gauge(context.Background())) + assert.NoError(t, r.Up(context.Background(), Options{File: composeFile, Stdout: io.Discard, Stderr: io.Discard})) + assert.NoError(t, r.Down(context.Background(), Options{File: composeFile, Stdout: io.Discard, Stderr: io.Discard})) +} + +// A project labelled for one owner is refused to another, and granted back to +// the owner that holds it. Needs a running docker engine. +func TestOwnerGuardsTheProject(t *testing.T) { + requireEngine(t) + + const project = "composerunner-owner-test" + + owner := domain.Owner{ + "foundry.signoz.io/managed-by": "foundry", + "foundry.signoz.io/kind": "Installation", + "foundry.signoz.io/name": project, } - if err := exec.Command("docker", "info").Run(); err != nil { - t.Skip("docker engine is not running") + labels := "" + for key, value := range owner { + labels += " " + key + ": " + value + "\n" } - dir := t.TempDir() - composeFile := filepath.Join(dir, "compose.yaml") - contents := "name: composerunner-test\nservices:\n ok:\n image: busybox:stable\n command: [\"sleep\", \"300\"]\n" + composeFile := filepath.Join(t.TempDir(), "compose.yaml") + contents := "name: " + project + "\nservices:\n ok:\n image: busybox:stable\n command: [\"sleep\", \"300\"]\n labels:\n" + labels assert.NoError(t, os.WriteFile(composeFile, []byte(contents), 0o644)) - r := New(slog.New(slog.DiscardHandler), Config{Stdout: io.Discard, Stderr: io.Discard}) + r := New(slog.New(slog.DiscardHandler)) + options := Options{ + File: composeFile, + Project: project, + Stdout: io.Discard, + Stderr: io.Discard, + } + + installation := options + installation.Owner = owner + + // One label of the set differing is a different owner. + agent := options + agent.Owner = maps.Clone(owner) + agent.Owner["foundry.signoz.io/kind"] = "CollectionAgent" + + assert.NoError(t, r.Up(context.Background(), installation)) + t.Cleanup(func() { _ = r.Down(context.Background(), options) }) - assert.NoError(t, r.Preflight(context.Background())) - assert.NoError(t, r.Up(context.Background(), composeFile, Options{})) - assert.NoError(t, r.Down(context.Background(), composeFile, Options{})) + assert.Error(t, r.Up(context.Background(), agent)) + assert.Error(t, r.Down(context.Background(), agent)) + assert.NoError(t, r.Down(context.Background(), installation)) } diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 81c9ef26..a1dc3773 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -3,23 +3,21 @@ // flavor speaks that tool. // // A runner is the interface between foundry and one tool. It is casting-, -// kind- and set-blind: no casting types, no ordering, no bindings. Runners -// hold no per-run state -- the root directory and usage options are always -// method arguments, never fields -- and all deployment state lives in the -// tool's own record. +// kind- and set-blind: no casting types, no ordering, no bindings. It holds +// nothing about any deployment either; a casting states everything about a +// call in the options it passes to the verb, so two castings of two Kinds can +// drive the same runner without either one's choices reaching the other. package runner import "context" -// Runner is the surface generic consumers need: gauge preflights every -// runner a casting is registered with. The tool's own operations live on -// each package's concrete type, which castings receive and use directly. +// Runner is the surface gauge needs. The tool's own operations live on each +// package's concrete type, which castings receive and use directly. type Runner interface { Name() string - // Preflight verifies the environment satisfies what the runner's - // configuration demands: the engine is present or can be pinned into - // place. It needs no root and no connection, which is what makes it - // runnable at gauge time. - Preflight(ctx context.Context) error + // Gauge reports whether the tool can run here, in the same vocabulary + // as tooler.Tooler. It needs nothing a casting knows, which is what + // makes it callable at gauge time. + Gauge(ctx context.Context) error }