diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 2ef1066f..317a83a7 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: - python-version: '3.x' + python-version: '3.12.0' check-latest: true - name: Extract Helm repo dependencies from Chart.yaml diff --git a/.github/workflows/server-develop.yaml b/.github/workflows/server-develop.yaml deleted file mode 100644 index 01adc329..00000000 --- a/.github/workflows/server-develop.yaml +++ /dev/null @@ -1,93 +0,0 @@ -name: Server - build and push develop -on: - workflow_dispatch: - push: - branches: [ develop ] - paths: - - "diode-server/**" - - "!diode-server/docker/**" - - "!diode-server/Makefile" - - "!diode-server/README.md" - -concurrency: - group: ${{ github.workflow }} - cancel-in-progress: false - -env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GO_VERSION: '1.24' - -permissions: - contents: write - issues: write - pull-requests: write - id-token: write - -jobs: - setup: - name: Setup - runs-on: ubuntu-latest - outputs: - apps: ${{ steps.apps.outputs.apps }} - steps: - - name: Get apps - id: apps - run: | - echo 'apps=["auth", "ingester", "reconciler"]' >> "$GITHUB_OUTPUT" - - build: - name: Build - ${{ matrix.app }} - needs: [ setup ] - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - app: ${{ fromJSON(needs.setup.outputs.apps) }} - env: - BUILD_VERSION: 0.0.0 - BUILD_COMMIT: ${{ github.sha }} - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up QEMU - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5 # v3 - - - name: Set build info - run: | - echo $BUILD_COMMIT > ./diode-server/version/BUILD_COMMIT.txt - echo $BUILD_VERSION > ./diode-server/version/BUILD_VERSION.txt - - - name: Setup JFrog CLI - id: setup-jfrog-cli - uses: jfrog/setup-jfrog-cli@ff5cb544114ffc152db9cea1cd3d5978d5074946 # v4.5.11 - env: - JF_URL: https://netboxlabs.jfrog.io - JF_PROJECT: obs - with: - oidc-provider-name: github-ci - - - name: Login to JFrog Artifactory - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 #v3 - with: - registry: netboxlabs.jfrog.io - username: ${{ steps.setup-jfrog-cli.outputs.oidc-user }} - password: ${{ steps.setup-jfrog-cli.outputs.oidc-token }} - - - name: Build image and push - uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6 - with: - context: diode-server - file: diode-server/docker/Dockerfile-build.${{ matrix.app }} - platforms: linux/amd64,linux/arm64 - push: true - cache-from: type=gha - cache-to: type=gha,mode=max - tags: | - netboxlabs.jfrog.io/obs-builds/diode-${{ matrix.app }}:develop - build-args: | - GO_VERSION=${{ env.GO_VERSION }} - SVC=${{ matrix.app }} diff --git a/.github/workflows/server-release.yaml b/.github/workflows/server-release.yaml index 6a1ca490..18fc69d9 100644 --- a/.github/workflows/server-release.yaml +++ b/.github/workflows/server-release.yaml @@ -17,6 +17,7 @@ env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} SEMANTIC_RELEASE_PACKAGE: ${{ github.repository }} GO_VERSION: '1.24' + JF_REPOSITORY: obs-builds permissions: contents: write @@ -119,6 +120,7 @@ jobs: echo $BUILD_VERSION > ./diode-server/version/BUILD_VERSION.txt - name: Build image and push + id: docker-build uses: docker/build-push-action@48aba3b46d1b1fec4febb7c5d0c644b249a11355 # v6 with: context: diode-server @@ -128,14 +130,20 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max tags: | - netboxlabs/diode-${{ matrix.app }}:latest - netboxlabs/diode-${{ matrix.app }}:${{ env.BUILD_VERSION }} - netboxlabs.jfrog.io/obs-builds/diode-${{ matrix.app }}:latest - netboxlabs.jfrog.io/obs-builds/diode-${{ matrix.app }}:${{ env.BUILD_VERSION }} + netboxlabs/${{ env.APP_NAME }}:latest + netboxlabs/${{ env.APP_NAME }}:${{ env.BUILD_VERSION }} + netboxlabs.jfrog.io/${{ env.JF_REPOSITORY }}/${{ env.APP_NAME }}:latest + netboxlabs.jfrog.io/${{ env.JF_REPOSITORY }}/${{ env.APP_NAME }}:${{ env.BUILD_VERSION }} build-args: | GO_VERSION=${{ env.GO_VERSION }} SVC=${{ matrix.app }} + - name: Create and publish JFrog build + run: | + echo '${{ steps.docker-build.outputs.metadata }}' > build-metadata + jf rt build-docker-create ${{ env.JF_REPOSITORY }} --image-file build-metadata --build-name ${{ env.APP_NAME }} --build-number ${{ env.BUILD_VERSION }} + jf rt build-publish ${{ env.APP_NAME }} ${{ env.BUILD_VERSION }} + semantic-release: name: Semantic release - ${{ matrix.app }} uses: netboxlabs/diode/.github/workflows/reusable_semantic_release.yaml@develop diff --git a/charts/diode/Chart.yaml b/charts/diode/Chart.yaml index ce71d516..1c8ff390 100644 --- a/charts/diode/Chart.yaml +++ b/charts/diode/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: diode description: A Helm chart for Diode type: application -version: 1.8.1 +version: "1.9.0" appVersion: "1.5.0" home: https://github.com/netboxlabs/diode sources: diff --git a/charts/diode/README.md b/charts/diode/README.md index b05357e1..95e9478b 100644 --- a/charts/diode/README.md +++ b/charts/diode/README.md @@ -2,7 +2,7 @@ A Helm chart for Diode -![Version: 1.8.1](https://img.shields.io/badge/Version-1.8.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5.0](https://img.shields.io/badge/AppVersion-1.5.0-informational?style=flat-square) +![Version: 1.9.0](https://img.shields.io/badge/Version-1.9.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.5.0](https://img.shields.io/badge/AppVersion-1.5.0-informational?style=flat-square) ## Prerequisites @@ -314,6 +314,11 @@ helm show values diode/diode | externalPostgresql.username | string | `"diode"` | username | | externalRedis.hostname | string | `"localhost"` | hostname | | externalRedis.port | int | `6379` | port | +| externalRedis.tls.caPath | string | `""` | path to CA certificate to verify server | +| externalRedis.tls.clientCertPath | string | `""` | path to client certificate for mutual TLS | +| externalRedis.tls.clientKeyPath | string | `""` | path to client private key for mutual TLS | +| externalRedis.tls.enabled | bool | `false` | enable TLS | +| externalRedis.tls.skipVerify | bool | `false` | skip TLS verify | | global.commonAnnotations | object | `{}` | common annotations for all resources | | global.commonLabels | object | `{}` | common labels for all resources | | global.diode | object | `{"busybox":{"image":"busybox:latest","imagePullPolicy":"IfNotPresent"},"hydra":{"waitForPostgres":true},"ingester":{"waitForRedis":true},"reconciler":{"waitForPostgres":true,"waitForRedis":true}}` | diode global configuration | diff --git a/charts/diode/templates/diode-ingester-configmap.yaml b/charts/diode/templates/diode-ingester-configmap.yaml index 65675527..be13a5d9 100644 --- a/charts/diode/templates/diode-ingester-configmap.yaml +++ b/charts/diode/templates/diode-ingester-configmap.yaml @@ -17,6 +17,13 @@ data: SENTRY_DSN: {{ $config.sentryDsn | quote }} REDIS_HOST: {{ include "diode.redis.hostname" . | quote }} REDIS_PORT: {{ include "diode.redis.port" . | quote }} + {{- if and .Values.externalRedis.tls .Values.externalRedis.tls.enabled }} + REDIS_TLS_ENABLED: "true" + REDIS_TLS_SKIP_VERIFY: {{ .Values.externalRedis.tls.skipVerify | default false | quote }} + REDIS_TLS_CA_PATH: {{ .Values.externalRedis.tls.caPath | default "" | quote }} + REDIS_TLS_CLIENT_KEY_PATH: {{ .Values.externalRedis.tls.clientKeyPath | default "" | quote }} + REDIS_TLS_CLIENT_CERT_PATH: {{ .Values.externalRedis.tls.clientCertPath | default "" | quote }} + {{- end }} REDIS_STREAM_DB: {{ $config.redisStreamDb | default "1" | quote }} TELEMETRY_METRICS_EXPORTER: {{ $config.telemetryMetricsExporter | quote }} TELEMETRY_TRACES_EXPORTER: {{ $config.telemetryTracesExporter | default "none" | quote }} diff --git a/charts/diode/templates/diode-reconciler-configmap.yaml b/charts/diode/templates/diode-reconciler-configmap.yaml index 0c06fbbe..fe911392 100644 --- a/charts/diode/templates/diode-reconciler-configmap.yaml +++ b/charts/diode/templates/diode-reconciler-configmap.yaml @@ -19,6 +19,13 @@ data: REDIS_PORT: {{ include "diode.redis.port" . | quote }} REDIS_DB: {{ $config.redisDb | default "0" | quote }} REDIS_STREAM_DB: {{ $config.redisStreamDb | default "1" | quote }} + {{- if and .Values.externalRedis.tls .Values.externalRedis.tls.enabled }} + REDIS_TLS_ENABLED: "true" + REDIS_TLS_SKIP_VERIFY: {{ .Values.externalRedis.tls.skipVerify | default false | quote }} + REDIS_TLS_CA_PATH: {{ .Values.externalRedis.tls.caPath | default "" | quote }} + REDIS_TLS_CLIENT_KEY_PATH: {{ .Values.externalRedis.tls.clientKeyPath | default "" | quote }} + REDIS_TLS_CLIENT_CERT_PATH: {{ .Values.externalRedis.tls.clientCertPath | default "" | quote }} + {{- end }} MIGRATION_ENABLED: {{ $config.migrationEnabled | default "true" | quote }} AUTO_APPLY_CHANGESETS: {{ $config.autoApplyChangesets | default "true" | quote }} RECONCILER_RATE_LIMITER_RPS: {{ $config.reconcilerRateLimiterRps | default "20" | quote }} diff --git a/charts/diode/values.yaml b/charts/diode/values.yaml index c2538362..c7e5e64e 100644 --- a/charts/diode/values.yaml +++ b/charts/diode/values.yaml @@ -57,6 +57,17 @@ externalRedis: hostname: localhost # -- port port: 6379 + tls: + # -- enable TLS + enabled: false + # -- skip TLS verify + skipVerify: false + # -- path to CA certificate to verify server + caPath: "" + # -- path to client private key for mutual TLS + clientKeyPath: "" + # -- path to client certificate for mutual TLS + clientCertPath: "" # Diode Auth configuration diodeAuth: diff --git a/diode-server/cmd/ingester/main.go b/diode-server/cmd/ingester/main.go index 710bde7e..0817482d 100644 --- a/diode-server/cmd/ingester/main.go +++ b/diode-server/cmd/ingester/main.go @@ -61,10 +61,18 @@ func main() { metricRecorder.SetServiceInfo(ctx, fmt.Sprintf("%s.%s", version.GetBuildVersion(), version.GetBuildCommit())) + redisTLSConfig, err := cfg.RedisTLS.ToTLSConfig() + if err != nil { + s.Logger().Error("failed to create TLS config for Redis", "error", err) + metricRecorder.RecordServiceStartupAttempt(ctx, false) + os.Exit(1) + } + redisStreamClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), - Password: cfg.RedisPassword, - DB: cfg.RedisStreamDB, + Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), + Password: cfg.RedisPassword, + DB: cfg.RedisStreamDB, + TLSConfig: redisTLSConfig, }) if _, err := redisStreamClient.Ping(ctx).Result(); err != nil { diff --git a/diode-server/cmd/reconciler/main.go b/diode-server/cmd/reconciler/main.go index d50c50fc..109d8dbd 100644 --- a/diode-server/cmd/reconciler/main.go +++ b/diode-server/cmd/reconciler/main.go @@ -77,10 +77,18 @@ func main() { } } + redisTLSConfig, err := cfg.RedisTLS.ToTLSConfig() + if err != nil { + s.Logger().Error("failed to create TLS config for Redis", "error", err) + metricRecorder.RecordServiceStartupAttempt(ctx, false) + os.Exit(1) + } + redisClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), - Password: cfg.RedisPassword, - DB: cfg.RedisDB, + Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), + Password: cfg.RedisPassword, + DB: cfg.RedisDB, + TLSConfig: redisTLSConfig, }) if _, err := redisClient.Ping(ctx).Result(); err != nil { @@ -90,9 +98,10 @@ func main() { } redisStreamClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), - Password: cfg.RedisPassword, - DB: cfg.RedisStreamDB, + Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), + Password: cfg.RedisPassword, + DB: cfg.RedisStreamDB, + TLSConfig: redisTLSConfig, }) if _, err := redisStreamClient.Ping(ctx).Result(); err != nil { diff --git a/diode-server/ingester/config.go b/diode-server/ingester/config.go index 6d38e863..ef716e00 100644 --- a/diode-server/ingester/config.go +++ b/diode-server/ingester/config.go @@ -1,6 +1,9 @@ package ingester -import "github.com/netboxlabs/diode/diode-server/telemetry" +import ( + "github.com/netboxlabs/diode/diode-server/telemetry" + "github.com/netboxlabs/diode/diode-server/tls" +) // Config is the configuration for the ingester service type Config struct { @@ -10,5 +13,6 @@ type Config struct { RedisPassword string `envconfig:"REDIS_PASSWORD" required:"true"` RedisStreamDB int `envconfig:"REDIS_STREAM_DB" default:"1"` + RedisTLS tls.Config `envconfig:"REDIS_TLS"` Telemetry telemetry.Config `envconfig:"TELEMETRY"` } diff --git a/diode-server/reconciler/config.go b/diode-server/reconciler/config.go index bc85436b..6ddd8c35 100644 --- a/diode-server/reconciler/config.go +++ b/diode-server/reconciler/config.go @@ -1,6 +1,9 @@ package reconciler -import "github.com/netboxlabs/diode/diode-server/telemetry" +import ( + "github.com/netboxlabs/diode/diode-server/telemetry" + "github.com/netboxlabs/diode/diode-server/tls" +) // Config is the configuration for the reconciler service type Config struct { @@ -29,5 +32,6 @@ type Config struct { DiodeToNetBoxClientID string `envconfig:"DIODE_TO_NETBOX_CLIENT_ID" required:"true"` DiodeToNetBoxClientSecret string `envconfig:"DIODE_TO_NETBOX_CLIENT_SECRET" required:"true"` + RedisTLS tls.Config `envconfig:"REDIS_TLS"` Telemetry telemetry.Config `envconfig:"TELEMETRY"` } diff --git a/diode-server/reconciler/server.go b/diode-server/reconciler/server.go index 6cdea70e..d9437ba3 100644 --- a/diode-server/reconciler/server.go +++ b/diode-server/reconciler/server.go @@ -32,10 +32,16 @@ func NewServer(ctx context.Context, logger *slog.Logger, repository Repository, var cfg Config envconfig.MustProcess("", &cfg) + redisTLSConfig, err := cfg.RedisTLS.ToTLSConfig() + if err != nil { + return nil, fmt.Errorf("failed to create TLS config for Redis: %v", err) + } + redisClient := redis.NewClient(&redis.Options{ - Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), - Password: cfg.RedisPassword, - DB: cfg.RedisDB, + Addr: fmt.Sprintf("%s:%s", cfg.RedisHost, cfg.RedisPort), + Password: cfg.RedisPassword, + DB: cfg.RedisDB, + TLSConfig: redisTLSConfig, }) if _, err := redisClient.Ping(ctx).Result(); err != nil { diff --git a/diode-server/tls/config.go b/diode-server/tls/config.go new file mode 100644 index 00000000..59be7eb2 --- /dev/null +++ b/diode-server/tls/config.go @@ -0,0 +1,55 @@ +package tls + +import ( + "crypto/tls" + "crypto/x509" + "fmt" + "os" +) + +// Config is the configuration for SSL/TLS settings +// Note the environment variables may be prefixed with REDIS_TLS_ +// based on the application's configuration structure +type Config struct { + Enabled bool `envconfig:"ENABLED" default:"false"` + SkipVerify bool `envconfig:"SKIP_VERIFY" default:"false"` + CaPath string `envconfig:"CA_PATH" default:""` + ClientKeyPath string `envconfig:"CLIENT_KEY_PATH" default:""` + ClientCertPath string `envconfig:"CLIENT_CERT_PATH" default:""` +} + +// ToTLSConfig converts the Diode TLS config to a Go Crypto TLS config +func (cfg *Config) ToTLSConfig() (*tls.Config, error) { + if !cfg.Enabled { + return nil, nil + } + + tlsConfig := &tls.Config{ + InsecureSkipVerify: cfg.SkipVerify, + MinVersion: tls.VersionTLS12, + } + + // Load CA certificate if provided + if cfg.CaPath != "" { + caCert, err := os.ReadFile(cfg.CaPath) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate: %w", err) + } + caCertPool := x509.NewCertPool() + if ok := caCertPool.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("failed to append CA certificate to pool") + } + tlsConfig.RootCAs = caCertPool + } + + // Load client certificate and key if provided + if cfg.ClientCertPath != "" && cfg.ClientKeyPath != "" { + cert, err := tls.LoadX509KeyPair(cfg.ClientCertPath, cfg.ClientKeyPath) + if err != nil { + return nil, fmt.Errorf("failed to load client certificate and key: %w", err) + } + tlsConfig.Certificates = []tls.Certificate{cert} + } + + return tlsConfig, nil +}