diff --git a/.github/workflows/configs/helm-ct.yaml b/.github/workflows/configs/helm-ct.yaml new file mode 100644 index 00000000..188561d2 --- /dev/null +++ b/.github/workflows/configs/helm-ct.yaml @@ -0,0 +1,2 @@ +validate-maintainers: false +skip-helm-dependencies: false \ No newline at end of file diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 9c8fda0a..2ef1066f 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -1,4 +1,5 @@ name: Helm - lint + on: push: branches: @@ -20,16 +21,44 @@ permissions: jobs: helm-lint: runs-on: ubuntu-latest - timeout-minutes: 10 - defaults: - run: - working-directory: charts steps: - name: Checkout - uses: actions/checkout@v4 - - name: Setup Helm - uses: azure/setup-helm@fe7b79cd5ee1e45176fcad797de68ecaf3ca4814 # v4 - - name: Update helm dependencies - run: helm dependency update diode - - name: Run helm lint - run: helm lint diode + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # 4.2.2 + with: + fetch-depth: 0 + + - name: Set up Helm + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 + with: + version: v3.17.0 + + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.x' + check-latest: true + + - name: Extract Helm repo dependencies from Chart.yaml + id: extract_repos + uses: mikefarah/yq@b534aa9ee5d38001fba3cd8fe254a037e4847b37 # v4.45.4 + with: + cmd: yq -o=tsv '.dependencies[] | [.name, .repository] | @tsv' charts/diode/Chart.yaml + - name: Add Helm repos from dependencies + run: | + echo "${{ steps.extract_repos.outputs.result }}" | while IFS=$'\t' read -r name repo; do + helm repo add "$name" "$repo" + done + + - name: Set up chart-testing + uses: helm/chart-testing-action@0d28d3144d3a25ea2cc349d6e59901c4ff469b3b # v2.7.0 + + - name: Run chart-testing (list-changed) + id: list-changed + run: | + changed=$(ct list-changed --target-branch release) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Run chart-testing (lint) + if: steps.list-changed.outputs.changed == 'true' + run: ct lint --config .github/workflows/configs/helm-ct.yaml --target-branch release \ No newline at end of file diff --git a/GET_STARTED.md b/GET_STARTED.md new file mode 100644 index 00000000..3e240180 --- /dev/null +++ b/GET_STARTED.md @@ -0,0 +1,194 @@ +# Getting Started with Diode + +This guide will help you set up and start using Diode to ingest data into NetBox. + +## Prerequisites + +Before you begin, ensure you have: + +- NetBox version 4.2.3 or later +- Docker version 27.0.3 or newer +- bash 4.x or newer +- jq + +## Installation Steps + +### Deploy Diode server + +> **Host**: These steps should be performed on the host where you want to run the Diode server. + +> **Note**: For the complete installation instructions, please refer to the [official Diode Server documentation](https://github.com/netboxlabs/diode/tree/develop/diode-server). + +We provide a `quickstart.sh` script to automate the setup process. The script will download and configure all necessary files: + +- `docker-compose.yaml` — Defines Diode server containers +- `.env` — Environment settings for customization +- `nginx.conf` — Nginx configuration for routing Diode endpoints +- `client-credentials.json` — Defines OAuth2 clients for secure communication + +1. Create a working directory: + ```bash + mkdir /opt/diode + cd /opt/diode + ``` + +2. Download and prepare the quickstart script: + ```bash + curl -sSfLo quickstart.sh https://raw.githubusercontent.com/netboxlabs/diode/release/diode-server/docker/scripts/quickstart.sh + chmod +x quickstart.sh + ``` + +3. Run the script with your NetBox server address: + ```bash + ./quickstart.sh https:// + ``` + This should have created an `.env` file for your environment. + +4. Start the Diode server: + ```bash + docker compose up -d + ``` +5. Extract the `netbox-to-diode` client secret. This will be needed for the Diode NetBox plugin installation: + ```bash + echo $(jq -r '.[] | select(.client_id == "netbox-to-diode") | .client_secret' /opt/diode/oauth2/client/client-credentials.json) + ``` + > **Note**: This will return a credential that will be used by the Diode NetBox plugin to connect to the Diode server. Store it safely. + +### Install Diode NetBox Plugin + +> **Host**: These steps should be performed on the host where NetBox is installed. + +> **Note**: For the complete installation instructions, please refer to the [official Diode NetBox Plugin documentation](https://github.com/netboxlabs/diode-netbox-plugin/blob/develop/README.md). + +1. **Source the NetBox Python Virtual Environment** + ```bash + cd /opt/netbox + source venv/bin/activate + ``` + +2. **Install the Plugin Package** + ```bash + pip install netboxlabs-diode-netbox-plugin + ``` + +3. **Configure NetBox Settings** + Add the following to your `configuration.py`: + ```python + PLUGINS = [ + "netbox_diode_plugin", + ] + + PLUGINS_CONFIG = { + "netbox_diode_plugin": { + # Diode gRPC target for communication with Diode server + "diode_target_override": "grpc:///diode", + # NetBox username associated with changes applied via plugin + "diode_username": "diode", + # netbox-to-diode client secret from earlier step + "netbox_to_diode_client_secret": "" + }, + } + ``` + +4. **Apply Database Migrations** + ```bash + cd /opt/netbox/netbox + ./manage.py migrate netbox_diode_plugin + ``` + +5. **Restart NetBox Services** + ```bash + sudo systemctl restart netbox netbox-rq + ``` + +6. **Generate Diode Client Credentials** + > **Note**: These credentials will be used by the Orb agent to send discovery results to NetBox via Diode. + + 1. Go to your NetBox instance (https://) + 2. In the left-hand pane, navigate to **Diode -> Client Credentials** + 3. Click on **+ Add a Credential** + 4. For Client Name, enter any name and click **Create** + 5. **IMPORTANT**: Copy the _Client ID_ and _Client Secret_ and save them securely + 6. Click **Return to List** + + You have now created your Diode client credentials. These will be used as environment variables when running the Orb agent. + +### Ingest Data with Orb Agent + +> **Host**: These steps should be performed on the host where you want to run the Orb agent for network discovery. + +> **Note**: For the complete installation instructions, please refer to the [official Orb Agent documentation](https://github.com/netboxlabs/orb-agent). + +1. **Export Client Credentials** + ```bash + # Export the client credentials you generated in NetBox + export DIODE_CLIENT_ID="" + export DIODE_CLIENT_SECRET="" + ``` + +2. **Create Agent Configuration File** + Create an `agent.yaml` file with the following content: + ```yaml + orb: + config_manager: + active: local + backends: + network_discovery: # Enable network discovery backend + common: + diode: + target: grpc:///diode + client_id: ${DIODE_CLIENT_ID} + client_secret: ${DIODE_CLIENT_SECRET} + agent_name: my_agent + policies: + network_discovery: + loopback_policy: + config: + scope: + targets: + - 127.0.0.1 + ``` + +3. **Run the Agent** + + Using host network mode (recommended): + ```bash + docker run --net=host \ + -v $(pwd):/opt/orb/ \ + -e DIODE_CLIENT_ID \ + -e DIODE_CLIENT_SECRET \ + netboxlabs/orb-agent:latest run -c /opt/orb/agent.yaml + ``` + + Alternative using root user: + ```bash + docker run -u root \ + -v $(pwd):/opt/orb/ \ + -e DIODE_CLIENT_ID \ + -e DIODE_CLIENT_SECRET \ + netboxlabs/orb-agent:latest run -c /opt/orb/agent.yaml + ``` + + > **Note**: The container needs sufficient permissions to send ICMP and TCP packets. This can be achieved either by: + > - Setting the network mode to `host` (recommended) + > - Running the container as root user + +4. **Verify Agent Operation** + - Check the agent logs for successful startup + - Verify data appears in NetBox + +## Troubleshooting + +### Common Issues + +1. **Connection Issues** + - Verify network connectivity between Diode and NetBox + - Check firewall rules + - Validate URLs and ports + +### Getting Help + +If you encounter issues: + +1. Search GitHub: [Issues](https://github.com/netboxlabs/diode/issues) +2. Find us in Slack: [NetDev Community #orb](https://https://netdev-community.slack.com/) \ No newline at end of file diff --git a/README.md b/README.md index 6b7e6801..459cfc7f 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,55 @@ # Diode -Diode is a NetBox data ingestion service that greatly simplifies and enhances the process to add and update network data -in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation -pipelines. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, -removing as much burden as possible from the user while shifting that effort to technology. - -To achieve this, Diode sits in front of NetBox and provides an API purpose built for ingestion of complex network data. -Diode eliminates the need to preprocess data to make it conform to the strict object hierarchy imposed by the NetBox -data model. This allows data to be sent to NetBox in a more freeform manner, in blocks that are intuitive for network -engineers (such as by device or by interface) with much of the related information treated as attributes or properties -of these components of interest. Then, Diode takes care of the heavy lifting, automatically transforming the data to -align it with NetBox’s structured and comprehensive data model. Diode can even create placeholder objects to compensate -for missing information, which means even fragmented information about the network can be captured in NetBox. - -## Project status - -The Diode project is currently in the _Public Preview_ stage. Please -see [NetBox Labs Product and Feature Lifecycle](https://netboxlabs.com/docs/console/product_feature_lifecycle/) for more -details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement. - -## Get started - -Diode runs as a sidecar service to NetBox and can run anywhere with network connectivity to NetBox, whether on the same -host or elsewhere. The overall Diode service is delivered through three main components (and a fourth optional -component): - -1. Diode plugin - see how to [install the Diode plugin](https://github.com/netboxlabs/diode-netbox-plugin) -2. Diode server - see how - to [run the Diode server](https://github.com/netboxlabs/diode/tree/develop/diode-server#readme) -3. Diode SDK - see how - to [install the Diode Python client SDK](https://github.com/netboxlabs/diode-sdk-python), [download Diode Python script examples](https://github.com/netboxlabs/netbox-learning/tree/develop/diode) - and [use the Diode SDK Go](https://github.com/netboxlabs/diode-sdk-go) -4. Diode agent (optional) - see how - to [install and run the Diode NAPALM discovery agent](https://github.com/netboxlabs/diode-agent/tree/develop/diode-napalm-agent) +Diode is a data ingestion service for NetBox that greatly simplifies and enhances the process of adding and updating data in NetBox, ensuring your network source of truth is always accurate and up to date. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, removing as much burden as possible from the user while shifting that effort to technology. + +## Project Status + +The Diode project is currently in the _Public Preview_ stage. Please see [NetBox Labs Product and Feature Lifecycle](https://netboxlabs.com/docs/console/product_feature_lifecycle/) for more details. We actively welcome feedback to help identify and prioritize bugs, new features and areas of improvement. + +## Prerequisites + +- NetBox 4.2.3 or later +- Python 3.8 or later (for Python SDK) +- Go 1.18 or later (for Go SDK) +- Network connectivity between Diode and NetBox + +## Quick Start + +For a quick step-by-step guide, see our [Getting Started Guide](./GET_STARTED.md). + +1. **Deploy the Diode Server** + See [deployment instructions](https://github.com/netboxlabs/diode/blob/develop/diode-server/README.md) + +2. **Install the Diode NetBox Plugin** + See [installation instructions](https://github.com/netboxlabs/diode-netbox-plugin/blob/develop/README.md) + +3. **Choose Your Data Ingestion Method** + - Use the NetBox Discovery agent for automated network discovery: see [instructions to run Orb agent](https://github.com/netboxlabs/orb-agent) + - Build custom integrations using our SDKs: + - [Python SDK](https://github.com/netboxlabs/diode-sdk-python) + - [Go SDK](https://github.com/netboxlabs/diode-sdk-go) + +## Documentation + +- [Diode Protocol Documentation](https://github.com/netboxlabs/diode/blob/develop/docs/diode-proto.md) ## Related Projects -- [diode-netbox-plugin](https://github.com/netboxlabs/diode-netbox-plugin) - The Diode NetBox plugin is a NetBox plugin - and a required component of the Diode ingestion service. -- [diode-sdk-python](https://github.com/netboxlabs/diode-sdk-python) - Diode SDK Python is a Python library for - interacting with the Diode ingestion service utilizing gRPC. -- [diode-sdk-go](https://github.com/netboxlabs/diode-sdk-go) - Diode SDK Go is a Go module for interacting with the - Diode ingestion service utilizing gRPC. -- [diode-agent](https://github.com/netboxlabs/diode-agent) - A collection of agents that leverage the Diode SDK to - interact with the Diode server. +- [diode-netbox-plugin](https://github.com/netboxlabs/diode-netbox-plugin) - The Diode NetBox plugin is a NetBox plugin and a required component of the Diode ingestion service. +- [diode-sdk-python](https://github.com/netboxlabs/diode-sdk-python) - Diode SDK Python is a Python library for interacting with the Diode ingestion service utilizing gRPC. +- [diode-sdk-go](https://github.com/netboxlabs/diode-sdk-go) - Diode SDK Go is a Go module for interacting with the Diode ingestion service utilizing gRPC. +- [orb-agent](https://github.com/netboxlabs/orb-agent) - The NetBox Discovery agent. + +## Support + +- [GitHub Issues](https://github.com/netboxlabs/diode/issues) +- [Slack NetDev Community (#orb channel)](https://https://netdev-community.slack.com/) ## License Distributed under the NetBox Limited Use License 1.0. See [LICENSE.md](./LICENSE.md) for more information. -Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for -more information. +Diode protocol buffers are distributed under the Apache 2.0 License. See [LICENSE.txt](./diode-proto/LICENSE.txt) for more information. ## Required Notice diff --git a/charts/diode/Chart.yaml b/charts/diode/Chart.yaml index 792aa9ba..5ab1f84f 100644 --- a/charts/diode/Chart.yaml +++ b/charts/diode/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: diode description: A Helm chart for Diode type: application -version: 1.5.0 +version: 1.6.0 appVersion: "1.2.0" home: https://github.com/netboxlabs/diode sources: diff --git a/charts/diode/README.md b/charts/diode/README.md index 181b1edf..8efd4683 100644 --- a/charts/diode/README.md +++ b/charts/diode/README.md @@ -2,7 +2,7 @@ A Helm chart for Diode -![Version: 1.5.0](https://img.shields.io/badge/Version-1.5.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) +![Version: 1.6.0](https://img.shields.io/badge/Version-1.6.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.2.0](https://img.shields.io/badge/AppVersion-1.2.0-informational?style=flat-square) ## Prerequisites @@ -227,8 +227,9 @@ helm show values diode/diode | certManager.prometheus | object | `{"enabled":false}` | prometheus enabled | | certManager.webhook | object | `{"enabled":true}` | webhook enabled | | diode.environment | string | `"development"` | environment name | +| diodeAuth.annotations | object | `{}` | annotations to add to the auth deployment | | diodeAuth.config.loggingLevel | string | `"INFO"` | logging level | -| diodeAuth.config.sentryDsn | string | `""` | sentry DSN | +| diodeAuth.config.sentryDsn | string | `""` | sentry DSN | | diodeAuth.config.telemetryEnvironment | string | `"dev"` | telemetry environment | | diodeAuth.config.telemetryMetricsExporter | string | `"prometheus"` | telemetry metrics exporter | | diodeAuth.config.telemetryTracesExporter | string | `"none"` | telemetry traces exporter | @@ -247,10 +248,12 @@ helm show values diode/diode | diodeAuthBootstrap.image.pullPolicy | string | `"IfNotPresent"` | pull policy | | diodeAuthBootstrap.image.repository | string | `"docker.io/netboxlabs/diode-auth"` | image repository | | diodeAuthBootstrap.image.tag | string | `"1.2.0"` | image tag | +| diodeAuthBootstrap.job.annotations | object | `{"helm.sh/hook":"post-install, post-upgrade","helm.sh/hook-weight":"2"}` | annotations to add to the auth bootstrap job | | diodeAuthBootstrap.job.backoffLimit | int | `20` | backoff limit | +| diodeIngester.annotations | object | `{}` | annotations to add to the ingester deployment | | diodeIngester.config.loggingLevel | string | `"INFO"` | logging level | | diodeIngester.config.redisStreamDb | int | `1` | redis stream db | -| diodeIngester.config.sentryDsn | string | `""` | sentry DSN | +| diodeIngester.config.sentryDsn | string | `""` | sentry DSN | | diodeIngester.config.telemetryEnvironment | string | `"dev"` | telemetry environment | | diodeIngester.config.telemetryMetricsExporter | string | `"prometheus"` | telemetry metrics exporter | | diodeIngester.config.telemetryTracesExporter | string | `"none"` | telemetry traces exporter | @@ -266,6 +269,7 @@ helm show values diode/diode | diodeIngester.replicaCount | int | `1` | replica count | | diodeIngester.resources | object | `{"limits":{"cpu":"500m","memory":"512Mi"},"requests":{"cpu":"100m","memory":"128Mi"}}` | resources | | diodeIngester.serviceAccount.create | bool | `true` | create service account | +| diodeReconciler.annotations | object | `{}` | annotations to add to the reconciler deployment | | diodeReconciler.config.autoApplyChangesets | string | `"true"` | auto apply changesets | | diodeReconciler.config.diodeToNetBoxClientId | string | `"diode-to-netbox"` | diode to netbox client id | | diodeReconciler.config.diodeToNetboxRateLimiterBurst | int | `1` | diode to netbox rate limiter burst | @@ -300,6 +304,8 @@ helm show values diode/diode | externalPostgresql.port | int | `5432` | port | | externalRedis.hostname | string | `"localhost"` | hostname | | externalRedis.port | int | `6379` | port | +| global.commonAnnotations | object | `{}` | common annotations for all resources | +| global.commonLabels | object | `{}` | common labels for all resources | | global.diode | object | `{"busybox":{"image":"busybox:latest","imagePullPolicy":"IfNotPresent"},"hydra":{"waitForPostgres":true}}` | diode global configuration | | global.diode.busybox | object | `{"image":"busybox:latest","imagePullPolicy":"IfNotPresent"}` | busybox image configuration | | global.diode.hydra | object | `{"waitForPostgres":true}` | hydra additional init containers configuration | diff --git a/charts/diode/templates/_helpers.tpl b/charts/diode/templates/_helpers.tpl index a94693a3..b03199bc 100644 --- a/charts/diode/templates/_helpers.tpl +++ b/charts/diode/templates/_helpers.tpl @@ -48,6 +48,9 @@ helm.sh/chart: {{ include "diode.chart" . }} app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- if .Values.global.commonLabels }} +{{- include "common.labels.standard" ( dict "customLabels" .Values.global.commonLabels "context" $ ) }} +{{- end }} {{- end }} {{/* diff --git a/charts/diode/templates/cert-issuer.yaml b/charts/diode/templates/cert-issuer.yaml index 15f96e29..6114b0b8 100644 --- a/charts/diode/templates/cert-issuer.yaml +++ b/charts/diode/templates/cert-issuer.yaml @@ -6,6 +6,9 @@ metadata: namespace: {{ include "diode.namespace" . }} labels: {{- include "diode.labels" . | nindent 4 }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: acme: # The ACME server URL diff --git a/charts/diode/templates/diode-auth-bootstrap-job.yaml b/charts/diode/templates/diode-auth-bootstrap-job.yaml index a0b0907c..55e2f078 100644 --- a/charts/diode/templates/diode-auth-bootstrap-job.yaml +++ b/charts/diode/templates/diode-auth-bootstrap-job.yaml @@ -7,6 +7,15 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.auth.bootstrapjobname" . }} + {{- if or .Values.global.commonAnnotations .Values.diodeAuthBootstrap.job.annotations }} + annotations: + {{- if .Values.diodeAuthBootstrap.job.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.diodeAuthBootstrap.job.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} spec: backoffLimit: {{ .Values.diodeAuthBootstrap.job.backoffLimit }} template: diff --git a/charts/diode/templates/diode-auth-configmap.yaml b/charts/diode/templates/diode-auth-configmap.yaml index a1cee002..299772ef 100644 --- a/charts/diode/templates/diode-auth-configmap.yaml +++ b/charts/diode/templates/diode-auth-configmap.yaml @@ -8,6 +8,9 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.auth.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} data: HTTP_PORT: {{ .Values.diodeAuth.containerPort | default "8080" | quote }} LOGGING_LEVEL: {{ $config.loggingLevel | default "INFO" | quote }} diff --git a/charts/diode/templates/diode-auth-deployment.yaml b/charts/diode/templates/diode-auth-deployment.yaml index d95a8f8d..bf0f162f 100644 --- a/charts/diode/templates/diode-auth-deployment.yaml +++ b/charts/diode/templates/diode-auth-deployment.yaml @@ -8,6 +8,15 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.auth.servicename" . }} app.kubernetes.io/component: {{ include "diode.auth.servicename" . }} + {{- if or .Values.global.commonAnnotations .Values.diodeAuth.annotations }} + annotations: + {{- if .Values.diodeAuth.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.diodeAuth.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} spec: replicas: {{ .Values.diodeAuth.replicaCount }} selector: diff --git a/charts/diode/templates/diode-auth-service.yaml b/charts/diode/templates/diode-auth-service.yaml index 0ee5e65c..a7a987aa 100644 --- a/charts/diode/templates/diode-auth-service.yaml +++ b/charts/diode/templates/diode-auth-service.yaml @@ -8,6 +8,9 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.auth.servicename" . }} app.kubernetes.io/component: {{ include "diode.auth.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: type: ClusterIP selector: diff --git a/charts/diode/templates/diode-auth-serviceaccount.yaml b/charts/diode/templates/diode-auth-serviceaccount.yaml index c34415eb..3682d65a 100644 --- a/charts/diode/templates/diode-auth-serviceaccount.yaml +++ b/charts/diode/templates/diode-auth-serviceaccount.yaml @@ -7,4 +7,7 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.auth.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/diode/templates/diode-hydra-extra-initcontainer-configmap.yaml b/charts/diode/templates/diode-hydra-extra-initcontainer-configmap.yaml index 87f87e50..743515e5 100644 --- a/charts/diode/templates/diode-hydra-extra-initcontainer-configmap.yaml +++ b/charts/diode/templates/diode-hydra-extra-initcontainer-configmap.yaml @@ -6,6 +6,9 @@ metadata: namespace: {{ include "diode.namespace" . }} labels: {{- include "diode.labels" . | nindent 4 }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} data: POSTGRES_HOST: {{ include "diode.postgresql.hostname" . | quote }} POSTGRES_PORT: {{ include "diode.postgresql.port" . | quote }} diff --git a/charts/diode/templates/diode-ingester-configmap.yaml b/charts/diode/templates/diode-ingester-configmap.yaml index 23f21a44..65675527 100644 --- a/charts/diode/templates/diode-ingester-configmap.yaml +++ b/charts/diode/templates/diode-ingester-configmap.yaml @@ -8,6 +8,9 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.ingester.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} data: GRPC_PORT: {{ .Values.diodeIngester.containerPort | default "8081" | quote }} LOGGING_LEVEL: {{ $config.loggingLevel | default "INFO" | quote }} diff --git a/charts/diode/templates/diode-ingester-deployment.yaml b/charts/diode/templates/diode-ingester-deployment.yaml index aed8db0d..7435f003 100644 --- a/charts/diode/templates/diode-ingester-deployment.yaml +++ b/charts/diode/templates/diode-ingester-deployment.yaml @@ -8,6 +8,15 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.ingester.servicename" . }} app.kubernetes.io/component: {{ include "diode.ingester.servicename" . }} + {{- if or .Values.global.commonAnnotations .Values.diodeIngester.annotations }} + annotations: + {{- if .Values.diodeIngester.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.diodeIngester.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} spec: replicas: {{ .Values.diodeIngester.replicaCount }} selector: diff --git a/charts/diode/templates/diode-ingester-service.yaml b/charts/diode/templates/diode-ingester-service.yaml index 42cf90a5..57b3ff5e 100644 --- a/charts/diode/templates/diode-ingester-service.yaml +++ b/charts/diode/templates/diode-ingester-service.yaml @@ -8,6 +8,9 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.ingester.servicename" . }} app.kubernetes.io/component: {{ include "diode.ingester.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: type: ClusterIP selector: diff --git a/charts/diode/templates/diode-ingester-serviceaccount.yaml b/charts/diode/templates/diode-ingester-serviceaccount.yaml index c1cd09c1..9e9e498e 100644 --- a/charts/diode/templates/diode-ingester-serviceaccount.yaml +++ b/charts/diode/templates/diode-ingester-serviceaccount.yaml @@ -7,4 +7,7 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.ingester.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/diode/templates/diode-postgresql-initdb-scriptsconfigmap.yaml b/charts/diode/templates/diode-postgresql-initdb-scriptsconfigmap.yaml index 55cba0dd..a034dbde 100644 --- a/charts/diode/templates/diode-postgresql-initdb-scriptsconfigmap.yaml +++ b/charts/diode/templates/diode-postgresql-initdb-scriptsconfigmap.yaml @@ -6,6 +6,9 @@ metadata: namespace: {{ include "diode.namespace" . }} labels: {{- include "diode.labels" . | nindent 4 }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} data: init_diode_databases.sh: | #!/bin/bash diff --git a/charts/diode/templates/diode-reconciler-configmap.yaml b/charts/diode/templates/diode-reconciler-configmap.yaml index e98eefef..a378652e 100644 --- a/charts/diode/templates/diode-reconciler-configmap.yaml +++ b/charts/diode/templates/diode-reconciler-configmap.yaml @@ -8,6 +8,9 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.reconciler.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} data: GRPC_PORT: {{ .Values.diodeReconciler.containerPort | default "8081" | quote }} LOGGING_LEVEL: {{ $config.loggingLevel | default "INFO" | quote }} diff --git a/charts/diode/templates/diode-reconciler-deployment.yaml b/charts/diode/templates/diode-reconciler-deployment.yaml index 156f1911..2c09b521 100644 --- a/charts/diode/templates/diode-reconciler-deployment.yaml +++ b/charts/diode/templates/diode-reconciler-deployment.yaml @@ -8,6 +8,15 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.reconciler.servicename" . }} app.kubernetes.io/component: {{ include "diode.reconciler.servicename" . }} + {{- if or .Values.global.commonAnnotations .Values.diodeReconciler.annotations }} + annotations: + {{- if .Values.diodeReconciler.annotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.diodeReconciler.annotations "context" $ ) | nindent 4 }} + {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} + {{- end }} spec: replicas: {{ .Values.diodeReconciler.replicaCount }} selector: diff --git a/charts/diode/templates/diode-reconciler-service.yaml b/charts/diode/templates/diode-reconciler-service.yaml index 8413968f..c1c0bb27 100644 --- a/charts/diode/templates/diode-reconciler-service.yaml +++ b/charts/diode/templates/diode-reconciler-service.yaml @@ -8,6 +8,9 @@ metadata: {{- include "diode.labels" . | nindent 4 }} app: {{ include "diode.reconciler.servicename" . }} app.kubernetes.io/component: {{ include "diode.reconciler.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: type: ClusterIP selector: diff --git a/charts/diode/templates/diode-reconciler-serviceaccount.yaml b/charts/diode/templates/diode-reconciler-serviceaccount.yaml index 1be6fcd5..85c8083f 100644 --- a/charts/diode/templates/diode-reconciler-serviceaccount.yaml +++ b/charts/diode/templates/diode-reconciler-serviceaccount.yaml @@ -7,4 +7,7 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.reconciler.servicename" . }} + {{- if .Values.global.commonAnnotations }} + annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} {{- end }} diff --git a/charts/diode/templates/ingress-grpc.yaml b/charts/diode/templates/ingress-grpc.yaml index b247e6ec..d6c7acfe 100644 --- a/charts/diode/templates/ingress-grpc.yaml +++ b/charts/diode/templates/ingress-grpc.yaml @@ -7,6 +7,9 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.ingress.name" . }} + {{- if .Values.global.commonLabels }} + {{- include "common.labels.standard" ( dict "customLabels" .Values.global.commonLabels "context" $ ) | nindent 4 }} + {{- end }} annotations: nginx.ingress.kubernetes.io/auth-url: "http://{{ include "diode.auth.hostname" . }}:{{ .Values.diodeAuth.containerPort }}/introspect" nginx.ingress.kubernetes.io/auth-method: "POST" @@ -15,6 +18,9 @@ metadata: {{- with $ingressNginx.grpcAnnotations }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: ingressClassName: {{ $ingressNginx.ingressClass | default "nginx" }} rules: diff --git a/charts/diode/templates/ingress-http.yaml b/charts/diode/templates/ingress-http.yaml index edfb288f..91d4a356 100644 --- a/charts/diode/templates/ingress-http.yaml +++ b/charts/diode/templates/ingress-http.yaml @@ -9,6 +9,9 @@ metadata: labels: {{- include "diode.labels" . | nindent 4 }} app.kubernetes.io/component: {{ include "diode.ingress.name" . }} + {{- if .Values.global.commonLabels }} + {{- include "common.labels.standard" ( dict "customLabels" .Values.global.commonLabels "context" $ ) | nindent 4 }} + {{- end }} annotations: nginx.ingress.kubernetes.io/backend-protocol: "HTTP" nginx.ingress.kubernetes.io/rewrite-target: /$1 @@ -22,6 +25,9 @@ metadata: cert-manager.io/cluster-issuer: {{ $certIssuer.name }} {{- end }} {{- end }} + {{- if .Values.global.commonAnnotations }} + {{- include "common.tplvalues.render" ( dict "value" .Values.global.commonAnnotations "context" $ ) | nindent 4 }} + {{- end }} spec: ingressClassName: {{ $ingressNginx.ingressClass | default "nginx" }} {{- if and $ingressNginx.tls $ingressNginx.hostname }} diff --git a/charts/diode/values.yaml b/charts/diode/values.yaml index edb0b0aa..e64ad687 100644 --- a/charts/diode/values.yaml +++ b/charts/diode/values.yaml @@ -10,6 +10,13 @@ global: # -- wait for PostgreSQL waitForPostgres: true + # -- common annotations for all resources + commonAnnotations: {} + + # -- common labels for all resources + commonLabels: {} + + diode: # -- environment name environment: development @@ -38,7 +45,7 @@ diodeAuth: # -- image tag tag: 1.2.0 # -- secrets with credentials to pull images from a private registry - imagePullSecrets: [ ] + imagePullSecrets: [] # -- pull policy pullPolicy: IfNotPresent # -- replica count @@ -58,10 +65,12 @@ diodeAuth: memory: 512Mi # -- extra environment variables to be set on containers' `env` section extraEnvs: [] + # -- annotations to add to the auth deployment + annotations: {} config: # -- logging level loggingLevel: INFO - # -- sentry DSN + # -- sentry DSN sentryDsn: "" # -- telemetry metrics exporter telemetryMetricsExporter: prometheus @@ -80,12 +89,16 @@ diodeAuthBootstrap: # -- image tag tag: 1.2.0 # -- secrets with credentials to pull images from a private registry - imagePullSecrets: [ ] + imagePullSecrets: [] # -- pull policy pullPolicy: IfNotPresent job: # -- backoff limit backoffLimit: 20 + # -- annotations to add to the auth bootstrap job + annotations: + "helm.sh/hook": post-install, post-upgrade + "helm.sh/hook-weight": "2" # Diode Ingester configuration diodeIngester: @@ -97,7 +110,7 @@ diodeIngester: # -- image tag tag: 1.2.0 # -- secrets with credentials to pull images from a private registry - imagePullSecrets: [ ] + imagePullSecrets: [] # -- pull policy pullPolicy: IfNotPresent # -- replica count @@ -122,10 +135,12 @@ diodeIngester: serviceName: diode.v1.IngesterService # -- extra environment variables to be set on containers' `env` section extraEnvs: [] + # -- annotations to add to the ingester deployment + annotations: {} config: # -- logging level loggingLevel: INFO - # -- sentry DSN + # -- sentry DSN sentryDsn: "" # -- redis stream db redisStreamDb: 1 @@ -146,7 +161,7 @@ diodeReconciler: # -- image tag tag: 1.2.0 # -- secrets with credentials to pull images from a private registry - imagePullSecrets: [ ] + imagePullSecrets: [] # -- pull policy pullPolicy: IfNotPresent # -- replica count @@ -171,6 +186,8 @@ diodeReconciler: serviceName: diode.v1.ReconcilerService # -- extra environment variables to be set on containers' `env` section extraEnvs: [] + # -- annotations to add to the reconciler deployment + annotations: {} config: # -- netbox diode plugin api base url netboxDiodePluginApiBaseUrl: http://localhost:8000/netbox/api/plugins/diode @@ -301,14 +318,14 @@ hydra: # -- public service enabled enabled: true # -- public service type - type: ClusterIP # Ensure service is only internal + type: ClusterIP # Ensure service is only internal # -- public service port port: 4444 admin: # -- admin service enabled enabled: true # -- admin service type - type: ClusterIP # Ensure service is only internal + type: ClusterIP # Ensure service is only internal # -- admin service port port: 4445 ingress: @@ -342,7 +359,7 @@ hydra: # -- automigration enabled enabled: true # -- dev mode - dev: true # Hydra is not exposed to the public internet + dev: true # Hydra is not exposed to the public internet deployment: # -- extra init containers extraInitContainers: |- diff --git a/diode-server/README.md b/diode-server/README.md index 5d49a6cc..7719b5e9 100644 --- a/diode-server/README.md +++ b/diode-server/README.md @@ -2,13 +2,7 @@ The Diode server is a required component of the [Diode](https://github.com/netboxlabs/diode) ingestion service. -Diode is a NetBox ingestion service that greatly simplifies and enhances the process to add and update network data -in NetBox, ensuring your network source of truth is always accurate and can be trusted to power your network automation -pipelines. - -More information about Diode can be found -at [https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/](https://netboxlabs.com/blog/introducing-diode-streamlining-data-ingestion-in-netbox/). - +Diode is a data ingestion service for NetBox that greatly simplifies and enhances the process of adding and updating data in NetBox, ensuring your network source of truth is always accurate and up to date. Our guiding principle in designing Diode has been to make it as easy as possible to get data into NetBox, removing as much burden as possible from the user while shifting that effort to technology. --- @@ -33,7 +27,7 @@ The Diode server is composed of three core services: ## Compatibility The Diode server has been tested with NetBox version **4.2.3**. -It also requires the [Diode NetBox Plugin](https://github.com/netboxlabs/diode-netbox-plugin) **1.x.x**. +It also requires the [Diode NetBox Plugin](https://github.com/netboxlabs/diode-netbox-plugin) **1.1.0**. --- diff --git a/diode-server/auth/cli/cli.go b/diode-server/auth/cli/cli.go index 1783ffbb..e34170ea 100644 --- a/diode-server/auth/cli/cli.go +++ b/diode-server/auth/cli/cli.go @@ -88,6 +88,7 @@ func (c *CreateClientCommand) Run() error { secret := cmd.String("client-secret", "", "client secret [generated if not provided]") owner := cmd.String("owner", "", "owner of the client") clientName := cmd.String("client-name", "", "name of the client") + audience := cmd.String("audience", "", "space separated list of audiences to allow") if err := cmd.Parse(os.Args[2:]); err != nil { return err @@ -109,6 +110,11 @@ func (c *CreateClientCommand) Run() error { } allScopes := strings.Join(scopes, " ") + audiences := []string{} + if *audience != "" { + audiences = strings.Split(*audience, " ") + } + secretProvided := true if *secret == "" { secretProvided = false @@ -133,6 +139,7 @@ func (c *CreateClientCommand) Run() error { ClientSecret: *secret, ClientName: *clientName, Owner: *owner, + Audience: audiences, }) if err != nil { return err diff --git a/diode-server/auth/manager.go b/diode-server/auth/manager.go index d7ee073a..5e0cbecb 100644 --- a/diode-server/auth/manager.go +++ b/diode-server/auth/manager.go @@ -13,12 +13,13 @@ import ( // ClientInfo is a struct that contains information about a client. type ClientInfo struct { - ClientID string `json:"client_id"` - Scope string `json:"scope"` - ClientSecret string `json:"client_secret,omitempty"` - Owner string `json:"owner,omitempty"` - ClientName string `json:"client_name,omitempty"` - CreatedAt string `json:"created_at,omitempty"` + ClientID string `json:"client_id"` + Scope string `json:"scope"` + ClientSecret string `json:"client_secret,omitempty"` + Owner string `json:"owner,omitempty"` + Audience []string `json:"audience,omitempty"` + ClientName string `json:"client_name,omitempty"` + CreatedAt string `json:"created_at,omitempty"` } // RetrieveClientsRequest is a struct that contains information about a request to retrieve clients. diff --git a/diode-server/auth/manager_hydra.go b/diode-server/auth/manager_hydra.go index 44a33cb4..2dc6fc01 100644 --- a/diode-server/auth/manager_hydra.go +++ b/diode-server/auth/manager_hydra.go @@ -59,6 +59,9 @@ func (h *HydraClientManager) CreateClient(ctx context.Context, clientInfo Client if clientInfo.ClientName != "" { newClient.ClientName = &clientInfo.ClientName } + if clientInfo.Audience != nil { + newClient.Audience = clientInfo.Audience + } createdClient, response, err := h.hydraAdmin.OAuth2API.CreateOAuth2Client(ctx).OAuth2Client(newClient).Execute() if response != nil { @@ -196,6 +199,9 @@ func clientInfoFromHydraClient(client *hydra.OAuth2Client) ClientInfo { if client.ClientSecret != nil { clientInfo.ClientSecret = *client.ClientSecret } + if client.Audience != nil { + clientInfo.Audience = client.Audience + } return clientInfo } diff --git a/diode-server/auth/manager_hydra_integration_test.go b/diode-server/auth/manager_hydra_integration_test.go index 285d3b03..2894f0cf 100644 --- a/diode-server/auth/manager_hydra_integration_test.go +++ b/diode-server/auth/manager_hydra_integration_test.go @@ -75,12 +75,14 @@ func TestHydraClientManager(t *testing.T) { ClientID: "diode-test-client-1", ClientSecret: "secret-material", Scope: "test:diode:1 test:diode:2", + Audience: []string{"aud:1", "aud:2"}, }) require.NoError(t, err) require.NotNil(t, createdClient) require.Equal(t, "diode-test-client-1", createdClient.ClientID) require.Equal(t, "secret-material", createdClient.ClientSecret) require.Equal(t, "test:diode:1 test:diode:2", createdClient.Scope) + require.Equal(t, []string{"aud:1", "aud:2"}, createdClient.Audience) // fetch the client by id client, err := manager.RetrieveClientByID(ctx, "diode-test-client-1") @@ -88,6 +90,7 @@ func TestHydraClientManager(t *testing.T) { require.Equal(t, "diode-test-client-1", client.ClientID) require.Equal(t, "", client.ClientSecret) require.Equal(t, "test:diode:1 test:diode:2", client.Scope) + require.Equal(t, []string{"aud:1", "aud:2"}, client.Audience) // client should be in the list of clients currentClients, err = manager.RetrieveClients(ctx, auth.RetrieveClientsRequest{}) diff --git a/diode-server/auth/server.go b/diode-server/auth/server.go index bb2a74e0..85aa5d6b 100644 --- a/diode-server/auth/server.go +++ b/diode-server/auth/server.go @@ -34,18 +34,20 @@ type Server struct { tokenParser TokenParser clientManager ClientManager tokenOwnership TokenOwnershipProvider + decorators []ClientInfoDecorator } // IntrospectResponse is the response for the introspect request type IntrospectResponse struct { - Active bool `json:"active"` - Subject string `json:"sub,omitempty"` - Scope string `json:"scope,omitempty"` - ExpiresAt int64 `json:"exp,omitempty"` - IssuedAt int64 `json:"iat,omitempty"` - Issuer string `json:"iss,omitempty"` - ClientID string `json:"client_id,omitempty"` - Username string `json:"username,omitempty"` + Active bool `json:"active"` + Subject string `json:"sub,omitempty"` + Scope string `json:"scope,omitempty"` + ExpiresAt int64 `json:"exp,omitempty"` + IssuedAt int64 `json:"iat,omitempty"` + Issuer string `json:"iss,omitempty"` + ClientID string `json:"client_id,omitempty"` + Username string `json:"username,omitempty"` + Audience []string `json:"aud,omitempty"` } // CreateClientRequest request to create client @@ -97,6 +99,11 @@ func (p *DefaultTokenOwner) TokenOwnerID(_ context.Context, _ string) (string, e return DefaultTokenOwnerID, nil } +// ClientInfoDecorator attaches additional information to a client info +type ClientInfoDecorator interface { + VisitClientInfo(ctx context.Context, clientInfo *ClientInfo) error +} + // NewServer creates a new auth server func NewServer(_ context.Context, logger *slog.Logger, tokenParser TokenParser, clientManager ClientManager, tokenOwnership TokenOwnershipProvider) (*Server, error) { var cfg Config @@ -171,6 +178,12 @@ func (s *Server) RegisterHandlers() { s.mux.HandleFunc("DELETE /clients/{clientID}", s.deleteClient) } +// AddClientInfoDecorator adds a ClientInfoDecorator to the server +// these are called prior to a user generated client being created. +func (s *Server) AddClientInfoDecorator(decorator ClientInfoDecorator) { + s.decorators = append(s.decorators, decorator) +} + // introspect handles the introspect request func (s *Server) introspect(w http.ResponseWriter, r *http.Request) { jwtToken, err := s.getAuthToken(r) @@ -198,6 +211,11 @@ func (s *Server) introspect(w http.ResponseWriter, r *http.Request) { Username: getStringClaim(claims, "username"), } + aud := getStringOrStringArrayClaim(claims, "aud") + if len(aud) > 0 { + resp.Audience = aud + } + err = writeJSON(w, http.StatusOK, resp) if err != nil { s.logger.Error("failed to write response", "error", err) @@ -294,6 +312,27 @@ func getStringClaim(claims jwt.MapClaims, key string) string { return "" } +func getStringOrStringArrayClaim(claims jwt.MapClaims, key string) []string { + // specification allows some standard claims to be either + // single strings or list of strings + if val, ok := claims[key].(string); ok { + if val == "" { + return []string{} + } + return []string{val} + } + if val, ok := claims[key].([]interface{}); ok { + out := make([]string, 0, len(val)) + for _, v := range val { + if s, ok := v.(string); ok { + out = append(out, s) + } + } + return out + } + return []string{} +} + func getInt64Claim(claims jwt.MapClaims, key string) int64 { switch val := claims[key].(type) { case float64: @@ -430,6 +469,15 @@ func (s *Server) createClient(w http.ResponseWriter, r *http.Request) { return } + for _, decorator := range s.decorators { + err = decorator.VisitClientInfo(r.Context(), &clientInfo) + if err != nil { + s.logger.Error("failed to decorate client info", "error", err) + w.WriteHeader(statusFromError(err)) + return + } + } + // Create the client created, err := s.clientManager.CreateClient(r.Context(), clientInfo) if err != nil { diff --git a/diode-server/auth/server_hydra_integration_test.go b/diode-server/auth/server_hydra_integration_test.go index 33c6b54d..01c61b56 100644 --- a/diode-server/auth/server_hydra_integration_test.go +++ b/diode-server/auth/server_hydra_integration_test.go @@ -101,13 +101,16 @@ func TestServerHydraIntegration(t *testing.T) { require.NoError(t, err) require.NotNil(t, server) + testAudience := []string{"aud:1", "aud:2"} + server.AddClientInfoDecorator(&audienceDecorator{aud: testAudience}) + testServer := httptest.NewServer(server.GetMux()) defer testServer.Close() client := &authTestClient{ endpoint: testServer.URL, } - client.authenticate(t, testClientID, testClientSecret, testClientScope) + client.authenticate(t, testClientID, testClientSecret, testClientScope, []string{}) // list clients (should be empty, only includes user created clients) result := client.listClients(t, "", 0) @@ -198,7 +201,7 @@ func TestServerHydraIntegration(t *testing.T) { tokenClientInfo := client.createClient(t, "test-client-token-auth", ingestClientScope) // call the token endpoint with the credentials and verify that a token comes back ... - resp := client.getToken(t, tokenClientInfo.ClientID, tokenClientInfo.ClientSecret, ingestClientScope) + resp := client.getToken(t, tokenClientInfo.ClientID, tokenClientInfo.ClientSecret, ingestClientScope, []string{"aud:2"}) defer func() { _ = resp.Body.Close() }() @@ -220,9 +223,21 @@ func TestServerHydraIntegration(t *testing.T) { scopeClaim, ok := claims["scope"] require.True(t, ok) require.Equal(t, ingestClientScope, scopeClaim) + audienceClaim, ok := claims["aud"] + require.True(t, ok) + // either of these is technically valid according to the spec + if audience, ok := audienceClaim.(string); ok { + require.Equal(t, "aud:2", audience) + } + if audiences, ok := audienceClaim.([]interface{}); ok { + require.Equal(t, 1, len(audiences)) + audience, ok := audiences[0].(string) + require.True(t, ok) + require.Equal(t, "aud:2", audience) + } // try to use the credentials to create a token with a different scope ... - resp = client.getToken(t, tokenClientInfo.ClientID, tokenClientInfo.ClientSecret, "netbox:read") + resp = client.getToken(t, tokenClientInfo.ClientID, tokenClientInfo.ClientSecret, "netbox:read", []string{"aud:2"}) require.Equal(t, http.StatusBadRequest, resp.StatusCode) } @@ -231,12 +246,15 @@ type authTestClient struct { token string } -func (c *authTestClient) getToken(t *testing.T, clientID string, clientSecret string, scope string) *http.Response { +func (c *authTestClient) getToken(t *testing.T, clientID string, clientSecret string, scope string, audience []string) *http.Response { data := url.Values{} data.Set("grant_type", "client_credentials") data.Set("client_id", clientID) data.Set("client_secret", clientSecret) data.Set("scope", scope) + if len(audience) > 0 { + data.Set("audience", strings.Join(audience, " ")) + } req, err := http.NewRequest(http.MethodPost, c.endpoint+"/token", strings.NewReader(data.Encode())) require.NoError(t, err) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -331,12 +349,15 @@ func (c *authTestClient) deleteClient(t *testing.T, clientID string) { require.Equal(t, http.StatusNoContent, resp.StatusCode) } -func (c *authTestClient) authenticate(t *testing.T, clientID string, clientSecret string, scope string) { +func (c *authTestClient) authenticate(t *testing.T, clientID string, clientSecret string, scope string, audience []string) { data := url.Values{} data.Set("grant_type", "client_credentials") data.Set("client_id", clientID) data.Set("client_secret", clientSecret) data.Set("scope", scope) + if len(audience) > 0 { + data.Set("audience", strings.Join(audience, " ")) + } req, err := http.NewRequest(http.MethodPost, c.endpoint+"/token", strings.NewReader(data.Encode())) require.NoError(t, err) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") @@ -359,3 +380,12 @@ func (c *authTestClient) authenticate(t *testing.T, clientID string, clientSecre c.token = result.AccessToken } + +type audienceDecorator struct { + aud []string +} + +func (d *audienceDecorator) VisitClientInfo(_ context.Context, clientInfo *auth.ClientInfo) error { + clientInfo.Audience = d.aud + return nil +} diff --git a/diode-server/auth/server_test.go b/diode-server/auth/server_test.go index 92b0e689..836ba8dd 100644 --- a/diode-server/auth/server_test.go +++ b/diode-server/auth/server_test.go @@ -31,27 +31,6 @@ func (p InvalidParser) Parse(_ string, _ jwt.Keyfunc) (*jwt.Token, error) { return nil, fmt.Errorf("invalid token") } -type ValidTokenParser struct{} - -func (p ValidTokenParser) Parse(_ string, _ jwt.Keyfunc) (*jwt.Token, error) { - claims := jwt.MapClaims{ - "iss": "https://auth.example.com", - "sub": "user123", - "aud": "api", - "exp": time.Now().Add(time.Hour).Unix(), - "iat": time.Now().Unix(), - "client_id": "client123", - "scope": "read write", - "username": "testuser", - } - - token := &jwt.Token{ - Claims: claims, - Valid: true, - } - return token, nil -} - type MockTokenParser struct { tokenMap map[string]jwt.Token } @@ -154,10 +133,75 @@ func TestIntrospectForInvalidTokens(t *testing.T) { } func TestIntrospectForValidTokens(t *testing.T) { - ctx := context.Background() - - setupEnv() - defer teardownEnv() + testToken := "eyJhbGciOiJSUzI1NiIsImtpZCI6InRlc3Qta2V5IiwidHlwIjoiSldUIn0.eyJpc3MiOiJodHRwczovL2F1dGguZXhhbXBsZS5jb20iLCJzdWIiOiJ1c2VyMTIzIiwiYXVkIjoiYXBpIiwiZXhwIjoxNjUwMDAwMDAwLCJpYXQiOjE1MDAwMDAwMDAsImNsaWVudF9pZCI6ImNsaWVudDEyMyIsInNjb3BlIjoicmVhZCB3cml0ZSIsInVzZXJuYW1lIjoidGVzdHVzZXIifQ.WcPGXClpKD7Bc1C0CCDA1060E2GGlTfamrd8-W0ghBE" + tests := []struct { + name string + token string + tokenParser auth.TokenParser + expectedStatus int + expectedAudience []string + expectedSubject string + expectedScope string + expectedIssuer string + expectedClientID string + expectedUsername string + }{ + { + name: "Valid Token", + token: testToken, + tokenParser: &MockTokenParser{ + tokenMap: map[string]jwt.Token{ + testToken: { + Claims: jwt.MapClaims{ + "iss": "https://auth.example.com", + "sub": "user123", + "aud": "api", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + "client_id": "client123", + "scope": "read write", + "username": "testuser", + }, + Valid: true, + }, + }, + }, + expectedStatus: http.StatusOK, + expectedAudience: []string{"api"}, + expectedSubject: "user123", + expectedScope: "read write", + expectedIssuer: "https://auth.example.com", + expectedClientID: "client123", + expectedUsername: "testuser", + }, + { + name: "Valid Token with empty audience", + tokenParser: &MockTokenParser{ + tokenMap: map[string]jwt.Token{ + testToken: { + Claims: jwt.MapClaims{ + "iss": "https://auth.example.com", + "sub": "user123", + "exp": time.Now().Add(time.Hour).Unix(), + "iat": time.Now().Unix(), + "client_id": "client123", + "scope": "read write", + "username": "testuser", + }, + Valid: true, + }, + }, + }, + token: testToken, + expectedStatus: http.StatusOK, + expectedAudience: nil, + expectedSubject: "user123", + expectedScope: "read write", + expectedIssuer: "https://auth.example.com", + expectedClientID: "client123", + expectedUsername: "testuser", + }, + } // Setup a test server to mock the OAuth2 server mockJWKSServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -180,47 +224,52 @@ func TestIntrospectForValidTokens(t *testing.T) { })) defer mockJWKSServer.Close() + setupEnv() + defer teardownEnv() + _ = os.Setenv("OAUTH2_PUBLIC_SERVER_URL", mockJWKSServer.URL) defer func() { _ = os.Unsetenv("OAUTH2_PUBLIC_SERVER_URL") }() logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) - server, err := auth.NewServer(ctx, logger, ValidTokenParser{}, nil, nil) - require.NoError(t, err) - require.NotNil(t, server) - // Create a test server using the server's mux - testServer := httptest.NewServer(server.GetMux()) - defer testServer.Close() + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := context.Background() - t.Run("Valid Token", func(t *testing.T) { - // This is just a dummy token for testing purposes - testToken := "eyJhbGciOiJSUzI1NiIsImtpZCI6InRlc3Qta2V5IiwidHlwIjoiSldUIn0.eyJpc3MiOiJodHRwczovL2F1dGguZXhhbXBsZS5jb20iLCJzdWIiOiJ1c2VyMTIzIiwiYXVkIjoiYXBpIiwiZXhwIjoxNjUwMDAwMDAwLCJpYXQiOjE1MDAwMDAwMDAsImNsaWVudF9pZCI6ImNsaWVudDEyMyIsInNjb3BlIjoicmVhZCB3cml0ZSIsInVzZXJuYW1lIjoidGVzdHVzZXIifQ.WcPGXClpKD7Bc1C0CCDA1060E2GGlTfamrd8-W0ghBE" + server, err := auth.NewServer(ctx, logger, test.tokenParser, nil, nil) + require.NoError(t, err) + require.NotNil(t, server) - data := url.Values{} - data.Set("token", testToken) + // Create a test server using the server's mux + testServer := httptest.NewServer(server.GetMux()) + defer testServer.Close() - resp, err := makeIntrospectRequest(testServer.URL, testToken) + data := url.Values{} + data.Set("token", test.token) - require.NoError(t, err) - require.Equal(t, http.StatusOK, resp.StatusCode) + resp, err := makeIntrospectRequest(testServer.URL, test.token) - defer func() { - _ = resp.Body.Close() - }() + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) - var introspectResp auth.IntrospectResponse - err = json.NewDecoder(resp.Body).Decode(&introspectResp) - require.NoError(t, err) - require.True(t, introspectResp.Active) - require.Equal(t, "user123", introspectResp.Subject) - require.Equal(t, "read write", introspectResp.Scope) - require.Equal(t, "https://auth.example.com", introspectResp.Issuer) - require.Equal(t, "client123", introspectResp.ClientID) + defer func() { + _ = resp.Body.Close() + }() - require.Equal(t, "testuser", introspectResp.Username) - }) + var introspectResp auth.IntrospectResponse + err = json.NewDecoder(resp.Body).Decode(&introspectResp) + require.NoError(t, err) + require.True(t, introspectResp.Active) + require.Equal(t, test.expectedSubject, introspectResp.Subject) + require.Equal(t, test.expectedScope, introspectResp.Scope) + require.Equal(t, test.expectedIssuer, introspectResp.Issuer) + require.Equal(t, test.expectedClientID, introspectResp.ClientID) + require.Equal(t, test.expectedAudience, introspectResp.Audience) + require.Equal(t, test.expectedUsername, introspectResp.Username) + }) + } } func TestCreateClient(t *testing.T) { @@ -614,7 +663,7 @@ func TestDeleteClient(t *testing.T) { accessToken string clientID string parsedToken jwt.Token - lookupResult auth.ClientInfo + lookupResult *auth.ClientInfo lookupErr error expectStatus int }{ @@ -623,7 +672,7 @@ func TestDeleteClient(t *testing.T) { accessToken: validAccessToken, clientID: "test-client-1-abcdef0123567890", parsedToken: writeToken, - lookupResult: auth.ClientInfo{ + lookupResult: &auth.ClientInfo{ ClientID: "test-client-1-abcdef0123567890", ClientName: "Test Client 1", Scope: "diode:ingest", @@ -659,7 +708,7 @@ func TestDeleteClient(t *testing.T) { accessToken: validAccessToken, clientID: "test-client-1-abcdef0123567890", parsedToken: writeToken, - lookupResult: auth.ClientInfo{ + lookupResult: &auth.ClientInfo{ ClientID: "test-client-1-abcdef0123567890", ClientName: "Test Client 1", Owner: "diode/system", @@ -704,8 +753,10 @@ func TestDeleteClient(t *testing.T) { testServer := httptest.NewServer(server.GetMux()) defer testServer.Close() - if test.lookupResult != (auth.ClientInfo{}) || test.lookupErr != nil { - mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(test.lookupResult, test.lookupErr) + if test.lookupResult != nil { + mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(*test.lookupResult, test.lookupErr) + } else if test.lookupErr != nil { + mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(auth.ClientInfo{}, test.lookupErr) } if test.expectStatus == http.StatusNoContent { @@ -748,7 +799,7 @@ func TestGetClient(t *testing.T) { accessToken string clientID string parsedToken jwt.Token - lookupResult auth.ClientInfo + lookupResult *auth.ClientInfo lookupErr error expectStatus int expect auth.ClientResponse @@ -758,7 +809,7 @@ func TestGetClient(t *testing.T) { accessToken: validAccessToken, clientID: "test-client-1-abcdef0123567890", parsedToken: readOnlyToken, - lookupResult: auth.ClientInfo{ + lookupResult: &auth.ClientInfo{ ClientID: "test-client-1-abcdef0123567890", ClientName: "Test Client 1", Scope: "diode:ingest", @@ -793,7 +844,7 @@ func TestGetClient(t *testing.T) { accessToken: validAccessToken, clientID: "test-client-1-abcdef0123567890", parsedToken: readOnlyToken, - lookupResult: auth.ClientInfo{ + lookupResult: &auth.ClientInfo{ ClientID: "test-client-1-abcdef0123567890", ClientName: "Test Client 1", Owner: "diode/system", @@ -838,8 +889,10 @@ func TestGetClient(t *testing.T) { testServer := httptest.NewServer(server.GetMux()) defer testServer.Close() - if test.lookupResult != (auth.ClientInfo{}) || test.lookupErr != nil { - mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(test.lookupResult, test.lookupErr) + if test.lookupResult != nil { + mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(*test.lookupResult, test.lookupErr) + } else if test.lookupErr != nil { + mockClientManager.EXPECT().RetrieveClientByID(mock.Anything, test.clientID).Return(auth.ClientInfo{}, test.lookupErr) } req, _ := http.NewRequest("GET", testServer.URL+"/clients/"+test.clientID, nil) diff --git a/diode-server/cmd/reconciler/main.go b/diode-server/cmd/reconciler/main.go index 0333240f..f28f810b 100644 --- a/diode-server/cmd/reconciler/main.go +++ b/diode-server/cmd/reconciler/main.go @@ -111,9 +111,17 @@ func main() { diodeToNetBoxMaxRetries := 3 - nbClient, err := netboxdiodeplugin.NewClient(s.Logger(), cfg.NetBoxDiodePluginAPIBaseURL, cfg.DiodeToNetBoxClientID, - cfg.DiodeToNetBoxClientSecret, cfg.DiodeAuthTokenURL, cfg.DiodeToNetBoxRateLimiterRPS, - cfg.DiodeToNetBoxRateLimiterBurst, diodeToNetBoxMaxRetries) + nbClient, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: s.Logger(), + BaseURL: cfg.NetBoxDiodePluginAPIBaseURL, + ClientID: cfg.DiodeToNetBoxClientID, + ClientSecret: cfg.DiodeToNetBoxClientSecret, + TokenURL: cfg.DiodeAuthTokenURL, + RateLimitRPS: cfg.DiodeToNetBoxRateLimiterRPS, + RateLimitBurstRPS: cfg.DiodeToNetBoxRateLimiterBurst, + MaxRetries: diodeToNetBoxMaxRetries, + }) if err != nil { s.Logger().Error("failed to create netbox diode plugin client", "error", err) os.Exit(1) diff --git a/diode-server/netboxdiodeplugin/client.go b/diode-server/netboxdiodeplugin/client.go index 6b71a2ef..82b781d5 100644 --- a/diode-server/netboxdiodeplugin/client.go +++ b/diode-server/netboxdiodeplugin/client.go @@ -133,18 +133,43 @@ func NewHTTPTransport() *http.Transport { } } +// ClientOptions represents the options for creating a new NetBox Diode plugin client +type ClientOptions struct { + // BaseURL is the base URL of the NetBox Diode plugin API + BaseURL string + + // ClientID is the client ID for the NetBox Diode plugin API + ClientID string + // ClientSecret is the client secret for the NetBox Diode plugin API + ClientSecret string + // TokenURL is the URL of the token endpoint + TokenURL string + // TokenEndpointParams are extra parameters provided to the token endpoint + TokenEndpointParams url.Values + + // Logger is the logger for the NetBox Diode plugin client + Logger *slog.Logger + + // RateLimitRPS is the rate limit for the NetBox Diode plugin client + RateLimitRPS int + // RateLimitBurstRPS is the rate limit burst for the NetBox Diode plugin client + RateLimitBurstRPS int + // MaxRetries is the maximum number of retries for the NetBox Diode plugin client + MaxRetries int +} + // NewClient creates a new NetBox Diode plugin client -func NewClient(logger *slog.Logger, baseURL, clientID, clientSecret, tokenURL string, rateLimitRps, rateLimitBurstRps int, maxRetries int) (*Client, error) { - u, err := url.Parse(baseURL) +func NewClient(options ClientOptions) (*Client, error) { + u, err := url.Parse(options.BaseURL) if err != nil { return nil, err } - if len(clientID) == 0 || len(clientSecret) == 0 { + if len(options.ClientID) == 0 || len(options.ClientSecret) == 0 { return nil, fmt.Errorf("client ID or secret not provided") } - if len(tokenURL) == 0 { + if len(options.TokenURL) == 0 { return nil, fmt.Errorf("token URL not provided") } @@ -153,11 +178,11 @@ func NewClient(logger *slog.Logger, baseURL, clientID, clientSecret, tokenURL st return nil, err } - if rateLimitRps <= 0 || rateLimitBurstRps <= 0 { - return nil, fmt.Errorf("invalid rate limit values: %d %d", rateLimitRps, rateLimitBurstRps) + if options.RateLimitRPS <= 0 || options.RateLimitBurstRPS <= 0 { + return nil, fmt.Errorf("invalid rate limit values: %d %d", options.RateLimitRPS, options.RateLimitBurstRPS) } - t, err := url.Parse(tokenURL) + t, err := url.Parse(options.TokenURL) if err != nil { return nil, err } @@ -169,18 +194,19 @@ func NewClient(logger *slog.Logger, baseURL, clientID, clientSecret, tokenURL st } rhttp := retryablehttp.NewClient() - rhttp.RetryMax = maxRetries + rhttp.RetryMax = options.MaxRetries rhttp.RetryWaitMin = 150 * time.Millisecond rhttp.RetryWaitMax = 2 * time.Second - rhttp.Logger = logger + rhttp.Logger = options.Logger rhttp.HTTPClient.Transport = headerRoundTripper oauthConfig := &clientcredentials.Config{ - ClientID: clientID, - ClientSecret: clientSecret, - TokenURL: t.String(), - Scopes: []string{authutil.ScopeNetBoxRead, authutil.ScopeNetBoxWrite}, + ClientID: options.ClientID, + ClientSecret: options.ClientSecret, + TokenURL: t.String(), + Scopes: []string{authutil.ScopeNetBoxRead, authutil.ScopeNetBoxWrite}, + EndpointParams: options.TokenEndpointParams, } oauthTransport := &oauth2.Transport{ @@ -194,11 +220,11 @@ func NewClient(logger *slog.Logger, baseURL, clientID, clientSecret, tokenURL st } client := &Client{ - logger: logger, + logger: options.Logger, http: httpClient, baseURL: u, userAgent: fmt.Sprintf("%s/%s", SDKName, SDKVersion), - limiter: rate.NewLimiter(rate.Limit(rateLimitRps), rateLimitBurstRps), + limiter: rate.NewLimiter(rate.Limit(options.RateLimitRPS), options.RateLimitBurstRPS), } return client, nil diff --git a/diode-server/netboxdiodeplugin/client_test.go b/diode-server/netboxdiodeplugin/client_test.go index bcce3536..2512ae4c 100644 --- a/diode-server/netboxdiodeplugin/client_test.go +++ b/diode-server/netboxdiodeplugin/client_test.go @@ -8,6 +8,7 @@ import ( "log/slog" "net/http" "net/http/httptest" + "net/url" "os" "testing" @@ -209,7 +210,17 @@ func TestNewClient(t *testing.T) { maxRetries := 3 - client, err := netboxdiodeplugin.NewClient(logger, tt.baseURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, tt.diodeAuthTokenURL, tt.rateLimiterRPS, tt.rateLimiterBurst, maxRetries) + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: tt.baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: tt.diodeAuthTokenURL, + RateLimitRPS: tt.rateLimiterRPS, + RateLimitBurstRPS: tt.rateLimiterBurst, + MaxRetries: maxRetries, + }) if tt.shouldError { require.Error(t, err) return @@ -351,7 +362,7 @@ func TestGenerateDiff(t *testing.T) { expectedToken := "mocked-token" authTokenURL := "/diode/auth/token" - mockOAuth2Server := newMockOAuth2Server(authTokenURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, expectedToken) + mockOAuth2Server := newMockOAuth2Server(authTokenURL, requireCredentials(tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret), expectedToken) defer mockOAuth2Server.Close() mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL @@ -381,7 +392,17 @@ func TestGenerateDiff(t *testing.T) { defer ts.Close() baseURL := fmt.Sprintf("%s/api/diode", ts.URL) - client, err := netboxdiodeplugin.NewClient(logger, baseURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, mockOAuth2ServerURL, tt.rateLimiterRPS, tt.rateLimiterBurst, tt.maxRetries) + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: tt.rateLimiterRPS, + RateLimitBurstRPS: tt.rateLimiterBurst, + MaxRetries: tt.maxRetries, + }) require.NoError(t, err) resp, err := client.GenerateDiff(context.Background(), tt.generateDiffRequest) if tt.shouldError { @@ -473,7 +494,7 @@ func TestGenerateDiffRateLimiting(t *testing.T) { expectedToken := "mocked-token" authTokenURL := "/diode/auth/token" - mockOAuth2Server := newMockOAuth2Server(authTokenURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, expectedToken) + mockOAuth2Server := newMockOAuth2Server(authTokenURL, requireCredentials(tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret), expectedToken) defer mockOAuth2Server.Close() mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL @@ -504,7 +525,17 @@ func TestGenerateDiffRateLimiting(t *testing.T) { defer ts.Close() baseURL := fmt.Sprintf("%s/api/diode", ts.URL) - client, err := netboxdiodeplugin.NewClient(logger, baseURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, mockOAuth2ServerURL, tt.rateLimiterRPS, tt.rateLimiterBurst, tt.maxRetries) + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: tt.rateLimiterRPS, + RateLimitBurstRPS: tt.rateLimiterBurst, + MaxRetries: tt.maxRetries, + }) require.NoError(t, err) resp, err := client.GenerateDiff(context.Background(), tt.generateDiffRequest) _, _ = client.GenerateDiff(context.Background(), tt.generateDiffRequest) @@ -521,6 +552,126 @@ func TestGenerateDiffRateLimiting(t *testing.T) { } } +func TestAuthenticationParams(t *testing.T) { + testRequest := netboxdiodeplugin.GenerateDiffRequest{ + ObjectType: "dcim.device", + Entity: &diodepb.Entity{ + Entity: &diodepb.Entity_Device{ + Device: &diodepb.Device{ + Name: strPtr("test"), + Site: &diodepb.Site{ + Name: "test-site", + }, + }, + }, + }, + } + mockServerResponse := `{"id": "00000000-0000-0000-0000-000000000001", "change_set": {"id": "00000000-0000-0000-0000-000000000001", "changes": [{"id": "00000000-0000-0000-0000-000000000002", "change_type": "create", "object_type": "dcim.device", "data": {"name": "test"}}]}}` + + tests := []struct { + name string + baseURL string + diodeAuthTokenURL string + diodeToNetBoxClientID string + diodeToNetBoxClientSecret string + extraParams map[string]string + requiredParams map[string]string + shouldError bool + expectedError error + }{ + { + name: "valid authentication params, no extra params", + baseURL: "http://", + diodeAuthTokenURL: "http://diode-auth:8000/diode/auth/token", + diodeToNetBoxClientID: "test_client", + diodeToNetBoxClientSecret: "test_secret", + requiredParams: requireCredentials("test_client", "test_secret"), + shouldError: false, + }, + { + name: "valid authentication params, with extra params", + baseURL: "http://", + diodeAuthTokenURL: "http://diode-auth:8000/diode/auth/token", + diodeToNetBoxClientID: "test_client", + diodeToNetBoxClientSecret: "test_secret", + extraParams: map[string]string{"test_param": "test_value"}, + requiredParams: map[string]string{ + "client_id": "test_client", + "client_secret": "test_secret", + "test_param": "test_value", + }, + shouldError: false, + }, + { + name: "valid authentication params, missing required params", + baseURL: "http://", + diodeAuthTokenURL: "http://diode-auth:8000/diode/auth/token", + diodeToNetBoxClientID: "test_client", + diodeToNetBoxClientSecret: "test_secret", + requiredParams: map[string]string{ + "client_id": "test_client", + "client_secret": "test_secret", + "test_param": "test_value", + }, + shouldError: true, + }, + } + + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cleanUpEnvVars() + + expectedToken := "mocked-token" + authTokenURL := "/diode/auth/token" + + mockOAuth2Server := newMockOAuth2Server(authTokenURL, tt.requiredParams, expectedToken) + defer mockOAuth2Server.Close() + + mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL + + handler := func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockServerResponse)) + } + mux := http.NewServeMux() + mux.HandleFunc("/api/diode/generate-diff/", handler) + ts := httptest.NewServer(mux) + defer ts.Close() + + baseURL := fmt.Sprintf("%s/api/diode", ts.URL) + + extraParams := url.Values{} + for k, v := range tt.extraParams { + extraParams.Set(k, v) + } + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + TokenEndpointParams: extraParams, + RateLimitRPS: 1, + RateLimitBurstRPS: 1, + MaxRetries: 0, + }) + require.NoError(t, err) + _, err = client.GenerateDiff(context.Background(), testRequest) + if tt.shouldError { + require.Error(t, err) + if tt.expectedError != nil { + assert.Equal(t, tt.expectedError, err) + } + return + } + require.NoError(t, err) + }) + } +} + func TestApplyChangeSet(t *testing.T) { tests := []struct { name string @@ -702,7 +853,7 @@ func TestApplyChangeSet(t *testing.T) { expectedToken := "mocked-token" authTokenURL := "/diode/auth/token" - mockOAuth2Server := newMockOAuth2Server(authTokenURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, expectedToken) + mockOAuth2Server := newMockOAuth2Server(authTokenURL, requireCredentials(tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret), expectedToken) defer mockOAuth2Server.Close() mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL @@ -727,7 +878,17 @@ func TestApplyChangeSet(t *testing.T) { defer ts.Close() baseURL := fmt.Sprintf("%s/api/diode", ts.URL) - client, err := netboxdiodeplugin.NewClient(logger, baseURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, mockOAuth2ServerURL, tt.rateLimiterRPS, tt.rateLimiterBurst, tt.maxRetries) + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: tt.rateLimiterRPS, + RateLimitBurstRPS: tt.rateLimiterBurst, + MaxRetries: tt.maxRetries, + }) require.NoError(t, err) resp, err := client.ApplyChangeSet(context.Background(), tt.changeSetRequest) if tt.shouldError { @@ -807,7 +968,7 @@ func TestApplyChangeSetRateLimiting(t *testing.T) { expectedToken := "mocked-token" authTokenURL := "/diode/auth/token" - mockOAuth2Server := newMockOAuth2Server(authTokenURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, expectedToken) + mockOAuth2Server := newMockOAuth2Server(authTokenURL, requireCredentials(tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret), expectedToken) defer mockOAuth2Server.Close() mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL @@ -834,7 +995,17 @@ func TestApplyChangeSetRateLimiting(t *testing.T) { baseURL := fmt.Sprintf("%s/api/diode", ts.URL) - client, err := netboxdiodeplugin.NewClient(logger, baseURL, tt.diodeToNetBoxClientID, tt.diodeToNetBoxClientSecret, mockOAuth2ServerURL, tt.rateLimiterRPS, tt.rateLimiterBurst, tt.maxRetries) + client, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: baseURL, + ClientID: tt.diodeToNetBoxClientID, + ClientSecret: tt.diodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: tt.rateLimiterRPS, + RateLimitBurstRPS: tt.rateLimiterBurst, + MaxRetries: tt.maxRetries, + }) require.NoError(t, err) resp, err := client.ApplyChangeSet(context.Background(), tt.changeSetRequest) _, _ = client.ApplyChangeSet(context.Background(), tt.changeSetRequest) @@ -864,7 +1035,14 @@ func strPtr(s string) *string { return &s } -func newMockOAuth2Server(authTokenURL, wantClientID, wantClientSecret, mockedToken string) *httptest.Server { +func requireCredentials(clientID, clientSecret string) map[string]string { + return map[string]string{ + "client_id": clientID, + "client_secret": clientSecret, + } +} + +func newMockOAuth2Server(authTokenURL string, requiredParams map[string]string, mockedToken string) *httptest.Server { handler := http.NewServeMux() handler.HandleFunc(authTokenURL, func(w http.ResponseWriter, r *http.Request) { @@ -873,18 +1051,19 @@ func newMockOAuth2Server(authTokenURL, wantClientID, wantClientSecret, mockedTok return } - // Optional: Validate client credentials - if r.PostForm.Get("client_id") != wantClientID || r.PostForm.Get("client_secret") != wantClientSecret { - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusUnauthorized) - if err := json.NewEncoder(w).Encode(map[string]string{ - "error": "unauthorized", - "error_description": "Authentication required", - }); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + for k, v := range requiredParams { + if r.PostForm.Get(k) != v { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnauthorized) + if err := json.NewEncoder(w).Encode(map[string]string{ + "error": "unauthorized", + "error_description": "Authentication required", + }); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } return } - return } // Simulate token response diff --git a/diode-server/reconciler/ingestion_processor.go b/diode-server/reconciler/ingestion_processor.go index 43e0c505..ebafde20 100644 --- a/diode-server/reconciler/ingestion_processor.go +++ b/diode-server/reconciler/ingestion_processor.go @@ -7,6 +7,7 @@ import ( "log/slog" "os" "strconv" + "sync" "github.com/google/uuid" "github.com/kelseyhightower/envconfig" @@ -62,6 +63,8 @@ type IngestionProcessor struct { redisConsumerGroup string ops IngestionProcessorOps metrics IngestionProcessorMetrics + cancel context.CancelFunc + mx sync.Mutex } // IngestionLogToProcess represents an ingestion log to process @@ -120,12 +123,22 @@ func (p *IngestionProcessor) Name() string { // Start starts the component func (p *IngestionProcessor) Start(ctx context.Context) error { p.logger.Info("starting component", "name", p.Name()) + p.mx.Lock() + ctx, cancel := context.WithCancel(ctx) + p.cancel = cancel + p.mx.Unlock() return p.consumeIngestionStream(ctx, p.redisStreamID, p.redisConsumerGroup, fmt.Sprintf("%s-%s", p.redisConsumerGroup, p.hostname)) } // Stop stops the component func (p *IngestionProcessor) Stop() error { p.logger.Info("stopping component", "name", p.Name()) + p.mx.Lock() + if p.cancel != nil { + p.cancel() + p.cancel = nil + } + p.mx.Unlock() redisClientErr := p.redisClient.Close() redisStreamErr := p.redisStreamClient.Close() @@ -139,6 +152,12 @@ func (p *IngestionProcessor) consumeIngestionStream(ctx context.Context, redisSt } for { + select { + case <-ctx.Done(): + p.logger.Debug("ingestion processor exiting consumer loop on request") + return nil + default: + } streams, err := p.redisStreamClient.XReadGroup(ctx, &redis.XReadGroupArgs{ Group: redisConsumerGroup, Consumer: redisConsumer, diff --git a/diode-server/reconciler/ingestion_processor_test.go b/diode-server/reconciler/ingestion_processor_test.go index ebb0fab8..4f465574 100644 --- a/diode-server/reconciler/ingestion_processor_test.go +++ b/diode-server/reconciler/ingestion_processor_test.go @@ -47,7 +47,17 @@ func TestNewIngestionProcessor(t *testing.T) { mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL - nbClient, err := netboxdiodeplugin.NewClient(logger, cfg.NetBoxDiodePluginAPIBaseURL, cfg.DiodeToNetBoxClientID, cfg.DiodeToNetBoxClientSecret, mockOAuth2ServerURL, cfg.DiodeToNetBoxRateLimiterRPS, cfg.DiodeToNetBoxRateLimiterBurst, 0) + nbClient, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: cfg.NetBoxDiodePluginAPIBaseURL, + ClientID: cfg.DiodeToNetBoxClientID, + ClientSecret: cfg.DiodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: cfg.DiodeToNetBoxRateLimiterRPS, + RateLimitBurstRPS: cfg.DiodeToNetBoxRateLimiterBurst, + MaxRetries: 0, + }) require.NoError(t, err) metrics := mocks.NewIngestionProcessorMetrics(t) @@ -97,7 +107,17 @@ func TestIngestionProcessorStart(t *testing.T) { mockOAuth2ServerURL := mockOAuth2Server.URL + authTokenURL maxRetries := 3 - nbClient, err := netboxdiodeplugin.NewClient(logger, cfg.NetBoxDiodePluginAPIBaseURL, cfg.DiodeToNetBoxClientID, cfg.DiodeToNetBoxClientSecret, mockOAuth2ServerURL, cfg.DiodeToNetBoxRateLimiterRPS, cfg.DiodeToNetBoxRateLimiterBurst, maxRetries) + nbClient, err := netboxdiodeplugin.NewClient( + netboxdiodeplugin.ClientOptions{ + Logger: logger, + BaseURL: cfg.NetBoxDiodePluginAPIBaseURL, + ClientID: cfg.DiodeToNetBoxClientID, + ClientSecret: cfg.DiodeToNetBoxClientSecret, + TokenURL: mockOAuth2ServerURL, + RateLimitRPS: cfg.DiodeToNetBoxRateLimiterRPS, + RateLimitBurstRPS: cfg.DiodeToNetBoxRateLimiterBurst, + MaxRetries: maxRetries, + }) require.NoError(t, err) mockMetrics := new(mocks.IngestionProcessorMetrics) mockMetrics.On("RecordHandleMessage", mock.Anything, mock.Anything).Return() @@ -286,7 +306,7 @@ func TestIngestionProcessorStart(t *testing.T) { // Start processor in a separate goroutine go func() { - err = processor.Start(ctx) + err := processor.Start(ctx) assert.NoError(t, err) }() // Wait server