diff --git a/.gitignore b/.gitignore index 17ecd33..cabce3e 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,12 @@ settings.yaml __debug_bin* *.code-workspace .history/ -vehicle-triggers-api +/vehicle-triggers-api +/nats-testpublisher +/triggers-bench +/triggers-state +/triggers-dlq +CLAUDE.md + +triggers-kvbench diff --git a/NATS_CONTRACT.md b/NATS_CONTRACT.md new file mode 100644 index 0000000..cb19d48 --- /dev/null +++ b/NATS_CONTRACT.md @@ -0,0 +1,152 @@ +# NATS JetStream Ingest Contract + +This document is the contract between **producers** (DIS / device-ingest) and +**vehicle-triggers-api** for the JetStream-based ingest path. It is what a +producer team needs to publish signals and events that this service will +evaluate against user webhooks. + +It applies when vehicle-triggers-api runs with `NATS_MODE=primary` (Kafka +bridges into NATS) or `NATS_MODE=exclusive` (NATS only, no Kafka). + +--- + +## ⚠️ Webhook receivers MUST dedup on `data.webhookId` + +This is the single most important thing to know if you build something that +*receives* DIMO webhooks. The pipeline guarantees **at-least-once** +delivery: the same logical fire can be delivered to your endpoint more than +once due to JetStream redelivery, replica races during cluster events, or +in-dispatcher retries on transient network failures. + +The CloudEvent ID (`data.webhookId` field, also surfaced as the CloudEvent +`id` header) is **deterministic**: `sha256(triggerID || sourceID)[:16]` +where `sourceID` is the inbound signal/event's CloudEvent ID. The same +logical fire always produces the same `data.webhookId` regardless of how +many times we deliver it. + +**Required receiver behavior:** + +``` +seen = persistent_set() // any KV / Redis / DB unique key +on receive(webhook): + id = webhook.data.webhookId + if id in seen: + return 200 // already processed; ack and move on + process(webhook) + seen.add(id, ttl=24h) // TTL > our redelivery window + return 200 +``` + +`vehicle_triggers_state_cas_conflicts_total{outcome="fallback"}` ticking on +our side is the public symptom that duplicates are reaching your endpoint. +If your endpoint isn't dedup'ing on `webhookId`, your downstream side-effects +WILL execute twice on any meaningful cluster event. + +--- + +## Streams + +The service provisions these on startup (idempotent `CreateOrUpdateStream`). +Producers only need to publish to the signal/event subjects; the service owns +the audit and DLQ streams. + +| Stream | Subjects | Written by | Read by | +|---|---|---|---| +| `DIMO_SIGNALS` | `dimo.signals.>` | **producer** | triggers-api | +| `DIMO_EVENTS` | `dimo.events.>` | **producer** | triggers-api | +| `DIMO_TRIGGER_AUDIT` | `dimo.trigger.fired.>` | triggers-api | billing/usage | +| `DIMO_TRIGGER_DLQ` | `dimo.dlq.>` | triggers-api | ops (`cmd/triggers-dlq`) | + +Stream names and subjects are configurable via `NATS_*` env (see `sample.env`), +but the defaults above are the contract. + +## Subjects + +Cardinality scales with the number of signal/event **names**, not the number +of vehicles. The vehicle identity travels in the payload, never the subject. + +``` +dimo.signals. e.g. dimo.signals.speed +dimo.events. e.g. dimo.events.harshBraking +``` + +`` / `` are the raw VSS names (no `vss.` prefix on the +subject). Characters illegal in NATS tokens (space, `.`, `*`, `>`, control +chars) must be replaced with `_`. The service applies the same sanitization on +its filter side, so a name containing a `.` (e.g. `powertrain.fuelLevel`) +becomes the subject token `powertrain_fuelLevel`. + +Publishing one message **per signal** (single-element CloudEvent) is preferred +so consumers can filter precisely. A multi-signal envelope on a single subject +also works — the service unpacks it — but then the subject name is whichever +signal you keyed on, which muddies filtering. Prefer one publish per signal. + +## Payload + +Standard DIMO CloudEvent, JSON-encoded, exactly as on the Kafka topics today. + +- Signals: `vss.SignalCloudEvent` (produced by `vss.PackSignals`) +- Events: `vss.EventCloudEvent` (produced by `vss.PackEvents`) + +### Required CloudEvent header fields + +| Field | Requirement | Notes | +|---|---|---| +| `subject` | **required** | Full ERC721 DID of the vehicle: `did:erc721:::`. The service decodes this to route to subscriptions. A bad DID fails the message (→ retried → DLQ). | +| `specversion` | `"1.0"` | | +| `type` | `dimo.signal` / `dimo.event` | | +| `time` | RFC3339 | Used for latency metrics. | +| `producer` | recommended | Surfaced in the webhook payload. | +| `source` | recommended | Surfaced in the webhook payload. | + +### Signal data fields (`vss.SignalData`) + +| Field | Notes | +|---|---| +| `name` | VSS signal name (e.g. `speed`). Must match the subject token (pre-sanitization). | +| `timestamp` | Signal observation time. | +| `valueNumber` / `valueString` / `valueLocation` | One set per the signal's value type. | + +## Delivery semantics + +- **At-least-once.** The service uses durable pull consumers with explicit + ack. A handler failure naks the message; JetStream redelivers per the + backoff ladder up to `NATS_MAX_DELIVER` (default 5). +- **After max deliveries**, the message is republished to + `dimo.dlq.` with headers `X-Original-Subject`, + `X-Failure-Reason`, `X-Delivered-Count`, then terminated so it stops + redelivering. Inspect/replay with `cmd/triggers-dlq`. +- **Duplicates are possible** (at-least-once + redelivery). Webhook evaluation + is idempotent w.r.t. cooldown: the `trigger_state` KV records the last fire + per (trigger, vehicle), so a redelivered message inside the cooldown window + does not re-fire. + +## Audit stream (consumers: billing/usage) + +Every successful webhook delivery publishes the full webhook CloudEvent +payload to: + +``` +dimo.trigger.fired. +``` + +`` is the developer's license address (hex, `0x...`). The +payload `data` carries `webhookId` (== the trigger ID), `assetDid`, +`metricName`, `service`, and the signal/event detail. Aggregate per developer +license for usage accounting. Publishes are async/best-effort and must never +be assumed to block delivery — treat the stream as eventually-consistent. + +## ID semantics + +- **Trigger ID == webhook ID.** The API returns `id` on webhook registration; + that same value is `triggers.id` in Postgres, the `webhookId` in webhook and + audit payloads, and the trigger-half of the `trigger_state` KV key + (`.`). + +## Versioning + +Payload schema follows `vss` from `github.com/DIMO-Network/model-garage`. A +breaking change to the CloudEvent shape requires coordinating the +model-garage version across producer and this service. The subject layout +(`dimo.signals.`) is independent of payload version and is not expected +to change. diff --git a/OPERATIONS.md b/OPERATIONS.md new file mode 100644 index 0000000..e53eba5 --- /dev/null +++ b/OPERATIONS.md @@ -0,0 +1,154 @@ +# vehicle-triggers-api Operations Runbook + +Operator-facing procedures. Read top to bottom in an incident. + +--- + +## Health quick check + +``` +GET /health +``` + +Returns 200 when the connection to NATS is up and every configured stream +(`DIMO_SIGNALS`, `DIMO_EVENTS`, `DIMO_TRIGGER_AUDIT`, `DIMO_TRIGGER_DLQ`) +returns a valid `Info` with a leader. Returns 503 with a JSON body listing +per-stream status when any of those probes fails. + +If 503: check `nats stream report` against the cluster; usually a node lost +quorum or a stream lost a replica. + +--- + +## Metric surfaces + +Prometheus namespace: `vehicle_triggers`. + +| Metric | Use | +|---|---| +| `vehicle_triggers_nats_publish_total{stream, outcome}` | Per-stream publish health (`ok` / `error`). | +| `vehicle_triggers_nats_consume_total{stream, outcome}` | Consume outcomes (`ack`, `nak`, `dlq`). A non-zero `dlq` rate means poison messages are accumulating - inspect with `triggers-dlq list`. | +| `vehicle_triggers_nats_eval_latency_seconds{stream, outcome}` | Histogram, JetStream timestamp to handler return. SLO surface. | +| `vehicle_triggers_state_cas_conflicts_total{bucket, outcome}` | Concurrent writers on the same `(trigger, vehicle)`. `retry` = resolved on retry; `fallback` = persistent, ended in unconditional Put. A steady non-zero rate means duplicate webhook fires are happening; receivers must dedup via the deterministic CloudEvent ID. | +| `vehicle_triggers_state_decode_errors_total{bucket}` | KV record JSON decode failures. Non-zero = schema mismatch or corruption. | +| `vehicle_triggers_tokenexchange_cache_total{outcome}` | Permission cache outcomes. Steady-state hit rate should be > 90%. Spikes in `miss` at autoscale events are expected. | + +--- + +## Inspection tools + +| Tool | Use | +|---|---| +| `triggers-state list / get / dump / watch` | Per-(trigger, vehicle) cooldown KV (`trigger_state` bucket). | +| `triggers-dlq list / get / replay / purge` | Dead-letter stream. Use `replay --all` after fixing the upstream cause; messages re-enter the normal evaluation path. | +| `triggers-bench` | Standalone load + latency probe. Useful for capacity tests on a fresh cluster. | + +--- + +## Daily JetStream backup (Helm) + +Set in `values.yaml`: + +```yaml +backup: + enabled: true + schedule: "0 6 * * *" + image: + s3Bucket: my-backup-bucket + s3Prefix: vehicle-triggers-api/jetstream +``` + +The CronJob runs `nats stream backup` for each stream and KV bucket, tarballs +the result, and uploads to `s3:////vehicle-triggers-backup-.tar.gz`. + +`natsio/nats-box` does not include the AWS CLI. For S3 uploads in prod, +bake an image that includes both and set `backup.image`. + +--- + +## Restore from backup + +Restoring is destructive: it deletes the existing stream/KV and re-creates +from the snapshot. Make sure traffic is paused first (scale the service to +0 replicas) so the consumer isn't fighting the restore. + +```sh +# 1. Pull the tarball +aws s3 cp s3://my-backup-bucket/vehicle-triggers-api/jetstream/vehicle-triggers-backup-.tar.gz . +tar -xzf vehicle-triggers-backup-.tar.gz +cd backup- + +# 2. Restore each stream +for d in stream-*; do + name="${d#stream-}" + nats stream restore --force "$name" "$d" +done + +# 3. Restore each KV (stored as KV_) +for d in kv-*; do + name="${d#kv-}" + nats stream restore --force "KV_${name}" "$d" +done + +# 4. Scale the service back up +kubectl scale deployment vehicle-triggers-api --replicas= +``` + +Verify with `/health` and `triggers-state dump` / `triggers-dlq list`. + +--- + +## Common incidents + +### Webhook deliveries silently failing + +1. Check `vehicle_triggers_nats_consume_total{outcome="dlq"}` rate. +2. `triggers-dlq list` to see what's poisoned, `triggers-dlq get ` for + one record. +3. If failures are receiver-side and resolved, `triggers-dlq replay --all`. +4. If failures look like our parsing, log into a pod and inspect the body. + +### CAS conflict counter rising + +This means two replicas are racing on the same `(trigger, vehicle)`. The +state store is still correct (last write wins) but webhooks are firing +twice. Receivers should dedup on the CloudEvent ID. If they can't, see +PROD_HARDENING.md P1 "decouple webhook delivery from eval" - that's the +fix. + +### Token-exchange cache hit rate drops + +`vehicle_triggers_tokenexchange_cache_total{outcome=miss}` rate spikes +typically happen at autoscale-up (cold caches on new pods). Sustained drop +in hit rate means either: + +- token-exchange-api increased its TTL → ours is still 15m default, mismatch +- subscriptions ballooned → cache is being evicted + +Tune `TOKEN_EXCHANGE_CACHE_EXPIRATION` if needed. + +### Latency histogram p99 climbs + +Buckets are 1ms-30s. p99 > 100ms typically means slow webhook receivers +(HTTP dispatch is in-line). Mitigate: get receiver to fix; or build the P1 +async dispatcher. + +### NATS cluster lost a node + +`/health` returns 503 with the failed stream named. JetStream auto-recovers +when the node returns. If permanent: rebalance replicas with `nats stream +edit --replicas`. + +--- + +## Cutover procedure + +Documented in PROD_HARDENING.md under "Cutover gates". Summary: + +- merge with `NATS_MODE=off` +- enable in dev (`NATS_MODE=primary`) +- soak 1 week +- enable in staging +- enable in prod +- once DIS publishes natively, flip to `NATS_MODE=exclusive` +- after exclusive is clean in prod, remove Kafka deps in a follow-up PR diff --git a/PROD_HARDENING.md b/PROD_HARDENING.md new file mode 100644 index 0000000..9c30915 --- /dev/null +++ b/PROD_HARDENING.md @@ -0,0 +1,159 @@ +# vehicle-triggers-api Prod Hardening Backlog + +Tracking list of items surfaced during the NATS JetStream migration review. +Ordered by priority. Effort estimates are rough. + +Categories: +- **Correctness**: could cause wrong webhook fires or data loss +- **Performance**: throughput / latency / resource ceilings +- **Operational**: ops can't recover, can't observe, can't tune +- **Architecture**: structural smells that block future scale +- **Scaling**: items needed when load grows past current ceiling + +--- + +## P0 — Land before / shortly after merge + +### Correctness + +- [x] **CAS-based `RecordFire`.** ~~Two replicas racing on same `(trigger, vehicle)` both pass cooldown KV read, both fire, both `Put`. Replace `kv.Put` with `kv.Update` on observed revision (or `kv.Create` for first write), retry once on conflict, skip fire on second conflict. Effort: 1-2h. File: `internal/services/triggerstate/triggerstate.go`.~~ Done: `writeWithCAS` retries once then falls back to `Put`, bumping `vehicle_triggers_state_cas_conflicts_total{bucket, outcome=retry|fallback}`. Full prevention requires receiver dedup via deterministic ID (P0-2). Race test in `tests/e2e/nats_state_cas_test.go`. +- [x] **Deterministic webhook ID for receiver dedup.** ~~Today `payload.ID = uuid.New().String()` regenerates per delivery~~ Done: `webhookID(triggerID, sourceID)` returns `sha256(triggerID|sourceID)[:16]` hex. `sourceID` = inbound CloudEvent ID from the signal/event (carried by DIS), stable across JS redelivery. UUID fallback only when source absent. Unit tested. +- [x] **KV decode error metric.** ~~`lookupPreviousSignal` / `lookupPreviousEvent` silently swallow JSON decode errors and return zero-value. Add `vehicle_triggers_kv_decode_errors_total{bucket}` so silent corruption shows up.~~ Done: `vehicle_triggers_state_decode_errors_total{bucket}` bumped from both evaluator lookup paths. + +### Performance — quick wins + +- [x] **Configure webhook HTTP Transport.** ~~Today `webhook_sender.go` uses Go's default Transport (`MaxIdleConnsPerHost=2`).~~ Done: cloned default with `MaxIdleConnsPerHost=64`, `MaxIdleConns=1024`, `IdleConnTimeout=90s`, `ForceAttemptHTTP2=true`. See `defaultTransport()`. + ```go + Transport: &http.Transport{ + MaxIdleConns: 1024, + MaxIdleConnsPerHost: 64, + IdleConnTimeout: 90 * time.Second, + ForceAttemptHTTP2: true, + TLSHandshakeTimeout: 10 * time.Second, + ResponseHeaderTimeout: 20 * time.Second, + DisableCompression: false, + } + ``` + Removes TLS handshake overhead at high fire rates. Effort: 30m. File: `internal/services/webhooksender/webhook_sender.go`. + +- [x] **End-to-end latency histogram.** ~~Add `vehicle_triggers_eval_latency_seconds{outcome}`~~ Done: `vehicle_triggers_nats_eval_latency_seconds{stream, outcome}` recorded from `meta.Timestamp` at ack/nak/dlq sites. Buckets 1ms-30s. + +- [x] **Token-exchange cache hit/miss counters.** ~~No way today to see whether the 15m permission cache is doing its job.~~ Done: `vehicle_triggers_tokenexchange_cache_total{outcome=hit|miss|error}` bumped in `Cache.HasVehiclePermissions`. + +### Operational + +- [x] **`Settings.Validate()` called from `main`.** ~~Catches misconfigurations at startup instead of at first failure.~~ Done: `Settings.Validate()` + `NATSSettings.Validate()` enforce mode enum, KAFKA-required-when-not-exclusive, MaxDeliver/MaxAckPending/AckWait/FetchBatch/StreamReplicas >= 1, retention >= AckWait * MaxDeliver. Called from `main` after `env.LoadSettings`. Unit-tested in `settings_test.go`. + - `MaxDeliver > len(BackOff)` (already mitigated in EnsureConsumer but config-level too) + - `AckWait < BackOff[0]` is a footgun + - `MaxAckPending > 0` + - `SignalsMaxAge >= AckWait * MaxDeliver` (else messages discarded mid-retry) + - `NATS_MODE` in {`off`, `primary`, `exclusive`} + - When `Mode != off`, `URL` non-empty + Effort: 1-2h. File: `internal/config/settings.go`. + +- [x] **Stream/KV write canary in `/health`.** ~~Connection alive ≠ JetStream writable.~~ Done: `Client.StreamHealth(ctx)` probes each configured stream via `js.Stream(name).Info()`, reports per-stream status, no-leader detected. `/health` returns 503 when any stream lookup/info fails. + +- [x] **Backup automation.** ~~Today recovery story is "you can `nats stream backup`."~~ Done: Helm CronJob templates (`backup-cronjob.yaml` + `backup-script-configmap.yaml`) run `nats stream backup` for each stream + KV bucket via the nats-box image, tarball + S3 upload. Default off; enabled with `backup.enabled=true` + `backup.s3Bucket`. Restore procedure documented in `OPERATIONS.md`. We deliberately did NOT build a Go binary - the standard `nats` CLI is the right tool for server-native snapshots. + +--- + +## P1 — Land before exclusive cutover in prod + +### Performance / architecture + +- [x] **Decouple webhook delivery from eval handler.** ~~Today `SendWebhook` is synchronous to the JetStream message handler — a slow receiver holds `MaxAckPending` slots and throttles consume.~~ Done: `internal/services/webhookdispatcher` is a pluggable pool. Workers own send + state + audit + circuit-breaker bookkeeping. `Enqueue` returns `ErrQueueFull` on overflow so the JetStream handler naks and JetStream retries. Default pool size 32 / queue 4096 via `NATS_DISPATCHER_WORKERS` / `NATS_DISPATCHER_QUEUE_SIZE`. `WithDispatcher` on the listener swaps the sync path out; sync path retained for the Kafka-only legacy mode. Metrics: queue depth gauge, queue_full counter, delivery_total+latency histogram. + +- [x] **Audit publish fire-and-forget queue.** ~~`PublishAsync` blocks at `MaxAckPending=4000`. We launch a 5s detached goroutine per fire — at 30k/s with 5s blocks that's potentially 150k goroutines.~~ Done: `internal/services/auditqueue` is a bounded buffer + small drainer pool. `Submit` drops on overflow (non-blocking). Drops surface via `vehicle_triggers_audit_dropped_total` — alarm on this. Adapter implements the `webhookdispatcher.AuditPublisher` interface so the dispatcher's `publishAudit` no longer spawns a goroutine. Default `NATS_AUDIT_WORKERS=4`, `NATS_AUDIT_QUEUE_SIZE=16384`. + +- [x] **Webhook cache distributed updates.** ~~Today each replica polls Postgres every 5 min.~~ Done: `internal/services/cachebroadcast` publishes change events on plain NATS subject `dimo.cache.webhook.changed`. `WebhookCache.ScheduleRefresh` publishes; receiver side calls `ScheduleRefreshSilent` to avoid echo loops. App wires the notifier on `WebhookCache` and subscribes in `CreateServers` when NATS is enabled. CRUD controllers untouched. 5-min poll stays as reconciliation safety net. + +- [x] **Webhook signing (HMAC).** ~~TODO in `webhook_sender.go:74`.~~ Done: migration `00006_trigger_signing_secret.sql` adds nullable `signing_secret`. `CreateTrigger` generates 32 random bytes per trigger; `RegisterWebhookResponse` exposes once. Sender adds `X-DIMO-Timestamp`, `X-DIMO-Signature`, `X-DIMO-Signature-Version: v1` headers when secret set. Algorithm documented in response: `HMAC-SHA256(timestamp + "." + body)`. + +### Operational + +- [x] **DLQ-headers include developer + vehicle context.** ~~Today DLQ records carry `X-Original-Subject` / `X-Failure-Reason` / `X-Delivered-Count`.~~ Done: `X-Source-Name` (subject suffix), `X-Asset-DID` (best-effort parsed from payload), `X-Recorded-At` (RFC3339Nano) added. Skipped `X-Developer-License` deliberately — one inbound message can match webhooks across multiple developers; tagging "the" developer would mislead. CLI `triggers-dlq list` updated. + +- [x] **Webhook config audit trail.** ~~Currently CRUD modifies `triggers` row in place with no history.~~ Done: new stream `DIMO_CONFIG_AUDIT` subject `dimo.config.changed.` provisioned alongside the others. `internal/services/configaudit` publishes on RegisterWebhook / UpdateWebhook / DeleteWebhook / AssignVehicleToWebhook / RemoveVehicleFromWebhook. `WithAudit` setters on both controllers; `configaudit.Noop` when NATS off. 90d retention default. Best-effort: failures logged, never block CRUD. + +--- + +## P2 — Nice-to-have / cleanup + +### Architecture + +- [~] **Failure count out of DB.** ~~`IncrementTriggerFailureCount` writes one row per delivery failure.~~ Deferred. Re-analysis: failure_count writes only occur on delivery failure, which is rare under healthy conditions. Once the circuit breaker trips, writes stop. Cross-replica CAS coordination is heavy for a slow-path write. Re-open if/when prod metrics show this is a real DB hotspot. + +- [x] **Drop `webhooks` KV bucket OR wire it.** ~~Provisioned in `EnsureBuckets`, never read or written.~~ Done: bucket + accessor removed. Provisioned in `EnsureBuckets`, never read or written. Either delete or commit to using it for distributed cache invalidation (paired with P1 webhook cache item). Effort: varies. File: `internal/nats/provision.go`. + +- [x] **Drop dead `triggersrepo` methods.** ~~`GetLastLogValue`, `GetLastLogForMetric`, `CreateTriggerLog` no longer called.~~ Done: removed + tests. `GetLastLogValue`, `GetLastLogForMetric`, `CreateTriggerLog` no longer called by anything. Either remove or leave with a `// Deprecated:` doc string pointing at the audit stream. Effort: 30m. + +- [~] **Drop `trigger_logs` table.** Deferred per its own pre-req: requires audit-stream consumer (owned by billing team, not us) and a soak period proving DB history is no longer queried. Re-open once those land. After audit-stream consumer is built and a soak period proves we don't need it. Effort: 30m migration + coordination. + +- [x] **Normalize one name for trigger ID.** ~~Audit and webhook payloads use both `webhookId` and `triggers.id` for the same value.~~ Done: Go field is `WebhookID`, JSON name stays `webhookId` (public name; comment documents the equivalence with triggers.id). Audit and webhook payloads use both `webhookId` and `triggers.id` for the same value. Pick one and use it everywhere. Effort: 1h. + +- [x] **Unify "do work when X enabled" pattern.** ~~Two styles in app.go.~~ Done (clarification, not rewrite): documented the convention in `buildListener` doc - settings.NATS.X decides what to construct; downstream functions take dependencies and check nil. Standard DI idiom; no code change. Two styles in app.go: `bridge != nil` (interface presence) vs `settings.NATS.PrimaryMode()` (config). Pick one. Effort: 1h. + +### Operational + +- [x] **Helm storage sizing guidance.** ~~`NATS_STREAM_REPLICAS=3` default, no comment on storage per `MaxAge`.~~ Done: rule of thumb and worked example in `values.yaml`. `NATS_STREAM_REPLICAS=3` default, no comment on storage per `MaxAge`. At 30k/s × 24h × ~600B = ~50GB/node/stream. Document in `values.yaml`. Effort: 30m. + +- [x] **CI smoke load test.** ~~`cmd/triggers-bench` is manual.~~ Done: `tests/e2e/load_smoke_test.go` (build tag `load`) runs ~300 msg/s for 5s against testcontainer NATS via `vtnats.Connect` + `PullLoop`, asserts zero drops and clean drain. CI runs `go test -tags=load ./tests/e2e/...` in a separate job. `cmd/triggers-bench` is manual. Add a `go test -tags=load` test that runs it briefly against a testcontainer cluster and asserts no drops at low rate. Catches wiring regressions. Effort: half day. + +- [~] **Reconciler pod for webhook cache.** Deferred. The 5-min DB poll in `webhookCache` is the existing reconciliation safety net for the new cachebroadcast pub/sub path (P1-3). Promote to a dedicated CronJob only if the poll proves insufficient in prod (e.g. propagation lag observed > 5 min sustained). + +--- + +## P3 — When load forces it + +### Scaling + +- [x] **Stream sharding.** ~~Today one `DIMO_SIGNALS` stream caps around ~50k/s in our bench~~ Documented in `SCALING.md` with concrete trigger (publish rate > 40k/s sustained) + design (shard by name prefix or hash) + risk notes. Wait to implement until the trigger fires. + +- [x] **Subject sharding for affinity routing.** ~~Hash `devLicense` (or `targetURL`) into N partition subjects~~ Documented in `SCALING.md` with trigger (cache miss rate > 5% + p99 climb) + design + risk. Defer until measured. + +- [x] **`signal_index` KV refcount + dynamic filter scoping.** Documented in `SCALING.md` with trigger (>1000 distinct metric names + low subscription hit rate) + design (CAS-protected refcount, consumer rebuilds FilterSubjects on watch). + +- [x] **Cross-region DR.** Active-passive design captured in `SCALING.md` with JetStream mirroring + Postgres logical replication + DNS failover. Active-active deliberately deferred until DIS owns the dual-publish semantics. Trigger: RTO requirement drops below AWS region recovery time. + +- [x] **Shared permissions cache in NATS KV.** Documented in `SCALING.md` with trigger (cache hit rate < 85% steady-state) + design (KV layer between in-process cache and gRPC, CAS-protected populate). Defer until the cache-hit metric we added in P0-6 shows we need it. + +--- + +## Cutover gates + +Before flipping `NATS_MODE=primary` in **dev**: +- All **P0** items done +- DIS publish-to-NATS PR ready (or accept bridge mode indefinitely) +- Backup CronJob running +- `/health` includes stream-write canary + +Before flipping `NATS_MODE=primary` in **prod**: +- 1-week soak in staging clean +- Audit consumer exists somewhere (billing team) +- Latency histogram + cache-hit counters wired +- All **P1** items done + +Before flipping `NATS_MODE=exclusive` in **prod**: +- DIS publishes natively, Kafka topic going empty +- 1-week soak in `primary` showing 0 nak/dlq drift +- Drop `KAFKA_*` from chart values +- Open follow-up PR to remove Kafka code paths + +--- + +## Open questions + +- What's our actual signal volume (peak msg/sec)? Current bench measures 30k/s/pod ceiling; we need to know if production is 1k or 100k. +- Who owns audit-stream consumer for billing? Stream is populated; nobody reads. +- What's the SLO for webhook delivery latency? Drives whether we need async dispatcher (P1) or current sync path is fine. +- What's the retention requirement for `DIMO_TRIGGER_AUDIT`? 90d default; finance/legal may need 7+ years (different storage entirely). + +--- + +## Reference + +- Branch: `nats-jetstream-migration` +- PR: https://github.com/DIMO-Network/vehicle-triggers-api/pull/129 +- Contract: `NATS_CONTRACT.md` +- Bench: `cmd/triggers-bench` +- Inspection: `cmd/triggers-state`, `cmd/triggers-dlq` diff --git a/PROD_HARDENING_V2.md b/PROD_HARDENING_V2.md new file mode 100644 index 0000000..8f49862 --- /dev/null +++ b/PROD_HARDENING_V2.md @@ -0,0 +1,149 @@ +# vehicle-triggers-api Prod Hardening Round 2 + +Findings from a fresh systems-engineer review after PROD_HARDENING.md items +landed. Items A through S. Same structure as the first backlog: priority, +trigger, effort. + +Goal: clear A through S so this branch is genuinely production-ready beyond +"all tests pass." The first review's items were about wiring + observability; +this round is about correctness gaps and operational sharp edges the first +pass left behind. + +--- + +## P0 — must land before exclusive cutover + +These are real bugs or readiness gaps that will bite on first cutover attempt. + +### Correctness + +- [x] **B. State TTL silently breaks long cooldowns.** `NATS_TRIGGER_STATE_TTL=168h` (7d). A trigger with `cooldownPeriod > 7d` loses its record after TTL → fires again as if first-time. Add to `Settings.Validate`: + ``` + TriggerStateTTL >= 2 * max(triggers.cooldown_period) + ``` + Or query DB at startup and assert. Without this, configuration drift becomes a silent correctness bug. Effort: 1h. File: `internal/config/settings.go`. + +- [x] **C. `ScheduleRefresh` from PermissionDenied broadcasts.** `metric_listener.go::processSignalWebhook` calls `webhookCache.ScheduleRefresh` after a permission-denied → DeleteVehicleSubscription. With cachebroadcast (P1-3) this now publishes a NATS message every time. At 30k signals/sec with a misconfigured developer, thousands of broadcasts per second across all replicas. Debounce inside ScheduleRefresh doesn't help because each call is a fresh NATS publish, only the local rebuild is debounced. Fix: subscription deletion doesn't need a full cache rebuild; it needs targeted invalidation OR rate-limit the broadcast at the cachebroadcast.NATSNotifier level. Effort: 30m. File: `internal/services/cachebroadcast/cachebroadcast.go` + `internal/controllers/metriclistener/signal.go`. + +- [x] **D. `ErrQueueFull` nak counts toward `MaxDeliver`.** Dispatcher backpressure → handler returns ErrQueueFull → nak → JetStream increments NumDelivered → 5 naks land message in DLQ. Transient saturation looks like poison. Fix: in `cmd/vehicle-triggers-api::RunNATSConsumer` callback (or `PullLoop`), unwrap the handler error; if it's `webhookdispatcher.ErrQueueFull`, call `NakWithDelay(longer)` and do NOT count toward MaxDeliver. JetStream doesn't natively support "don't count this nak" so we either: + - Implement a separate redelivery counter in-process and bypass the consumer's MaxDeliver + - Use a longer NakWithDelay and accept slower retries + Effort: 1h. File: `internal/nats/consumer_loop.go`. + +### Operational + +- [x] **I. `/health` synchronous stream probes can blow K8s probe timeout.** 4 streams × 500ms timeout each = up to 2s. K8s `probe.timeoutSeconds=3`, `periodSeconds=5`. At cluster degradation, /health itself becomes a tarpit and the probe times out → pod marked unready while it's actually fine. Fix: probe streams in parallel with `errgroup`; cap total at 1s. Effort: 30m. File: `internal/nats/client.go::StreamHealth`. + +--- + +## P1 — land before primary cutover in prod + +### Performance + +- [x] **F. No inline webhook retry.** Single HTTP attempt → fail → return error → JS redelivers → re-evaluates from scratch (parse + CEL + perms + KV). Wasteful. Worker-internal retry loop with capped backoff (3 tries, 100ms/500ms/2s) handles the common transient receiver hiccup without re-running eval. Trade-off: failure semantics live with us, not JS. Pair with deterministic webhook ID (already shipped). Effort: 4h. File: `internal/services/webhookdispatcher/dispatcher.go`. + +- [x] **G. KV write throughput not benchmarked.** At 30k fire/sec we do 60k KV writes/sec (`trigger_state` + `signal_history`). The cluster bench measured stream publish + pull throughput, not KV writes. KV is its own JetStream-backed stream with its own ceiling. Extend `cmd/triggers-bench` with a `-kv-writes` mode or write a separate kv-bench. Effort: half day. Files: `cmd/triggers-bench/main.go` or new `cmd/triggers-kvbench/`. + +- [x] **H. Stream `MaxBytes` not set.** Storage bounded only by `MaxAge`. One pathological signal name dominating traffic can consume the whole disk before age expires. Add `MaxBytes` per stream with `Discard: DiscardOld`. Defaults: configurable via `NATS_*_MAX_BYTES`. Effort: 30m. File: `internal/nats/provision.go`. + +### Architecture + +- [x] **Q. No global rate limiting to receivers.** N pods × 32 workers each = up to 320 concurrent POSTs to one receiver. If they rate-limit us at 10 RPS, we burn through `MaxFailureCount` on every fire. Per-receiver token bucket, ideally shared across replicas via a KV semaphore. Without shared state: per-pod bucket + accept that aggregate rate is `pods * bucket_rate`. Effort: 1 day. Files: new `internal/services/ratelimit/` + dispatcher worker integration. + +### Observability + +- [x] **N. Trace propagation via JS message ID.** Following one fire from `dimo.signals.speed` → CEL eval → dispatcher → receiver across multiple pods needs a span ID. JS message ID is natural. Thread through: handler logs include `js_msg_id`, dispatcher logs include it, outbound webhook gets `X-DIMO-Trace-ID` header. Receiver-side trace integration optional. Effort: half day. Files: `internal/nats/consumer_loop.go`, `internal/services/webhookdispatcher/dispatcher.go`, `internal/services/webhooksender/webhook_sender.go`. + +### Test coverage + +- [x] **O. Replica=3 testcontainer NATS in CI.** All testcontainer tests use single-node. Real bugs (replica election, lost writes during failover) won't surface. Add a `tests/e2e/nats_cluster_test.go` that spins 3 NATS containers on a podman network (we did this manually for the cluster bench; mechanize it). Effort: 1 day. + +--- + +## P2 — cleanup that we should not ship without + +### Operational + +- [x] **E. Audit-side PublishAsync block telemetry.** Our queue is bounded; but once a worker calls `js.PublishAsync`, that can **block** when JetStream's own `max_pending=4000` is exhausted. Worker goroutines park. We don't track time-blocked. Add a histogram `vehicle_triggers_audit_publish_blocked_seconds`. Effort: 1h. File: `internal/services/auditqueue/auditqueue.go`. + +- [x] **J. `webhookcache` 5-min refresh goroutine leaks after shutdown.** `range ticker.C` with no ctx check; never exits. Minor but signals shutdown discipline gap. Fix: select on ctx.Done() inside the loop. Effort: 15m. File: `internal/services/webhookcache/webhook_cache.go` (or actually `internal/app/app.go::startWebhookCache`). + +- [x] **K. Decode error counter without payload context.** `vehicle_triggers_state_decode_errors_total` ticks; nothing logs the malformed body. Add a sampled debug log (1 in 100) with first 200 bytes. Effort: 30m. Files: `internal/services/triggerevaluator/trigger_evaluator.go::lookupPreviousSignal` / `lookupPreviousEvent`. + +### Security + +- [x] **L. No signing-secret rotation.** Lose the secret → must delete + recreate webhook → vehicle subscription list gone. Add `POST /v1/webhooks/:id/rotate-secret` returning the new secret once, marking old secret as expired. Optional: dual-secret window (old valid for N minutes after rotation). Effort: 4h. Files: `internal/controllers/webhook/webhook_controller.go`, `internal/services/triggersrepo/triggersrepo.go`. + +- [x] **M. Signing secret stored plaintext in Postgres.** DB compromise = every webhook secret compromised at once. Two paths: + - Envelope encryption via AWS KMS: encrypt at write, decrypt at read (small per-call latency) + - Store in AWS Secrets Manager per trigger ID, DB holds only a reference + Picking the cheaper of the two. Effort: half day (KMS) to 1 day (Secrets Manager). Files: `internal/services/triggersrepo/triggersrepo.go`, new `internal/services/secrets/` package. + +### Correctness + +- [x] **A. CAS protects state writes, not fire decisions.** Two evaluators on different replicas both pass `lookupLastFireTime` (zero), both fire, both reach RecordFire. CAS conflict is recorded but both webhooks already sent. Receiver dedup via deterministic ID (P0-2 shipped) is the only real protection. Reality: this isn't a code fix, it's a documentation + contract item: + - Move the receiver-dedup callout from buried in NATS_CONTRACT.md to the TOP, in bold + - Add an example of how to dedup in the receiver doc + - Add a "we tested duplicates happen" e2e test asserting that two parallel writers produce two webhook deliveries with identical IDs + Effort: 2h. Files: `NATS_CONTRACT.md`, new `tests/e2e/nats_duplicate_fire_test.go`. + +--- + +## P3 — known issues, low cost-of-deferral + +- [x] **P. `payload.Source = "vehicle-triggers-api"` hardcoded.** TODO in `metric_listener.go` says "should be 0x of the storageNode." Receivers verifying CloudEvent provenance can't tell which DIMO node served the event. Blocked on storage-node identity being available at runtime (which it isn't yet). Effort: depends on upstream; keep TODO visible. + +- [x] **R. Audit drops are silent billing miscounts.** Counter exists; no alert. Add a Prometheus alert rule template in `OPERATIONS.md`: + ``` + rate(vehicle_triggers_audit_dropped_total[5m]) > 0 + ``` + fires at any drop. Effort: 30m. File: `OPERATIONS.md` + the chart's ServiceMonitor. + +- [x] **S. Bridge mode does 2N JetStream ops per Kafka message.** Acceptable transitional cost; defined cutover acceptance criteria here so it doesn't become permanent: + + - **Trigger A**: `NATS_MODE=primary` runs in prod >= 90 days. + - **Trigger B**: aggregate publish + ack JetStream operations driven by the bridge cost more than 30% of the cluster's measured budget (compare `rate(vehicle_triggers_nats_publish_total{stream="DIMO_SIGNALS"})` × 2 against published cluster ceiling). + + When either trigger fires, escalate to DIS team for native-publish cutover. The work on our side is just flipping `NATS_MODE=exclusive`; the gating action is DIS publishing to `dimo.signals.>` directly. Documented this checkpoint in `SCALING.md` cross-region section neighborhood. + +--- + +## Cutover gates (revised) + +Before flipping `NATS_MODE=primary` in **dev**: +- All **P0** items in this doc (B, C, D, I) done + +Before flipping `NATS_MODE=primary` in **prod**: +- 1 week soak in staging clean (cas_conflicts ≤ baseline, queue_full = 0, audit_dropped = 0, dlq = 0) +- All **P1** items done +- Receiver-dedup contract explicitly communicated to the largest 3 consumers + +Before flipping `NATS_MODE=exclusive` in **prod**: +- DIS publishing natively to NATS +- 1 week soak in `primary` clean +- All **P2** items done + +--- + +## Metric reference (already shipped — for trigger conditions) + +| Metric | Watch for | +|---|---| +| `vehicle_triggers_state_cas_conflicts_total` | non-zero rate → replica race; tells customers to dedup | +| `vehicle_triggers_state_decode_errors_total` | non-zero → schema mismatch / corruption | +| `vehicle_triggers_dispatcher_queue_full_total` | non-zero → bump workers / queue / scale out | +| `vehicle_triggers_audit_dropped_total` | non-zero → billing data loss; alert | +| `vehicle_triggers_nats_consume_total{outcome="dlq"}` | non-zero → poison or D-class queue-full nak chain | +| `vehicle_triggers_nats_eval_latency_seconds` p99 | SLO surface | +| `vehicle_triggers_tokenexchange_cache_total{outcome="hit"}` rate | < 85% → permissions cache cold; consider shared KV cache | + +--- + +## Reference + +- Previous backlog: `PROD_HARDENING.md` (24 done + 5 deferred with triggers) +- Scaling levers: `SCALING.md` +- DIS contract: `NATS_CONTRACT.md` +- Ops runbook: `OPERATIONS.md` +- PR: https://github.com/DIMO-Network/vehicle-triggers-api/pull/129 +- Branch: `nats-jetstream-migration` diff --git a/SCALING.md b/SCALING.md new file mode 100644 index 0000000..d471419 --- /dev/null +++ b/SCALING.md @@ -0,0 +1,291 @@ +# vehicle-triggers-api Scaling Playbook + +Companion to `PROD_HARDENING.md`. The P3 items in that document are +scaling levers we'd pull when prod metrics show we've outgrown the current +design. Each section here has: + +1. **Trigger** — what observable condition justifies the work +2. **Design** — what to build +3. **Risk** — what can go wrong +4. **Out-of-scope** — what we explicitly aren't solving + +These are not "do now" items. They're "do when the metric crosses the +trigger" items. Capturing them now means the future engineer has a +starting line, not a blank page. + +--- + +## Stream sharding + +### Trigger + +`vehicle_triggers_nats_publish_total{stream="DIMO_SIGNALS"}` rate sustained +above **~40k msg/s** for more than 1h, **or** `nats stream report` shows +`bytes` growth outpacing storage budget. + +The bench measured ~50k/s as the single-stream ceiling on replicas=3 file +storage. Plan to act when we're at 80% of that. + +### Design + +Shard `DIMO_SIGNALS` by signal-name prefix: + +``` +DIMO_SIGNALS_A subjects: dimo.signals.a* +DIMO_SIGNALS_B subjects: dimo.signals.b* +DIMO_SIGNALS_C subjects: dimo.signals.c* (etc.) +``` + +Or hash: + +``` +DIMO_SIGNALS_0 subjects: dimo.signals.0.* +DIMO_SIGNALS_N subjects: dimo.signals..* with DIS picking shard by hash(signalName) % N +``` + +Wiring on our side: + +- `config.NATSSettings.SignalsStreamShards int` (default 0 = unsharded) +- `EnsureStreams` loops over shards +- `EnsureConsumer` creates one durable per shard with the appropriate + `FilterSubjects` +- `app.CreateServers` runs one `PullLoop` per shard + +DIS side: pick shard during publish. Subject convention is the contract; +hash function lives in `internal/nats/subjects.go` for both sides to share. + +### Risk + +- **Rebalancing during sharding.** Migrating from 1 → N shards needs a + drain-and-restart, or a transitional "write to both" period. Plan a + weekend window. +- **Hot shard.** If one signal name dominates traffic and lands in a + single shard, that shard hits the ceiling first. Mitigation: split that + shard further, or hash with a salt to spread. + +### Out-of-scope + +Per-tenant or per-developer sharding — that's affinity routing (next). + +--- + +## Subject sharding for affinity routing + +### Trigger + +`vehicle_triggers_tokenexchange_cache_total{outcome="miss"}` rate stays +above ~5% **and** webhook delivery p99 climbs because connection pools +keep going cold. + +Today the pull consumer load-balances arbitrarily across pods, so the +same `(trigger, vehicle)` fire can land on different pods. Each pod +maintains its own HTTP keep-alive pool to each receiver; pool reuse only +happens when the same pod evaluates back-to-back fires for the same +receiver. At scale this means cold connection opens dominate latency. + +### Design + +Hash `devLicense` (or `targetURL`) into N partition subjects: + +``` +dimo.signals.speed (kept as today, for backwards compat) ++ dimo.signals.p.speed (sharded form: = hash(devLicense) % shards) +``` + +DIS keeps publishing to the unsharded subject. A bridge inside this +service (or a separate dispatcher) re-publishes to the partition subject +based on the matched trigger's developer license. Each pod consumes only +one (or a few) partition subjects. + +Result: the same pod always evaluates all fires for the same developer, +so its HTTP keep-alive pool stays warm and its permissions cache always +hits. + +### Risk + +- **Pod churn.** Autoscale events or rolling deploys cause cold pools + briefly. Acceptable trade-off if rare. +- **Uneven load.** A heavy developer pins one pod. Mitigation: include + trigger ID in the hash so a single dev with many triggers spreads. +- **Rebalancing.** Same drain-and-restart story as stream sharding. + +### Out-of-scope + +Consistent hashing across N pods with online rebalancing. NATS doesn't +give us that natively; if we need it we're probably looking at a +separate dispatcher service. + +--- + +## `signal_index` KV resurrection + +### Trigger + +Number of distinct webhook `metricName` values exceeds ~1,000 **and** +`vehicle_triggers_nats_consume_total{stream="DIMO_SIGNALS"}` shows most +messages have no matching subscription (i.e. we're spending CPU on +payloads no one cares about). + +Today's `dimo.signals.>` wildcard pulls every signal regardless of +subscription. Fine at ~30 signal names. Bad at thousands. + +### Design + +A KV bucket `signal_index` keyed by metric name, value = active +subscription count: + +``` +speed -> 12 (12 webhooks subscribed) +fuelLevel -> 4 +deepObscure -> 0 (no subs; consumer can skip) +``` + +CRUD on webhook config updates the refcount (increment on +RegisterWebhook, decrement on DeleteWebhook). The consumer rebuilds its +`FilterSubjects` list periodically (or on KV watch) from the keys with +count > 0: + +``` +FilterSubjects: ["dimo.signals.speed", "dimo.signals.fuelLevel", ...] +``` + +Subjects with no active subscribers are filtered out at the JetStream +level — we never pull them, never pay parse cost. + +### Risk + +- **Filter cap.** `MaxAckPending` interacts with filter-subject count; + large filter lists can be expensive for the server. Capped at + `FILTER_SUBJECT_CAP=2048` in config today. +- **Race on refcount.** Use KV CAS (same pattern as triggerstate) to + avoid lost increments. At low CRUD rate this is fine. + +### Out-of-scope + +Vehicle-level filtering. We don't want one subject per `(metric, vehicle)` +— that's the cardinality explosion we avoided in the initial subject +design. + +--- + +## Shared permissions cache in NATS KV + +### Trigger + +`vehicle_triggers_tokenexchange_cache_total{outcome="hit"}` cache hit +rate drops below ~85% steady state, **and** autoscale-up events cause +visible token-exchange-api gRPC RTT spikes for >2 minutes. + +Each pod today maintains an independent 15-minute permission cache. +Cold-fill on a new pod is N grpc calls; at autoscale-up with M new pods +it's M×N redundant calls. + +### Design + +KV bucket `permissions` with key +`::` and value +`{allowed: bool, expiresAt: time}`. TTL on the bucket is 15 minutes, +matching the in-process cache TTL. + +Read path: + +1. Check in-process cache → hit, return. +2. Miss → check KV → hit, populate in-process, return. +3. Miss both → gRPC to token-exchange-api, populate both. + +Write to KV via `Update` on observed revision so concurrent populators +don't conflict. + +### Risk + +- **Stale grant after revocation.** Today: 15 min worst case (per pod + cache). With shared KV: 15 min worst case (TTL). No regression, but no + improvement either — revocation-aware cache invalidation is a separate + item. +- **KV outage.** Falls back to in-process cache then gRPC; same as + today. +- **Cost.** Adds one KV roundtrip per cache miss. At expected ~10% miss + rate this is negligible. + +### Out-of-scope + +Revocation propagation — that's a token-exchange-api feature (push +invalidation), not ours. + +--- + +## Cross-region DR + +### Trigger + +Recovery time objective (RTO) for the service drops below the AWS region +recovery time (typically hours), **or** a regulatory requirement mandates +multi-region failover. + +### Design + +**Active-passive (recommended starting point):** + +``` +us-east-1 (active) us-west-2 (passive) + +------------+ +------------+ + | NATS | --------> | NATS | + | 3 nodes | mirror | 3 nodes | + +------------+ +------------+ + | vehicle- | | vehicle- | + | triggers | | triggers | + | api | | api | + +------------+ +------------+ + | Postgres | --------> | Postgres | + | primary | replica | read repl | + +------------+ +------------+ +``` + +- JetStream stream `mirror` config replicates each stream cross-region. +- Postgres logical replication keeps webhook config in sync. +- DNS failover or Route53 health check fronts the API. +- During failover, passive becomes active. Manual confirmation step + prevents split-brain (no automatic promotion). + +**Active-active** is significantly harder: DIS must publish to both +regions, which means it owns the conflict-resolution semantics. We +don't recommend until the trigger above is firm. + +### Risk + +- **DIS coordination.** Both designs need DIS to know about the + passive region. +- **Postgres replica lag.** Webhook config changes during the failover + window may be lost. Mitigation: maintenance window for the cutover. +- **Cost.** Doubles JetStream + Postgres + pod fleet costs. + +### Out-of-scope + +Multi-cloud (AWS + GCP). One cloud at a time. + +--- + +## What we deliberately don't plan for + +- **Per-message ordering guarantees beyond per-subject.** JetStream + gives ordering within a subject only. Our pipeline relies on this. + Cross-subject ordering needs a different architecture (single + consumer, single stream) and we've explicitly traded that for + throughput. +- **Exactly-once webhook delivery.** Receivers must dedup on the + deterministic CloudEvent ID (`payload.ID`). This is documented in + `NATS_CONTRACT.md`. +- **Synchronous response to webhook receivers.** The dispatcher is + fire-and-forget from the consumer's perspective. We do not wait for + receiver-side processing. + +--- + +## Reference + +- Bench tool: `cmd/triggers-bench` +- Per-stream sizing math: see `charts/vehicle-triggers-api/values.yaml` + comment near `nats:` block. +- DIS publish contract: `NATS_CONTRACT.md` +- Operational procedures: `OPERATIONS.md` +- Backlog tracking: `PROD_HARDENING.md` diff --git a/charts/vehicle-triggers-api/templates/backup-cronjob.yaml b/charts/vehicle-triggers-api/templates/backup-cronjob.yaml new file mode 100644 index 0000000..72eab18 --- /dev/null +++ b/charts/vehicle-triggers-api/templates/backup-cronjob.yaml @@ -0,0 +1,76 @@ +{{- if .Values.backup.enabled }} +# Daily JetStream backup CronJob. Snapshots each configured stream and KV +# bucket with `nats stream backup` / `nats kv backup` from the nats-box image +# and uploads the tarballs to an S3 prefix. Restore procedure lives in +# OPERATIONS.md. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ include "vehicle-triggers-api.fullname" . }}-backup + namespace: {{ .Release.Namespace }} + labels: + {{- include "vehicle-triggers-api.labels" . | nindent 4 }} +spec: + schedule: {{ .Values.backup.schedule | quote }} + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 5 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: OnFailure + serviceAccountName: {{ include "vehicle-triggers-api.serviceAccountName" . }} + {{- if .Values.nats.credsEnabled }} + volumes: + - name: nats-creds + secret: + secretName: {{ include "vehicle-triggers-api.fullname" . }}-nats-creds + - name: script + configMap: + name: {{ include "vehicle-triggers-api.fullname" . }}-backup-script + defaultMode: 0755 + {{- else }} + volumes: + - name: script + configMap: + name: {{ include "vehicle-triggers-api.fullname" . }}-backup-script + defaultMode: 0755 + {{- end }} + containers: + - name: backup + image: {{ .Values.backup.image | default "natsio/nats-box:latest" | quote }} + command: ["/bin/sh", "/script/backup.sh"] + env: + - name: NATS_URL + value: {{ .Values.env.NATS_URL | quote }} + - name: BACKUP_S3_BUCKET + value: {{ .Values.backup.s3Bucket | quote }} + - name: BACKUP_S3_PREFIX + value: {{ .Values.backup.s3Prefix | quote }} + - name: STREAMS + value: {{ list .Values.env.NATS_SIGNALS_STREAM .Values.env.NATS_EVENTS_STREAM .Values.env.NATS_AUDIT_STREAM .Values.env.NATS_DLQ_STREAM | join "," | quote }} + - name: BUCKETS + value: {{ list .Values.env.NATS_TRIGGER_STATE_BUCKET .Values.env.NATS_SIGNAL_HISTORY_BUCKET | join "," | quote }} + {{- if .Values.nats.credsEnabled }} + - name: NATS_CREDS + value: {{ printf "%s/nats.creds" .Values.nats.credsMountPath | quote }} + {{- end }} + volumeMounts: + - name: script + mountPath: /script + readOnly: true + {{- if .Values.nats.credsEnabled }} + - name: nats-creds + mountPath: {{ .Values.nats.credsMountPath }} + readOnly: true + {{- end }} + resources: + limits: + cpu: 500m + memory: 256Mi + requests: + cpu: 50m + memory: 64Mi +{{- end }} diff --git a/charts/vehicle-triggers-api/templates/backup-script-configmap.yaml b/charts/vehicle-triggers-api/templates/backup-script-configmap.yaml new file mode 100644 index 0000000..55c8971 --- /dev/null +++ b/charts/vehicle-triggers-api/templates/backup-script-configmap.yaml @@ -0,0 +1,61 @@ +{{- if .Values.backup.enabled }} +# Shell script the backup CronJob runs. Kept inline so operators can tweak +# the retention policy / S3 path without redeploying the chart. +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "vehicle-triggers-api.fullname" . }}-backup-script + namespace: {{ .Release.Namespace }} + labels: + {{- include "vehicle-triggers-api.labels" . | nindent 4 }} +data: + backup.sh: | + #!/bin/sh + # vehicle-triggers-api JetStream backup script. + # Snapshots every configured stream + KV bucket and uploads to S3. + # Exits non-zero on the first failure so the CronJob is marked failed. + set -eu + + ts=$(date -u +%Y%m%dT%H%M%SZ) + workdir="/tmp/backup-${ts}" + mkdir -p "$workdir" + + auth="" + if [ -n "${NATS_CREDS:-}" ] && [ -f "$NATS_CREDS" ]; then + auth="--creds=${NATS_CREDS}" + fi + + # Streams + IFS=, + for stream in $STREAMS; do + [ -z "$stream" ] && continue + echo "[backup] snapshotting stream $stream" + nats stream backup $auth -s "$NATS_URL" "$stream" "$workdir/stream-$stream" + done + + # KV buckets are backed by streams named KV_. + for bucket in $BUCKETS; do + [ -z "$bucket" ] && continue + echo "[backup] snapshotting kv $bucket" + nats stream backup $auth -s "$NATS_URL" "KV_${bucket}" "$workdir/kv-$bucket" + done + unset IFS + + # Tarball + upload. aws CLI may not be present in nats-box; the chart + # operator should bake it in via .Values.backup.image when needed. + tarball="/tmp/vehicle-triggers-backup-${ts}.tar.gz" + tar -C /tmp -czf "$tarball" "$(basename "$workdir")" + echo "[backup] tarball $(ls -lh "$tarball" | awk '{print $5}')" + + if command -v aws >/dev/null 2>&1 && [ -n "${BACKUP_S3_BUCKET:-}" ]; then + dest="s3://${BACKUP_S3_BUCKET}/${BACKUP_S3_PREFIX}/$(basename "$tarball")" + echo "[backup] uploading to $dest" + aws s3 cp "$tarball" "$dest" + else + echo "[backup] aws cli or BACKUP_S3_BUCKET missing; backup left at $tarball" + ls -lh "$tarball" + fi + + rm -rf "$workdir" + echo "[backup] done" +{{- end }} diff --git a/charts/vehicle-triggers-api/templates/deployment.yaml b/charts/vehicle-triggers-api/templates/deployment.yaml index f6cbf34..08d3498 100644 --- a/charts/vehicle-triggers-api/templates/deployment.yaml +++ b/charts/vehicle-triggers-api/templates/deployment.yaml @@ -36,6 +36,12 @@ spec: serviceAccountName: {{ include "vehicle-triggers-api.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + # The app tears the ingest->dispatch->audit->NATS pipeline down in + # ordered phases on SIGTERM (drain consumers, then dispatcher queue, + # then audit queue, then a 30s NATS connection close). That can exceed + # the 30s k8s default, which would SIGKILL mid-drain and drop webhooks + # that were already acked off JetStream. Give it room. + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds | default 120 }} containers: - name: {{ .Chart.Name }} securityContext: @@ -43,11 +49,22 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} command: {{ toYaml .Values.command | nindent 12 }} + # NOTE: no preStop sleep hook - the runtime image is + # gcr.io/distroless/static, which has no shell, so an exec hook + # would fail. Endpoint-removal races are bounded by the readiness + # probe; the app's SIGTERM handler drains in-flight work within + # terminationGracePeriodSeconds above. envFrom: - configMapRef: name: {{ include "vehicle-triggers-api.fullname" . }}-config - secretRef: name: {{ include "vehicle-triggers-api.fullname" . }}-secret + {{- if .Values.nats.credsEnabled }} + volumeMounts: + - name: nats-creds + mountPath: {{ .Values.nats.credsMountPath }} + readOnly: true + {{- end }} ports: {{ toYaml .Values.ports | indent 12 }} livenessProbe: @@ -70,6 +87,12 @@ spec: successThreshold: 1 resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if .Values.nats.credsEnabled }} + volumes: + - name: nats-creds + secret: + secretName: {{ include "vehicle-triggers-api.fullname" . }}-nats-creds + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/charts/vehicle-triggers-api/templates/envconfigmap.yaml b/charts/vehicle-triggers-api/templates/envconfigmap.yaml index 87c4d81..6676651 100644 --- a/charts/vehicle-triggers-api/templates/envconfigmap.yaml +++ b/charts/vehicle-triggers-api/templates/envconfigmap.yaml @@ -8,4 +8,7 @@ metadata: data: {{- range $key, $val := .Values.env }} {{ $key }} : {{ $val | quote}} +{{- end}} +{{- if .Values.nats.credsEnabled }} + NATS_CREDS_FILE: {{ printf "%s/nats.creds" .Values.nats.credsMountPath | quote }} {{- end}} \ No newline at end of file diff --git a/charts/vehicle-triggers-api/templates/nats-secret.yaml b/charts/vehicle-triggers-api/templates/nats-secret.yaml new file mode 100644 index 0000000..2f7ae1b --- /dev/null +++ b/charts/vehicle-triggers-api/templates/nats-secret.yaml @@ -0,0 +1,22 @@ +{{- if .Values.nats.credsEnabled }} +# NATS credentials file, pulled from AWS Secrets Manager into a k8s Secret and +# mounted into the pod. The application reads it via NATS_CREDS_FILE. Only +# rendered when nats.credsEnabled is true. +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ include "vehicle-triggers-api.fullname" . }}-nats-creds + namespace: {{ .Release.Namespace }} +spec: + data: + - remoteRef: + key: {{ .Values.nats.credsSecretKey }} + secretKey: nats.creds + secretStoreRef: + kind: ClusterSecretStore + name: aws-secretsmanager-secret-store + target: + name: {{ include "vehicle-triggers-api.fullname" . }}-nats-creds + template: + metadata: {} +{{- end }} diff --git a/charts/vehicle-triggers-api/templates/prometheusrules.yaml b/charts/vehicle-triggers-api/templates/prometheusrules.yaml new file mode 100644 index 0000000..e16b694 --- /dev/null +++ b/charts/vehicle-triggers-api/templates/prometheusrules.yaml @@ -0,0 +1,62 @@ +{{- if .Values.serviceMonitor.alertsEnabled }} +# PrometheusRule covering the operational SLOs documented in OPERATIONS.md. +# Enable with .Values.serviceMonitor.alertsEnabled = true. Routes are not +# specified here - Alertmanager owns the destination map per env. +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: {{ include "vehicle-triggers-api.fullname" . }}-rules + namespace: {{ .Release.Namespace }} + labels: + {{- include "vehicle-triggers-api.labels" . | nindent 4 }} +spec: + groups: + - name: vehicle-triggers-api + rules: + # R from PROD_HARDENING_V2: any audit drop is a billing miscount. + # Alert on any non-zero rate over a 5-minute window so we catch + # sustained underrun without firing on single-tick blips. + - alert: VehicleTriggersAuditDropped + expr: rate(vehicle_triggers_audit_dropped_total[5m]) > 0 + for: 2m + labels: + severity: warning + service: vehicle-triggers-api + annotations: + summary: "Audit records dropping (billing miscount)" + description: "vehicle_triggers_audit_dropped_total has a non-zero rate; downstream billing aggregation will undercount. Raise NATS_AUDIT_QUEUE_SIZE or NATS_AUDIT_WORKERS." + # Dispatcher queue-full is backpressure made visible. Sustained + # rate means we should scale workers / pods or rate-limit the + # offending receiver. + - alert: VehicleTriggersDispatcherQueueFull + expr: rate(vehicle_triggers_dispatcher_queue_full_total[5m]) > 0 + for: 5m + labels: + severity: warning + service: vehicle-triggers-api + annotations: + summary: "Webhook dispatcher backpressure" + description: "Dispatcher Enqueue is rejecting jobs; messages are being naked and risk DLQing. Raise NATS_DISPATCHER_WORKERS / NATS_DISPATCHER_QUEUE_SIZE or scale out." + # DLQ accumulation = sustained poison or a chain of queue-full naks + # (mitigated by ErrBackpressure handling but still possible). + - alert: VehicleTriggersDLQAccumulating + expr: rate(vehicle_triggers_nats_consume_total{outcome="dlq"}[10m]) > 0 + for: 10m + labels: + severity: warning + service: vehicle-triggers-api + annotations: + summary: "DLQ entries accumulating" + description: "Messages are landing in the DLQ stream. Inspect with `triggers-dlq list`; replay or purge as appropriate." + # /health failure means stream-level health check is failing - + # JetStream cluster degradation, not just connection liveness. + - alert: VehicleTriggersHealthCheckFailing + expr: probe_success{job=~".*vehicle-triggers-api.*"} == 0 + for: 5m + labels: + severity: critical + service: vehicle-triggers-api + annotations: + summary: "/health probe failing" + description: "vehicle-triggers-api /health is returning 503 sustained. Check NATS stream leaders via `nats stream report`." +{{- end }} diff --git a/charts/vehicle-triggers-api/values-prod.yaml b/charts/vehicle-triggers-api/values-prod.yaml index e52d660..6af1006 100644 --- a/charts/vehicle-triggers-api/values-prod.yaml +++ b/charts/vehicle-triggers-api/values-prod.yaml @@ -15,13 +15,44 @@ env: SERVICE_NAME: vehicle-triggers-api JWK_KEY_SET_URL: https://auth.dimo.zone/keys IDENTITY_API_URL: https://identity-api.dimo.zone/query - KAFKA_BROKERS: kafka-prod-dimo-kafka-kafka-brokers:9092 TOKEN_EXCHANGE_GRPC_ADDR: token-exchange-api-prod:8086 DIMO_REGISTRY_CHAIN_ID: 137 VEHICLE_NFT_ADDRESS: '0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF' MAX_IN_FLIGHT: 50 TOKEN_EXCHANGE_CACHE_EXPIRATION: 10m TOKEN_EXCHANGE_CACHE_CLEANUP_INTERVAL: 2m + # NATS JetStream ingest. This is what activates the consume+dispatch + # pipeline in prod; without NATS_MODE=exclusive the service runs API-only. + # DELIVER_POLICY=new is the safe cutover default (no replay of retained + # telemetry). Switch to "all" only for a deliberate backfill. + NATS_MODE: exclusive + NATS_URL: nats://nats-prod.dimo-platform.svc.cluster.local:4222 + NATS_NAME: vehicle-triggers-api + NATS_DELIVER_POLICY: new + NATS_SIGNALS_STREAM: DIMO_SIGNALS + NATS_EVENTS_STREAM: DIMO_EVENTS + NATS_AUDIT_STREAM: DIMO_TRIGGER_AUDIT + NATS_DLQ_STREAM: DIMO_TRIGGER_DLQ + NATS_SIGNALS_SUBJECT: dimo.signals.> + NATS_EVENTS_SUBJECT: dimo.events.> + NATS_AUDIT_SUBJECT: dimo.trigger.fired.> + NATS_DLQ_SUBJECT: dimo.dlq.> + NATS_SIGNALS_DURABLE: triggers-signals + NATS_EVENTS_DURABLE: triggers-events + NATS_STREAM_REPLICAS: '3' + NATS_ACK_WAIT: 45s + NATS_MAX_DELIVER: '5' + NATS_MAX_ACK_PENDING: '5000' + NATS_FETCH_BATCH: '100' + NATS_PUBLISH_ASYNC_MAX_PENDING: '4000' + NATS_DLQ_MAX_AGE: 168h + NATS_TRIGGER_STATE_BUCKET: trigger_state + NATS_SIGNAL_HISTORY_BUCKET: signal_history + # Cooldown TTL invariant: TTL >= 2*MAX_ALLOWED_COOLDOWN_PERIOD or startup fails. + NATS_TRIGGER_STATE_TTL: 1512h + MAX_ALLOWED_COOLDOWN_PERIOD: '2592000' +# Ordered pipeline drain needs > 30s default (see deployment.yaml note). +terminationGracePeriodSeconds: 120 ingress: enabled: true className: nginx diff --git a/charts/vehicle-triggers-api/values.yaml b/charts/vehicle-triggers-api/values.yaml index 4f0140a..170efa8 100644 --- a/charts/vehicle-triggers-api/values.yaml +++ b/charts/vehicle-triggers-api/values.yaml @@ -37,9 +37,6 @@ env: SERVICE_NAME: vehicle-triggers-api JWK_KEY_SET_URL: https://auth.dev.dimo.zone/keys IDENTITY_API_URL: https://identity-api.dev.dimo.zone/query - KAFKA_BROKERS: kafka-dev-dimo-kafka-kafka-brokers:9092 - DEVICE_SIGNALS_TOPIC: topic.device.signals - DEVICE_EVENTS_TOPIC: topic.device.events VEHICLE_NFT_ADDRESS: '0x45fbCD3ef7361d156e8b16F5538AE36DEdf61Da8' DIMO_REGISTRY_CHAIN_ID: 80002 MAX_WEBHOOK_FAILURE_COUNT: 5 @@ -47,6 +44,75 @@ env: CACHE_DEBOUNCE_TIME: 5s TOKEN_EXCHANGE_CACHE_EXPIRATION: 15m TOKEN_EXCHANGE_CACHE_CLEANUP_INTERVAL: 5m + # NATS JetStream ingest. Modes: + # off - API only, no ingest pipeline. + # primary | exclusive - NATS owns evaluation + dispatch (equivalent + # post-Kafka rip; "exclusive" is canonical prod). + # + # Storage planning rule of thumb (file storage, replicas=3): + # per_node_disk_per_stream = msgs_per_sec * avg_msg_bytes * max_age_seconds + # At ~30k msg/s steady state, 600B avg signal envelope, 24h MaxAge: + # 30000 * 600 * 86400 ~= 50 GB per node per signals stream. + # Audit and DLQ are sized off fire rate (typically 10-100x lower than + # signal rate); CONFIG_AUDIT is sized off CRUD rate (negligible). + # Bump NATS_*_MAX_AGE down to reduce storage, up to reduce risk of + # discarding mid-retry messages (see Settings.Validate). + NATS_MODE: off + NATS_URL: nats://nats-dev.dimo-platform.svc.cluster.local:4222 + NATS_NAME: vehicle-triggers-api + NATS_SIGNALS_STREAM: DIMO_SIGNALS + NATS_EVENTS_STREAM: DIMO_EVENTS + NATS_AUDIT_STREAM: DIMO_TRIGGER_AUDIT + NATS_DLQ_STREAM: DIMO_TRIGGER_DLQ + NATS_SIGNALS_SUBJECT: dimo.signals.> + NATS_EVENTS_SUBJECT: dimo.events.> + NATS_AUDIT_SUBJECT: dimo.trigger.fired.> + NATS_DLQ_SUBJECT: dimo.dlq.> + NATS_SIGNALS_DURABLE: triggers-signals + NATS_EVENTS_DURABLE: triggers-events + NATS_STREAM_REPLICAS: '3' + NATS_ACK_WAIT: 45s + NATS_MAX_DELIVER: '5' + NATS_MAX_ACK_PENDING: '5000' + NATS_FETCH_BATCH: '100' + NATS_PUBLISH_ASYNC_MAX_PENDING: '4000' + NATS_DLQ_MAX_AGE: 168h + # KV buckets for distributed cooldown state and signal history. Named + # explicitly because the backup CronJob snapshots them by name - if they're + # absent the backup silently skips them and a restore comes up with empty + # cooldown state (every trigger re-fires). + NATS_TRIGGER_STATE_BUCKET: trigger_state + NATS_SIGNAL_HISTORY_BUCKET: signal_history + # Cooldown TTL invariant: NATS_TRIGGER_STATE_TTL must be >= + # 2*MAX_ALLOWED_COOLDOWN_PERIOD or the service refuses to start + # (Settings.Validate). 63d TTL covers the 30d cooldown ceiling. + NATS_TRIGGER_STATE_TTL: 1512h + MAX_ALLOWED_COOLDOWN_PERIOD: '2592000' +# Grace period for the ordered pipeline drain on SIGTERM (consumers -> +# dispatcher -> audit -> 30s NATS close). Must exceed the 30s default or the +# pod is SIGKILLed mid-drain, dropping acked-but-unsent webhooks. +terminationGracePeriodSeconds: 120 +# NATS authentication. When credsEnabled is true an ExternalSecret pulls the +# creds file from AWS Secrets Manager (credsSecretKey) into a k8s Secret, +# mounts it at credsMountPath, and sets NATS_CREDS_FILE for the app. Leave +# disabled for unauthenticated local/dev NATS. +nats: + credsEnabled: false + credsSecretKey: dev/vehicle-triggers-api/nats/creds + credsMountPath: /etc/nats + +# Daily JetStream backup. Snapshots every configured stream + KV bucket via +# `nats stream backup` from a nats-box container and (if aws CLI is present) +# uploads the tarball to s3:////. Restore procedure in +# OPERATIONS.md. Image must include both `nats` CLI and `aws` CLI - the +# default natsio/nats-box only has nats; override with an image that bakes +# aws in for prod use. +backup: + enabled: false + schedule: "0 6 * * *" # daily 06:00 UTC + image: natsio/nats-box:latest + s3Bucket: "" + s3Prefix: "vehicle-triggers-api/jetstream" service: type: ClusterIP ports: @@ -103,6 +169,10 @@ autoscaling: nodeSelector: {} tolerations: [] affinity: {} +# minAvailable: 0 is intentional for the single-replica dev deployment so a +# node drain isn't blocked by the lone pod. Multi-replica environments MUST +# override this to >=1 (see values-prod.yaml) or a drain can take all +# replicas down at once. podDisruptionBudget: minAvailable: 0 serviceMonitor: @@ -110,3 +180,7 @@ serviceMonitor: path: /metrics port: mon-http interval: 30s + # PrometheusRule wiring for the SLOs documented in OPERATIONS.md. + # Disabled by default so envs without prometheus-operator don't fail + # on the missing CRD; flip to true when the operator is installed. + alertsEnabled: false diff --git a/cmd/nats-testpublisher/main.go b/cmd/nats-testpublisher/main.go new file mode 100644 index 0000000..981d162 --- /dev/null +++ b/cmd/nats-testpublisher/main.go @@ -0,0 +1,372 @@ +// nats-testpublisher is a developer tool that stands in for the DIS service. +// It synthesizes vss SignalCloudEvent / EventCloudEvent payloads and publishes +// them to the JetStream subjects the vehicle-triggers-api service consumes. +// +// Example: +// +// nats-testpublisher -url nats://localhost:4222 \ +// -rate 50 -vehicles 100 -signals speed,fuelLevel,odometer \ +// -mode both +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "math/big" + "math/rand/v2" + "os" + "os/signal" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/model-garage/pkg/vss" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type mode string + +const ( + modeSignals mode = "signals" + modeEvents mode = "events" + modeBoth mode = "both" +) + +type config struct { + URL string + CredsFile string + + Rate float64 + Vehicles int + ChainID uint64 + Contract string + + Signals []string + Events []string + Mode mode + + Duration time.Duration + Burst int + Jitter time.Duration + + EnsureStreams bool + SignalsStream string + EventsStream string + SignalsSubject string + EventsSubject string + MaxAge time.Duration + + ReplayFrom string +} + +func parseConfig() (*config, error) { + cfg := &config{} + var sigList, evtList string + var m string + + flag.StringVar(&cfg.URL, "url", "nats://localhost:4222", "NATS URL") + flag.StringVar(&cfg.CredsFile, "creds", "", "NATS credentials file") + flag.Float64Var(&cfg.Rate, "rate", 10, "Messages per second (per mode). 0 = as fast as possible") + flag.IntVar(&cfg.Vehicles, "vehicles", 10, "Number of distinct synthetic vehicle token IDs") + flag.Uint64Var(&cfg.ChainID, "chain-id", 137, "EVM chain ID to stamp in the ERC721 DID") + flag.StringVar(&cfg.Contract, "contract", "0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF", "Vehicle NFT contract address") + flag.StringVar(&sigList, "signals", "speed,fuelLevel,odometer,batteryVoltage", "Comma-separated signal names") + flag.StringVar(&evtList, "events", "harshBraking,ignitionOn,ignitionOff", "Comma-separated event names") + flag.StringVar(&m, "mode", "both", "signals|events|both") + flag.DurationVar(&cfg.Duration, "duration", 0, "Run for this long then exit. 0 = run forever until Ctrl-C") + flag.IntVar(&cfg.Burst, "burst", 1, "Messages emitted per tick") + flag.DurationVar(&cfg.Jitter, "jitter", 0, "Random extra delay 0..jitter between ticks") + flag.BoolVar(&cfg.EnsureStreams, "ensure-streams", true, "Create/update DIMO_SIGNALS and DIMO_EVENTS streams before publishing") + flag.StringVar(&cfg.SignalsStream, "signals-stream", "DIMO_SIGNALS", "Signals stream name") + flag.StringVar(&cfg.EventsStream, "events-stream", "DIMO_EVENTS", "Events stream name") + flag.StringVar(&cfg.SignalsSubject, "signals-subject", "dimo.signals.>", "Signals stream subject filter") + flag.StringVar(&cfg.EventsSubject, "events-subject", "dimo.events.>", "Events stream subject filter") + flag.DurationVar(&cfg.MaxAge, "max-age", 24*time.Hour, "Stream MaxAge when ensuring streams") + flag.StringVar(&cfg.ReplayFrom, "replay-from", "", "Path to a jsonl file of captured payloads to replay") + flag.Parse() + + cfg.Signals = splitCSV(sigList) + cfg.Events = splitCSV(evtList) + cfg.Mode = mode(strings.ToLower(m)) + switch cfg.Mode { + case modeSignals, modeEvents, modeBoth: + default: + return nil, fmt.Errorf("invalid -mode %q (signals|events|both)", m) + } + if cfg.Vehicles < 1 { + return nil, errors.New("-vehicles must be >= 1") + } + if cfg.Mode != modeEvents && len(cfg.Signals) == 0 { + return nil, errors.New("at least one -signals name required") + } + if cfg.Mode != modeSignals && len(cfg.Events) == 0 { + return nil, errors.New("at least one -events name required") + } + if !common.IsHexAddress(cfg.Contract) { + return nil, fmt.Errorf("-contract %q is not a hex address", cfg.Contract) + } + return cfg, nil +} + +func splitCSV(s string) []string { + parts := strings.Split(s, ",") + out := parts[:0] + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func main() { + cfg, err := parseConfig() + if err != nil { + fmt.Fprintln(os.Stderr, "config:", err) + flag.Usage() + os.Exit(2) + } + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + if cfg.Duration > 0 { + ctx2, cancel2 := context.WithTimeout(ctx, cfg.Duration) + defer cancel2() + ctx = ctx2 + } + + opts := []nc.Option{nc.Name("nats-testpublisher"), nc.Timeout(5 * time.Second)} + if cfg.CredsFile != "" { + opts = append(opts, nc.UserCredentials(cfg.CredsFile)) + } + conn, err := nc.Connect(cfg.URL, opts...) + if err != nil { + fmt.Fprintln(os.Stderr, "nats connect:", err) + os.Exit(1) + } + defer conn.Drain() //nolint:errcheck + + js, err := jetstream.New(conn) + if err != nil { + fmt.Fprintln(os.Stderr, "jetstream:", err) + os.Exit(1) + } + + if cfg.EnsureStreams { + if err := ensureStreams(ctx, js, cfg); err != nil { + fmt.Fprintln(os.Stderr, "ensure streams:", err) + os.Exit(1) + } + } + + if cfg.ReplayFrom != "" { + if err := replayFile(ctx, js, cfg); err != nil { + fmt.Fprintln(os.Stderr, "replay:", err) + os.Exit(1) + } + return + } + + if err := loop(ctx, js, cfg); err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) { + fmt.Fprintln(os.Stderr, "loop:", err) + os.Exit(1) + } +} + +func ensureStreams(ctx context.Context, js jetstream.JetStream, cfg *config) error { + specs := []jetstream.StreamConfig{ + {Name: cfg.SignalsStream, Subjects: []string{cfg.SignalsSubject}, Retention: jetstream.LimitsPolicy, Discard: jetstream.DiscardOld, Storage: jetstream.FileStorage, MaxAge: cfg.MaxAge, Replicas: 1}, + {Name: cfg.EventsStream, Subjects: []string{cfg.EventsSubject}, Retention: jetstream.LimitsPolicy, Discard: jetstream.DiscardOld, Storage: jetstream.FileStorage, MaxAge: cfg.MaxAge, Replicas: 1}, + } + for _, s := range specs { + if _, err := js.CreateOrUpdateStream(ctx, s); err != nil { + return fmt.Errorf("stream %s: %w", s.Name, err) + } + } + return nil +} + +func loop(ctx context.Context, js jetstream.JetStream, cfg *config) error { + var interval time.Duration + if cfg.Rate > 0 { + interval = time.Duration(float64(time.Second) / cfg.Rate) + } + contract := common.HexToAddress(cfg.Contract) + + var sent atomic.Uint64 + go func() { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + last := uint64(0) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + cur := sent.Load() + fmt.Printf("[testpublisher] sent=%d rate=%.1f/s\n", cur, float64(cur-last)/5) + last = cur + } + } + }() + + publish := func() error { + tokenID := rand.IntN(cfg.Vehicles) + 1 + did := cloudevent.ERC721DID{ChainID: cfg.ChainID, ContractAddress: contract, TokenID: big.NewInt(int64(tokenID))} + switch cfg.Mode { + case modeSignals: + return publishSignalBatch(ctx, js, cfg, did) + case modeEvents: + return publishEventBatch(ctx, js, cfg, did) + case modeBoth: + if rand.IntN(2) == 0 { + return publishSignalBatch(ctx, js, cfg, did) + } + return publishEventBatch(ctx, js, cfg, did) + } + return nil + } + + tick := func() error { + for i := 0; i < cfg.Burst; i++ { + if err := publish(); err != nil { + return err + } + sent.Add(1) + } + return nil + } + + if interval == 0 { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + if err := tick(); err != nil { + return err + } + } + } + } + + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-t.C: + if err := tick(); err != nil { + return err + } + if cfg.Jitter > 0 { + time.Sleep(time.Duration(rand.Int64N(int64(cfg.Jitter)))) + } + } + } +} + +func publishSignalBatch(ctx context.Context, js jetstream.JetStream, cfg *config, did cloudevent.ERC721DID) error { + name := cfg.Signals[rand.IntN(len(cfg.Signals))] + now := time.Now().UTC() + hdr := cloudevent.CloudEventHeader{ + SpecVersion: "1.0", + Type: "dimo.signal", + Source: "nats-testpublisher", + Subject: did.String(), + ID: fmt.Sprintf("tp-sig-%d-%d", now.UnixNano(), rand.Uint32()), + Time: now, + DataContentType: "application/json", + Producer: "nats-testpublisher/synthetic", + } + sig := vss.Signal{ + CloudEventHeader: hdr, + Data: vss.SignalData{ + Timestamp: now, + Name: name, + ValueNumber: rand.Float64() * 120, + }, + } + ce := vss.PackSignals(hdr, []vss.Signal{sig}) + payload, err := json.Marshal(ce) + if err != nil { + return err + } + subject := vtnats.SignalSubject(name) + _, err = js.Publish(ctx, subject, payload) + return err +} + +func publishEventBatch(ctx context.Context, js jetstream.JetStream, cfg *config, did cloudevent.ERC721DID) error { + name := cfg.Events[rand.IntN(len(cfg.Events))] + now := time.Now().UTC() + hdr := cloudevent.CloudEventHeader{ + SpecVersion: "1.0", + Type: "dimo.event", + Source: "nats-testpublisher", + Subject: did.String(), + ID: fmt.Sprintf("tp-evt-%d-%d", now.UnixNano(), rand.Uint32()), + Time: now, + DataContentType: "application/json", + Producer: "nats-testpublisher/synthetic", + } + evt := vss.Event{ + CloudEventHeader: hdr, + Data: vss.EventData{ + Name: name, + Timestamp: now, + DurationNs: uint64(rand.IntN(5_000_000_000)), + }, + } + ce := vss.PackEvents(hdr, []vss.Event{evt}) + payload, err := json.Marshal(ce) + if err != nil { + return err + } + subject := vtnats.EventSubject(name) + _, err = js.Publish(ctx, subject, payload) + return err +} + +// replayFile reads a jsonl file of captured payloads and re-publishes them. +// Each line: {"subject":"dimo.signals.123.speed","payload":} +// If "payload" is a JSON object, it is re-marshaled; otherwise it must be a string +// (interpreted as the raw body to publish). +func replayFile(ctx context.Context, js jetstream.JetStream, cfg *config) error { + f, err := os.Open(cfg.ReplayFrom) + if err != nil { + return err + } + defer f.Close() //nolint:errcheck // read-only file, close error is uninteresting + + dec := json.NewDecoder(f) + var count int + for dec.More() { + var rec struct { + Subject string `json:"subject"` + Payload json.RawMessage `json:"payload"` + } + if err := dec.Decode(&rec); err != nil { + return fmt.Errorf("decode line %d: %w", count, err) + } + if rec.Subject == "" || len(rec.Payload) == 0 { + continue + } + if _, err := js.Publish(ctx, rec.Subject, rec.Payload); err != nil { + return fmt.Errorf("publish %s: %w", rec.Subject, err) + } + count++ + } + fmt.Printf("[testpublisher] replayed %d messages\n", count) + return nil +} diff --git a/cmd/triggers-bench/main.go b/cmd/triggers-bench/main.go new file mode 100644 index 0000000..120cd41 --- /dev/null +++ b/cmd/triggers-bench/main.go @@ -0,0 +1,615 @@ +// triggers-bench is a load + latency probe for the vehicle-triggers-api NATS +// JetStream ingest path. It publishes synthetic vss SignalCloudEvent payloads +// at a target rate, pulls them back through a JetStream consumer that mirrors +// the production filter subjects, and reports throughput plus publish/round- +// trip latency percentiles. +// +// Example: +// +// triggers-bench -url nats://localhost:4222 \ +// -rate 5000 -duration 30s -signals speed,fuelLevel,odometer \ +// -consumers 4 +// +// Use a transient stream name so concurrent runs don't trample each other. +package main + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "flag" + "fmt" + "math/big" + "math/rand/v2" + "os" + "os/signal" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/model-garage/pkg/vss" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type config struct { + URL string + CredsFile string + + Rate float64 + Duration time.Duration + Vehicles int + Signals []string + Workers int + Publishers int + Replicas int + Async bool + + StreamName string + Subject string + ConsumerN string + + NoConsume bool + + // KVMode replaces the stream publish + consume loop with a pure KV + // write loop against a temporary bucket. Used to measure the trigger_ + // state bucket's write ceiling, which dominates the hot-path cost at + // scale (one Put per fire per trigger). + KVMode bool + KVBucket string +} + +func parseConfig() (*config, error) { + cfg := &config{} + var sigList string + flag.StringVar(&cfg.URL, "url", "nats://localhost:4222", "NATS URL") + flag.StringVar(&cfg.CredsFile, "creds", "", "NATS credentials file (optional)") + flag.Float64Var(&cfg.Rate, "rate", 1000, "Target publishes per second") + flag.DurationVar(&cfg.Duration, "duration", 30*time.Second, "How long to publish") + flag.IntVar(&cfg.Vehicles, "vehicles", 10000, "Number of synthetic vehicle token IDs") + flag.StringVar(&sigList, "signals", "speed,fuelLevel,odometer,batteryVoltage", "Comma-separated signal names") + flag.IntVar(&cfg.Workers, "consumers", 1, "Concurrent NATS pull loops") + flag.IntVar(&cfg.Publishers, "publishers", 1, "Concurrent publisher goroutines (each gets rate/N)") + flag.IntVar(&cfg.Replicas, "replicas", 1, "Stream replication factor (1, 3, 5...)") + flag.BoolVar(&cfg.Async, "async", false, "Use PublishAsync instead of sync Publish for higher throughput") + flag.StringVar(&cfg.StreamName, "stream", "BENCH_SIGNALS", "JetStream stream name (created/updated)") + flag.StringVar(&cfg.Subject, "subject", "bench.signals.>", "Stream subject") + flag.StringVar(&cfg.ConsumerN, "consumer-name", "bench-cons", "Durable consumer name (random suffix appended)") + flag.BoolVar(&cfg.NoConsume, "no-consume", false, "Skip the consumer side (publish-only benchmark)") + flag.BoolVar(&cfg.KVMode, "kv-writes", false, "Bench KV writes instead of stream publish/consume (measures trigger_state ceiling)") + flag.StringVar(&cfg.KVBucket, "kv-bucket", "BENCH_KV", "KV bucket name when -kv-writes is set") + flag.Parse() + cfg.Signals = splitCSV(sigList) + if len(cfg.Signals) == 0 { + return nil, errors.New("at least one -signals name required") + } + if cfg.Rate <= 0 { + return nil, errors.New("-rate must be > 0") + } + return cfg, nil +} + +func splitCSV(s string) []string { + parts := strings.Split(s, ",") + out := parts[:0] + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func main() { + cfg, err := parseConfig() + if err != nil { + fmt.Fprintln(os.Stderr, "config:", err) + flag.Usage() + os.Exit(2) + } + if err := run(cfg); err != nil { + fmt.Fprintln(os.Stderr, "bench:", err) + os.Exit(1) + } +} + +func run(cfg *config) error { + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + opts := []nc.Option{nc.Name("triggers-bench"), nc.Timeout(5 * time.Second)} + if cfg.CredsFile != "" { + opts = append(opts, nc.UserCredentials(cfg.CredsFile)) + } + + if cfg.KVMode { + return runKV(ctx, cfg, opts) + } + + conn, err := nc.Connect(cfg.URL, opts...) + if err != nil { + return fmt.Errorf("nats connect: %w", err) + } + defer conn.Drain() //nolint:errcheck // benchmark teardown, best-effort + + js, err := jetstream.New(conn, jetstream.WithPublishAsyncMaxPending(50_000)) + if err != nil { + return fmt.Errorf("jetstream: %w", err) + } + + if _, err := js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: cfg.StreamName, + Subjects: []string{cfg.Subject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: 10 * time.Minute, + Replicas: cfg.Replicas, + }); err != nil { + return fmt.Errorf("ensure stream: %w", err) + } + defer js.DeleteStream(context.Background(), cfg.StreamName) //nolint:errcheck // benchmark teardown + + // Stats collectors. publishSamples records ack latency; rtSamples records + // publish-to-consume latency. Both cap at duration*rate, then sample. + pubSamples := newSamples(int(cfg.Rate*cfg.Duration.Seconds()) + 1024) + rtSamples := newSamples(int(cfg.Rate*cfg.Duration.Seconds()) + 1024) + + var publishedOK, publishedErr, consumed uint64 + + var wg sync.WaitGroup + consumerDone := make(chan struct{}) + if !cfg.NoConsume { + consumerName := fmt.Sprintf("%s-%d", cfg.ConsumerN, time.Now().UnixNano()) + cons, err := js.CreateOrUpdateConsumer(ctx, cfg.StreamName, jetstream.ConsumerConfig{ + Durable: consumerName, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + FilterSubject: cfg.Subject, + MaxAckPending: 50_000, + }) + if err != nil { + return fmt.Errorf("ensure consumer: %w", err) + } + defer js.DeleteConsumer(context.Background(), cfg.StreamName, consumerName) //nolint:errcheck // benchmark teardown + + for i := 0; i < cfg.Workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + runConsumer(ctx, cons, rtSamples, &consumed) + }() + } + go func() { wg.Wait(); close(consumerDone) }() + } else { + close(consumerDone) + } + + // Periodic progress line so the operator can see liveness during long runs. + progressDone := make(chan struct{}) + go func() { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + var lastPub uint64 + var lastCons uint64 + for { + select { + case <-ctx.Done(): + close(progressDone) + return + case <-t.C: + p := atomic.LoadUint64(&publishedOK) + c := atomic.LoadUint64(&consumed) + fmt.Printf("[bench] pub=%d (+%d) cons=%d (+%d) errs=%d\n", + p, p-lastPub, c, c-lastCons, atomic.LoadUint64(&publishedErr)) + lastPub = p + lastCons = c + } + } + }() + + pubCtx, pubCancel := context.WithTimeout(ctx, cfg.Duration) + defer pubCancel() + + start := time.Now() + var pubWG sync.WaitGroup + pubCount := cfg.Publishers + if pubCount < 1 { + pubCount = 1 + } + perPubRate := cfg.Rate / float64(pubCount) + for i := 0; i < pubCount; i++ { + pubWG.Add(1) + go func() { + defer pubWG.Done() + runPublisher(pubCtx, js, cfg, perPubRate, pubSamples, &publishedOK, &publishedErr) + }() + } + pubWG.Wait() + if cfg.Async { + // Wait for all async acks to land so we measure to-server-ack, not + // just to-network. Bounded by a hard ceiling so a stuck server + // doesn't hang the bench. + select { + case <-js.PublishAsyncComplete(): + case <-time.After(30 * time.Second): + fmt.Fprintln(os.Stderr, "[bench] async publish drain timed out") + } + } + pubElapsed := time.Since(start) + + // Drain consumer: wait briefly for trailing messages to land, then stop. + if !cfg.NoConsume { + drainDeadline := time.Now().Add(10 * time.Second) + for time.Now().Before(drainDeadline) { + before := atomic.LoadUint64(&consumed) + time.Sleep(500 * time.Millisecond) + after := atomic.LoadUint64(&consumed) + if after == before && after >= atomic.LoadUint64(&publishedOK) { + break + } + } + cancel() + select { + case <-consumerDone: + case <-time.After(5 * time.Second): + fmt.Fprintln(os.Stderr, "[bench] consumers did not exit cleanly") + } + } + <-progressDone + + report(cfg, pubElapsed, &publishedOK, &publishedErr, &consumed, pubSamples, rtSamples) + return nil +} + +func runPublisher(ctx context.Context, js jetstream.JetStream, cfg *config, rate float64, samples *samples, ok, errs *uint64) { + contract := common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF") + interval := time.Duration(float64(time.Second) / rate) + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + tokenID := rand.IntN(cfg.Vehicles) + 1 + signal := cfg.Signals[rand.IntN(len(cfg.Signals))] + payload := buildPayload(cfg, contract, tokenID, signal) + subject := strings.Replace(cfg.Subject, ".>", "."+signal, 1) + t0 := time.Now() + if cfg.Async { + future, err := js.PublishAsync(subject, payload) + if err != nil { + atomic.AddUint64(errs, 1) + continue + } + atomic.AddUint64(ok, 1) + go func() { + select { + case <-future.Ok(): + samples.add(time.Since(t0)) + case <-future.Err(): + atomic.AddUint64(errs, 1) + } + }() + continue + } + _, err := js.Publish(ctx, subject, payload) + lat := time.Since(t0) + if err != nil { + atomic.AddUint64(errs, 1) + continue + } + atomic.AddUint64(ok, 1) + samples.add(lat) + } + } +} + +// buildPayload synthesizes a SignalCloudEvent with the publish timestamp +// embedded in the Producer field so the consumer can compute roundtrip time +// without needing message headers. +func buildPayload(_ *config, contract common.Address, tokenID int, name string) []byte { + now := time.Now().UTC() + did := cloudevent.ERC721DID{ChainID: 137, ContractAddress: contract, TokenID: big.NewInt(int64(tokenID))} + hdr := cloudevent.CloudEventHeader{ + SpecVersion: "1.0", + Type: "dimo.signal", + Source: "triggers-bench", + Subject: did.String(), + ID: fmt.Sprintf("bench-%d", now.UnixNano()), + Time: now, + DataContentType: "application/json", + Producer: encodeProducer(now), + } + sig := vss.Signal{CloudEventHeader: hdr, Data: vss.SignalData{ + Timestamp: now, Name: name, ValueNumber: rand.Float64() * 120, + }} + ce := vss.PackSignals(hdr, []vss.Signal{sig}) + b, _ := json.Marshal(ce) + return b +} + +// encodeProducer stashes the publish wall time inside the Producer string so +// the consumer can recover it without a separate header parse. We use raw +// little-endian int64 (UnixNano) base16'd to avoid JSON escaping issues. +func encodeProducer(t time.Time) string { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(t.UnixNano())) + return "bench/" + hex(buf[:]) +} + +func decodeProducer(s string) (time.Time, bool) { + const prefix = "bench/" + if !strings.HasPrefix(s, prefix) { + return time.Time{}, false + } + raw, ok := unhex(s[len(prefix):]) + if !ok || len(raw) != 8 { + return time.Time{}, false + } + return time.Unix(0, int64(binary.LittleEndian.Uint64(raw))), true +} + +const hexDigits = "0123456789abcdef" + +func hex(b []byte) string { + out := make([]byte, len(b)*2) + for i, x := range b { + out[i*2] = hexDigits[x>>4] + out[i*2+1] = hexDigits[x&0x0f] + } + return string(out) +} + +func unhex(s string) ([]byte, bool) { + if len(s)%2 != 0 { + return nil, false + } + out := make([]byte, len(s)/2) + for i := 0; i < len(out); i++ { + hi := hexVal(s[2*i]) + lo := hexVal(s[2*i+1]) + if hi < 0 || lo < 0 { + return nil, false + } + out[i] = byte(hi<<4 | lo) + } + return out, true +} + +func hexVal(b byte) int { + switch { + case b >= '0' && b <= '9': + return int(b - '0') + case b >= 'a' && b <= 'f': + return int(b-'a') + 10 + case b >= 'A' && b <= 'F': + return int(b-'A') + 10 + } + return -1 +} + +func runConsumer(ctx context.Context, cons jetstream.Consumer, samples *samples, consumed *uint64) { + iter, err := cons.Messages(jetstream.PullMaxMessages(500)) + if err != nil { + return + } + defer iter.Stop() + go func() { + <-ctx.Done() + iter.Stop() + }() + for { + msg, err := iter.Next() + if err != nil { + return + } + var ce vss.SignalCloudEvent + if json.Unmarshal(msg.Data(), &ce) == nil { + if pubT, ok := decodeProducer(ce.Producer); ok { + samples.add(time.Since(pubT)) + } + } + _ = msg.Ack() + atomic.AddUint64(consumed, 1) + } +} + +func report(cfg *config, pubElapsed time.Duration, ok, errs, consumed *uint64, pubS *samples, rtS *samples) { + pubCount := atomic.LoadUint64(ok) + errCount := atomic.LoadUint64(errs) + consCount := atomic.LoadUint64(consumed) + throughput := float64(pubCount) / pubElapsed.Seconds() + fmt.Println() + fmt.Println("=== triggers-bench results ===") + fmt.Printf("target rate : %.0f msg/s for %s (publishers=%d, replicas=%d, async=%v)\n", cfg.Rate, cfg.Duration, cfg.Publishers, cfg.Replicas, cfg.Async) + fmt.Printf("publisher elapsed : %s\n", pubElapsed) + fmt.Printf("published ok / err : %d / %d\n", pubCount, errCount) + fmt.Printf("publish throughput : %.1f msg/s\n", throughput) + if !cfg.NoConsume { + fmt.Printf("consumed : %d (delta=%d)\n", consCount, int64(pubCount)-int64(consCount)) + } + fmt.Println() + fmt.Println("publish-ack latency:") + printLatency(pubS) + if !cfg.NoConsume { + fmt.Println("\npublish->consume roundtrip latency:") + printLatency(rtS) + } +} + +func printLatency(s *samples) { + xs := s.snapshot() + if len(xs) == 0 { + fmt.Println(" (no samples)") + return + } + sort.Slice(xs, func(i, j int) bool { return xs[i] < xs[j] }) + pct := func(p float64) time.Duration { + idx := int(float64(len(xs)-1) * p) + return xs[idx] + } + fmt.Printf(" n=%d min=%s p50=%s p90=%s p95=%s p99=%s max=%s\n", + len(xs), xs[0], pct(0.50), pct(0.90), pct(0.95), pct(0.99), xs[len(xs)-1]) +} + +// samples is a lock-protected slice of latency observations. Caller passes a +// hint for max capacity to avoid runaway memory; once full it down-samples by +// keeping every other entry on overflow. +type samples struct { + mu sync.Mutex + data []time.Duration + maxCap int + stride int + counter int +} + +func newSamples(cap int) *samples { + if cap < 1024 { + cap = 1024 + } + return &samples{data: make([]time.Duration, 0, cap), maxCap: cap, stride: 1} +} + +func (s *samples) add(d time.Duration) { + s.mu.Lock() + defer s.mu.Unlock() + s.counter++ + if s.counter%s.stride != 0 { + return + } + if len(s.data) >= s.maxCap { + // Halve resolution: drop every other sample, double stride. + half := s.data[:0] + for i := 0; i < len(s.data); i += 2 { + half = append(half, s.data[i]) + } + s.data = half + s.stride *= 2 + } + s.data = append(s.data, d) +} + +func (s *samples) snapshot() []time.Duration { + s.mu.Lock() + defer s.mu.Unlock() + out := make([]time.Duration, len(s.data)) + copy(out, s.data) + return out +} + +// ensure vtnats is used (subjects derived from production helpers when +// callers want exact production parity in -subject flags they construct +// from outside this tool). +var _ = vtnats.SignalSubject + +// runKV measures the KV-write ceiling. Each "publish" is a Put into the +// configured bucket, mirroring what triggerstate.RecordFire does on every +// fire. We don't model the read side - the production path reads + writes, +// and the bench focuses on the write half since that's the bottleneck +// observed in real load. +func runKV(ctx context.Context, cfg *config, opts []nc.Option) error { + conn, err := nc.Connect(cfg.URL, opts...) + if err != nil { + return fmt.Errorf("nats connect: %w", err) + } + defer conn.Drain() //nolint:errcheck + js, err := jetstream.New(conn, jetstream.WithPublishAsyncMaxPending(50_000)) + if err != nil { + return fmt.Errorf("jetstream: %w", err) + } + + kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: cfg.KVBucket, + History: 1, + Replicas: cfg.Replicas, + TTL: 10 * time.Minute, + }) + if err != nil { + return fmt.Errorf("kv setup: %w", err) + } + defer js.DeleteKeyValue(context.Background(), cfg.KVBucket) //nolint:errcheck + + samples := newSamples(int(cfg.Rate*cfg.Duration.Seconds()) + 1024) + var ok, errs uint64 + + pubCtx, cancelPub := context.WithTimeout(ctx, cfg.Duration) + defer cancelPub() + + progress := make(chan struct{}) + go func() { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + var last uint64 + for { + select { + case <-pubCtx.Done(): + close(progress) + return + case <-t.C: + cur := atomic.LoadUint64(&ok) + fmt.Printf("[kv-bench] ok=%d (+%d) errs=%d\n", cur, cur-last, atomic.LoadUint64(&errs)) + last = cur + } + } + }() + var wg sync.WaitGroup + pubCount := cfg.Publishers + if pubCount < 1 { + pubCount = 1 + } + perPub := cfg.Rate / float64(pubCount) + start := time.Now() + for i := 0; i < pubCount; i++ { + wg.Add(1) + go func(id int) { + defer wg.Done() + interval := time.Duration(float64(time.Second) / perPub) + t := time.NewTicker(interval) + defer t.Stop() + payload := []byte(`{"lastFiredAt":"2026-01-01T00:00:00Z","triggerId":"x","assetDid":"y"}`) + seq := 0 + for { + select { + case <-pubCtx.Done(): + return + case <-t.C: + seq++ + key := fmt.Sprintf("p%d.k%d", id, seq) + t0 := time.Now() + _, err := kv.Put(pubCtx, key, payload) + lat := time.Since(t0) + if err != nil { + atomic.AddUint64(&errs, 1) + continue + } + atomic.AddUint64(&ok, 1) + samples.add(lat) + } + } + }(i) + } + wg.Wait() + elapsed := time.Since(start) + cancelPub() // signals progress to exit if it's still ticking + _ = progress + + fmt.Println() + fmt.Println("=== triggers-bench KV results ===") + fmt.Printf("target rate : %.0f put/s for %s (publishers=%d, replicas=%d)\n", cfg.Rate, cfg.Duration, cfg.Publishers, cfg.Replicas) + fmt.Printf("elapsed : %s\n", elapsed) + fmt.Printf("ok / errs : %d / %d\n", atomic.LoadUint64(&ok), atomic.LoadUint64(&errs)) + fmt.Printf("throughput : %.1f put/s\n", float64(atomic.LoadUint64(&ok))/elapsed.Seconds()) + fmt.Println("\nKV put latency:") + printLatency(samples) + return nil +} diff --git a/cmd/triggers-dlq/main.go b/cmd/triggers-dlq/main.go new file mode 100644 index 0000000..264ebbf --- /dev/null +++ b/cmd/triggers-dlq/main.go @@ -0,0 +1,302 @@ +// triggers-dlq is an operator tool for the dead-letter stream +// (DIMO_TRIGGER_DLQ). Messages land here after exceeding MaxDeliver retries +// on the signal/event consumers. Each carries headers describing why it +// failed and where it came from. +// +// Subcommands: +// +// triggers-dlq list # summarize every DLQ message +// triggers-dlq get # print one message (headers + body) +// triggers-dlq replay [--all|] # republish to the original subject, then drop from DLQ +// triggers-dlq purge [--yes] # delete the entire DLQ stream contents +// +// Replay re-publishes to the X-Original-Subject header so the message +// re-enters the normal evaluation path. Use after fixing whatever caused the +// failure (a webhook endpoint that was down, a transient bug). +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strconv" + "time" + + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type flags struct { + URL string + CredsFile string + Stream string + DLQPrefix string +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + sub := os.Args[1] + fs := flag.NewFlagSet(sub, flag.ExitOnError) + f := flags{} + fs.StringVar(&f.URL, "url", "nats://localhost:4222", "NATS URL") + fs.StringVar(&f.CredsFile, "creds", "", "NATS credentials file (optional)") + fs.StringVar(&f.Stream, "stream", "DIMO_TRIGGER_DLQ", "DLQ stream name") + fs.StringVar(&f.DLQPrefix, "dlq-prefix", "dimo.dlq.", "DLQ subject prefix (stripped on replay if X-Original-Subject absent)") + + all := fs.Bool("all", false, "replay: replay every message") + yes := fs.Bool("yes", false, "purge: skip confirmation") + + switch sub { + case "list": + _ = fs.Parse(os.Args[2:]) + mustRun(cmdList(f)) + case "get": + _ = fs.Parse(os.Args[2:]) + if fs.NArg() < 1 { + fmt.Fprintln(os.Stderr, "usage: triggers-dlq get ") + os.Exit(2) + } + mustRun(cmdGet(f, fs.Arg(0))) + case "replay": + _ = fs.Parse(os.Args[2:]) + mustRun(cmdReplay(f, *all, fs.Args())) + case "purge": + _ = fs.Parse(os.Args[2:]) + mustRun(cmdPurge(f, *yes)) + default: + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: triggers-dlq {list|get |replay [--all|...]|purge [--yes]} [flags]") +} + +func mustRun(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func open(f flags) (*nc.Conn, jetstream.JetStream, jetstream.Stream, error) { + opts := []nc.Option{nc.Name("triggers-dlq"), nc.Timeout(5 * time.Second)} + if f.CredsFile != "" { + opts = append(opts, nc.UserCredentials(f.CredsFile)) + } + conn, err := nc.Connect(f.URL, opts...) + if err != nil { + return nil, nil, nil, fmt.Errorf("nats connect: %w", err) + } + js, err := jetstream.New(conn) + if err != nil { + conn.Close() + return nil, nil, nil, fmt.Errorf("jetstream: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + stream, err := js.Stream(ctx, f.Stream) + if err != nil { + conn.Close() + return nil, nil, nil, fmt.Errorf("stream %q: %w", f.Stream, err) + } + return conn, js, stream, nil +} + +// eachMessage walks the stream from seq 1 using a short-lived ordered consumer +// and calls fn for each message. Stops at the current last sequence so it +// terminates on a live stream. +func eachMessage(ctx context.Context, stream jetstream.Stream, fn func(jetstream.Msg) error) error { + info, err := stream.Info(ctx) + if err != nil { + return fmt.Errorf("stream info: %w", err) + } + last := info.State.LastSeq + if info.State.Msgs == 0 || last == 0 { + return nil + } + cons, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{}) + if err != nil { + return fmt.Errorf("ordered consumer: %w", err) + } + for { + msg, err := cons.Next() + if err != nil { + return fmt.Errorf("next: %w", err) + } + if err := fn(msg); err != nil { + return err + } + meta, err := msg.Metadata() + if err == nil && meta.Sequence.Stream >= last { + return nil + } + } +} + +func cmdList(f flags) error { + conn, _, stream, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + count := 0 + err = eachMessage(ctx, stream, func(m jetstream.Msg) error { + meta, _ := m.Metadata() + seq := uint64(0) + var ts time.Time + if meta != nil { + seq = meta.Sequence.Stream + ts = meta.Timestamp + } + fmt.Printf("seq=%-6d subject=%s original=%q source=%q vehicle=%q reason=%q delivered=%s at=%s\n", + seq, m.Subject(), + m.Headers().Get("X-Original-Subject"), + m.Headers().Get("X-Source-Name"), + m.Headers().Get("X-Asset-DID"), + truncate(m.Headers().Get("X-Failure-Reason"), 80), + m.Headers().Get("X-Delivered-Count"), + ts.Format(time.RFC3339)) + count++ + return nil + }) + if err != nil { + return err + } + fmt.Printf("\n%d message(s) in %s\n", count, f.Stream) + return nil +} + +func cmdGet(f flags, seqStr string) error { + seq, err := strconv.ParseUint(seqStr, 10, 64) + if err != nil { + return fmt.Errorf("invalid seq %q: %w", seqStr, err) + } + conn, _, stream, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + raw, err := stream.GetMsg(ctx, seq) + if err != nil { + return fmt.Errorf("get seq %d: %w", seq, err) + } + fmt.Printf("seq: %d\n", raw.Sequence) + fmt.Printf("subject: %s\n", raw.Subject) + fmt.Printf("time: %s\n", raw.Time.Format(time.RFC3339)) + fmt.Println("headers:") + for k, vs := range raw.Header { + for _, v := range vs { + fmt.Printf(" %s: %s\n", k, v) + } + } + fmt.Printf("body:\n%s\n", string(raw.Data)) + return nil +} + +func cmdReplay(f flags, all bool, args []string) error { + conn, js, stream, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + replayOne := func(raw *jetstream.RawStreamMsg) error { + original := "" + if raw.Header != nil { + original = raw.Header.Get("X-Original-Subject") + } + if original == "" { + return fmt.Errorf("seq %d has no X-Original-Subject header; cannot replay", raw.Sequence) + } + out := &nc.Msg{Subject: original, Data: raw.Data} + if _, err := js.PublishMsg(ctx, out); err != nil { + return fmt.Errorf("republish seq %d to %q: %w", raw.Sequence, original, err) + } + if err := stream.DeleteMsg(ctx, raw.Sequence); err != nil { + return fmt.Errorf("delete seq %d after replay: %w", raw.Sequence, err) + } + fmt.Printf("replayed seq=%d -> %s\n", raw.Sequence, original) + return nil + } + + if all { + // Snapshot sequences first so deletes don't disturb iteration. + var seqs []uint64 + if err := eachMessage(ctx, stream, func(m jetstream.Msg) error { + if meta, err := m.Metadata(); err == nil { + seqs = append(seqs, meta.Sequence.Stream) + } + return nil + }); err != nil { + return err + } + for _, seq := range seqs { + raw, err := stream.GetMsg(ctx, seq) + if err != nil { + fmt.Fprintf(os.Stderr, "skip seq %d: %v\n", seq, err) + continue + } + if err := replayOne(raw); err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + } + } + return nil + } + + if len(args) == 0 { + return errors.New("replay needs --all or one or more args") + } + for _, a := range args { + seq, err := strconv.ParseUint(a, 10, 64) + if err != nil { + return fmt.Errorf("invalid seq %q: %w", a, err) + } + raw, err := stream.GetMsg(ctx, seq) + if err != nil { + return fmt.Errorf("get seq %d: %w", seq, err) + } + if err := replayOne(raw); err != nil { + return err + } + } + return nil +} + +func cmdPurge(f flags, yes bool) error { + if !yes { + fmt.Fprintln(os.Stderr, "refusing to purge without --yes (this deletes all DLQ messages)") + os.Exit(2) + } + conn, _, stream, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := stream.Purge(ctx); err != nil { + return fmt.Errorf("purge: %w", err) + } + fmt.Printf("purged %s\n", f.Stream) + return nil +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} diff --git a/cmd/triggers-kvbench/main.go b/cmd/triggers-kvbench/main.go new file mode 100644 index 0000000..8bdda1e --- /dev/null +++ b/cmd/triggers-kvbench/main.go @@ -0,0 +1,190 @@ +// triggers-kvbench measures sustained Put rate against a JetStream KV bucket. +// +// Background: PROD_HARDENING_V2.md item G calls out that at 30k fires/sec +// the dispatcher writes 60k KV puts/sec (trigger_state + signal_history). +// The cluster bench in cmd/triggers-bench measures stream publish + pull +// throughput; KV writes are a different code path with their own ceiling +// (each Put is a JetStream message + key index update). This binary +// isolates that path so the cutover plan has a measured number, not a +// model. +// +// Usage: +// +// go run ./cmd/triggers-kvbench \ +// -url=nats://localhost:4222 \ +// -bucket=bench_kv \ +// -keys=10000 \ +// -concurrency=64 \ +// -duration=30s +// +// Output: rolling Put/sec, p50/p95/p99 latency, error count. The bucket is +// created with TTL=10m and replicas=1 by default - adjust per cluster +// shape. Cleans up on exit. +package main + +import ( + "context" + "flag" + "fmt" + "log" + "math/rand" + "os" + "os/signal" + "sort" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type config struct { + url string + bucket string + keys int + concurrency int + duration time.Duration + ttl time.Duration + replicas int + valueSize int + keep bool +} + +func main() { + cfg := config{} + flag.StringVar(&cfg.url, "url", "nats://localhost:4222", "NATS URL") + flag.StringVar(&cfg.bucket, "bucket", "bench_kv", "KV bucket name") + flag.IntVar(&cfg.keys, "keys", 10000, "key cardinality (cycle through this many keys)") + flag.IntVar(&cfg.concurrency, "concurrency", 64, "parallel writer goroutines") + flag.DurationVar(&cfg.duration, "duration", 30*time.Second, "test duration") + flag.DurationVar(&cfg.ttl, "ttl", 10*time.Minute, "bucket TTL") + flag.IntVar(&cfg.replicas, "replicas", 1, "bucket replicas") + flag.IntVar(&cfg.valueSize, "value-size", 256, "value size in bytes") + flag.BoolVar(&cfg.keep, "keep", false, "leave bucket behind on exit") + flag.Parse() + + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + + conn, err := nats.Connect(cfg.url) + if err != nil { + log.Fatalf("connect: %s", err) + } + defer conn.Close() + js, err := jetstream.New(conn) + if err != nil { + log.Fatalf("jetstream: %s", err) + } + + kv, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: cfg.bucket, + TTL: cfg.ttl, + Replicas: cfg.replicas, + }) + if err != nil { + log.Fatalf("create kv: %s", err) + } + if !cfg.keep { + defer func() { + delCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _ = js.DeleteKeyValue(delCtx, cfg.bucket) + }() + } + + value := make([]byte, cfg.valueSize) + for i := range value { + value[i] = byte(i % 256) + } + + deadline, cancelDeadline := context.WithTimeout(ctx, cfg.duration) + defer cancelDeadline() + + var ( + writes atomic.Uint64 + errs atomic.Uint64 + latNanos = make(chan int64, cfg.concurrency*256) + latencies []int64 + latMu sync.Mutex + ) + + go func() { + for ns := range latNanos { + latMu.Lock() + latencies = append(latencies, ns) + latMu.Unlock() + } + }() + + start := time.Now() + var wg sync.WaitGroup + for w := 0; w < cfg.concurrency; w++ { + wg.Add(1) + go func(seed int64) { + defer wg.Done() + r := rand.New(rand.NewSource(seed)) //nolint:gosec // bench-only PRNG + for deadline.Err() == nil { + key := fmt.Sprintf("k%d", r.Intn(cfg.keys)) + putStart := time.Now() + _, err := kv.Put(deadline, key, value) + lat := time.Since(putStart).Nanoseconds() + if err != nil { + if deadline.Err() == nil { + errs.Add(1) + } + continue + } + writes.Add(1) + select { + case latNanos <- lat: + default: + // shedding latency samples is fine - we keep a rolling + // average via the count; the histogram is best-effort. + } + } + }(int64(w)) + } + + // Per-second progress so the run is observable. + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + last := uint64(0) + for deadline.Err() == nil { + select { + case <-deadline.Done(): + case <-ticker.C: + cur := writes.Load() + fmt.Printf("[%s] writes=%d (+%d /s) errs=%d\n", + time.Since(start).Round(time.Second), cur, cur-last, errs.Load()) + last = cur + } + } + wg.Wait() + close(latNanos) + + elapsed := time.Since(start) + total := writes.Load() + failed := errs.Load() + + fmt.Println() + fmt.Println("== KV write bench ==") + fmt.Printf("bucket=%s keys=%d concurrency=%d ttl=%s replicas=%d value=%dB\n", + cfg.bucket, cfg.keys, cfg.concurrency, cfg.ttl, cfg.replicas, cfg.valueSize) + fmt.Printf("duration=%s writes=%d errors=%d throughput=%.0f /s\n", + elapsed.Round(10*time.Millisecond), total, failed, float64(total)/elapsed.Seconds()) + + latMu.Lock() + sort.Slice(latencies, func(i, j int) bool { return latencies[i] < latencies[j] }) + if n := len(latencies); n > 0 { + fmt.Printf("latency p50=%s p95=%s p99=%s max=%s (%d samples)\n", + time.Duration(latencies[n/2]), + time.Duration(latencies[n*95/100]), + time.Duration(latencies[n*99/100]), + time.Duration(latencies[n-1]), + n, + ) + } + latMu.Unlock() +} diff --git a/cmd/triggers-state/main.go b/cmd/triggers-state/main.go new file mode 100644 index 0000000..3ca9a43 --- /dev/null +++ b/cmd/triggers-state/main.go @@ -0,0 +1,274 @@ +// triggers-state is an operator tool for reading the trigger_state KV +// bucket. It is the canonical way to inspect distributed cooldown state - +// "did this trigger fire recently for this vehicle?" - across the running +// vehicle-triggers-api replicas without touching Postgres. +// +// Subcommands: +// +// triggers-state list # print every key in the bucket +// triggers-state get # print one record (JSON) +// triggers-state dump # print every key+value (JSONL) +// triggers-state watch # tail bucket updates until Ctrl-C +// +// All commands accept -url and -bucket. The DID argument for `get` is the +// full ERC721 DID string (did:erc721:::). +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "math/big" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type flags struct { + URL string + CredsFile string + Bucket string +} + +func main() { + if len(os.Args) < 2 { + usage() + os.Exit(2) + } + sub := os.Args[1] + rest := os.Args[2:] + + fs := flag.NewFlagSet(sub, flag.ExitOnError) + f := flags{} + fs.StringVar(&f.URL, "url", "nats://localhost:4222", "NATS URL") + fs.StringVar(&f.CredsFile, "creds", "", "NATS credentials file (optional)") + fs.StringVar(&f.Bucket, "bucket", "trigger_state", "KV bucket name") + + switch sub { + case "list": + _ = fs.Parse(rest) + mustRun(cmdList(f)) + case "get": + _ = fs.Parse(rest) + args := fs.Args() + if len(args) < 2 { + fmt.Fprintln(os.Stderr, "usage: triggers-state get [flags] ") + os.Exit(2) + } + mustRun(cmdGet(f, args[0], args[1])) + case "dump": + _ = fs.Parse(rest) + mustRun(cmdDump(f)) + case "watch": + _ = fs.Parse(rest) + mustRun(cmdWatch(f)) + default: + usage() + os.Exit(2) + } +} + +func usage() { + fmt.Fprintln(os.Stderr, "usage: triggers-state {list|get|dump|watch} [flags]") +} + +func mustRun(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } +} + +func open(f flags) (*nc.Conn, jetstream.KeyValue, error) { + opts := []nc.Option{nc.Name("triggers-state"), nc.Timeout(5 * time.Second)} + if f.CredsFile != "" { + opts = append(opts, nc.UserCredentials(f.CredsFile)) + } + conn, err := nc.Connect(f.URL, opts...) + if err != nil { + return nil, nil, fmt.Errorf("nats connect: %w", err) + } + js, err := jetstream.New(conn) + if err != nil { + conn.Close() + return nil, nil, fmt.Errorf("jetstream: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + kv, err := js.KeyValue(ctx, f.Bucket) + if err != nil { + conn.Close() + return nil, nil, fmt.Errorf("kv bucket %q: %w", f.Bucket, err) + } + return conn, kv, nil +} + +func cmdList(f flags) error { + conn, kv, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + keyLister, err := kv.ListKeys(ctx) + if err != nil { + return fmt.Errorf("list keys: %w", err) + } + defer func() { _ = keyLister.Stop() }() + for k := range keyLister.Keys() { + fmt.Println(k) + } + return nil +} + +func cmdGet(f flags, triggerID, did string) error { + conn, kv, err := open(f) + if err != nil { + return err + } + defer conn.Close() + + parsed, err := parseDID(did) + if err != nil { + return err + } + key := triggerstate.Key(triggerID, parsed) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + entry, err := kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return fmt.Errorf("no record for %s (trigger=%s vehicle=%s)", key, triggerID, did) + } + return fmt.Errorf("kv get %q: %w", key, err) + } + var rec triggerstate.Record + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return fmt.Errorf("decode: %w", err) + } + out := map[string]any{ + "key": key, + "revision": entry.Revision(), + "created": entry.Created(), + "lastFiredAt": rec.LastFiredAt, + "triggerId": rec.TriggerID, + "assetDid": rec.AssetDID, + "ageSeconds": time.Since(rec.LastFiredAt).Seconds(), + "lastSnapshot": json.RawMessage(rec.LastSnapshot), + } + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(out) +} + +func cmdDump(f flags) error { + conn, kv, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + keyLister, err := kv.ListKeys(ctx) + if err != nil { + return fmt.Errorf("list keys: %w", err) + } + defer func() { _ = keyLister.Stop() }() + enc := json.NewEncoder(os.Stdout) + for k := range keyLister.Keys() { + entry, err := kv.Get(ctx, k) + if err != nil { + fmt.Fprintf(os.Stderr, "skip %q: %v\n", k, err) + continue + } + var rec triggerstate.Record + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + fmt.Fprintf(os.Stderr, "skip %q (decode): %v\n", k, err) + continue + } + _ = enc.Encode(map[string]any{ + "key": k, + "revision": entry.Revision(), + "created": entry.Created(), + "lastFiredAt": rec.LastFiredAt, + "triggerId": rec.TriggerID, + "assetDid": rec.AssetDID, + }) + } + return nil +} + +func cmdWatch(f flags) error { + conn, kv, err := open(f) + if err != nil { + return err + } + defer conn.Close() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer cancel() + watcher, err := kv.WatchAll(ctx) + if err != nil { + return fmt.Errorf("watch: %w", err) + } + defer func() { _ = watcher.Stop() }() + enc := json.NewEncoder(os.Stdout) + for entry := range watcher.Updates() { + if entry == nil { + // Initial replay finished. Continue waiting for live updates. + continue + } + var rec triggerstate.Record + _ = json.Unmarshal(entry.Value(), &rec) + _ = enc.Encode(map[string]any{ + "op": entry.Operation().String(), + "key": entry.Key(), + "revision": entry.Revision(), + "updated": entry.Created(), + "lastFiredAt": rec.LastFiredAt, + }) + } + return nil +} + +// parseDID accepts the standard ERC721 DID string +// "did:erc721:::" and returns the parsed struct. +// The DID grammar is strict enough that we can do this without pulling in the +// cloudevent package's full parser. +func parseDID(s string) (cloudevent.ERC721DID, error) { + const prefix = "did:erc721:" + if !strings.HasPrefix(s, prefix) { + return cloudevent.ERC721DID{}, fmt.Errorf("invalid DID (missing %q prefix): %q", prefix, s) + } + parts := strings.Split(s[len(prefix):], ":") + if len(parts) != 3 { + return cloudevent.ERC721DID{}, fmt.Errorf("invalid DID (need chain:contract:tokenId): %q", s) + } + chain := new(big.Int) + if _, ok := chain.SetString(parts[0], 10); !ok { + return cloudevent.ERC721DID{}, fmt.Errorf("invalid chain id %q", parts[0]) + } + if !common.IsHexAddress(parts[1]) { + return cloudevent.ERC721DID{}, fmt.Errorf("invalid contract address %q", parts[1]) + } + token := new(big.Int) + if _, ok := token.SetString(parts[2], 10); !ok { + return cloudevent.ERC721DID{}, fmt.Errorf("invalid token id %q", parts[2]) + } + return cloudevent.ERC721DID{ + ChainID: chain.Uint64(), + ContractAddress: common.HexToAddress(parts[1]), + TokenID: token, + }, nil +} diff --git a/cmd/vehicle-triggers-api/main.go b/cmd/vehicle-triggers-api/main.go index 13f3736..fcbecf6 100644 --- a/cmd/vehicle-triggers-api/main.go +++ b/cmd/vehicle-triggers-api/main.go @@ -7,22 +7,22 @@ import ( "fmt" "log" "net" - "net/http" "os" "os/signal" "strconv" "strings" + "sync" "syscall" "time" "github.com/DIMO-Network/server-garage/pkg/env" "github.com/DIMO-Network/server-garage/pkg/logging" "github.com/DIMO-Network/server-garage/pkg/monserver" - "github.com/DIMO-Network/server-garage/pkg/runner" "github.com/DIMO-Network/vehicle-triggers-api/internal/app" "github.com/DIMO-Network/vehicle-triggers-api/internal/config" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/migrations" - "github.com/DIMO-Network/vehicle-triggers-api/internal/kafka" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" "golang.org/x/sync/errgroup" ) @@ -71,8 +71,7 @@ func main() { if *migrationCommand == "" { *migrationCommand = "up -v" } - err := migrations.RunGoose(mainCtx, strings.Fields(*migrationCommand), settings.DB) - if err != nil { + if err := migrations.RunGoose(mainCtx, strings.Fields(*migrationCommand), settings.DB); err != nil { logger.Fatal().Err(err).Msg("Failed to run migrations") } if *migrateOnly { @@ -80,97 +79,104 @@ func main() { } } + // Migrations don't need full config; validate everything else only + // when we're about to start the service. + if err := settings.Validate(); err != nil { + log.Fatalf("settings.Validate: %s", err) + } + monApp := monserver.NewMonitoringServer(&logger, settings.EnablePprof) logger.Info().Str("port", strconv.Itoa(settings.MonPort)).Msgf("Starting monitoring server") - runHandlerWithLogging(runnerCtx, runnerGroup, &logger, "monitoring", monApp, net.JoinHostPort("0.0.0.0", strconv.Itoa(settings.MonPort))) + runHTTPServer(runnerCtx, runnerGroup, &logger, "monitoring", monApp, net.JoinHostPort("0.0.0.0", strconv.Itoa(settings.MonPort))) servers, err := app.CreateServers(runnerCtx, &settings, logger) if err != nil { logger.Fatal().Err(err).Msg("Failed to create servers") } logger.Info().Str("port", strconv.Itoa(settings.Port)).Msgf("Starting web server") - runFiberWithLogging(runnerCtx, runnerGroup, &logger, servers.Application, net.JoinHostPort("0.0.0.0", strconv.Itoa(settings.Port))) - RunConsumer(runnerCtx, runnerGroup, &logger, servers.SignalConsumer) - RunConsumer(runnerCtx, runnerGroup, &logger, servers.EventConsumer) - - if err := runnerGroup.Wait(); err != nil { - logger.Fatal().Err(err).Msg("Server failed.") - } - logger.Info().Msg("Server stopped.") -} - -// RunConsumer starts the Kafka consumer in a single goroutine. Stop is -// deferred until Start returns so the subscriber cannot be closed before -// Subscribe runs (which would mask the real Subscribe error as -// "subscriber closed"). Entry/exit is logged with the consumer's name. -func RunConsumer(ctx context.Context, group *errgroup.Group, logger *zerolog.Logger, consumer *kafka.Consumer) { - name := consumer.Name() - topic := consumer.Topic() - group.Go(func() error { - logger.Info().Str("consumer", name).Str("topic", topic).Msg("consumer goroutine: start enter") - err := consumer.Start(ctx) - logger.Info().Str("consumer", name).Str("topic", topic).Err(err).Msg("consumer goroutine: start exit") - - stopErr := consumer.Stop(context.Background()) - logger.Info().Str("consumer", name).Str("topic", topic).Err(stopErr).Msg("consumer goroutine: stop exit") - - if err != nil { - return fmt.Errorf("consumer %q (topic %q) start: %w", name, topic, err) + runFiber(runnerCtx, runnerGroup, &logger, servers.Application, net.JoinHostPort("0.0.0.0", strconv.Itoa(settings.Port))) + + // Phased shutdown for the ingest -> dispatch -> NATS pipeline. A webhook is + // acked off JetStream as soon as it is enqueued (async dispatch), so an + // unordered teardown would drop acked-but-unsent webhooks on every deploy. + // We tear the pipeline down strictly in dependency order: + // 1. signal cancels runnerCtx -> pull loops stop pulling/acking and + // drain their in-flight handlers + // 2. once consumers have exited -> stop the dispatcher; its workers + // drain the queue (no new Enqueue can + // race them now) + // 3. once the dispatcher has drained -> stop the audit queue; it flushes + // the dispatcher's final audit records + // 4. once audit has drained -> close the NATS connection last so + // RecordFire / audit publishes landed + if servers.NATSClient != nil { + client := servers.NATSClient + dispatcherCtx, stopDispatcher := context.WithCancel(context.Background()) + auditCtx, stopAudit := context.WithCancel(context.Background()) + var dispatcherWG, auditWG, consumerWG sync.WaitGroup + + if servers.AuditQueue != nil { + aq := servers.AuditQueue + auditWG.Go(func() { + logger.Info().Msg("audit queue: starting drainer pool") + err := aq.Run(auditCtx) + logger.Info().Err(err).Msg("audit queue: drainer pool exited") + }) } - if stopErr != nil { - return fmt.Errorf("consumer %q (topic %q) stop: %w", name, topic, stopErr) + if servers.Dispatcher != nil { + disp := servers.Dispatcher + dispatcherWG.Go(func() { + logger.Info().Msg("dispatcher: starting workers") + err := disp.Run(dispatcherCtx) + logger.Info().Err(err).Msg("dispatcher: workers exited") + }) } - return nil - }) -} -// runFiberWithLogging mirrors runner.RunFiber but logs goroutine -// enter/exit so we can see which subsystem returned first. -func runFiberWithLogging(ctx context.Context, group *errgroup.Group, logger *zerolog.Logger, fiberApp runner.FiberApp, addr string) { - group.Go(func() error { - logger.Info().Str("addr", addr).Msg("fiber goroutine: listen enter") - err := fiberApp.Listen(addr) - logger.Info().Str("addr", addr).Err(err).Msg("fiber goroutine: listen exit") - if err != nil { - return fmt.Errorf("fiber listen %q: %w", addr, err) + startPullLoop := func(name string, cons jetstream.Consumer, handler vtnats.PayloadHandler) { + consumerWG.Add(1) + runnerGroup.Go(func() error { + defer consumerWG.Done() + slog := logger.With().Str("subsystem", name).Logger() + slog.Info().Msg("pull loop: start") + err := client.PullLoop(runnerCtx, cons, settings.MaxInFlight, handler) + slog.Info().Err(err).Msg("pull loop: exit") + if err != nil && !errors.Is(err, context.Canceled) { + return fmt.Errorf("%s: %w", name, err) + } + return nil + }) } - return nil - }) - group.Go(func() error { - <-ctx.Done() - logger.Info().Str("addr", addr).Msg("fiber goroutine: shutdown enter") - err := fiberApp.Shutdown() - logger.Info().Str("addr", addr).Err(err).Msg("fiber goroutine: shutdown exit") - if err != nil { - return fmt.Errorf("fiber shutdown %q: %w", addr, err) + if servers.NATSSignalConsumer != nil { + startPullLoop("nats-signals", servers.NATSSignalConsumer, servers.NATSListener.HandleSignalPayload) } - return nil - }) -} - -// runHandlerWithLogging mirrors runner.RunHandler but logs goroutine -// enter/exit so we can see which subsystem returned first. -func runHandlerWithLogging(ctx context.Context, group *errgroup.Group, logger *zerolog.Logger, name string, handler http.Handler, addr string) { - srv := &http.Server{Addr: addr, Handler: handler} - group.Go(func() error { - logger.Info().Str("server", name).Str("addr", addr).Msg("http goroutine: listen enter") - err := srv.ListenAndServe() - logger.Info().Str("server", name).Str("addr", addr).Err(err).Msg("http goroutine: listen exit") - if err != nil && !errors.Is(err, http.ErrServerClosed) { - return fmt.Errorf("%s listen %q: %w", name, addr, err) - } - return nil - }) - group.Go(func() error { - <-ctx.Done() - logger.Info().Str("server", name).Str("addr", addr).Msg("http goroutine: shutdown enter") - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - err := srv.Shutdown(shutdownCtx) - logger.Info().Str("server", name).Str("addr", addr).Err(err).Msg("http goroutine: shutdown exit") - if err != nil { - return fmt.Errorf("%s shutdown %q: %w", name, addr, err) + if servers.NATSEventConsumer != nil { + startPullLoop("nats-events", servers.NATSEventConsumer, servers.NATSListener.HandleEventPayload) } - return nil - }) + + // Shutdown coordinator: the single errgroup member that owns the + // ordered teardown. It blocks the group from returning until every + // phase completes. + runnerGroup.Go(func() error { + <-runnerCtx.Done() + consumerWG.Wait() // phase 1: consumers + in-flight handlers done + logger.Info().Msg("shutdown: consumers drained; stopping dispatcher") + stopDispatcher() // phase 2 + dispatcherWG.Wait() + logger.Info().Msg("shutdown: dispatcher drained; stopping audit queue") + stopAudit() // phase 3 + auditWG.Wait() + logger.Info().Msg("shutdown: audit drained; closing NATS") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := client.Shutdown(shutdownCtx); err != nil { // phase 4 + return fmt.Errorf("nats shutdown: %w", err) + } + return nil + }) + } + + if err := runnerGroup.Wait(); err != nil { + logger.Fatal().Err(err).Msg("Server failed.") + } + logger.Info().Msg("Server stopped.") } diff --git a/cmd/vehicle-triggers-api/runners.go b/cmd/vehicle-triggers-api/runners.go new file mode 100644 index 0000000..fdec09f --- /dev/null +++ b/cmd/vehicle-triggers-api/runners.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/DIMO-Network/server-garage/pkg/runner" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +// supervised runs `start` in the errgroup with structured enter/exit logs and +// optional graceful `stop` once ctx cancels. Replaces the duplicated +// run* helpers main.go used to carry. All shared the same shape: log enter -> +// start -> log exit -> wrap error with the subsystem name; only the inner +// call differed. +// +// The `stop` callback may be nil for subsystems that exit on ctx alone (NATS +// pull loop). When non-nil it runs in a parallel goroutine wired to +// ctx.Done(); its error joins the group like any other. +func supervised( + ctx context.Context, + group *errgroup.Group, + logger *zerolog.Logger, + name string, + fields map[string]string, + start func(ctx context.Context) error, + stop func(ctx context.Context) error, +) { + group.Go(func() error { + log := logger.With().Str("subsystem", name).Logger() + for k, v := range fields { + log = log.With().Str(k, v).Logger() + } + log.Info().Msg("supervisor: start enter") + err := start(ctx) + log.Info().Err(err).Msg("supervisor: start exit") + if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("%s start: %w", name, err) + } + return nil + }) + if stop == nil { + return + } + group.Go(func() error { + <-ctx.Done() + log := logger.With().Str("subsystem", name).Logger() + stopCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + log.Info().Msg("supervisor: stop enter") + err := stop(stopCtx) + log.Info().Err(err).Msg("supervisor: stop exit") + if err != nil { + return fmt.Errorf("%s stop: %w", name, err) + } + return nil + }) +} + +// NATS pull loops are wired directly in main with phased-shutdown tracking +// (see the shutdown coordinator there), so they intentionally bypass the +// generic supervisor helper. + +// runFiber wires a Fiber app's Listen/Shutdown pair through the supervisor. +func runFiber(ctx context.Context, group *errgroup.Group, logger *zerolog.Logger, fiberApp runner.FiberApp, addr string) { + supervised(ctx, group, logger, "fiber", map[string]string{"addr": addr}, + func(_ context.Context) error { return fiberApp.Listen(addr) }, + func(_ context.Context) error { return fiberApp.Shutdown() }, + ) +} + +// runHTTPServer wires an http.Handler through the supervisor as an +// http.Server. Used for the monitoring server. +func runHTTPServer(ctx context.Context, group *errgroup.Group, logger *zerolog.Logger, name string, handler http.Handler, addr string) { + srv := &http.Server{Addr: addr, Handler: handler} + supervised(ctx, group, logger, name, map[string]string{"addr": addr}, + func(_ context.Context) error { return srv.ListenAndServe() }, + func(stopCtx context.Context) error { return srv.Shutdown(stopCtx) }, + ) +} diff --git a/docker-compose.yml b/docker-compose.yml index 9605854..ef9372e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,3 +7,20 @@ services: - POSTGRES_USER=dimo - POSTGRES_PASSWORD=dimo - POSTGRES_DB=vehicle_triggers_api + + nats: + image: nats:2.11-alpine + command: + - "-js" + - "-sd" + - "/data" + - "-m" + - "8222" + ports: + - "4222:4222" + - "8222:8222" + volumes: + - nats-data:/data + +volumes: + nats-data: diff --git a/go.mod b/go.mod index 3bc06d2..a3a6d60 100644 --- a/go.mod +++ b/go.mod @@ -8,9 +8,6 @@ require ( github.com/DIMO-Network/server-garage v0.0.5 github.com/DIMO-Network/shared v1.0.7 github.com/DIMO-Network/token-exchange-api v0.3.8-0.20251015163407-530d08daccd8 - github.com/IBM/sarama v1.46.2 - github.com/ThreeDotsLabs/watermill v1.5.1 - github.com/ThreeDotsLabs/watermill-kafka/v3 v3.1.2 github.com/aarondl/null/v8 v8.1.3 github.com/aarondl/sqlboiler/v4 v4.19.5 github.com/aarondl/strmangle v0.0.9 @@ -25,16 +22,19 @@ require ( github.com/google/uuid v1.6.0 github.com/jftuga/geodist v1.0.0 github.com/lib/pq v1.10.9 + github.com/nats-io/nats.go v1.52.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pressly/goose/v3 v3.27.0 + github.com/prometheus/client_golang v1.23.0 github.com/rs/zerolog v1.34.0 github.com/stretchr/testify v1.11.1 github.com/swaggo/swag v1.16.6 - github.com/testcontainers/testcontainers-go v0.40.0 - github.com/testcontainers/testcontainers-go/modules/kafka v0.39.0 - github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 + github.com/testcontainers/testcontainers-go v0.42.0 + github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 go.uber.org/mock v0.6.0 - golang.org/x/sync v0.19.0 + golang.org/x/sync v0.20.0 + golang.org/x/time v0.11.0 google.golang.org/grpc v1.79.1 ) @@ -70,14 +70,9 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/dnwe/otelsarama v0.0.0-20240308230250-9388d9d40bc0 // indirect - github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/eapache/go-resiliency v1.7.0 // indirect - github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect - github.com/eapache/queue v1.1.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/go-sysinfo v1.15.4 // indirect github.com/elastic/go-windows v1.0.2 // indirect @@ -98,10 +93,6 @@ require ( github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect - github.com/golang/snappy v1.0.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/holiman/uint256 v1.3.2 // indirect github.com/huandu/xstrings v1.5.0 // indirect @@ -110,16 +101,10 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgx/v5 v5.8.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jcmturner/aescts/v2 v2.0.0 // indirect - github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect - github.com/jcmturner/gofork v1.7.6 // indirect - github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect - github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/joho/godotenv v1.5.1 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.18.4 // indirect - github.com/lithammer/shortuuid/v3 v3.0.7 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -134,15 +119,17 @@ require ( github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect + github.com/moby/moby/api v1.54.1 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect - github.com/morikuni/aec v1.1.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nkeys v0.4.15 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/oklog/ulid v1.3.1 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.12.0 // indirect @@ -152,17 +139,15 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/prometheus/client_golang v1.23.0 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.65.0 // indirect github.com/prometheus/procfs v0.19.2 // indirect - github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect - github.com/shirou/gopsutil/v4 v4.26.2 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect github.com/shopspring/decimal v1.4.0 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/spf13/afero v1.10.0 // indirect @@ -196,12 +181,12 @@ require ( go.opentelemetry.io/otel/trace v1.42.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/crypto v0.48.0 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/mod v0.33.0 // indirect - golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/tools v0.42.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect diff --git a/go.sum b/go.sum index bc2dc37..0efad0b 100644 --- a/go.sum +++ b/go.sum @@ -76,8 +76,6 @@ github.com/DIMO-Network/shared v1.0.7 h1:LfSgsqJ6R7EUyfo2GTfuhrCpoDcweJqe7eVOa4j github.com/DIMO-Network/shared v1.0.7/go.mod h1:lDHUKwwT2LW6Zvd42Nb33dXklRNTmfyOlbUNx2dQfGY= github.com/DIMO-Network/token-exchange-api v0.3.8-0.20251015163407-530d08daccd8 h1:LqcoqvfMFzXmLLArwabwu4NRgwZiitskakhXlVyIwjY= github.com/DIMO-Network/token-exchange-api v0.3.8-0.20251015163407-530d08daccd8/go.mod h1:gKoB1Zi3EXJqIyfLnTn1GV1NSzTMBDkRleChBU/EQv8= -github.com/IBM/sarama v1.46.2 h1:65JJmZpxKUWe/7HEHmc56upTfAvgoxuyu4Ek+TcevDE= -github.com/IBM/sarama v1.46.2/go.mod h1:PDOGmVeKmW744c/0d4CZ0MfrzmcIYtpmS5+KIWs1zHQ= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= @@ -92,10 +90,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 h1:1zYrtlhrZ6/b6SAjLSfKzWtdgqK0U+HtH/VcBWh1BaU= github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6/go.mod h1:ioLG6R+5bUSO1oeGSDxOV3FADARuMoytZCSX6MEMQkI= -github.com/ThreeDotsLabs/watermill v1.5.1 h1:t5xMivyf9tpmU3iozPqyrCZXHvoV1XQDfihas4sV0fY= -github.com/ThreeDotsLabs/watermill v1.5.1/go.mod h1:Uop10dA3VeJWsSvis9qO3vbVY892LARrKAdki6WtXS4= -github.com/ThreeDotsLabs/watermill-kafka/v3 v3.1.2 h1:lLmrzZnl8o8U5uLVhMLSFHGSuWLcsqhW1MOtltx2CbQ= -github.com/ThreeDotsLabs/watermill-kafka/v3 v3.1.2/go.mod h1:o1GcoF/1CSJ9JSmQzUkULvpZeO635pZe+WWrYNFlJNk= github.com/aarondl/inflect v0.0.2 h1:XvH8K5g1wKS921tMmDOUsZ3zS1Eo8WwK5RHC0IGGT2s= github.com/aarondl/inflect v0.0.2/go.mod h1:zjmCfdXHUDQ9jFOV6SeHknpo0Au6rQhV8GchS4Vzv/0= github.com/aarondl/null/v8 v8.1.3 h1:ZJcvvj34BkXAguqU7xzDqEmzG86cSBgM8HYxcqeK0+8= @@ -118,8 +112,6 @@ github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5m github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -152,8 +144,8 @@ github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHf github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc= github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -164,22 +156,12 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etly github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dnwe/otelsarama v0.0.0-20240308230250-9388d9d40bc0 h1:R2zQhFwSCyyd7L43igYjDrH0wkC/i+QBPELuY0HOu84= -github.com/dnwe/otelsarama v0.0.0-20240308230250-9388d9d40bc0/go.mod h1:2MqLKYJfjs3UriXXF9Fd0Qmh/lhxi/6tHXkqtXxyIHc= -github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= -github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= -github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= -github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= -github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= -github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/elastic/go-sysinfo v1.8.1/go.mod h1:JfllUnzoQV/JRYymbH3dO1yggI3mV2oTKSXsDHM+uIM= @@ -201,8 +183,6 @@ github.com/ethereum/go-ethereum v1.17.1 h1:IjlQDjgxg2uL+GzPRkygGULPMLzcYWncEI7wb github.com/ethereum/go-ethereum v1.17.1/go.mod h1:7UWOVHL7K3b8RfVRea022btnzLCaanwHtBuH1jUCH/I= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/friendsofgo/errors v0.9.2 h1:X6NYxef4efCBdwI7BgS820zFaN7Cphrmb+Pljdzjtgk= @@ -286,8 +266,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= -github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/cel-go v0.26.1 h1:iPbVVEdkhTX++hpe3lzSk7D3G3QSYqLGoHOcEio+UXQ= @@ -323,26 +301,12 @@ github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17k github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= -github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= -github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= @@ -366,18 +330,6 @@ github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= -github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= -github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= -github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= -github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= -github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= -github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= -github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= -github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= -github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= -github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= -github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jftuga/geodist v1.0.0 h1:PFPQlZtj10u8ETAYTyxE0DWMl1bwA+Xzrqb4+oLkkC0= github.com/jftuga/geodist v1.0.0/go.mod h1:BohEDxpZ8S5ADAxW/9EKPSKWOVl0+3wHENIT40m4UO4= @@ -393,8 +345,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= -github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= @@ -408,8 +360,6 @@ github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= -github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88 h1:PTw+yKnXcOFCR6+8hHTyWBeQ/P4Nb7dd4/0ohEcWQuM= github.com/lufia/plan9stats v0.0.0-20260216142805-b3301c5f2a88/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= @@ -443,10 +393,12 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -456,14 +408,16 @@ github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcY github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/morikuni/aec v1.1.0 h1:vBBl0pUnvi/Je71dsRrhMBtreIqNMYErSAbEeb8jrXQ= -github.com/morikuni/aec v1.1.0/go.mod h1:xDRgiq/iw5l+zkao76YTKzKttOp2cwPEne25HDkJnBw= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= +github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -503,8 +457,6 @@ github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGI github.com/prometheus/procfs v0.0.0-20190425082905-87a4384529e0/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= -github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= -github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rekby/fixenv v0.6.1 h1:jUFiSPpajT4WY2cYuc++7Y1zWrnCxnovGCIX72PZniM= github.com/rekby/fixenv v0.6.1/go.mod h1:/b5LRc06BYJtslRtHKxsPWFT/ySpHV+rWvzTg+XWk4c= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= @@ -525,8 +477,8 @@ github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/shirou/gopsutil/v4 v4.26.2 h1:X8i6sicvUFih4BmYIGT1m2wwgw2VG9YgrDTi7cIRGUI= -github.com/shirou/gopsutil/v4 v4.26.2/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= @@ -548,19 +500,14 @@ github.com/spf13/viper v1.12.0/go.mod h1:b6COn30jlNxbm/V2IqWiNWkJ+vZNiMNksliPCiu github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= @@ -569,12 +516,12 @@ github.com/swaggo/files/v2 v2.0.2 h1:Bq4tgS/yxLB/3nwOMcul5oLEUKa877Ykgz3CJMVbQKU github.com/swaggo/files/v2 v2.0.2/go.mod h1:TVqetIzZsO9OhHX1Am9sRf9LdrFZqoK49N37KON/jr0= github.com/swaggo/swag v1.16.6 h1:qBNcx53ZaX+M5dxVyTrgQ0PJ/ACK+NzhwcbieTt+9yI= github.com/swaggo/swag v1.16.6/go.mod h1:ngP2etMK5a0P3QBizic5MEwpRmluJZPHjXcMoj4Xesg= -github.com/testcontainers/testcontainers-go v0.40.0 h1:pSdJYLOVgLE8YdUY2FHQ1Fxu+aMnb6JfVz1mxk7OeMU= -github.com/testcontainers/testcontainers-go v0.40.0/go.mod h1:FSXV5KQtX2HAMlm7U3APNyLkkap35zNLxukw9oBi/MY= -github.com/testcontainers/testcontainers-go/modules/kafka v0.39.0 h1:Nkrk5fjoHbj1bqE8OkMT25Y8bcSDgS5smdVaX3Xkfyc= -github.com/testcontainers/testcontainers-go/modules/kafka v0.39.0/go.mod h1:9Si8E8u8DWMUPQpHSSDseA3lXfhyMgVnCfdMWjoqNNw= -github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0 h1:REJz+XwNpGC/dCgTfYvM4SKqobNqDBfvhq74s2oHTUM= -github.com/testcontainers/testcontainers-go/modules/postgres v0.39.0/go.mod h1:4K2OhtHEeT+JSIFX4V8DkGKsyLa96Y2vLdd3xsxD5HE= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 h1:WQR0+1r4GkM5QgOBoLxlP41empovt5PxtaiDpC0G7ow= +github.com/testcontainers/testcontainers-go/modules/nats v0.42.0/go.mod h1:ZAI9iisjDNJmcRcycQFKSLpiBN9u2g1v9AJRq1afriE= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= +github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= @@ -614,7 +561,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= @@ -632,10 +578,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:Oyrsyzu go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= @@ -645,8 +587,6 @@ go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= -go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -661,12 +601,10 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -702,7 +640,6 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -737,11 +674,8 @@ golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -762,9 +696,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -804,19 +737,15 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -825,14 +754,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -882,7 +810,6 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1054,6 +981,8 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/internal/app/app.go b/internal/app/app.go index 040efc0..e734e4f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -2,10 +2,13 @@ package app import ( "context" + "encoding/hex" "fmt" "strings" "time" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/secrets" + "github.com/DIMO-Network/server-garage/pkg/fibercommon" "github.com/DIMO-Network/shared/pkg/db" _ "github.com/DIMO-Network/vehicle-triggers-api/docs" // Import Swagger docs @@ -15,22 +18,42 @@ import ( "github.com/DIMO-Network/vehicle-triggers-api/internal/config" "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/metriclistener" "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" - "github.com/DIMO-Network/vehicle-triggers-api/internal/kafka" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/auditqueue" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/cachebroadcast" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/configaudit" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookdispatcher" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhooksender" - "github.com/IBM/sarama" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/redirect" "github.com/gofiber/swagger" + "github.com/nats-io/nats.go/jetstream" "github.com/rs/zerolog" ) type Servers struct { - Application *fiber.App - SignalConsumer *kafka.Consumer - EventConsumer *kafka.Consumer + Application *fiber.App + + // NATSClient is non-nil when NATSSettings.Enabled(). main owns shutdown. + NATSClient *vtnats.Client + // NATSSignalConsumer / NATSEventConsumer are non-nil when NATS owns the + // evaluation path (NATSSettings.PrimaryMode()). + NATSSignalConsumer jetstream.Consumer + NATSEventConsumer jetstream.Consumer + // NATSListener is the listener used to dispatch messages pulled off the + // NATS consumers. + NATSListener *metriclistener.MetricListener + // Dispatcher decouples webhook delivery from the JetStream handler. Non- + // nil whenever NATS owns the evaluation path; main is responsible for + // calling Run on it in the errgroup. + Dispatcher *webhookdispatcher.Dispatcher + // AuditQueue fronts the audit stream with a fire-and-forget pool. Same + // lifecycle as the dispatcher. + AuditQueue *auditqueue.Queue } func CreateServers(ctx context.Context, settings *config.Settings, logger zerolog.Logger) (*Servers, error) { @@ -44,20 +67,124 @@ func CreateServers(ctx context.Context, settings *config.Settings, logger zerolo tokenExchangeCache := tokenexchange.NewCache(settings.TokenExchangeCacheExpiration, settings.TokenExchangeCacheCleanupInterval, tokenExchangeAPI) repo := triggersrepo.NewRepository(store.DBS().Writer.DB) + if hexKey := strings.TrimSpace(settings.SigningSecretKeyHex); hexKey != "" { + key, err := hex.DecodeString(hexKey) + if err != nil { + return nil, fmt.Errorf("SIGNING_SECRET_KEY_HEX: %w", err) + } + cipher, err := secrets.NewAESGCM(key) + if err != nil { + return nil, fmt.Errorf("AES-GCM cipher: %w", err) + } + repo.SetCipher(cipher) + } webhookCache, err := startWebhookCache(ctx, settings, tokenExchangeCache, repo) if err != nil { return nil, fmt.Errorf("failed to start webhook cache: %w", err) } - signalConsumer, err := createSignalConsumer(ctx, settings, tokenExchangeCache, repo, webhookCache) - if err != nil { - return nil, fmt.Errorf("failed to create signal consumer: %w", err) - } - - eventConsumer, err := createEventConsumer(ctx, settings, tokenExchangeCache, repo, webhookCache) - if err != nil { - return nil, fmt.Errorf("failed to create event consumer: %w", err) + var ( + natsClient *vtnats.Client + natsListener *metriclistener.MetricListener + natsSigCons jetstream.Consumer + natsEvtCons jetstream.Consumer + dispatcher *webhookdispatcher.Dispatcher + auditQ *auditqueue.Queue + ) + if settings.NATS.Enabled() { + natsClient, err = vtnats.Connect(ctx, settings.NATS, logger) + if err != nil { + return nil, fmt.Errorf("failed to connect to nats: %w", err) + } + if err := natsClient.EnsureStreams(ctx); err != nil { + return nil, fmt.Errorf("failed to ensure nats streams: %w", err) + } + if err := natsClient.EnsureBuckets(ctx); err != nil { + return nil, fmt.Errorf("failed to ensure nats buckets: %w", err) + } + // Wire the webhook cache to publish + subscribe to invalidation + // notifications so cross-replica CRUD propagation drops from 5 min + // (poll) to sub-second. + webhookCache.SetNotifier(cachebroadcast.NewNATSNotifier(natsClient.Conn, logger)) + if _, err := cachebroadcast.Subscribe(natsClient.Conn, ctx, webhookCache, logger); err != nil { + return nil, fmt.Errorf("failed to subscribe to cache invalidate: %w", err) + } + if settings.NATS.PrimaryMode() { + stateKV, err := natsClient.TriggerState(ctx) + if err != nil { + return nil, fmt.Errorf("failed to open trigger-state bucket: %w", err) + } + historyKV, err := natsClient.SignalHistory(ctx) + if err != nil { + return nil, fmt.Errorf("failed to open signal-history bucket: %w", err) + } + stateStore := triggerstate.New(stateKV, historyKV) + webhookSender := webhooksender.NewWebhookSender(nil) + maxFailureCount := int(settings.MaxWebhookFailureCount) + if maxFailureCount < 1 { + maxFailureCount = 1 + } + auditQ = auditqueue.New(auditqueue.Config{ + Workers: settings.Audit.Workers, + Buffer: settings.Audit.QueueSize, + }, natsClient, logger) + dispatcher = webhookdispatcher.New(webhookdispatcher.Config{ + Workers: settings.Dispatcher.Workers, + QueueSize: settings.Dispatcher.QueueSize, + MaxFailureCount: maxFailureCount, + RetryAttempts: settings.Dispatcher.RetryAttempts, + RetryInitialDelay: settings.Dispatcher.RetryInitialDelay, + PerHostRPS: settings.Dispatcher.PerHostRPS, + PerHostBurst: settings.Dispatcher.PerHostBurst, + }, webhookSender, stateStore, auditQ, repo, logger) + // Promote the dispatcher's per-pod limiter to a cluster-shared + // one when RATE_LIMIT_BUCKET is configured and reachable. KV + // hiccups degrade gracefully to the per-pod limiter inside + // clusterLimiter.Wait; an empty bucket name skips it entirely. + if settings.Dispatcher.PerHostRPS > 0 && settings.NATS.RateLimitBucket != "" { + if rlKV, err := natsClient.RateLimit(ctx); err != nil { + logger.Warn().Err(err).Msg("cluster rate limit KV unavailable; using per-pod limiter") + } else { + dispatcher = dispatcher.WithClusterLimiter(rlKV) + } + } + natsListener = buildListener(settings, tokenExchangeCache, repo, webhookCache, stateStore, dispatcher) + // At cutover a brand-new durable defaults to DeliverNew so the + // service doesn't replay retained telemetry and fire stale + // webhooks; "all" is opt-in for backfill/replay. Ignored once the + // durable exists (JetStream fixes deliver policy at creation). + deliverPolicy := jetstream.DeliverNewPolicy + if settings.NATS.DeliverPolicy == "all" { + deliverPolicy = jetstream.DeliverAllPolicy + } + natsSigCons, err = natsClient.EnsureConsumer(ctx, vtnats.ConsumerSpec{ + Stream: settings.NATS.SignalsStream, + Durable: settings.NATS.SignalsDurable, + FilterSubjects: []string{vtnats.AllSignalsFilter()}, + DeliverPolicy: deliverPolicy, + AckWait: settings.NATS.AckWait, + MaxDeliver: settings.NATS.MaxDeliver, + MaxAckPending: settings.NATS.MaxAckPending, + Description: "vehicle-triggers signals evaluator", + }) + if err != nil { + return nil, fmt.Errorf("failed to ensure signals consumer: %w", err) + } + natsEvtCons, err = natsClient.EnsureConsumer(ctx, vtnats.ConsumerSpec{ + Stream: settings.NATS.EventsStream, + Durable: settings.NATS.EventsDurable, + FilterSubjects: []string{vtnats.AllEventsFilter()}, + DeliverPolicy: deliverPolicy, + AckWait: settings.NATS.AckWait, + MaxDeliver: settings.NATS.MaxDeliver, + MaxAckPending: settings.NATS.MaxAckPending, + Description: "vehicle-triggers events evaluator", + }) + if err != nil { + return nil, fmt.Errorf("failed to ensure events consumer: %w", err) + } + } } identityClient, err := identity.New(settings, logger) @@ -65,14 +192,18 @@ func CreateServers(ctx context.Context, settings *config.Settings, logger zerolo return nil, fmt.Errorf("failed to create identity client: %w", err) } - app, err := CreateFiberApp(logger, repo, webhookCache, tokenExchangeAPI, identityClient, settings) + app, err := CreateFiberApp(logger, repo, webhookCache, tokenExchangeAPI, identityClient, settings, natsClient) if err != nil { return nil, fmt.Errorf("failed to create fiber app: %w", err) } return &Servers{ - Application: app, - SignalConsumer: signalConsumer, - EventConsumer: eventConsumer, + Application: app, + NATSClient: natsClient, + NATSSignalConsumer: natsSigCons, + NATSEventConsumer: natsEvtCons, + NATSListener: natsListener, + Dispatcher: dispatcher, + AuditQueue: auditQ, }, nil } @@ -81,7 +212,8 @@ func CreateFiberApp(logger zerolog.Logger, repo *triggersrepo.Repository, webhookCache *webhookcache.WebhookCache, tokenExchangeClient *tokenexchange.Client, identityClient *identity.Client, - settings *config.Settings) (*fiber.App, error) { + settings *config.Settings, + natsClient *vtnats.Client) (*fiber.App, error) { app := fiber.New(fiber.Config{ ErrorHandler: func(c *fiber.Ctx, err error) error { @@ -108,16 +240,53 @@ func CreateFiberApp(logger zerolog.Logger, repo *triggersrepo.Repository, // settings.IdentityAPIURL is loaded from your settings.yaml. // Register Webhook routes. - webhookController, err := webhook.NewWebhookController(repo, webhookCache) + audit := configaudit.New(natsClient) + webhookController, err := webhook.NewWebhookController(repo, webhookCache, settings.MaxAllowedCooldownPeriod) if err != nil { return nil, fmt.Errorf("failed to create webhook controller: %w", err) } - vehicleSubscriptionController := webhook.NewVehicleSubscriptionController(repo, identityClient, tokenExchangeClient, webhookCache) + webhookController.WithAudit(audit) + vehicleSubscriptionController := webhook.NewVehicleSubscriptionController(repo, identityClient, tokenExchangeClient, webhookCache).WithAudit(audit) app.Get("/health", func(c *fiber.Ctx) error { - return c.JSON(fiber.Map{ + body := fiber.Map{ "data": "Server is up and running", - }) + } + healthy := true + // DB ping comes first. Without it a Postgres outage leaves the + // pod in service routing webhook creates that will fail at write, + // surfacing as 500s instead of a readiness-probe-driven drain. + ctx, cancel := context.WithTimeout(c.UserContext(), 2*time.Second) + defer cancel() + if err := repo.Ping(ctx); err != nil { + body["db"] = map[string]any{"healthy": false, "error": err.Error()} + healthy = false + } else { + body["db"] = map[string]any{"healthy": true} + } + if natsClient != nil { + natsHealthy := natsClient.Healthy() + streams := natsClient.StreamHealth(ctx) + streamsOK := natsHealthy + for _, status := range streams { + if status != "ok" { + streamsOK = false + } + } + body["nats"] = map[string]any{ + "enabled": true, + "healthy": natsHealthy, + "streams": streams, + "mode": settings.NATS.Mode, + } + if !streamsOK { + healthy = false + } + } + if !healthy { + return c.Status(fiber.StatusServiceUnavailable).JSON(body) + } + return c.JSON(body) }) jwtMiddleware := auth.Middleware(settings) @@ -130,6 +299,7 @@ func CreateFiberApp(logger zerolog.Logger, repo *triggersrepo.Repository, devJWTAuth.Get("/v1/webhooks/:webhookId", vehicleSubscriptionController.ListVehiclesForWebhook) devJWTAuth.Put("/v1/webhooks/:webhookId", webhookController.UpdateWebhook) devJWTAuth.Delete("/v1/webhooks/:webhookId", webhookController.DeleteWebhook) + devJWTAuth.Post("/v1/webhooks/:webhookId/rotate-secret", webhookController.RotateSigningSecret) // Vehicle subscriptions devJWTAuth.Post("/v1/webhooks/:webhookId/subscribe/list", vehicleSubscriptionController.SubscribeVehiclesFromList) @@ -162,62 +332,29 @@ func startWebhookCache(ctx context.Context, settings *config.Settings, tokenExch go func() { ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() - for range ticker.C { - webhookCache.ScheduleRefresh(ctx) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + webhookCache.ScheduleRefresh(ctx) + } } }() - logger.Info().Msgf("Device signals consumer started on topic: %s", settings.DeviceSignalsTopic) + logger.Info().Msg("webhook cache started") return webhookCache, nil } -func createSignalConsumer(ctx context.Context, settings *config.Settings, tokenExchangeCache *tokenexchange.Cache, repo *triggersrepo.Repository, webhookCache *webhookcache.WebhookCache) (*kafka.Consumer, error) { - clusterConfig := sarama.NewConfig() - clusterConfig.Version = sarama.V2_8_1_0 - clusterConfig.Consumer.Offsets.Initial = sarama.OffsetOldest - webhookSender := webhooksender.NewWebhookSender(nil) - triggerEvaluator := triggerevaluator.NewTriggerEvaluator(repo, tokenExchangeCache) - vehicleProcessor := metriclistener.NewMetricsListener(webhookCache, repo, webhookSender, triggerEvaluator, settings) - consumerConfig := &kafka.Config{ - ClusterConfig: clusterConfig, - BrokerAddresses: strings.Split(settings.KafkaBrokers, ","), - Topic: settings.DeviceSignalsTopic, - GroupID: "vehicle-triggers", - MaxInFlight: int64(settings.MaxInFlight), - Processor: vehicleProcessor.ProcessSignalMessages, - Name: "signals", - } - - consumer, err := kafka.NewConsumer(consumerConfig) - if err != nil { - return nil, fmt.Errorf("failed to create device signals consumer: %w", err) - } - - return consumer, nil -} - -func createEventConsumer(ctx context.Context, settings *config.Settings, tokenExchangeCache *tokenexchange.Cache, repo *triggersrepo.Repository, webhookCache *webhookcache.WebhookCache) (*kafka.Consumer, error) { - clusterConfig := sarama.NewConfig() - clusterConfig.Version = sarama.V2_8_1_0 - clusterConfig.Consumer.Offsets.Initial = sarama.OffsetOldest - webhookSender := webhooksender.NewWebhookSender(nil) - triggerEvaluator := triggerevaluator.NewTriggerEvaluator(repo, tokenExchangeCache) - vehicleProcessor := metriclistener.NewMetricsListener(webhookCache, repo, webhookSender, triggerEvaluator, settings) - consumerConfig := &kafka.Config{ - ClusterConfig: clusterConfig, - BrokerAddresses: strings.Split(settings.KafkaBrokers, ","), - Topic: settings.DeviceEventsTopic, - GroupID: "vehicle-triggers", - MaxInFlight: int64(settings.MaxInFlight), - Processor: vehicleProcessor.ProcessEventMessages, - Name: "events", +// buildListener builds a listener whose evaluator consults the supplied +// state store for cooldown / previousValue lookups and hands fires off to +// the supplied dispatcher. State and dispatcher are required for the +// production path; only the unit test harness passes nil. +func buildListener(settings *config.Settings, tokenExchangeCache *tokenexchange.Cache, repo *triggersrepo.Repository, webhookCache *webhookcache.WebhookCache, state triggerevaluator.StateStore, dispatcher metriclistener.WebhookDispatcher) *metriclistener.MetricListener { + evaluator := triggerevaluator.NewTriggerEvaluator(tokenExchangeCache) + if state != nil { + evaluator = evaluator.WithStateStore(state) } - - consumer, err := kafka.NewConsumer(consumerConfig) - if err != nil { - return nil, fmt.Errorf("failed to create device events consumer: %w", err) - } - - return consumer, nil + return metriclistener.NewMetricsListener(webhookCache, repo, evaluator, dispatcher, settings) } diff --git a/internal/celcondition/celcondition.go b/internal/celcondition/celcondition.go index c55fd30..0fb720e 100644 --- a/internal/celcondition/celcondition.go +++ b/internal/celcondition/celcondition.go @@ -37,7 +37,11 @@ func buildSignalEnv() (*cel.Env, error) { cel.Variable("value.longitude", cel.DynType), cel.Variable("value.hdop", cel.DynType), geoDistanceOpt(), - cel.Variable("source", cel.DoubleType), + // source is the signal's origin string (e.g. device address); it is + // populated from signal.Source (a string) at eval time, so it must be + // declared StringType. Declaring it DoubleType made any condition + // referencing source fail type-checking at prepare time. + cel.Variable("source", cel.StringType), cel.Variable("previousValueNumber", cel.DoubleType), cel.Variable("previousValueString", cel.StringType), cel.Variable("previousValue", cel.DynType), diff --git a/internal/clients/tokenexchange/cache.go b/internal/clients/tokenexchange/cache.go index 86578fa..fdbe449 100644 --- a/internal/clients/tokenexchange/cache.go +++ b/internal/clients/tokenexchange/cache.go @@ -37,14 +37,17 @@ func (c *Cache) HasVehiclePermissions(ctx context.Context, assetDid cloudevent.E cacheKey := accessRequestCacheKey(&req) if hasAccess, found := c.cache.Get(cacheKey); found { + metricsCache("hit") return hasAccess.(bool), nil } hasAccess, err := c.tokenExchangeClient.HasVehiclePermissions(ctx, assetDid, devLicense, permissions) if err != nil { + metricsCache("error") return false, fmt.Errorf("failed to check access: %w", err) } c.cache.Set(cacheKey, hasAccess, 0) + metricsCache("miss") return hasAccess, nil } diff --git a/internal/clients/tokenexchange/metrics.go b/internal/clients/tokenexchange/metrics.go new file mode 100644 index 0000000..8cc423d --- /dev/null +++ b/internal/clients/tokenexchange/metrics.go @@ -0,0 +1,21 @@ +package tokenexchange + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Cache observability. The permission cache hit rate is the single biggest +// lever on hot-path latency at scale: a cold miss is a gRPC RTT to +// token-exchange, a hit is a map lookup. We track outcomes so capacity +// planning can see warm-up spikes after autoscale events. +var cacheTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "tokenexchange", + Name: "cache_total", + Help: "Token-exchange permission cache outcomes. hit = served from local cache, miss = backed gRPC succeeded and cached, error = gRPC failed.", +}, []string{"outcome"}) + +func metricsCache(outcome string) { + cacheTotal.WithLabelValues(outcome).Inc() +} diff --git a/internal/config/settings.go b/internal/config/settings.go index c8788e5..033924c 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -1,6 +1,9 @@ package config import ( + "errors" + "fmt" + "strings" "time" "github.com/DIMO-Network/shared/pkg/db" @@ -16,9 +19,6 @@ type Settings struct { ServiceName string `env:"SERVICE_NAME"` JWKKeySetURL string `env:"JWK_KEY_SET_URL"` IdentityAPIURL string `env:"IDENTITY_API_URL"` - KafkaBrokers string `env:"KAFKA_BROKERS"` - DeviceSignalsTopic string `env:"DEVICE_SIGNALS_TOPIC"` - DeviceEventsTopic string `env:"DEVICE_EVENTS_TOPIC"` TokenExchangeGRPCAddr string `env:"TOKEN_EXCHANGE_GRPC_ADDR"` TokenExchangeCacheExpiration time.Duration `env:"TOKEN_EXCHANGE_CACHE_EXPIRATION"` TokenExchangeCacheCleanupInterval time.Duration `env:"TOKEN_EXCHANGE_CACHE_CLEANUP_INTERVAL"` @@ -27,14 +27,278 @@ type Settings struct { MaxWebhookFailureCount uint `env:"MAX_WEBHOOK_FAILURE_COUNT"` // MaxInFlight is the maximum number of messages to process concurrently per consumer MaxInFlight int `env:"MAX_IN_FLIGHT" envDefault:"50"` - // CacheDebounceTime wait time betweeen to successive cache refreshes - CacheDebounceTime time.Duration `env:"CACHE_DEBOUNCE_TIME"` - // CacheBuildWorkers caps the parallelism of the per-trigger fetch+CEL - // compile loop in webhookcache.PopulateCache. Each worker takes one DB - // roundtrip plus one CEL compile at a time, so it doubles as a DB - // connection-pool guard. Defaults to 2 because the prod pod is pinned - // to ~1 CPU; raise it on multi-core nodes. - CacheBuildWorkers int `env:"CACHE_BUILD_WORKERS" envDefault:"2"` + + // MaxAllowedCooldownPeriod is the hard upper bound (seconds) on + // triggers.cooldown_period. It exists so the cooldown KV bucket TTL is + // guaranteed to outlive any configured cooldown - silent fires + // otherwise once TTL expires the state record mid-cooldown. Default + // 30 days; service refuses to start unless TriggerStateTTL is at least + // 2x this value (see Settings.Validate). + MaxAllowedCooldownPeriod int `env:"MAX_ALLOWED_COOLDOWN_PERIOD" envDefault:"2592000"` + + // SigningSecretKeyHex is a 64-char hex (32-byte) key used to AES-256-GCM + // encrypt per-trigger HMAC signing secrets at rest in Postgres. Empty + // disables encryption (legacy plaintext storage). Rows written before + // the key is enabled remain readable - the cipher falls back to + // plaintext when the stored value doesn't look encrypted. + SigningSecretKeyHex string `env:"SIGNING_SECRET_KEY_HEX"` + + NATS NATSSettings `envPrefix:"NATS_"` + // Env prefix kept as NATS_DISPATCHER_*, NATS_AUDIT_*, CACHE_* so the + // regroup is a pure code-organisation change and prod env files don't + // have to move. The semantics live with the subsystem; the namespace + // reflects history. + Dispatcher DispatcherSettings `envPrefix:"NATS_DISPATCHER_"` + Audit AuditSettings `envPrefix:"NATS_AUDIT_"` + Cache CacheSettings `envPrefix:"CACHE_"` DB db.Settings `envPrefix:"DB_"` } + +// DispatcherSettings tunes the webhook delivery worker pool. Owned by +// webhookdispatcher.Dispatcher; lived under NATSSettings historically +// because the env prefix was the path of least resistance. +type DispatcherSettings struct { + // Workers=0 keeps the legacy synchronous behavior; >0 spins up a worker + // pool that owns delivery + state + audit + failure-count bookkeeping + // so a slow receiver can't throttle the consumer. + Workers int `env:"WORKERS" envDefault:"32"` + QueueSize int `env:"QUEUE_SIZE" envDefault:"4096"` + RetryAttempts int `env:"RETRY_ATTEMPTS" envDefault:"2"` + RetryInitialDelay time.Duration `env:"RETRY_INITIAL_DELAY" envDefault:"100ms"` + PerHostRPS float64 `env:"PER_HOST_RPS" envDefault:"0"` + PerHostBurst int `env:"PER_HOST_BURST" envDefault:"0"` +} + +// AuditSettings tunes the fire-and-forget audit queue fronting JetStream +// PublishAsync. Sized to absorb a stream-side stall without spawning one +// goroutine per fire. +type AuditSettings struct { + Workers int `env:"WORKERS" envDefault:"4"` + QueueSize int `env:"QUEUE_SIZE" envDefault:"16384"` +} + +// CacheSettings tunes the webhookcache rebuild loop. +type CacheSettings struct { + // DebounceTime is the wait between successive cache refreshes so a + // CRUD burst collapses into one rebuild. + DebounceTime time.Duration `env:"DEBOUNCE_TIME"` + // BuildWorkers caps the parallelism of the per-trigger fetch+CEL + // compile loop. Each worker takes one DB roundtrip plus one CEL + // compile at a time, so it doubles as a DB connection-pool guard. + // Defaults to 2 because the prod pod is pinned to ~1 CPU; raise it + // on multi-core nodes. + BuildWorkers int `env:"BUILD_WORKERS" envDefault:"2"` +} + +// NATSSettings holds NATS JetStream wiring. +// +// Mode is now effectively a two-valued switch since Kafka was ripped out: +// - off (default): service runs the HTTP API only, no ingest. +// - primary | exclusive: NATS owns the evaluation path. (The two values +// are kept distinct for backwards-compat with prod env files but +// behave identically post-Kafka deletion.) +// +// When Mode != off the service refuses to start if a NATS connection cannot +// be established. +type NATSSettings struct { + Mode string `env:"MODE" envDefault:"off"` + + URL string `env:"URL" envDefault:"nats://localhost:4222"` + CredsFile string `env:"CREDS_FILE"` + Name string `env:"NAME" envDefault:"vehicle-triggers-api"` + + SignalsStream string `env:"SIGNALS_STREAM" envDefault:"DIMO_SIGNALS"` + EventsStream string `env:"EVENTS_STREAM" envDefault:"DIMO_EVENTS"` + AuditStream string `env:"AUDIT_STREAM" envDefault:"DIMO_TRIGGER_AUDIT"` + DLQStream string `env:"DLQ_STREAM" envDefault:"DIMO_TRIGGER_DLQ"` + ConfigAuditStream string `env:"CONFIG_AUDIT_STREAM" envDefault:"DIMO_CONFIG_AUDIT"` + SignalsSubject string `env:"SIGNALS_SUBJECT" envDefault:"dimo.signals.>"` + EventsSubject string `env:"EVENTS_SUBJECT" envDefault:"dimo.events.>"` + AuditSubject string `env:"AUDIT_SUBJECT" envDefault:"dimo.trigger.fired.>"` + DLQSubject string `env:"DLQ_SUBJECT" envDefault:"dimo.dlq.>"` + ConfigAuditSubject string `env:"CONFIG_AUDIT_SUBJECT" envDefault:"dimo.config.changed.>"` + + SignalsDurable string `env:"SIGNALS_DURABLE" envDefault:"triggers-signals"` + EventsDurable string `env:"EVENTS_DURABLE" envDefault:"triggers-events"` + + // DeliverPolicy controls where a NEWLY created signals/events consumer + // starts: + // - "new" (default): deliver only messages published after the consumer + // is created. The safe choice at cutover so the service doesn't replay + // hours of retained telemetry and fire stale webhooks at receivers. + // - "all": replay the stream from the beginning (backfill / replay). + // Ignored for an already-existing durable: JetStream fixes deliver policy + // at consumer-creation time, so flipping this only affects the first boot + // that creates the durable. + DeliverPolicy string `env:"DELIVER_POLICY" envDefault:"new"` + + StreamReplicas int `env:"STREAM_REPLICAS" envDefault:"1"` + SignalsMaxAge time.Duration `env:"SIGNALS_MAX_AGE" envDefault:"24h"` + EventsMaxAge time.Duration `env:"EVENTS_MAX_AGE" envDefault:"24h"` + AuditMaxAge time.Duration `env:"AUDIT_MAX_AGE" envDefault:"2160h"` // 90d + DLQMaxAge time.Duration `env:"DLQ_MAX_AGE" envDefault:"168h"` // 7d + ConfigAuditMaxAge time.Duration `env:"CONFIG_AUDIT_MAX_AGE" envDefault:"2160h"` // 90d + // Per-stream hard storage caps. 0 = unlimited (rely on MaxAge). Pair + // with Discard:DiscardOld so the oldest data is evicted when the cap + // is hit. Defaults sized for the prod load envelope; raise per + // SCALING.md sizing math if disk usage tells you to. + SignalsMaxBytes int64 `env:"SIGNALS_MAX_BYTES" envDefault:"107374182400"` // 100 GiB + EventsMaxBytes int64 `env:"EVENTS_MAX_BYTES" envDefault:"10737418240"` // 10 GiB + AuditMaxBytes int64 `env:"AUDIT_MAX_BYTES" envDefault:"107374182400"` // 100 GiB + DLQMaxBytes int64 `env:"DLQ_MAX_BYTES" envDefault:"10737418240"` // 10 GiB + ConfigAuditMaxBytes int64 `env:"CONFIG_AUDIT_MAX_BYTES" envDefault:"1073741824"` // 1 GiB + FetchBatch int `env:"FETCH_BATCH" envDefault:"100"` + AckWait time.Duration `env:"ACK_WAIT" envDefault:"45s"` + MaxDeliver int `env:"MAX_DELIVER" envDefault:"5"` + MaxAckPending int `env:"MAX_ACK_PENDING" envDefault:"5000"` + FilterSubjectCap int `env:"FILTER_SUBJECT_CAP" envDefault:"2048"` + PublishAsyncMaxPending int `env:"PUBLISH_ASYNC_MAX_PENDING" envDefault:"4000"` + + // Dispatcher and Audit live as their own subsystem-typed Settings + // off the root struct; see Settings.Dispatcher and Settings.Audit. + // They keep NATS_DISPATCHER_* and NATS_AUDIT_* env prefixes for + // backwards compatibility with prod env files. + + TriggerStateBucket string `env:"TRIGGER_STATE_BUCKET" envDefault:"trigger_state"` + // TriggerStateTTL must be >= 2*MaxAllowedCooldownPeriod (see + // Settings.Validate). Default 63d keeps that invariant satisfied against + // the default 30d cooldown ceiling so enabling NATS with stock defaults + // doesn't fail startup. Lower it only if you also lower the ceiling. + TriggerStateTTL time.Duration `env:"TRIGGER_STATE_TTL" envDefault:"1512h"` // 63d + SignalHistoryBucket string `env:"SIGNAL_HISTORY_BUCKET" envDefault:"signal_history"` + SignalHistoryTTL time.Duration `env:"SIGNAL_HISTORY_TTL" envDefault:"168h"` // 7d + + // RateLimitBucket holds the cluster-shared per-host token bucket + // state for outbound webhook delivery. Empty disables cluster + // sharing (dispatcher falls back to the per-pod hostLimiter). + RateLimitBucket string `env:"RATE_LIMIT_BUCKET" envDefault:"webhook_rate_limit"` + RateLimitTTL time.Duration `env:"RATE_LIMIT_TTL" envDefault:"1h"` +} + +// Enabled reports whether any NATS wiring should run. +func (n NATSSettings) Enabled() bool { return n.Mode != "" && n.Mode != "off" } + +// PrimaryMode reports whether NATS owns the evaluation path. Equivalent to +// Enabled() post-Kafka-deletion; kept as a separate method so the wiring +// reads naturally ("if NATS is the primary ingest path") and a future split +// (e.g. "evaluator-only, no consumer" Mode) has a place to land. +func (n NATSSettings) PrimaryMode() bool { return n.Enabled() } + +// Validate returns an error describing any misconfiguration. Called from +// main at startup so we fail fast rather than discovering trouble at first +// signal. Sweeps the cross-field constraints that env tags can't express: +// mode enum, NATS URL when enabled, Kafka brokers when Kafka is in play, +// and the retry/backoff/retention math. +func (s Settings) Validate() error { + var errs []string + + if err := s.NATS.Validate(); err != nil { + errs = append(errs, err.Error()) + } + + if s.MaxInFlight < 1 { + errs = append(errs, fmt.Sprintf("MAX_IN_FLIGHT=%d must be >= 1", s.MaxInFlight)) + } + + // Critical HTTP-serving dependencies. These are always required to serve + // the API (auth, identity lookups, permission checks) regardless of NATS + // mode. Without them the service boots and only fails on the first + // request - fail fast at startup instead. + if s.Port < 1 { + errs = append(errs, fmt.Sprintf("PORT=%d must be >= 1", s.Port)) + } + if strings.TrimSpace(s.JWKKeySetURL) == "" { + errs = append(errs, "JWK_KEY_SET_URL is required") + } + if strings.TrimSpace(s.IdentityAPIURL) == "" { + errs = append(errs, "IDENTITY_API_URL is required") + } + if strings.TrimSpace(s.TokenExchangeGRPCAddr) == "" { + errs = append(errs, "TOKEN_EXCHANGE_GRPC_ADDR is required") + } + + if s.MaxAllowedCooldownPeriod < 1 { + errs = append(errs, fmt.Sprintf("MAX_ALLOWED_COOLDOWN_PERIOD=%d must be >= 1", s.MaxAllowedCooldownPeriod)) + } + + // Distributed-cooldown invariant: the trigger_state KV bucket TTL must + // outlive the longest cooldown by a comfortable factor. Otherwise a + // long-cooldown trigger silently re-fires once TTL expires mid-window. + // Only enforced when NATS is wired in (the in-memory fallback isn't + // TTL-bounded). + if s.NATS.Enabled() && s.MaxAllowedCooldownPeriod > 0 { + maxCooldown := time.Duration(s.MaxAllowedCooldownPeriod) * time.Second + minTTL := 2 * maxCooldown + if s.NATS.TriggerStateTTL < minTTL { + errs = append(errs, fmt.Sprintf( + "NATS_TRIGGER_STATE_TTL=%s must be >= 2*MAX_ALLOWED_COOLDOWN_PERIOD (=%s); raise the TTL or lower the cooldown ceiling", + s.NATS.TriggerStateTTL, minTTL, + )) + } + } + + if len(errs) == 0 { + return nil + } + return errors.New("invalid configuration: " + strings.Join(errs, "; ")) +} + +// Validate checks the NATS configuration's internal consistency. +func (n NATSSettings) Validate() error { + var errs []string + + switch n.Mode { + case "", "off", "primary", "exclusive": + default: + errs = append(errs, fmt.Sprintf("NATS_MODE=%q must be one of off|primary|exclusive", n.Mode)) + } + + switch n.DeliverPolicy { + case "", "new", "all": + default: + errs = append(errs, fmt.Sprintf("NATS_DELIVER_POLICY=%q must be one of new|all", n.DeliverPolicy)) + } + + if !n.Enabled() { + // Nothing else matters until NATS is turned on. + if len(errs) == 0 { + return nil + } + return errors.New(strings.Join(errs, "; ")) + } + + if strings.TrimSpace(n.URL) == "" { + errs = append(errs, "NATS_URL empty but mode != off") + } + if n.MaxDeliver < 1 { + errs = append(errs, fmt.Sprintf("NATS_MAX_DELIVER=%d must be >= 1", n.MaxDeliver)) + } + if n.MaxAckPending < 1 { + errs = append(errs, fmt.Sprintf("NATS_MAX_ACK_PENDING=%d must be >= 1", n.MaxAckPending)) + } + if n.AckWait <= 0 { + errs = append(errs, fmt.Sprintf("NATS_ACK_WAIT=%s must be > 0", n.AckWait)) + } + if n.FetchBatch < 1 { + errs = append(errs, fmt.Sprintf("NATS_FETCH_BATCH=%d must be >= 1", n.FetchBatch)) + } + if n.StreamReplicas < 1 { + errs = append(errs, fmt.Sprintf("NATS_STREAM_REPLICAS=%d must be >= 1", n.StreamReplicas)) + } + + // Retention must outlive the worst-case retry window: AckWait * MaxDeliver. + // Otherwise messages can be discarded mid-retry. + worstCase := n.AckWait * time.Duration(n.MaxDeliver) + if n.SignalsMaxAge > 0 && n.SignalsMaxAge < worstCase { + errs = append(errs, fmt.Sprintf("NATS_SIGNALS_MAX_AGE=%s shorter than AckWait*MaxDeliver=%s; messages may be discarded mid-retry", n.SignalsMaxAge, worstCase)) + } + if n.EventsMaxAge > 0 && n.EventsMaxAge < worstCase { + errs = append(errs, fmt.Sprintf("NATS_EVENTS_MAX_AGE=%s shorter than AckWait*MaxDeliver=%s; messages may be discarded mid-retry", n.EventsMaxAge, worstCase)) + } + + if len(errs) == 0 { + return nil + } + return errors.New(strings.Join(errs, "; ")) +} diff --git a/internal/config/settings_test.go b/internal/config/settings_test.go new file mode 100644 index 0000000..c29c1c8 --- /dev/null +++ b/internal/config/settings_test.go @@ -0,0 +1,89 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func validBaseSettings() Settings { + return Settings{ + Port: 8080, + JWKKeySetURL: "https://auth.example.test/keys", + IdentityAPIURL: "https://identity.example.test/query", + TokenExchangeGRPCAddr: "token-exchange:8086", + MaxInFlight: 50, + MaxAllowedCooldownPeriod: 2592000, // 30 days + NATS: NATSSettings{ + Mode: "off", + URL: "nats://localhost:4222", + MaxDeliver: 5, + MaxAckPending: 5000, + AckWait: 45 * time.Second, + FetchBatch: 100, + StreamReplicas: 1, + SignalsMaxAge: 24 * time.Hour, + EventsMaxAge: 24 * time.Hour, + TriggerStateTTL: 60 * 24 * time.Hour, // 60d, > 2 * 30d + }, + } +} + +func TestSettingsValidate(t *testing.T) { + t.Parallel() + + t.Run("happy path off", func(t *testing.T) { + require.NoError(t, validBaseSettings().Validate()) + }) + + t.Run("happy path exclusive", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "exclusive" + require.NoError(t, s.Validate()) + }) + + t.Run("rejects unknown mode", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "bogus" + require.ErrorContains(t, s.Validate(), "NATS_MODE") + }) + + t.Run("rejects MaxDeliver < 1 when nats enabled", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "primary" + s.NATS.MaxDeliver = 0 + require.ErrorContains(t, s.Validate(), "NATS_MAX_DELIVER") + }) + + t.Run("flags max-age shorter than retry window", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "primary" + s.NATS.AckWait = 60 * time.Second + s.NATS.MaxDeliver = 5 + s.NATS.SignalsMaxAge = time.Minute // 1m < 60s * 5 = 5m + require.ErrorContains(t, s.Validate(), "SIGNALS_MAX_AGE") + }) + + t.Run("flags MaxInFlight 0", func(t *testing.T) { + s := validBaseSettings() + s.MaxInFlight = 0 + require.ErrorContains(t, s.Validate(), "MAX_IN_FLIGHT") + }) + + t.Run("rejects TriggerStateTTL shorter than 2*max cooldown when nats enabled", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "primary" + s.MaxAllowedCooldownPeriod = 7 * 24 * 60 * 60 // 7d + s.NATS.TriggerStateTTL = 7 * 24 * time.Hour // = max cooldown; fails 2x check + require.ErrorContains(t, s.Validate(), "NATS_TRIGGER_STATE_TTL") + }) + + t.Run("accepts TriggerStateTTL >= 2 * max cooldown", func(t *testing.T) { + s := validBaseSettings() + s.NATS.Mode = "primary" + s.MaxAllowedCooldownPeriod = 7 * 24 * 60 * 60 + s.NATS.TriggerStateTTL = 15 * 24 * time.Hour + require.NoError(t, s.Validate()) + }) +} diff --git a/internal/controllers/metriclistener/events.go b/internal/controllers/metriclistener/events.go index 223f625..fdfd667 100644 --- a/internal/controllers/metriclistener/events.go +++ b/internal/controllers/metriclistener/events.go @@ -12,15 +12,13 @@ import ( "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" - "github.com/ThreeDotsLabs/watermill/message" - "github.com/rs/zerolog" - "golang.org/x/sync/errgroup" ) -func (m *MetricListener) processEventMessage(msg *message.Message) error { +// HandleEventPayload parses an EventCloudEvent and evaluates triggers +// against each unpacked event. The NATS pull-loop entry point. +func (m *MetricListener) HandleEventPayload(ctx context.Context, payload []byte) error { var eventCE vss.EventCloudEvent - if err := json.Unmarshal(msg.Payload, &eventCE); err != nil { + if err := json.Unmarshal(payload, &eventCE); err != nil { return fmt.Errorf("failed to parse event CloudEvent: %w", err) } @@ -33,7 +31,7 @@ func (m *MetricListener) processEventMessage(msg *message.Message) error { errs = errors.Join(errs, fmt.Errorf("failed to marshal event: %w", err)) continue } - if err := m.processSingleEvent(msg.Context(), event, eventData); err != nil { + if err := m.processSingleEvent(ctx, event, eventData); err != nil { errs = errors.Join(errs, err) } } @@ -41,60 +39,22 @@ func (m *MetricListener) processEventMessage(msg *message.Message) error { } func (m *MetricListener) processSingleEvent(ctx context.Context, event vss.Event, rawPayload json.RawMessage) error { - eventEval := triggerevaluator.EventEvaluationData{ - Event: event, - RawData: rawPayload, - } - - var err error - eventEval.VehicleDID, err = cloudevent.DecodeERC721DID(event.Subject) + vehicleDID, err := cloudevent.DecodeERC721DID(event.Subject) if err != nil { return fmt.Errorf("failed to decode ERC721DID: %w", err) } - - webhooks := m.webhookCache.GetWebhooks(eventEval.VehicleDID.String(), triggersrepo.ServiceEvent, event.Data.Name) - if len(webhooks) == 0 { - return nil + eval := &triggerevaluator.EventEvaluationData{ + Event: event, + VehicleDID: vehicleDID, + RawData: rawPayload, } - - group, groupCtx := errgroup.WithContext(ctx) - group.SetLimit(100) - for _, wh := range webhooks { - group.Go(func() error { - if err := m.processEventWebhook(groupCtx, wh, &eventEval); err != nil { - zerolog.Ctx(groupCtx).Error().Str("trigger_id", wh.Trigger.ID).Err(err).Msg("failed to process webhook") - } - return nil - }) - } - return group.Wait() + webhooks := m.webhookCache.GetWebhooks(vehicleDID.String(), triggersrepo.ServiceEvent, event.Data.Name) + return fanoutAndFire(ctx, m, webhooks, vehicleDID, rawPayload, eval, + m.triggerEvaluator.EvaluateEventTrigger, m.createEventPayload) } -func (m *MetricListener) processEventWebhook(ctx context.Context, wh *webhookcache.Webhook, eventEval *triggerevaluator.EventEvaluationData) error { - // Evaluate the trigger using the new service - result, err := m.triggerEvaluator.EvaluateEventTrigger(ctx, wh.Trigger, wh.Program, eventEval) - if err != nil { - return fmt.Errorf("failed to evaluate event trigger: %w", err) - } - - if !result.ShouldFire { - // Handle permission denied - unsubscribe from the trigger - if result.PermissionDenied { - _, err := m.repo.DeleteVehicleSubscription(ctx, wh.Trigger.ID, eventEval.VehicleDID) - if err != nil { - return fmt.Errorf("failed to delete vehicle subscription: %w", err) - } - m.webhookCache.ScheduleRefresh(ctx) - } - return nil - } - - payload := m.createEventPayload(wh.Trigger, eventEval) - return m.handleTriggeredWebhook(ctx, wh.Trigger, eventEval.RawData, payload) - -} -func (m *MetricListener) createEventPayload(trigger *models.Trigger, eventEval *triggerevaluator.EventEvaluationData) *cloudevent.CloudEvent[webhook.WebhookPayload] { - payload := m.createWebhookPayload(trigger, eventEval.VehicleDID) +func (m *MetricListener) createEventPayload(trigger *models.Trigger, eventEval *triggerevaluator.EventEvaluationData) (*cloudevent.CloudEvent[webhook.WebhookPayload], error) { + payload := m.createWebhookPayload(trigger, eventEval.VehicleDID, eventEval.Event.ID) payload.Data.Event = &webhook.EventData{ Name: eventEval.Event.Data.Name, Timestamp: eventEval.Event.Data.Timestamp, @@ -103,5 +63,5 @@ func (m *MetricListener) createEventPayload(trigger *models.Trigger, eventEval * DurationNs: eventEval.Event.Data.DurationNs, Metadata: eventEval.Event.Data.Metadata, } - return payload + return payload, nil } diff --git a/internal/controllers/metriclistener/fire.go b/internal/controllers/metriclistener/fire.go new file mode 100644 index 0000000..78e4dfa --- /dev/null +++ b/internal/controllers/metriclistener/fire.go @@ -0,0 +1,102 @@ +package metriclistener + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" + "github.com/google/cel-go/cel" + "github.com/rs/zerolog" + "golang.org/x/sync/errgroup" +) + +// fanoutLimit caps concurrent CEL evaluations per inbound message. Above this +// a single noisy vehicle could spawn thousands of goroutines deep into the +// trigger evaluator + permissions cache; the cache itself is thread-safe but +// the goroutine churn becomes the bottleneck. +const fanoutLimit = 100 + +// payloadBuilder turns an evaluation result into the outbound CloudEvent +// payload. Returning an error keeps the err-trace contract: a builder failure +// (unsupported signal type, etc.) surfaces as a per-webhook failure rather +// than poisoning the whole inbound message. +type payloadBuilder[E any] func(trigger *models.Trigger, eval *E) (*cloudevent.CloudEvent[webhook.WebhookPayload], error) + +// triggerEvalFn is the per-type evaluator wired to TriggerEvaluator. Signal +// and event paths differ in input type but produce the same Result. +type triggerEvalFn[E any] func(ctx context.Context, trigger *models.Trigger, program cel.Program, eval *E) (*triggerevaluator.TriggerEvaluationResult, error) + +// fanoutAndFire runs eval + post-check + fire across every webhook for the +// inbound message. Shared by the signal and event paths because the eval -> +// permission-denied -> fire flow is identical once the per-type evaluation +// data is built. +// +// Returns the aggregated error from group.Wait but the inner goroutines +// always return nil after logging - one bad webhook should never poison the +// whole fanout. Aggregation stays so the signature carries an error future- +// proofly when we want per-webhook errors back. +func fanoutAndFire[E any]( + ctx context.Context, + m *MetricListener, + webhooks []*webhookcache.Webhook, + vehicleDID cloudevent.ERC721DID, + raw json.RawMessage, + eval *E, + evaluate triggerEvalFn[E], + build payloadBuilder[E], +) error { + if len(webhooks) == 0 { + return nil + } + group, groupCtx := errgroup.WithContext(ctx) + group.SetLimit(fanoutLimit) + for _, wh := range webhooks { + group.Go(func() error { + if err := evalAndFire(groupCtx, m, wh, vehicleDID, raw, eval, evaluate, build); err != nil { + zerolog.Ctx(groupCtx).Error().Str("trigger_id", wh.Trigger.ID).Err(err).Msg("failed to process webhook") + } + return nil + }) + } + return group.Wait() +} + +func evalAndFire[E any]( + ctx context.Context, + m *MetricListener, + wh *webhookcache.Webhook, + vehicleDID cloudevent.ERC721DID, + raw json.RawMessage, + eval *E, + evaluate triggerEvalFn[E], + build payloadBuilder[E], +) error { + result, err := evaluate(ctx, wh.Trigger, wh.Program, eval) + if err != nil { + return fmt.Errorf("failed to evaluate trigger: %w", err) + } + if !result.ShouldFire { + if result.PermissionDenied { + if _, err := m.repo.DeleteVehicleSubscription(ctx, wh.Trigger.ID, vehicleDID); err != nil { + return fmt.Errorf("failed to delete vehicle subscription: %w", err) + } + // Surgical local invalidation; no broadcast. A permission-denied + // signal stream can fire this thousands of times per second on a + // misconfigured developer and broadcasting each one would thrash + // every replica. Other replicas catch up via the periodic poll or + // the next permission-denied on their side. + m.webhookCache.InvalidateVehicleTrigger(vehicleDID.String(), wh.Trigger.ID) + } + return nil + } + payload, err := build(wh.Trigger, eval) + if err != nil { + return fmt.Errorf("failed to create webhook payload: %w", err) + } + return m.handleTriggeredWebhook(ctx, wh.Trigger, raw, payload) +} diff --git a/internal/controllers/metriclistener/metric_listener.go b/internal/controllers/metriclistener/metric_listener.go index 527f14b..cfe1dfe 100644 --- a/internal/controllers/metriclistener/metric_listener.go +++ b/internal/controllers/metriclistener/metric_listener.go @@ -2,42 +2,27 @@ package metriclistener import ( "context" - "encoding/json" - "fmt" + "crypto/sha256" + "encoding/hex" "time" "github.com/DIMO-Network/cloudevent" - "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/config" "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhooksender" - "github.com/ThreeDotsLabs/watermill/message" - "github.com/aarondl/sqlboiler/v4/types" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookdispatcher" "github.com/google/cel-go/cel" "github.com/google/uuid" - "github.com/rs/zerolog" - "golang.org/x/sync/semaphore" ) +// TriggerRepo is the listener's narrowed view of triggersrepo. The listener +// only deletes vehicle subscriptions when permissions are revoked; failure- +// count writes and reset live with the dispatcher post-Kafka deletion. type TriggerRepo interface { - CreateTriggerLog(ctx context.Context, triggerLog *models.TriggerLog) error DeleteVehicleSubscription(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (int64, error) - ResetTriggerFailureCount(ctx context.Context, trigger *models.Trigger) error - IncrementTriggerFailureCount(ctx context.Context, trigger *models.Trigger, failureReason error, maxFailureCount int) error -} - -type WebhookSender interface { - SendWebhook(ctx context.Context, trigger *models.Trigger, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error -} - -type WebhookFailureManager interface { - ShouldAttemptWebhook(trigger *models.Trigger) bool - HandleWebhookSuccess(ctx context.Context, trigger *models.Trigger) error - HandleWebhookFailure(ctx context.Context, trigger *models.Trigger, failureReason error) error } type TriggerEvaluator interface { @@ -48,21 +33,39 @@ type TriggerEvaluator interface { type WebhookCache interface { GetWebhooks(vehicleDID string, service string, metricName string) []*webhookcache.Webhook ScheduleRefresh(ctx context.Context) + InvalidateVehicleTrigger(assetDID, triggerID string) } +// WebhookDispatcher is the sink for evaluated triggers. The production +// implementation lives in internal/services/webhookdispatcher and owns +// send + state + audit + failure-count bookkeeping. +type WebhookDispatcher interface { + Enqueue(ctx context.Context, j webhookdispatcher.Job) error +} + +// MetricListener parses inbound CloudEvent payloads, runs each matching +// trigger's CEL condition against the per-vehicle state snapshot, and hands +// firing triggers off to the dispatcher. +// +// Post-Kafka rip the listener owns only evaluation; delivery, state, audit, +// and circuit-breaker live in webhookdispatcher.Dispatcher. The dispatcher +// is required - inline delivery has been removed because it duplicated +// every responsibility the dispatcher already owned. type MetricListener struct { webhookCache WebhookCache repo TriggerRepo - webhookSender WebhookSender triggerEvaluator TriggerEvaluator + dispatcher WebhookDispatcher maxFailureCount int } -// NewMetricsListener creates a new MetrticListener. +// NewMetricsListener constructs a listener wired to the supplied dispatcher. +// The dispatcher may be nil in unit tests that don't exercise the fire path; +// fanoutAndFire short-circuits when no webhook subscribers match. func NewMetricsListener(wc WebhookCache, repo TriggerRepo, - webhookSender WebhookSender, triggerEvaluator TriggerEvaluator, + dispatcher WebhookDispatcher, settings *config.Settings, ) *MetricListener { failureCount := int(settings.MaxWebhookFailureCount) @@ -72,122 +75,54 @@ func NewMetricsListener(wc WebhookCache, return &MetricListener{ webhookCache: wc, repo: repo, - webhookSender: webhookSender, triggerEvaluator: triggerEvaluator, + dispatcher: dispatcher, maxFailureCount: failureCount, } } -func (m *MetricListener) ProcessSignalMessages(ctx context.Context, messages <-chan *message.Message, maxInFlight int) error { - return processMessage(ctx, messages, m.processSignalMessage, maxInFlight) -} - -func (m *MetricListener) ProcessEventMessages(ctx context.Context, messages <-chan *message.Message, maxInFlight int) error { - return processMessage(ctx, messages, m.processEventMessage, maxInFlight) -} - -func processMessage(ctx context.Context, messages <-chan *message.Message, processor func(msg *message.Message) error, maxInFlight int) error { - logger := zerolog.Ctx(ctx) - sem := semaphore.NewWeighted(int64(maxInFlight)) - - // waitForInFlight waits for all in-flight goroutines to complete - waitForInFlight := func() { - _ = sem.Acquire(context.Background(), int64(maxInFlight)) - sem.Release(int64(maxInFlight)) - } - - for { - select { - case <-ctx.Done(): - waitForInFlight() - return ctx.Err() - case msg, ok := <-messages: - if !ok { - // channel is closed, wait for all in-flight messages to complete - waitForInFlight() - return nil - } - if ctx.Err() != nil { - // check context since select is not deterministic when multiple cases are ready - waitForInFlight() - return ctx.Err() - } - - // Acquire semaphore slot before processing - if err := sem.Acquire(ctx, 1); err != nil { - // Context cancelled while waiting for slot, wait for in-flight to complete - waitForInFlight() - return ctx.Err() - } - - msg.SetContext(ctx) - go func(m *message.Message) { - defer sem.Release(1) - if err := processor(m); err != nil { - logger.Error().Err(err).Msg("error processing message") - } - m.Ack() - }(msg) - } - } -} - -func (m *MetricListener) handleTriggeredWebhook(ctx context.Context, trigger *models.Trigger, metricData json.RawMessage, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error { - // Check if we should attempt the webhook (circuit breaker logic) +// handleTriggeredWebhook is the post-eval hand-off: the circuit-breaker +// check stays here so we never enqueue work for disabled/failed triggers, +// then we Enqueue. ErrQueueFull bubbles back to the JetStream handler which +// nak's it with the long backpressure delay. +func (m *MetricListener) handleTriggeredWebhook(ctx context.Context, trigger *models.Trigger, metricData []byte, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error { if !m.ShouldAttemptWebhook(trigger) { return nil } - - // Send the webhook - err := m.webhookSender.SendWebhook(ctx, trigger, payload) - if err != nil { - // Check if it's a webhook-specific failure - if richError, ok := richerrors.AsRichError(err); ok && richError.Code == webhooksender.WebhookFailureCode { - if failErr := m.repo.IncrementTriggerFailureCount(ctx, trigger, err, m.maxFailureCount); failErr != nil { - zerolog.Ctx(ctx).Error().Err(failErr).Str("triggerId", trigger.ID).Msg("failed to handle webhook failure") - } - return fmt.Errorf("webhook delivery failed: %w", err) - } - return fmt.Errorf("failed to send webhook: %w", err) - } - - if trigger.FailureCount > 0 { - // If this webhook was previously failed, reset the failure count. - if err := m.repo.ResetTriggerFailureCount(ctx, trigger); err != nil { - zerolog.Ctx(ctx).Error().Err(err).Str("triggerId", trigger.ID).Msg("failed to handle webhook success") - } - } - - // Log the successful trigger - if err := m.logWebhookTrigger(ctx, payload, metricData); err != nil { - return fmt.Errorf("failed to log webhook trigger: %w", err) - } - - return nil -} - -func (m *MetricListener) logWebhookTrigger(ctx context.Context, payload *cloudevent.CloudEvent[webhook.WebhookPayload], metricData json.RawMessage) error { - now := time.Now().UTC() - eventLog := &models.TriggerLog{ - ID: payload.ID, - TriggerID: payload.Data.WebhookId, - AssetDid: payload.Data.AssetDID.String(), - SnapshotData: types.JSON(metricData), - LastTriggeredAt: now, - CreatedAt: now, - } - if err := m.repo.CreateTriggerLog(ctx, eventLog); err != nil { - return fmt.Errorf("failed to create trigger log: %w", err) + if m.dispatcher == nil { + // Test mode: no dispatcher wired. Silently swallow so tests that + // exercise only the eval path don't have to thread a mock through. + return nil } - return nil + return m.dispatcher.Enqueue(ctx, webhookdispatcher.Job{ + Trigger: trigger, + Payload: payload, + Snapshot: metricData, + MetricName: trigger.MetricName, + VehicleDID: payload.Data.AssetDID, + }) } -// createWebhookPayload creates a standardized webhook payload following industry best practices -func (m *MetricListener) createWebhookPayload(trigger *models.Trigger, assetDid cloudevent.ERC721DID) *cloudevent.CloudEvent[webhook.WebhookPayload] { +// createWebhookPayload creates a standardized webhook payload. The CloudEvent +// ID is derived deterministically from (triggerID, sourceID) so receivers +// can dedup across JetStream redelivery: the same source signal/event +// re-evaluated by another replica produces the same webhook ID. Falls back +// to a random UUID when sourceID is empty (defensive; the call sites all +// pass a CloudEvent ID from the inbound payload). +func (m *MetricListener) createWebhookPayload(trigger *models.Trigger, assetDid cloudevent.ERC721DID, sourceID string) *cloudevent.CloudEvent[webhook.WebhookPayload] { + id := webhookID(trigger.ID, sourceID) payload := &cloudevent.CloudEvent[webhook.WebhookPayload]{ CloudEventHeader: cloudevent.CloudEventHeader{ - ID: uuid.New().String(), - Source: "vehicle-triggers-api", //TODO(kevin): Should be 0x of the storageNode + ID: id, + // Source is hardcoded today. The right value is the 0x address + // of the DIMO storage node that emitted the signal/event we + // fired on - that's the only identity that lets receivers + // verify CloudEvent provenance. Blocked on storage-node + // identity being available to the evaluator at runtime; tracked + // in PROD_HARDENING_V2.md item P. Until that lands, "vehicle- + // triggers-api" stamps the dispatcher service identity, which + // is at least stable. + Source: "vehicle-triggers-api", Subject: assetDid.String(), Time: time.Now().UTC(), DataContentType: "application/json", @@ -198,7 +133,7 @@ func (m *MetricListener) createWebhookPayload(trigger *models.Trigger, assetDid Data: webhook.WebhookPayload{ Service: trigger.Service, MetricName: trigger.MetricName, - WebhookId: trigger.ID, + WebhookID: trigger.ID, WebhookName: trigger.DisplayName, AssetDID: assetDid, Condition: trigger.Condition, @@ -207,17 +142,24 @@ func (m *MetricListener) createWebhookPayload(trigger *models.Trigger, assetDid return payload } -// ShouldAttemptWebhook checks if a webhook should be attempted based on its current state +// webhookID returns a deterministic CloudEvent ID for a fire. The output is +// stable across JetStream redelivery so receivers can deduplicate. +func webhookID(triggerID, sourceID string) string { + if sourceID == "" { + return uuid.New().String() + } + sum := sha256.Sum256([]byte(triggerID + "|" + sourceID)) + return hex.EncodeToString(sum[:16]) // 128 bits is plenty for collision-safe dedup +} + +// ShouldAttemptWebhook is the circuit-breaker check: skip enqueue when the +// trigger is disabled, deleted, or at the failure threshold. func (m *MetricListener) ShouldAttemptWebhook(trigger *models.Trigger) bool { - // Don't attempt if webhook is disabled or failed if trigger.Status != triggersrepo.StatusEnabled { return false } - - // Don't attempt if already at failure threshold if trigger.FailureCount >= m.maxFailureCount { return false } - return true } diff --git a/internal/controllers/metriclistener/metric_listener_mock_test.go b/internal/controllers/metriclistener/metric_listener_mock_test.go deleted file mode 100644 index 577757e..0000000 --- a/internal/controllers/metriclistener/metric_listener_mock_test.go +++ /dev/null @@ -1,312 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: metric_listener.go -// -// Generated by this command: -// -// mockgen -source=metric_listener.go -destination=metric_listener_mock_test.go -package=metriclistener -// - -// Package metriclistener is a generated GoMock package. -package metriclistener - -import ( - context "context" - reflect "reflect" - - cloudevent "github.com/DIMO-Network/cloudevent" - webhook "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" - models "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" - triggerevaluator "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" - webhookcache "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" - cel "github.com/google/cel-go/cel" - gomock "go.uber.org/mock/gomock" -) - -// MockTriggerRepo is a mock of TriggerRepo interface. -type MockTriggerRepo struct { - ctrl *gomock.Controller - recorder *MockTriggerRepoMockRecorder - isgomock struct{} -} - -// MockTriggerRepoMockRecorder is the mock recorder for MockTriggerRepo. -type MockTriggerRepoMockRecorder struct { - mock *MockTriggerRepo -} - -// NewMockTriggerRepo creates a new mock instance. -func NewMockTriggerRepo(ctrl *gomock.Controller) *MockTriggerRepo { - mock := &MockTriggerRepo{ctrl: ctrl} - mock.recorder = &MockTriggerRepoMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTriggerRepo) EXPECT() *MockTriggerRepoMockRecorder { - return m.recorder -} - -// CreateTriggerLog mocks base method. -func (m *MockTriggerRepo) CreateTriggerLog(ctx context.Context, triggerLog *models.TriggerLog) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "CreateTriggerLog", ctx, triggerLog) - ret0, _ := ret[0].(error) - return ret0 -} - -// CreateTriggerLog indicates an expected call of CreateTriggerLog. -func (mr *MockTriggerRepoMockRecorder) CreateTriggerLog(ctx, triggerLog any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateTriggerLog", reflect.TypeOf((*MockTriggerRepo)(nil).CreateTriggerLog), ctx, triggerLog) -} - -// DeleteVehicleSubscription mocks base method. -func (m *MockTriggerRepo) DeleteVehicleSubscription(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DeleteVehicleSubscription", ctx, triggerID, assetDid) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// DeleteVehicleSubscription indicates an expected call of DeleteVehicleSubscription. -func (mr *MockTriggerRepoMockRecorder) DeleteVehicleSubscription(ctx, triggerID, assetDid any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteVehicleSubscription", reflect.TypeOf((*MockTriggerRepo)(nil).DeleteVehicleSubscription), ctx, triggerID, assetDid) -} - -// IncrementTriggerFailureCount mocks base method. -func (m *MockTriggerRepo) IncrementTriggerFailureCount(ctx context.Context, trigger *models.Trigger, failureReason error, maxFailureCount int) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IncrementTriggerFailureCount", ctx, trigger, failureReason, maxFailureCount) - ret0, _ := ret[0].(error) - return ret0 -} - -// IncrementTriggerFailureCount indicates an expected call of IncrementTriggerFailureCount. -func (mr *MockTriggerRepoMockRecorder) IncrementTriggerFailureCount(ctx, trigger, failureReason, maxFailureCount any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IncrementTriggerFailureCount", reflect.TypeOf((*MockTriggerRepo)(nil).IncrementTriggerFailureCount), ctx, trigger, failureReason, maxFailureCount) -} - -// ResetTriggerFailureCount mocks base method. -func (m *MockTriggerRepo) ResetTriggerFailureCount(ctx context.Context, trigger *models.Trigger) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ResetTriggerFailureCount", ctx, trigger) - ret0, _ := ret[0].(error) - return ret0 -} - -// ResetTriggerFailureCount indicates an expected call of ResetTriggerFailureCount. -func (mr *MockTriggerRepoMockRecorder) ResetTriggerFailureCount(ctx, trigger any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ResetTriggerFailureCount", reflect.TypeOf((*MockTriggerRepo)(nil).ResetTriggerFailureCount), ctx, trigger) -} - -// MockWebhookSender is a mock of WebhookSender interface. -type MockWebhookSender struct { - ctrl *gomock.Controller - recorder *MockWebhookSenderMockRecorder - isgomock struct{} -} - -// MockWebhookSenderMockRecorder is the mock recorder for MockWebhookSender. -type MockWebhookSenderMockRecorder struct { - mock *MockWebhookSender -} - -// NewMockWebhookSender creates a new mock instance. -func NewMockWebhookSender(ctrl *gomock.Controller) *MockWebhookSender { - mock := &MockWebhookSender{ctrl: ctrl} - mock.recorder = &MockWebhookSenderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockWebhookSender) EXPECT() *MockWebhookSenderMockRecorder { - return m.recorder -} - -// SendWebhook mocks base method. -func (m *MockWebhookSender) SendWebhook(ctx context.Context, trigger *models.Trigger, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SendWebhook", ctx, trigger, payload) - ret0, _ := ret[0].(error) - return ret0 -} - -// SendWebhook indicates an expected call of SendWebhook. -func (mr *MockWebhookSenderMockRecorder) SendWebhook(ctx, trigger, payload any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SendWebhook", reflect.TypeOf((*MockWebhookSender)(nil).SendWebhook), ctx, trigger, payload) -} - -// MockWebhookFailureManager is a mock of WebhookFailureManager interface. -type MockWebhookFailureManager struct { - ctrl *gomock.Controller - recorder *MockWebhookFailureManagerMockRecorder - isgomock struct{} -} - -// MockWebhookFailureManagerMockRecorder is the mock recorder for MockWebhookFailureManager. -type MockWebhookFailureManagerMockRecorder struct { - mock *MockWebhookFailureManager -} - -// NewMockWebhookFailureManager creates a new mock instance. -func NewMockWebhookFailureManager(ctrl *gomock.Controller) *MockWebhookFailureManager { - mock := &MockWebhookFailureManager{ctrl: ctrl} - mock.recorder = &MockWebhookFailureManagerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockWebhookFailureManager) EXPECT() *MockWebhookFailureManagerMockRecorder { - return m.recorder -} - -// HandleWebhookFailure mocks base method. -func (m *MockWebhookFailureManager) HandleWebhookFailure(ctx context.Context, trigger *models.Trigger, failureReason error) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HandleWebhookFailure", ctx, trigger, failureReason) - ret0, _ := ret[0].(error) - return ret0 -} - -// HandleWebhookFailure indicates an expected call of HandleWebhookFailure. -func (mr *MockWebhookFailureManagerMockRecorder) HandleWebhookFailure(ctx, trigger, failureReason any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleWebhookFailure", reflect.TypeOf((*MockWebhookFailureManager)(nil).HandleWebhookFailure), ctx, trigger, failureReason) -} - -// HandleWebhookSuccess mocks base method. -func (m *MockWebhookFailureManager) HandleWebhookSuccess(ctx context.Context, trigger *models.Trigger) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "HandleWebhookSuccess", ctx, trigger) - ret0, _ := ret[0].(error) - return ret0 -} - -// HandleWebhookSuccess indicates an expected call of HandleWebhookSuccess. -func (mr *MockWebhookFailureManagerMockRecorder) HandleWebhookSuccess(ctx, trigger any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "HandleWebhookSuccess", reflect.TypeOf((*MockWebhookFailureManager)(nil).HandleWebhookSuccess), ctx, trigger) -} - -// ShouldAttemptWebhook mocks base method. -func (m *MockWebhookFailureManager) ShouldAttemptWebhook(trigger *models.Trigger) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ShouldAttemptWebhook", trigger) - ret0, _ := ret[0].(bool) - return ret0 -} - -// ShouldAttemptWebhook indicates an expected call of ShouldAttemptWebhook. -func (mr *MockWebhookFailureManagerMockRecorder) ShouldAttemptWebhook(trigger any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ShouldAttemptWebhook", reflect.TypeOf((*MockWebhookFailureManager)(nil).ShouldAttemptWebhook), trigger) -} - -// MockTriggerEvaluator is a mock of TriggerEvaluator interface. -type MockTriggerEvaluator struct { - ctrl *gomock.Controller - recorder *MockTriggerEvaluatorMockRecorder - isgomock struct{} -} - -// MockTriggerEvaluatorMockRecorder is the mock recorder for MockTriggerEvaluator. -type MockTriggerEvaluatorMockRecorder struct { - mock *MockTriggerEvaluator -} - -// NewMockTriggerEvaluator creates a new mock instance. -func NewMockTriggerEvaluator(ctrl *gomock.Controller) *MockTriggerEvaluator { - mock := &MockTriggerEvaluator{ctrl: ctrl} - mock.recorder = &MockTriggerEvaluatorMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTriggerEvaluator) EXPECT() *MockTriggerEvaluatorMockRecorder { - return m.recorder -} - -// EvaluateEventTrigger mocks base method. -func (m *MockTriggerEvaluator) EvaluateEventTrigger(ctx context.Context, trigger *models.Trigger, program cel.Program, ev *triggerevaluator.EventEvaluationData) (*triggerevaluator.TriggerEvaluationResult, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EvaluateEventTrigger", ctx, trigger, program, ev) - ret0, _ := ret[0].(*triggerevaluator.TriggerEvaluationResult) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EvaluateEventTrigger indicates an expected call of EvaluateEventTrigger. -func (mr *MockTriggerEvaluatorMockRecorder) EvaluateEventTrigger(ctx, trigger, program, ev any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EvaluateEventTrigger", reflect.TypeOf((*MockTriggerEvaluator)(nil).EvaluateEventTrigger), ctx, trigger, program, ev) -} - -// EvaluateSignalTrigger mocks base method. -func (m *MockTriggerEvaluator) EvaluateSignalTrigger(ctx context.Context, trigger *models.Trigger, program cel.Program, signal *triggerevaluator.SignalEvaluationData) (*triggerevaluator.TriggerEvaluationResult, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "EvaluateSignalTrigger", ctx, trigger, program, signal) - ret0, _ := ret[0].(*triggerevaluator.TriggerEvaluationResult) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// EvaluateSignalTrigger indicates an expected call of EvaluateSignalTrigger. -func (mr *MockTriggerEvaluatorMockRecorder) EvaluateSignalTrigger(ctx, trigger, program, signal any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EvaluateSignalTrigger", reflect.TypeOf((*MockTriggerEvaluator)(nil).EvaluateSignalTrigger), ctx, trigger, program, signal) -} - -// MockWebhookCache is a mock of WebhookCache interface. -type MockWebhookCache struct { - ctrl *gomock.Controller - recorder *MockWebhookCacheMockRecorder - isgomock struct{} -} - -// MockWebhookCacheMockRecorder is the mock recorder for MockWebhookCache. -type MockWebhookCacheMockRecorder struct { - mock *MockWebhookCache -} - -// NewMockWebhookCache creates a new mock instance. -func NewMockWebhookCache(ctrl *gomock.Controller) *MockWebhookCache { - mock := &MockWebhookCache{ctrl: ctrl} - mock.recorder = &MockWebhookCacheMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockWebhookCache) EXPECT() *MockWebhookCacheMockRecorder { - return m.recorder -} - -// GetWebhooks mocks base method. -func (m *MockWebhookCache) GetWebhooks(vehicleDID, service, metricName string) []*webhookcache.Webhook { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetWebhooks", vehicleDID, service, metricName) - ret0, _ := ret[0].([]*webhookcache.Webhook) - return ret0 -} - -// GetWebhooks indicates an expected call of GetWebhooks. -func (mr *MockWebhookCacheMockRecorder) GetWebhooks(vehicleDID, service, metricName any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetWebhooks", reflect.TypeOf((*MockWebhookCache)(nil).GetWebhooks), vehicleDID, service, metricName) -} - -// ScheduleRefresh mocks base method. -func (m *MockWebhookCache) ScheduleRefresh(ctx context.Context) { - m.ctrl.T.Helper() - m.ctrl.Call(m, "ScheduleRefresh", ctx) -} - -// ScheduleRefresh indicates an expected call of ScheduleRefresh. -func (mr *MockWebhookCacheMockRecorder) ScheduleRefresh(ctx any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScheduleRefresh", reflect.TypeOf((*MockWebhookCache)(nil).ScheduleRefresh), ctx) -} diff --git a/internal/controllers/metriclistener/metric_listener_test.go b/internal/controllers/metriclistener/metric_listener_test.go deleted file mode 100644 index 1194499..0000000 --- a/internal/controllers/metriclistener/metric_listener_test.go +++ /dev/null @@ -1,478 +0,0 @@ -//go:generate go tool mockgen -source=metric_listener.go -destination=metric_listener_mock_test.go -package=metriclistener - -package metriclistener - -import ( - "context" - "encoding/json" - "math/big" - "testing" - "time" - - "github.com/DIMO-Network/cloudevent" - "github.com/DIMO-Network/model-garage/pkg/vss" - "github.com/DIMO-Network/vehicle-triggers-api/internal/config" - "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" - - "github.com/ThreeDotsLabs/watermill/message" - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/mock/gomock" -) - -func TestNewMetricsListener(t *testing.T) { - t.Parallel() - - t.Run("creates listener with correct settings", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - settings := createTestSettings() - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, settings) - - require.NotNil(t, listener) - assert.Equal(t, int(settings.MaxWebhookFailureCount), listener.maxFailureCount) - }) -} - -func TestMetricListener_ShouldAttemptWebhook(t *testing.T) { - t.Parallel() - - listener := &MetricListener{maxFailureCount: 5} - - tests := []struct { - name string - status string - failureCount int - expected bool - }{ - { - name: "should attempt when enabled and under threshold", - status: triggersrepo.StatusEnabled, - failureCount: 2, - expected: true, - }, - { - name: "should not attempt when disabled", - status: triggersrepo.StatusDisabled, - failureCount: 2, - expected: false, - }, - { - name: "should not attempt when failed", - status: triggersrepo.StatusFailed, - failureCount: 2, - expected: false, - }, - { - name: "should not attempt when at failure threshold", - status: triggersrepo.StatusEnabled, - failureCount: 5, - expected: false, - }, - { - name: "should not attempt when beyond failure threshold", - status: triggersrepo.StatusEnabled, - failureCount: 6, - expected: false, - }, - { - name: "should attempt when exactly under threshold", - status: triggersrepo.StatusEnabled, - failureCount: 4, - expected: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - trigger := &models.Trigger{ - Status: tt.status, - FailureCount: tt.failureCount, - } - result := listener.ShouldAttemptWebhook(trigger) - assert.Equal(t, tt.expected, result) - }) - } -} - -func TestMetricListener_ProcessSignalMessages(t *testing.T) { - t.Parallel() - - t.Run("processes messages until channel is closed", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx := context.Background() - - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - - testSignal := vss.Signal{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now().UTC(), - Name: "speed", - ValueNumber: 25.0, - }, - } - signalCE := vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Signal{testSignal}) - signalJSON, err := json.Marshal(signalCE) - require.NoError(t, err) - - // Mock expectations - no webhooks found for this signal - mockCache.EXPECT(). - GetWebhooks(vehicleDID.String(), triggersrepo.ServiceSignal, "vss.speed"). - Return([]*webhookcache.Webhook{}). - Times(1) - - messages := make(chan *message.Message, 1) - msg := message.NewMessage(uuid.New().String(), signalJSON) - messages <- msg - close(messages) - - err = listener.ProcessSignalMessages(ctx, messages, 1) - require.NoError(t, err) - }) - - t.Run("processes signal with webhook trigger", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx := context.Background() - - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - - testSignal := vss.Signal{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now().UTC(), - Name: "speed", - ValueNumber: 25.0, - }, - } - signalCE := vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Signal{testSignal}) - signalJSON, err := json.Marshal(signalCE) - require.NoError(t, err) - - mockTrigger := &models.Trigger{ - ID: "test-trigger-id", - Status: triggersrepo.StatusEnabled, - Service: triggersrepo.ServiceSignal, - MetricName: "vss.speed", - Condition: "valueNumber > 20", - DisplayName: "Speed Alert", - FailureCount: 0, - } - - mockWebhook := &webhookcache.Webhook{ - Trigger: mockTrigger, - } - - // Mock expectations - webhook found for this signal - mockCache.EXPECT(). - GetWebhooks(vehicleDID.String(), triggersrepo.ServiceSignal, "vss.speed"). - Return([]*webhookcache.Webhook{mockWebhook}). - Times(1) - - mockTriggerEvaluator.EXPECT(). - EvaluateSignalTrigger(gomock.Any(), mockTrigger, gomock.Any(), gomock.Any()). - Return(&triggerevaluator.TriggerEvaluationResult{ - ShouldFire: true, - PermissionDenied: false, - }, nil). - Times(1) - mockWebhookSender.EXPECT(). - SendWebhook(gomock.Any(), mockTrigger, gomock.Any()). - Return(nil). - Times(1) - - mockRepo.EXPECT(). - CreateTriggerLog(gomock.Any(), gomock.Any()). - Return(nil). - Times(1) - - messages := make(chan *message.Message, 1) - msg := message.NewMessage(uuid.New().String(), signalJSON) - messages <- msg - close(messages) - - err = listener.ProcessSignalMessages(ctx, messages, 1) - require.NoError(t, err) - }) - - t.Run("stops processing when context is cancelled", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - messages := make(chan *message.Message, 1) - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - testSignal := vss.Signal{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - }, - Data: vss.SignalData{ - Timestamp: time.Now().UTC(), - Name: "speed", - ValueNumber: 25.0, - }, - } - signalCE := vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Signal{testSignal}) - signalJSON, err := json.Marshal(signalCE) - require.NoError(t, err) - - msg := message.NewMessage(uuid.New().String(), signalJSON) - messages <- msg - - err = listener.ProcessSignalMessages(ctx, messages, 1) - require.Error(t, err) - assert.Equal(t, context.Canceled, err) - }) -} - -func TestMetricListener_ProcessEventMessages(t *testing.T) { - t.Parallel() - - t.Run("processes messages until channel is closed", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx := context.Background() - - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - testEvent := vss.Event{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, - Data: vss.EventData{ - Name: "behavior.harshBraking", - Timestamp: time.Now().UTC(), - DurationNs: 1000000, - }, - } - eventCE := vss.PackEvents(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Event{testEvent}) - eventJSON, err := json.Marshal(eventCE) - require.NoError(t, err) - - mockCache.EXPECT(). - GetWebhooks(vehicleDID.String(), triggersrepo.ServiceEvent, "behavior.harshBraking"). - Return([]*webhookcache.Webhook{}). - Times(1) - - messages := make(chan *message.Message, 1) - msg := message.NewMessage(uuid.New().String(), eventJSON) - messages <- msg - close(messages) - - err = listener.ProcessEventMessages(ctx, messages, 1) - require.NoError(t, err) - }) - - t.Run("processes event with webhook trigger", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx := context.Background() - - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - testEvent := vss.Event{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, - Data: vss.EventData{ - Name: "behavior.harshBraking", - Timestamp: time.Now().UTC(), - DurationNs: 2000000, - }, - } - eventCE := vss.PackEvents(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Event{testEvent}) - eventJSON, err := json.Marshal(eventCE) - require.NoError(t, err) - - mockTrigger := &models.Trigger{ - ID: "test-event-trigger-id", - Status: triggersrepo.StatusEnabled, - Service: triggersrepo.ServiceEvent, - MetricName: "behavior.harshBraking", - Condition: "durationNs > 1000000", - DisplayName: "Harsh Braking Alert", - FailureCount: 0, - } - - mockWebhook := &webhookcache.Webhook{ - Trigger: mockTrigger, - } - - mockCache.EXPECT(). - GetWebhooks(vehicleDID.String(), triggersrepo.ServiceEvent, "behavior.harshBraking"). - Return([]*webhookcache.Webhook{mockWebhook}). - Times(1) - - mockTriggerEvaluator.EXPECT(). - EvaluateEventTrigger(gomock.Any(), mockTrigger, gomock.Any(), gomock.Any()). - Return(&triggerevaluator.TriggerEvaluationResult{ - ShouldFire: true, - PermissionDenied: false, - }, nil). - Times(1) - - mockWebhookSender.EXPECT(). - SendWebhook(gomock.Any(), mockTrigger, gomock.Any()). - Return(nil). - Times(1) - - mockRepo.EXPECT(). - CreateTriggerLog(gomock.Any(), gomock.Any()). - Return(nil). - Times(1) - - messages := make(chan *message.Message, 1) - msg := message.NewMessage(uuid.New().String(), eventJSON) - messages <- msg - close(messages) - - err = listener.ProcessEventMessages(ctx, messages, 1) - require.NoError(t, err) - }) - - t.Run("stops processing when context is cancelled", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockCache := NewMockWebhookCache(ctrl) - mockRepo := NewMockTriggerRepo(ctrl) - mockWebhookSender := NewMockWebhookSender(ctrl) - mockTriggerEvaluator := NewMockTriggerEvaluator(ctrl) - - listener := NewMetricsListener(mockCache, mockRepo, mockWebhookSender, mockTriggerEvaluator, createTestSettings()) - ctx, cancel := context.WithCancel(context.Background()) - cancel() // Cancel immediately - - messages := make(chan *message.Message, 1) - vehicleDID := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - testEvent := vss.Event{ - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, - Data: vss.EventData{ - Name: "behavior.harshBraking", - Timestamp: time.Now().UTC(), - DurationNs: 1000000, - }, - } - eventCE := vss.PackEvents(cloudevent.CloudEventHeader{ - Subject: vehicleDID.String(), - Source: "test-source", - }, []vss.Event{testEvent}) - eventJSON, err := json.Marshal(eventCE) - require.NoError(t, err) - - msg := message.NewMessage(uuid.New().String(), eventJSON) - messages <- msg - - err = listener.ProcessEventMessages(ctx, messages, 1) - require.Error(t, err) - assert.Equal(t, context.Canceled, err) - }) -} - -// Helper functions for creating test data -func createTestSettings() *config.Settings { - return &config.Settings{ - VehicleNFTAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - DIMORegistryChainID: 137, - MaxWebhookFailureCount: 5, - } -} diff --git a/internal/controllers/metriclistener/signal.go b/internal/controllers/metriclistener/signal.go index ab29017..6a199ed 100644 --- a/internal/controllers/metriclistener/signal.go +++ b/internal/controllers/metriclistener/signal.go @@ -12,16 +12,14 @@ import ( "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerevaluator" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhookcache" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" - "github.com/ThreeDotsLabs/watermill/message" - "github.com/rs/zerolog" - "golang.org/x/sync/errgroup" ) -func (m *MetricListener) processSignalMessage(msg *message.Message) error { +// HandleSignalPayload parses a SignalCloudEvent and evaluates triggers +// against each unpacked signal. The NATS pull-loop entry point. +func (m *MetricListener) HandleSignalPayload(ctx context.Context, payload []byte) error { var signalCE vss.SignalCloudEvent - if err := json.Unmarshal(msg.Payload, &signalCE); err != nil { + if err := json.Unmarshal(payload, &signalCE); err != nil { return fmt.Errorf("failed to parse signal CloudEvent: %w", err) } @@ -39,7 +37,7 @@ func (m *MetricListener) processSignalMessage(msg *message.Message) error { errs = errors.Join(errs, fmt.Errorf("failed to marshal signal: %w", err)) continue } - if err := m.processSingleSignal(msg.Context(), sig, vehicleDID, sigData); err != nil { + if err := m.processSingleSignal(ctx, sig, vehicleDID, sigData); err != nil { errs = errors.Join(errs, err) } } @@ -59,58 +57,17 @@ func inferSignalValueType(sig vss.Signal) string { func (m *MetricListener) processSingleSignal(ctx context.Context, sig vss.Signal, vehicleDID cloudevent.ERC721DID, rawPayload json.RawMessage) error { signalDef := signals.GetSignalDefinitionOrDefault(sig.Data.Name, inferSignalValueType(sig)) - - sigAndRaw := triggerevaluator.SignalEvaluationData{ + eval := &triggerevaluator.SignalEvaluationData{ Signal: sig, VehicleDID: vehicleDID, Def: signalDef, RawData: rawPayload, } - webhooks := m.webhookCache.GetWebhooks(vehicleDID.String(), triggersrepo.ServiceSignal, signals.VSSPrefix+sig.Data.Name) - if len(webhooks) == 0 { - return nil - } - - group, groupCtx := errgroup.WithContext(ctx) - group.SetLimit(100) - for _, wh := range webhooks { - group.Go(func() error { - if err := m.processSignalWebhook(groupCtx, wh, &sigAndRaw); err != nil { - zerolog.Ctx(groupCtx).Error().Str("trigger_id", wh.Trigger.ID).Err(err).Msg("failed to process webhook") - } - return nil - }) - } - return group.Wait() + return fanoutAndFire(ctx, m, webhooks, vehicleDID, rawPayload, eval, + m.triggerEvaluator.EvaluateSignalTrigger, m.createSignalPayload) } -func (m *MetricListener) processSignalWebhook(ctx context.Context, wh *webhookcache.Webhook, sigAndRaw *triggerevaluator.SignalEvaluationData) error { - // Evaluate the trigger using the new service - result, err := m.triggerEvaluator.EvaluateSignalTrigger(ctx, wh.Trigger, wh.Program, sigAndRaw) - if err != nil { - return fmt.Errorf("failed to evaluate signal trigger: %w", err) - } - - if !result.ShouldFire { - // Handle permission denied - unsubscribe from the trigger - if result.PermissionDenied { - _, err := m.repo.DeleteVehicleSubscription(ctx, wh.Trigger.ID, sigAndRaw.VehicleDID) - if err != nil { - return fmt.Errorf("failed to delete vehicle subscription: %w", err) - } - m.webhookCache.ScheduleRefresh(ctx) - } - return nil - } - - payload, err := m.createSignalPayload(wh.Trigger, sigAndRaw) - if err != nil { - return fmt.Errorf("failed to create webhook payload: %w", err) - } - - return m.handleTriggeredWebhook(ctx, wh.Trigger, sigAndRaw.RawData, payload) -} func (m *MetricListener) createSignalPayload(trigger *models.Trigger, sigEval *triggerevaluator.SignalEvaluationData) (*cloudevent.CloudEvent[webhook.WebhookPayload], error) { var signalValue any switch sigEval.Def.ValueType { @@ -123,7 +80,7 @@ func (m *MetricListener) createSignalPayload(trigger *models.Trigger, sigEval *t default: return nil, fmt.Errorf("unsupported signal type: %s", sigEval.Def.ValueType) } - payload := m.createWebhookPayload(trigger, sigEval.VehicleDID) + payload := m.createWebhookPayload(trigger, sigEval.VehicleDID, sigEval.Signal.ID) payload.Data.Signal = &webhook.SignalData{ Name: sigEval.Signal.Data.Name, Source: sigEval.Signal.Source, diff --git a/internal/controllers/metriclistener/webhook_id_test.go b/internal/controllers/metriclistener/webhook_id_test.go new file mode 100644 index 0000000..af95706 --- /dev/null +++ b/internal/controllers/metriclistener/webhook_id_test.go @@ -0,0 +1,33 @@ +package metriclistener + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestWebhookID(t *testing.T) { + t.Parallel() + + t.Run("deterministic across calls", func(t *testing.T) { + a := webhookID("trig-123", "src-abc") + b := webhookID("trig-123", "src-abc") + require.Equal(t, a, b, "same (trigger, source) must produce the same id") + require.Len(t, a, 32, "128-bit hex id expected") + }) + + t.Run("distinguishes triggers", func(t *testing.T) { + require.NotEqual(t, webhookID("trig-a", "src"), webhookID("trig-b", "src")) + }) + + t.Run("distinguishes sources", func(t *testing.T) { + require.NotEqual(t, webhookID("trig", "src-1"), webhookID("trig", "src-2")) + }) + + t.Run("falls back to uuid when source empty", func(t *testing.T) { + a := webhookID("trig", "") + b := webhookID("trig", "") + require.NotEqual(t, a, b, "no source -> uuid -> non-deterministic") + require.NotEmpty(t, a) + }) +} diff --git a/internal/controllers/webhook/types.go b/internal/controllers/webhook/types.go index f705a0a..2efda62 100644 --- a/internal/controllers/webhook/types.go +++ b/internal/controllers/webhook/types.go @@ -32,12 +32,31 @@ type RegisterWebhookRequest struct { VerificationToken string `json:"verificationToken" validate:"required" example:"1234567890"` } +// RotateSigningSecretResponse is returned after a webhook signing secret is +// rotated. The new secret is surfaced exactly once; subsequent reads only +// see the old delivery behavior with the new secret applied. +type RotateSigningSecretResponse struct { + ID string `json:"id"` + Message string `json:"message"` + SigningSecret string `json:"signingSecret"` + SignatureAlgorithm string `json:"signatureAlgorithm"` +} + // RegisterWebhookResponse is returned after a webhook is successfully created. type RegisterWebhookResponse struct { // ID is the unique identifier of the created webhook. ID string `json:"id"` // Message provides a brief status message for the operation. Message string `json:"message"` + // SigningSecret is the per-trigger HMAC-SHA256 key used to sign outbound + // webhook bodies. Returned ONCE at registration time - store it on the + // receiver side and verify the X-DIMO-Signature header on every webhook. + // The server does not surface this value again; rotation is delete + + // recreate. + SigningSecret string `json:"signingSecret"` + // SignatureAlgorithm describes how to verify the signature header: + // X-DIMO-Signature = hex(HMAC-SHA256(SigningSecret, X-DIMO-Timestamp + "." + body)) + SignatureAlgorithm string `json:"signatureAlgorithm"` } // UpdateWebhookRequest represents the fields that can be modified on an existing webhook. @@ -111,8 +130,10 @@ type WebhookPayload struct { // MetricName is the fully qualified signal/metric monitored by the webhook. MetricName string `json:"metricName"` - // WebhookId is the ID of the webhook trigger that fired - WebhookId string `json:"webhookId"` + // WebhookID is the ID of the webhook trigger that fired. Same value as + // triggers.id in the database; surfaced under the public name + // "webhookId" in the JSON payload. + WebhookID string `json:"webhookId"` // WebhookName is the user-friendly display name of the trigger WebhookName string `json:"webhookName"` diff --git a/internal/controllers/webhook/validation.go b/internal/controllers/webhook/validation.go index 1b74f71..a02906b 100644 --- a/internal/controllers/webhook/validation.go +++ b/internal/controllers/webhook/validation.go @@ -12,11 +12,23 @@ import ( "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/celcondition" + "github.com/DIMO-Network/vehicle-triggers-api/internal/safetransport" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" "github.com/gofiber/fiber/v2" ) +// verifyClient is the SSRF-guarded HTTP client for the registration +// verification probe. The guard refuses to dial internal/loopback/metadata +// addresses so a developer can't point a webhook at the cluster's internal +// network or the instance metadata endpoint. +var verifyClient = safetransport.Client(10 * time.Second) + +// maxVerifyBodySize caps the verification response read. The token is a +// short opaque string; a larger body is a misbehaving receiver and must not +// be allowed to exhaust the API server heap. +const maxVerifyBodySize = 4096 + // verifyWebhookURL verifies that the target URL is valid and returns the verification token. // It sends a POST request to the target URL with a dummy payload and verifies that the response contains the expected verification token. func verifyWebhookURL(ctx context.Context, targetURL string, verificationToken string) error { @@ -31,7 +43,7 @@ func verifyWebhookURL(ctx context.Context, targetURL string, verificationToken s } } req.Header.Set("Content-Type", "application/json") - resp, err := http.DefaultClient.Do(req) + resp, err := verifyClient.Do(req) if err != nil { return richerrors.Error{ ExternalMsg: "Failed to call target URL", @@ -47,7 +59,7 @@ func verifyWebhookURL(ctx context.Context, targetURL string, verificationToken s } } - bodyBytes, err := io.ReadAll(resp.Body) + bodyBytes, err := io.ReadAll(io.LimitReader(resp.Body, maxVerifyBodySize)) if err != nil { return richerrors.Error{ ExternalMsg: "Failed to read response from target URL", @@ -116,13 +128,19 @@ func validateCondition(serviceName, condition, valueType string) error { return nil } -func validateCoolDownPeriod(coolDownPeriod int) error { +func validateCoolDownPeriod(coolDownPeriod, maxAllowed int) error { if coolDownPeriod < 0 { return richerrors.Error{ ExternalMsg: "Cool down period must be greater than or equal to 0", Code: fiber.StatusBadRequest, } } + if maxAllowed > 0 && coolDownPeriod > maxAllowed { + return richerrors.Error{ + ExternalMsg: fmt.Sprintf("Cool down period %d exceeds MAX_ALLOWED_COOLDOWN_PERIOD=%d; the state KV bucket's TTL cannot cover longer cooldowns", coolDownPeriod, maxAllowed), + Code: fiber.StatusBadRequest, + } + } return nil } diff --git a/internal/controllers/webhook/vehicle_subscription_controller.go b/internal/controllers/webhook/vehicle_subscription_controller.go index 84db466..bbb2f77 100644 --- a/internal/controllers/webhook/vehicle_subscription_controller.go +++ b/internal/controllers/webhook/vehicle_subscription_controller.go @@ -12,6 +12,7 @@ import ( "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/auth" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/configaudit" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" "github.com/ethereum/go-ethereum/common" @@ -34,6 +35,7 @@ type VehicleSubscriptionController struct { tokenExchangeClient TokenExchangeClient cache WebhookCache repo Repository + audit ConfigAudit } // NewVehicleSubscriptionController creates a new VehicleSubscriptionController. @@ -43,9 +45,32 @@ func NewVehicleSubscriptionController(repo Repository, identityClient IdentityCl tokenExchangeClient: tokenExchangeClient, cache: cache, repo: repo, + audit: configaudit.Noop{}, } } +// WithAudit wires the controller to publish subscription change events. +func (v *VehicleSubscriptionController) WithAudit(a ConfigAudit) *VehicleSubscriptionController { + if a == nil { + a = configaudit.Noop{} + } + v.audit = a + return v +} + +// publishSubscriptionAudit emits a subscription change event. Best-effort. +func (v *VehicleSubscriptionController) publishSubscriptionAudit(ctx context.Context, op configaudit.Op, webhookID, assetDID, devLicense string) { + if v.audit == nil { + return + } + _ = v.audit.Publish(ctx, configaudit.Event{ + Op: op, + WebhookID: webhookID, + AssetDID: assetDID, + DevLicense: devLicense, + }) +} + // AssignVehicleToWebhook godoc // @Summary Assign a vehicle to a webhook // @Description Associates a vehicle with a specific event webhook. @@ -104,6 +129,7 @@ func (v *VehicleSubscriptionController) AssignVehicleToWebhook(c *fiber.Ctx) err } v.cache.ScheduleRefresh(c.Context()) + v.publishSubscriptionAudit(c.Context(), configaudit.OpSubscribeVehicle, webhookID, assetDid.String(), dl.Hex()) return c.Status(http.StatusCreated).JSON(GenericResponse{Message: "Vehicle assigned successfully"}) } @@ -247,6 +273,7 @@ func (v *VehicleSubscriptionController) RemoveVehicleFromWebhook(c *fiber.Ctx) e return richerrors.Error{ExternalMsg: "Failed to unsubscribe", Err: err, Code: http.StatusInternalServerError} } v.cache.ScheduleRefresh(c.Context()) + v.publishSubscriptionAudit(c.Context(), configaudit.OpUnsubscribeVehicle, webhookID, assetDid.String(), dl.Hex()) return c.JSON(GenericResponse{Message: "Vehicle unsubscribed successfully"}) } diff --git a/internal/controllers/webhook/webhook_controller.go b/internal/controllers/webhook/webhook_controller.go index aab1070..fc92bf0 100644 --- a/internal/controllers/webhook/webhook_controller.go +++ b/internal/controllers/webhook/webhook_controller.go @@ -8,6 +8,7 @@ import ( "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/auth" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/configaudit" "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" "github.com/aarondl/null/v8" @@ -21,6 +22,7 @@ type Repository interface { GetTriggerByIDAndDeveloperLicense(ctx context.Context, triggerID string, developerLicense common.Address) (*models.Trigger, error) UpdateTrigger(ctx context.Context, trigger *models.Trigger) error DeleteTrigger(ctx context.Context, triggerID string, developerLicense common.Address) error + RotateSigningSecret(ctx context.Context, triggerID string, developerLicense common.Address) (string, error) // subscriptions CreateVehicleSubscription(ctx context.Context, assetDID cloudevent.ERC721DID, triggerID string) (*models.VehicleSubscription, error) @@ -34,22 +36,59 @@ type WebhookCache interface { ScheduleRefresh(ctx context.Context) } +// ConfigAudit publishes config-change events for downstream compliance and +// change-management. Best-effort: failures are logged, never block the API. +type ConfigAudit interface { + Publish(ctx context.Context, e configaudit.Event) error +} + // WebhookController is the controller for creating and managing webhooks. type WebhookController struct { - repo Repository - signalDefs []signals.SignalDefinition - cache WebhookCache + repo Repository + signalDefs []signals.SignalDefinition + cache WebhookCache + audit ConfigAudit + maxCooldownPeriod int } -// NewWebhookController creates a new WebhookController. -func NewWebhookController(repo Repository, cache WebhookCache) (*WebhookController, error) { +// NewWebhookController creates a new WebhookController. maxCooldownPeriod +// caps trigger.cooldown_period at registration; the cap is required so the +// trigger_state KV bucket TTL can be sized to cover any allowed cooldown +// (enforced by Settings.Validate). +func NewWebhookController(repo Repository, cache WebhookCache, maxCooldownPeriod int) (*WebhookController, error) { return &WebhookController{ - repo: repo, - signalDefs: signals.GetAllSignalDefinitions(), - cache: cache, + repo: repo, + signalDefs: signals.GetAllSignalDefinitions(), + cache: cache, + audit: configaudit.Noop{}, + maxCooldownPeriod: maxCooldownPeriod, }, nil } +// WithAudit wires the controller to publish config-change events. Returns +// the same controller for fluent chaining at construction time. +func (w *WebhookController) WithAudit(a ConfigAudit) *WebhookController { + if a == nil { + a = configaudit.Noop{} + } + w.audit = a + return w +} + +// publishAudit emits a config-change event for the audit trail. Failures are +// logged via the audit publisher's own logging; the controller continues. +func (w *WebhookController) publishAudit(ctx context.Context, op configaudit.Op, webhookID, devLicense string, snapshot map[string]any) { + if w.audit == nil { + return + } + _ = w.audit.Publish(ctx, configaudit.Event{ + Op: op, + WebhookID: webhookID, + DevLicense: devLicense, + Snapshot: snapshot, + }) +} + // RegisterWebhook godoc // @Summary Register a new webhook // @Description Registers a new webhook with the specified configuration. The target URI is validated to ensure it is a valid URL, responds with 200 within a timeout, and returns a verification token. @@ -80,7 +119,7 @@ func (w *WebhookController) RegisterWebhook(c *fiber.Ctx) error { return err } - if err := validateCoolDownPeriod(payload.CoolDownPeriod); err != nil { + if err := validateCoolDownPeriod(payload.CoolDownPeriod, w.maxCooldownPeriod); err != nil { return err } @@ -118,7 +157,23 @@ func (w *WebhookController) RegisterWebhook(c *fiber.Ctx) error { } } - return c.Status(fiber.StatusCreated).JSON(RegisterWebhookResponse{ID: trigger.ID, Message: "Webhook registered successfully"}) + secret := "" + if trigger.SigningSecret.Valid { + secret = trigger.SigningSecret.String + } + w.publishAudit(c.Context(), configaudit.OpWebhookCreate, trigger.ID, token.EthereumAddress.Hex(), map[string]any{ + "service": trigger.Service, + "metricName": trigger.MetricName, + "targetUri": trigger.TargetURI, + "status": trigger.Status, + "cooldown": trigger.CooldownPeriod, + }) + return c.Status(fiber.StatusCreated).JSON(RegisterWebhookResponse{ + ID: trigger.ID, + Message: "Webhook registered successfully", + SigningSecret: secret, + SignatureAlgorithm: "HMAC-SHA256(timestamp + \".\" + body)", + }) } // ListWebhooks godoc @@ -220,7 +275,7 @@ func (w *WebhookController) UpdateWebhook(c *fiber.Ctx) error { event.Condition = *payload.Condition } if payload.CoolDownPeriod != nil { - if err := validateCoolDownPeriod(*payload.CoolDownPeriod); err != nil { + if err := validateCoolDownPeriod(*payload.CoolDownPeriod, w.maxCooldownPeriod); err != nil { return err } event.CooldownPeriod = *payload.CoolDownPeriod @@ -239,10 +294,53 @@ func (w *WebhookController) UpdateWebhook(c *fiber.Ctx) error { } w.cache.ScheduleRefresh(c.Context()) + w.publishAudit(c.Context(), configaudit.OpWebhookUpdate, event.ID, common.BytesToAddress(event.DeveloperLicenseAddress).Hex(), map[string]any{ + "status": event.Status, + "targetUri": event.TargetURI, + "condition": event.Condition, + "cooldown": event.CooldownPeriod, + }) return c.Status(fiber.StatusOK).JSON(UpdateWebhookResponse{ID: event.ID, Message: "Webhook updated successfully"}) } +// RotateSigningSecret godoc +// @Summary Rotate a webhook signing secret +// @Description Generates and persists a new HMAC-SHA256 signing secret for +// @Description the webhook. The new secret is returned exactly once - store +// @Description it on the receiver side before the response leaves the wire. +// @Tags Webhooks +// @Produce json +// @Param webhookId path string true "Webhook ID" +// @Success 200 {object} RotateSigningSecretResponse "New signing secret" +// @Failure 404 "Webhook not found" +// @Failure 500 "Internal server error" +// @Security BearerAuth +// @Router /v1/webhooks/{webhookId}/rotate-secret [post] +func (w *WebhookController) RotateSigningSecret(c *fiber.Ctx) error { + webhookID, err := getWebhookID(c) + if err != nil { + return err + } + devLicense, err := getDevLicense(c) + if err != nil { + return err + } + secret, err := w.repo.RotateSigningSecret(c.Context(), webhookID, devLicense) + if err != nil { + return err + } + w.publishAudit(c.Context(), configaudit.OpWebhookUpdate, webhookID, devLicense.Hex(), map[string]any{ + "secretRotated": true, + }) + return c.Status(fiber.StatusOK).JSON(RotateSigningSecretResponse{ + ID: webhookID, + Message: "Signing secret rotated", + SigningSecret: secret, + SignatureAlgorithm: "HMAC-SHA256(timestamp + \".\" + body)", + }) +} + // DeleteWebhook godoc // @Summary Delete a webhook // @Description Deletes a webhook by its ID. @@ -273,6 +371,7 @@ func (w *WebhookController) DeleteWebhook(c *fiber.Ctx) error { return fmt.Errorf("failed to delete webhook: %w", err) } w.cache.ScheduleRefresh(c.Context()) + w.publishAudit(c.Context(), configaudit.OpWebhookDelete, webhookID, devLicense.Hex(), nil) return c.Status(fiber.StatusOK).JSON(GenericResponse{Message: "Webhook deleted successfully"}) } diff --git a/internal/controllers/webhook/webhook_controller_mock_test.go b/internal/controllers/webhook/webhook_controller_mock_test.go index 8459253..7efd954 100644 --- a/internal/controllers/webhook/webhook_controller_mock_test.go +++ b/internal/controllers/webhook/webhook_controller_mock_test.go @@ -15,6 +15,7 @@ import ( cloudevent "github.com/DIMO-Network/cloudevent" models "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + configaudit "github.com/DIMO-Network/vehicle-triggers-api/internal/services/configaudit" triggersrepo "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" common "github.com/ethereum/go-ethereum/common" gomock "go.uber.org/mock/gomock" @@ -178,6 +179,21 @@ func (mr *MockRepositoryMockRecorder) GetVehicleSubscriptionsByVehicleAndDevelop return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVehicleSubscriptionsByVehicleAndDeveloperLicense", reflect.TypeOf((*MockRepository)(nil).GetVehicleSubscriptionsByVehicleAndDeveloperLicense), ctx, assetDID, developerLicense) } +// RotateSigningSecret mocks base method. +func (m *MockRepository) RotateSigningSecret(ctx context.Context, triggerID string, developerLicense common.Address) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RotateSigningSecret", ctx, triggerID, developerLicense) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RotateSigningSecret indicates an expected call of RotateSigningSecret. +func (mr *MockRepositoryMockRecorder) RotateSigningSecret(ctx, triggerID, developerLicense any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RotateSigningSecret", reflect.TypeOf((*MockRepository)(nil).RotateSigningSecret), ctx, triggerID, developerLicense) +} + // UpdateTrigger mocks base method. func (m *MockRepository) UpdateTrigger(ctx context.Context, trigger *models.Trigger) error { m.ctrl.T.Helper() @@ -227,3 +243,41 @@ func (mr *MockWebhookCacheMockRecorder) ScheduleRefresh(ctx any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ScheduleRefresh", reflect.TypeOf((*MockWebhookCache)(nil).ScheduleRefresh), ctx) } + +// MockConfigAudit is a mock of ConfigAudit interface. +type MockConfigAudit struct { + ctrl *gomock.Controller + recorder *MockConfigAuditMockRecorder + isgomock struct{} +} + +// MockConfigAuditMockRecorder is the mock recorder for MockConfigAudit. +type MockConfigAuditMockRecorder struct { + mock *MockConfigAudit +} + +// NewMockConfigAudit creates a new mock instance. +func NewMockConfigAudit(ctrl *gomock.Controller) *MockConfigAudit { + mock := &MockConfigAudit{ctrl: ctrl} + mock.recorder = &MockConfigAuditMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockConfigAudit) EXPECT() *MockConfigAuditMockRecorder { + return m.recorder +} + +// Publish mocks base method. +func (m *MockConfigAudit) Publish(ctx context.Context, e configaudit.Event) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Publish", ctx, e) + ret0, _ := ret[0].(error) + return ret0 +} + +// Publish indicates an expected call of Publish. +func (mr *MockConfigAuditMockRecorder) Publish(ctx, e any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Publish", reflect.TypeOf((*MockConfigAudit)(nil).Publish), ctx, e) +} diff --git a/internal/controllers/webhook/webhook_controller_test.go b/internal/controllers/webhook/webhook_controller_test.go index 18b2890..93057d6 100644 --- a/internal/controllers/webhook/webhook_controller_test.go +++ b/internal/controllers/webhook/webhook_controller_test.go @@ -13,6 +13,7 @@ import ( "net/http/httptest" "os" "testing" + "time" "github.com/DIMO-Network/server-garage/pkg/fibercommon" "github.com/DIMO-Network/vehicle-triggers-api/internal/auth" @@ -42,6 +43,18 @@ func TestMain(m *testing.M) { transport.TLSClientConfig = tlsCfg } + // The registration verification probe normally uses an SSRF-guarded + // client that refuses to dial loopback. Tests point webhooks at httptest + // servers on loopback, so swap in an unguarded client that also trusts + // the self-signed test cert. The SSRF guard is covered by safetransport's + // own tests. + verifyClient = &http.Client{ + Timeout: 10 * time.Second, + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // test-only + }, + } + flag.Parse() os.Exit(m.Run()) } @@ -473,7 +486,7 @@ func newWebhookControllerAndMocks(t *testing.T) (*WebhookController, *MockReposi ctrl := gomock.NewController(t) mockRepo := NewMockRepository(ctrl) mockCache := NewMockWebhookCache(ctrl) - controller, err := NewWebhookController(mockRepo, mockCache) + controller, err := NewWebhookController(mockRepo, mockCache, 0) require.NoError(t, err) return controller, mockRepo, mockCache } diff --git a/internal/db/migrations/00006_trigger_signing_secret.sql b/internal/db/migrations/00006_trigger_signing_secret.sql new file mode 100644 index 0000000..427e755 --- /dev/null +++ b/internal/db/migrations/00006_trigger_signing_secret.sql @@ -0,0 +1,26 @@ +-- +goose Up +-- +goose StatementBegin + +-- Per-trigger HMAC signing secret. The webhook sender uses this to sign +-- (timestamp || body) so receivers can verify the request was issued by us +-- and hasn't been tampered with in flight. Nullable for backwards compat +-- with existing rows; new rows fill on create. +-- +-- There is NO automatic backfill: pre-existing webhooks (signing_secret IS +-- NULL) continue to deliver UNSIGNED indefinitely until their owner calls +-- POST /v1/webhooks/:id/rotate-secret. If signed delivery is required for +-- all existing webhooks, run a one-off rotation over the affected rows as a +-- separate operational step. + +ALTER TABLE triggers + ADD COLUMN IF NOT EXISTS signing_secret VARCHAR(64); + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin + +ALTER TABLE triggers + DROP COLUMN IF EXISTS signing_secret; + +-- +goose StatementEnd diff --git a/internal/db/models/triggers.go b/internal/db/models/triggers.go index 0572b0b..5ad2b45 100644 --- a/internal/db/models/triggers.go +++ b/internal/db/models/triggers.go @@ -37,6 +37,7 @@ type Trigger struct { Description null.String `boil:"description" json:"description,omitempty" toml:"description" yaml:"description,omitempty"` FailureCount int `boil:"failure_count" json:"failure_count" toml:"failure_count" yaml:"failure_count"` DisplayName string `boil:"display_name" json:"display_name" toml:"display_name" yaml:"display_name"` + SigningSecret null.String `boil:"signing_secret" json:"signing_secret,omitempty" toml:"signing_secret" yaml:"signing_secret,omitempty"` R *triggerR `boil:"-" json:"-" toml:"-" yaml:"-"` L triggerL `boil:"-" json:"-" toml:"-" yaml:"-"` @@ -56,6 +57,7 @@ var TriggerColumns = struct { Description string FailureCount string DisplayName string + SigningSecret string }{ ID: "id", Service: "service", @@ -70,6 +72,7 @@ var TriggerColumns = struct { Description: "description", FailureCount: "failure_count", DisplayName: "display_name", + SigningSecret: "signing_secret", } var TriggerTableColumns = struct { @@ -86,6 +89,7 @@ var TriggerTableColumns = struct { Description string FailureCount string DisplayName string + SigningSecret string }{ ID: "triggers.id", Service: "triggers.service", @@ -100,6 +104,7 @@ var TriggerTableColumns = struct { Description: "triggers.description", FailureCount: "triggers.failure_count", DisplayName: "triggers.display_name", + SigningSecret: "triggers.signing_secret", } // Generated where @@ -150,6 +155,7 @@ var TriggerWhere = struct { Description whereHelpernull_String FailureCount whereHelperint DisplayName whereHelperstring + SigningSecret whereHelpernull_String }{ ID: whereHelperstring{field: "\"vehicle_triggers_api\".\"triggers\".\"id\""}, Service: whereHelperstring{field: "\"vehicle_triggers_api\".\"triggers\".\"service\""}, @@ -164,6 +170,7 @@ var TriggerWhere = struct { Description: whereHelpernull_String{field: "\"vehicle_triggers_api\".\"triggers\".\"description\""}, FailureCount: whereHelperint{field: "\"vehicle_triggers_api\".\"triggers\".\"failure_count\""}, DisplayName: whereHelperstring{field: "\"vehicle_triggers_api\".\"triggers\".\"display_name\""}, + SigningSecret: whereHelpernull_String{field: "\"vehicle_triggers_api\".\"triggers\".\"signing_secret\""}, } // TriggerRels is where relationship names are stored. @@ -222,9 +229,9 @@ func (r *triggerR) GetVehicleSubscriptions() VehicleSubscriptionSlice { type triggerL struct{} var ( - triggerAllColumns = []string{"id", "service", "metric_name", "condition", "target_uri", "cooldown_period", "developer_license_address", "created_at", "updated_at", "status", "description", "failure_count", "display_name"} + triggerAllColumns = []string{"id", "service", "metric_name", "condition", "target_uri", "cooldown_period", "developer_license_address", "created_at", "updated_at", "status", "description", "failure_count", "display_name", "signing_secret"} triggerColumnsWithoutDefault = []string{"id", "service", "metric_name", "condition", "target_uri", "developer_license_address", "status"} - triggerColumnsWithDefault = []string{"cooldown_period", "created_at", "updated_at", "description", "failure_count", "display_name"} + triggerColumnsWithDefault = []string{"cooldown_period", "created_at", "updated_at", "description", "failure_count", "display_name", "signing_secret"} triggerPrimaryKeyColumns = []string{"id"} triggerGeneratedColumns = []string{} ) diff --git a/internal/kafka/consumer.go b/internal/kafka/consumer.go deleted file mode 100644 index e8fd17b..0000000 --- a/internal/kafka/consumer.go +++ /dev/null @@ -1,99 +0,0 @@ -package kafka - -import ( - "context" - "fmt" - - "github.com/IBM/sarama" - "github.com/ThreeDotsLabs/watermill" - wmkafka "github.com/ThreeDotsLabs/watermill-kafka/v3/pkg/kafka" - "github.com/ThreeDotsLabs/watermill/message" - "github.com/rs/zerolog" -) - -type ProcessorFunc func(ctx context.Context, messages <-chan *message.Message, maxInFlight int) error - -type Config struct { - ClusterConfig *sarama.Config - BrokerAddresses []string - Topic string - GroupID string - MaxInFlight int64 - Processor ProcessorFunc - // Name is a human-readable identifier used in log lines (e.g. "signals", "events"). - Name string -} - -type Consumer struct { - subscriber *wmkafka.Subscriber - topic string - name string - Processor ProcessorFunc - maxInFlight int -} - -// Name returns the consumer's identifier used in log lines. -func (c *Consumer) Name() string { return c.name } - -// Topic returns the topic this consumer subscribes to. -func (c *Consumer) Topic() string { return c.topic } - -func NewConsumer(cfg *Config) (*Consumer, error) { - saramaSubscriberConfig := wmkafka.DefaultSaramaSubscriberConfig() - - saramaSubscriberConfig.Version = cfg.ClusterConfig.Version - saramaSubscriberConfig.Consumer.Offsets.Initial = cfg.ClusterConfig.Consumer.Offsets.Initial - - subscriber, err := wmkafka.NewSubscriber( - wmkafka.SubscriberConfig{ - Brokers: cfg.BrokerAddresses, - Unmarshaler: wmkafka.DefaultMarshaler{}, - OverwriteSaramaConfig: saramaSubscriberConfig, - ConsumerGroup: cfg.GroupID, - }, - watermill.NewStdLogger(false, false), - ) - if err != nil { - return nil, err - } - - maxInFlight := int(cfg.MaxInFlight) - if maxInFlight < 1 { - maxInFlight = 1 - } - - name := cfg.Name - if name == "" { - name = cfg.Topic - } - - return &Consumer{ - subscriber: subscriber, - topic: cfg.Topic, - name: name, - Processor: cfg.Processor, - maxInFlight: maxInFlight, - }, nil -} - -func (c *Consumer) Start(ctx context.Context) error { - logger := zerolog.Ctx(ctx).With().Str("consumer", c.name).Str("topic", c.topic).Logger() - logger.Info().Msg("kafka consumer: subscribing") - messages, err := c.subscriber.Subscribe(ctx, c.topic) - if err != nil { - logger.Error().Err(err).Msg("kafka consumer: subscribe failed") - return fmt.Errorf("could not subscribe to topic %q: %w", c.topic, err) - } - logger.Info().Msg("kafka consumer: subscribed, entering processor") - if c.Processor == nil { - return fmt.Errorf("processor function is nil") - } - - err = c.Processor(ctx, messages, c.maxInFlight) - logger.Info().Err(err).Msg("kafka consumer: processor returned") - return err -} - -func (c *Consumer) Stop(ctx context.Context) error { - return c.subscriber.Close() -} diff --git a/internal/nats/audit.go b/internal/nats/audit.go new file mode 100644 index 0000000..1ca9d7c --- /dev/null +++ b/internal/nats/audit.go @@ -0,0 +1,24 @@ +package nats + +import ( + "context" + "fmt" +) + +// PublishTriggerFired writes a per-fire audit record to the audit stream +// keyed on developer license. Uses async publish so a stalled audit stream +// can't backpressure the evaluation path; the in-flight ceiling is governed +// by NATS_PUBLISH_ASYNC_MAX_PENDING. +// +// Lives in audit.go (was bridge.go) since the Kafka->NATS republish helpers +// were ripped out post-cutover. The audit stream is the only publish path +// the service owns now; everything else is a consume + dispatch. +func (c *Client) PublishTriggerFired(ctx context.Context, devLicense string, record []byte) error { + subject := AuditSubject(devLicense) + if _, err := c.JS.PublishAsync(subject, record); err != nil { + MetricsPublish(c.cfg.AuditStream, "error") + return fmt.Errorf("audit publish %q: %w", subject, err) + } + MetricsPublish(c.cfg.AuditStream, "ok") + return nil +} diff --git a/internal/nats/client.go b/internal/nats/client.go new file mode 100644 index 0000000..7219833 --- /dev/null +++ b/internal/nats/client.go @@ -0,0 +1,202 @@ +// Package nats provides a thin wrapper around nats.go + JetStream for the +// vehicle-triggers-api service. It exposes a Client that owns the connection, +// a JetStream context, and KV buckets, plus helpers for publish / pull-consume +// and declarative stream+consumer+bucket provisioning. +package nats + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +// Client owns a NATS connection and JetStream context. +type Client struct { + Conn *nats.Conn + JS jetstream.JetStream + + cfg config.NATSSettings + log zerolog.Logger +} + +// Connect establishes a NATS connection and initializes JetStream. +// The caller owns Close(). +func Connect(ctx context.Context, cfg config.NATSSettings, log zerolog.Logger) (*Client, error) { + opts := []nats.Option{ + nats.Name(cfg.Name), + nats.RetryOnFailedConnect(true), + nats.MaxReconnects(-1), + nats.ReconnectWait(time.Second), + nats.Timeout(10 * time.Second), + nats.DrainTimeout(30 * time.Second), + nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { + log.Warn().Err(err).Msg("nats disconnected") + }), + nats.ReconnectHandler(func(c *nats.Conn) { + log.Info().Str("url", c.ConnectedUrl()).Msg("nats reconnected") + }), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { + log.Error().Err(err).Msg("nats async error") + }), + } + if cfg.CredsFile != "" { + opts = append(opts, nats.UserCredentials(cfg.CredsFile)) + } + + nc, err := nats.Connect(cfg.URL, opts...) + if err != nil { + return nil, fmt.Errorf("nats connect: %w", err) + } + + jsOpts := []jetstream.JetStreamOpt{} + if cfg.PublishAsyncMaxPending > 0 { + jsOpts = append(jsOpts, jetstream.WithPublishAsyncMaxPending(cfg.PublishAsyncMaxPending)) + } + // Audit records are published async (PublishTriggerFired). Without this + // handler a server-side NACK surfaces only on the discarded PubAckFuture, + // so a rejected billing record would vanish silently. Surface it as a log + // + error metric so ops can alert on lost audit publishes. + jsOpts = append(jsOpts, jetstream.WithPublishAsyncErrHandler( + func(_ jetstream.JetStream, m *nats.Msg, err error) { + subject := "" + if m != nil { + subject = m.Subject + } + MetricsPublish(cfg.AuditStream, "async_error") + log.Error().Err(err).Str("subject", subject).Msg("nats async publish failed (audit record lost)") + }, + )) + js, err := jetstream.New(nc, jsOpts...) + if err != nil { + nc.Close() + return nil, fmt.Errorf("jetstream init: %w", err) + } + + // Sanity check JetStream reachability with the supplied context. + if _, err := js.AccountInfo(ctx); err != nil { + nc.Close() + return nil, fmt.Errorf("jetstream account info: %w", err) + } + + return &Client{Conn: nc, JS: js, cfg: cfg, log: log}, nil +} + +// Close drains the NATS connection. Prefer Shutdown when the caller has issued +// async publishes that need to flush first. +func (c *Client) Close() error { + if c == nil || c.Conn == nil { + return nil + } + if err := c.Conn.Drain(); err != nil && !errors.Is(err, nats.ErrConnectionClosed) { + return err + } + return nil +} + +// Shutdown waits for any in-flight async publishes to be acked (bounded by +// ctx), then drains the NATS connection. Call this on service stop. +func (c *Client) Shutdown(ctx context.Context) error { + if c == nil || c.Conn == nil { + return nil + } + if c.JS != nil { + select { + case <-c.JS.PublishAsyncComplete(): + case <-ctx.Done(): + c.log.Warn().Err(ctx.Err()).Msg("nats shutdown: async publishes did not complete before deadline") + } + } + return c.Close() +} + +// Config returns the settings the client was constructed with. +func (c *Client) Config() config.NATSSettings { return c.cfg } + +// Healthy reports whether the underlying NATS connection is currently +// connected. Reconnect-in-flight returns false so /health surfaces transient +// disconnects to liveness probes. +func (c *Client) Healthy() bool { + if c == nil || c.Conn == nil { + return false + } + return c.Conn.Status() == nats.CONNECTED +} + +// StreamHealth probes each of the configured streams and returns a per-stream +// status. A stream is "ok" when it exists and reports a current leader. A +// stream that's read-only or has lost replicas surfaces as an error string +// in the returned map. Used by /health so a degraded JetStream doesn't pass +// liveness while accepting reads but refusing writes. +// +// Probes run in parallel and the whole call is capped at 1s so /health +// always returns well within the kube probe timeout, even when one stream +// is wedged. +func (c *Client) StreamHealth(ctx context.Context) map[string]string { + if c == nil || c.JS == nil { + return map[string]string{"_client": "nil"} + } + names := []string{ + c.cfg.SignalsStream, + c.cfg.EventsStream, + c.cfg.AuditStream, + c.cfg.DLQStream, + } + + probeCtx, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + + type result struct { + name, status string + } + results := make(chan result, len(names)) + var wg sync.WaitGroup + for _, name := range names { + if name == "" { + continue + } + wg.Add(1) + go func(n string) { + defer wg.Done() + stream, err := c.JS.Stream(probeCtx, n) + if err != nil { + results <- result{n, "stream lookup failed: " + err.Error()} + return + } + info, err := stream.Info(probeCtx) + if err != nil { + results <- result{n, "stream info failed: " + err.Error()} + return + } + if info.Cluster != nil && info.Cluster.Leader == "" { + results <- result{n, "no leader"} + return + } + results <- result{n, "ok"} + }(name) + } + wg.Wait() + close(results) + + out := make(map[string]string, len(names)) + for r := range results { + out[r.name] = r.status + } + return out +} + +// StreamsOK reports whether every probed stream is healthy. +func (c *Client) StreamsOK(ctx context.Context) bool { + for _, status := range c.StreamHealth(ctx) { + if status != "ok" { + return false + } + } + return true +} diff --git a/internal/nats/client_test.go b/internal/nats/client_test.go new file mode 100644 index 0000000..37f2128 --- /dev/null +++ b/internal/nats/client_test.go @@ -0,0 +1,158 @@ +package nats_test + +import ( + "context" + "os" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// testNATSURL returns a URL from NATS_TEST_URL or skips the test. +// Run locally with: NATS_TEST_URL=nats://localhost:4222 go test ./internal/nats/... +func testNATSURL(t *testing.T) string { + t.Helper() + u := os.Getenv("NATS_TEST_URL") + if u == "" { + t.Skip("NATS_TEST_URL not set") + } + return u +} + +func testSettings(url, prefix string) config.NATSSettings { + suffix := itoa(time.Now().UnixNano()) + return config.NATSSettings{ + URL: url, + Name: "vt-test", + SignalsStream: "T_" + prefix + "_SIG_" + suffix, + EventsStream: "T_" + prefix + "_EVT_" + suffix, + AuditStream: "T_" + prefix + "_AUD_" + suffix, + DLQStream: "T_" + prefix + "_DLQ_" + suffix, + ConfigAuditStream: "T_" + prefix + "_CFG_" + suffix, + SignalsSubject: "t." + prefix + "." + suffix + ".signals.>", + EventsSubject: "t." + prefix + "." + suffix + ".events.>", + AuditSubject: "t." + prefix + "." + suffix + ".audit.>", + DLQSubject: "t." + prefix + "." + suffix + ".dlq.>", + ConfigAuditSubject: "t." + prefix + "." + suffix + ".cfg.>", + SignalsDurable: "sig-" + suffix, + EventsDurable: "evt-" + suffix, + StreamReplicas: 1, + SignalsMaxAge: time.Minute, + EventsMaxAge: time.Minute, + AuditMaxAge: time.Minute, + FetchBatch: 10, + AckWait: 5 * time.Second, + MaxDeliver: 3, + MaxAckPending: 100, + FilterSubjectCap: 128, + TriggerStateBucket: "tb_state_" + suffix, + SignalHistoryBucket: "tb_hist_" + suffix, + TriggerStateTTL: time.Minute, + } +} + +func itoa(n int64) string { + const digits = "0123456789" + if n == 0 { + return "0" + } + neg := n < 0 + if neg { + n = -n + } + var buf [20]byte + i := len(buf) + for n > 0 { + i-- + buf[i] = digits[n%10] + n /= 10 + } + if neg { + i-- + buf[i] = '-' + } + return string(buf[i:]) +} + +func TestConnectAndProvision(t *testing.T) { + url := testNATSURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cfg := testSettings(url, "prov") + + c, err := vtnats.Connect(ctx, cfg, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + + require.NoError(t, c.EnsureStreams(ctx)) + require.NoError(t, c.EnsureBuckets(ctx)) + + // trigger_state bucket is reachable. + tsKV, err := c.TriggerState(ctx) + require.NoError(t, err) + _, err = tsKV.PutString(ctx, "probe", "ok") + require.NoError(t, err) + got, err := tsKV.Get(ctx, "probe") + require.NoError(t, err) + require.Equal(t, "ok", string(got.Value())) + + // Cleanup: delete streams we created. + t.Cleanup(func() { + _ = c.JS.DeleteStream(ctx, cfg.SignalsStream) + _ = c.JS.DeleteStream(ctx, cfg.EventsStream) + _ = c.JS.DeleteStream(ctx, cfg.AuditStream) + _ = c.JS.DeleteKeyValue(ctx, cfg.TriggerStateBucket) + _ = c.JS.DeleteKeyValue(ctx, cfg.SignalHistoryBucket) + }) +} + +func TestPublishSubscribeRoundtrip(t *testing.T) { + url := testNATSURL(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + cfg := testSettings(url, "rt") + + c, err := vtnats.Connect(ctx, cfg, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = c.Close() }) + + require.NoError(t, c.EnsureStreams(ctx)) + t.Cleanup(func() { + _ = c.JS.DeleteStream(ctx, cfg.SignalsStream) + _ = c.JS.DeleteStream(ctx, cfg.EventsStream) + _ = c.JS.DeleteStream(ctx, cfg.AuditStream) + }) + + prefix := cfg.SignalsSubject[:len(cfg.SignalsSubject)-1] // strip trailing ">" + subject := prefix + "speed" + filter := prefix + "speed" + seq, err := c.Publish(ctx, subject, []byte(`{"hello":"world"}`)) + require.NoError(t, err) + require.Greater(t, seq, uint64(0)) + + cons, err := c.EnsureConsumer(ctx, vtnats.ConsumerSpec{ + Stream: cfg.SignalsStream, + Durable: cfg.SignalsDurable, + FilterSubjects: []string{filter}, + DeliverPolicy: jetstream.DeliverAllPolicy, + }) + require.NoError(t, err) + + msgs, err := cons.Fetch(1, jetstream.FetchMaxWait(3*time.Second)) + require.NoError(t, err) + var got []byte + for m := range msgs.Messages() { + got = m.Data() + require.NoError(t, m.Ack()) + break + } + require.NoError(t, msgs.Error()) + require.Equal(t, `{"hello":"world"}`, string(got)) +} diff --git a/internal/nats/consumer.go b/internal/nats/consumer.go new file mode 100644 index 0000000..a2c370d --- /dev/null +++ b/internal/nats/consumer.go @@ -0,0 +1,95 @@ +package nats + +import ( + "context" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// ConsumerSpec describes a durable pull consumer bound to a stream. +type ConsumerSpec struct { + Stream string + Durable string + FilterSubjects []string + DeliverPolicy jetstream.DeliverPolicy + AckWait time.Duration + MaxDeliver int + MaxAckPending int + BackOff []time.Duration + Description string +} + +// DefaultBackOff is the retry backoff ladder for webhook dispatch failures. +var DefaultBackOff = []time.Duration{ + 1 * time.Second, + 5 * time.Second, + 30 * time.Second, + 2 * time.Minute, + 10 * time.Minute, +} + +// EnsureConsumer creates-or-updates a durable pull consumer for the given spec +// and returns the consumer handle. Callers must set DeliverPolicy explicitly — +// jetstream.DeliverAllPolicy is the zero value and we must not silently +// override it. +func (c *Client) EnsureConsumer(ctx context.Context, spec ConsumerSpec) (jetstream.Consumer, error) { + if spec.AckWait == 0 { + spec.AckWait = 45 * time.Second + } + if spec.MaxDeliver == 0 { + spec.MaxDeliver = 5 + } + if spec.MaxAckPending == 0 { + spec.MaxAckPending = 5000 + } + if len(spec.BackOff) == 0 { + spec.BackOff = DefaultBackOff + } + // JetStream requires MaxDeliver > len(BackOff). Trim the ladder so a low + // MaxDeliver configured via env doesn't crash startup. + if len(spec.BackOff) >= spec.MaxDeliver { + spec.BackOff = spec.BackOff[:spec.MaxDeliver-1] + } + + cfg := jetstream.ConsumerConfig{ + Durable: spec.Durable, + Name: spec.Durable, + DeliverPolicy: spec.DeliverPolicy, + AckPolicy: jetstream.AckExplicitPolicy, + AckWait: spec.AckWait, + MaxDeliver: spec.MaxDeliver, + MaxAckPending: spec.MaxAckPending, + BackOff: spec.BackOff, + FilterSubjects: spec.FilterSubjects, + Description: spec.Description, + } + + // DeliverPolicy (and its OptStart* companions) is immutable once a durable + // exists: JetStream fixes it at creation. Passing a different value on a + // subsequent boot (e.g. an operator flips NATS_DELIVER_POLICY new<->all) + // makes CreateOrUpdateConsumer reject the update and crash startup. Reuse + // the existing consumer's policy so re-provisioning is a no-op on that + // field; the configured policy only takes effect on first creation. + if existing, err := c.JS.Consumer(ctx, spec.Stream, spec.Durable); err == nil { + ec := existing.CachedInfo().Config + if ec.DeliverPolicy != cfg.DeliverPolicy { + c.log.Warn(). + Str("consumer", spec.Durable). + Str("configured", cfg.DeliverPolicy.String()). + Str("existing", ec.DeliverPolicy.String()). + Msg("nats consumer deliver policy is immutable; keeping existing policy") + } + cfg.DeliverPolicy = ec.DeliverPolicy + cfg.OptStartSeq = ec.OptStartSeq + cfg.OptStartTime = ec.OptStartTime + } + + cons, err := c.JS.CreateOrUpdateConsumer(ctx, spec.Stream, cfg) + if err != nil { + return nil, fmt.Errorf("ensure consumer %s/%s: %w", spec.Stream, spec.Durable, err) + } + c.log.Info().Str("stream", spec.Stream).Str("consumer", spec.Durable).Int("filters", len(spec.FilterSubjects)).Msg("nats consumer ready") + return cons, nil +} diff --git a/internal/nats/consumer_loop.go b/internal/nats/consumer_loop.go new file mode 100644 index 0000000..10bef8f --- /dev/null +++ b/internal/nats/consumer_loop.go @@ -0,0 +1,179 @@ +package nats + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + "golang.org/x/sync/semaphore" +) + +// traceCtxKey carries a per-message trace identifier through the handler +// pipeline so downstream components (listener, dispatcher, sender) can stamp +// it on logs and outbound webhook headers. The format is ":"; +// the JetStream message sequence is the natural span id - stable across +// redelivery on the same stream, distinct across streams, and lets ops +// grep one fire end-to-end. +type traceCtxKey struct{} + +// WithTrace attaches the supplied trace id to ctx. +func WithTrace(ctx context.Context, id string) context.Context { + return context.WithValue(ctx, traceCtxKey{}, id) +} + +// TraceFromContext returns the trace id attached to ctx, or "" if none. +func TraceFromContext(ctx context.Context) string { + if v, ok := ctx.Value(traceCtxKey{}).(string); ok { + return v + } + return "" +} + +// PayloadHandler processes the raw JetStream message body. Returning nil acks +// the message; a non-nil error nak's it so JetStream redelivers per the +// consumer's BackOff ladder. +// +// Wrap with ErrBackpressure to signal that the failure is transient and +// purely a load-shed - the message itself isn't poison. The PullLoop will +// use BackpressureNakDelay instead of the consumer's BackOff, which buys +// time for the downstream queue to drain without burning MaxDeliver budget +// on a self-inflicted condition. +type PayloadHandler func(ctx context.Context, payload []byte) error + +// ErrBackpressure marks a handler error as load-shed, not poison. Wrap with +// fmt.Errorf("%w: ...", ErrBackpressure) or errors.Join(ErrBackpressure, ...) +// so the PullLoop classifier picks it up. +var ErrBackpressure = errors.New("nats: handler backpressure") + +// BackpressureNakDelay is how long PullLoop holds a backpressure-flagged +// message before redelivery. Default 30s - long enough that we don't churn +// the queue, short enough that a brief spike clears within MaxDeliver +// attempts (e.g. 5 attempts * 30s = 2.5 min total runway). +var BackpressureNakDelay = 30 * time.Second + +// PullLoop pulls in batches from the consumer and dispatches each message to +// handler in a bounded worker pool. Returns when ctx is cancelled or the +// consumer's Messages() iterator is closed. +// +// JetStream pull-consumer semantics: +// - Ack on success. +// - NakWithDelay on error using the next BackOff step from MaxDeliver state +// (the server picks the step; we just ask for a redelivery). +// - Up to maxInFlight handler goroutines run concurrently per pull loop. +func (c *Client) PullLoop(ctx context.Context, cons jetstream.Consumer, maxInFlight int, handler PayloadHandler) error { + if maxInFlight < 1 { + maxInFlight = 1 + } + log := c.log.With().Str("component", "nats.pullloop").Logger() + + sem := semaphore.NewWeighted(int64(maxInFlight)) + waitForInFlight := func() { + _ = sem.Acquire(context.Background(), int64(maxInFlight)) + sem.Release(int64(maxInFlight)) + } + + iter, err := cons.Messages(jetstream.PullMaxMessages(c.cfg.FetchBatch)) + if err != nil { + return err + } + defer iter.Stop() + + // Cancel iterator when ctx cancels. + go func() { + <-ctx.Done() + iter.Stop() + }() + + for { + msg, err := iter.Next() + if err != nil { + if errors.Is(err, jetstream.ErrMsgIteratorClosed) || ctx.Err() != nil { + waitForInFlight() + return ctx.Err() + } + log.Error().Err(err).Msg("pull iterator error") + continue + } + + if err := sem.Acquire(ctx, 1); err != nil { + // Context cancelled. Nak so the message redelivers on the next leader. + _ = msg.Nak() + waitForInFlight() + return ctx.Err() + } + + go func(m jetstream.Msg) { + defer sem.Release(1) + meta, _ := m.Metadata() + stream := "" + var numDelivered uint64 + var arrived time.Time + var seq uint64 + if meta != nil { + stream = meta.Stream + numDelivered = meta.NumDelivered + arrived = meta.Timestamp + seq = meta.Sequence.Stream + } + traceID := fmt.Sprintf("%s:%d", stream, seq) + handlerCtx := WithTrace(ctx, traceID) + payload := m.Data() + if err := handler(handlerCtx, payload); err != nil { + // Backpressure: handler refused work because a downstream + // queue was full. Nak with a long delay so we don't burn + // through MaxDeliver while the queue drains. We still count + // these against MaxDeliver (JetStream doesn't let us not), + // but the long delay gives a default 5x30s = 2.5min window + // before DLQ, which is plenty for a transient spike. + if errors.Is(err, ErrBackpressure) { + log.Warn().Err(err).Str("subject", m.Subject()).Uint64("attempt", numDelivered).Msg("backpressure; nak with delay") + _ = m.NakWithDelay(BackpressureNakDelay) + MetricsConsume(stream, "nak_backpressure") + MetricsEvalLatency(stream, "nak_backpressure", arrived) + return + } + // Last attempt? Park it in the DLQ and terminally fail so the + // stream doesn't keep redelivering forever or silently drop + // after MaxDeliver expires. + if c.cfg.MaxDeliver > 0 && numDelivered >= uint64(c.cfg.MaxDeliver) { + if dlqErr := c.publishDLQ(m, err); dlqErr != nil { + log.Error().Err(dlqErr).Str("subject", m.Subject()).Msg("dlq publish failed; falling back to nak") + _ = m.NakWithDelay(0) + MetricsConsume(stream, "nak") + MetricsEvalLatency(stream, "nak", arrived) + return + } + _ = m.Term() + MetricsConsume(stream, "dlq") + MetricsEvalLatency(stream, "dlq", arrived) + return + } + log.Error().Err(err).Str("subject", m.Subject()).Uint64("attempt", numDelivered).Msg("handler failed; nak") + _ = m.NakWithDelay(0) + MetricsConsume(stream, "nak") + MetricsEvalLatency(stream, "nak", arrived) + return + } + if err := m.Ack(); err != nil { + log.Warn().Err(err).Str("subject", m.Subject()).Msg("ack failed") + } + MetricsConsume(stream, "ack") + MetricsEvalLatency(stream, "ack", arrived) + }(msg) + } +} + +// MustWaitFor blocks up to d for the JetStream account info round-trip. Used +// for startup sanity checks where the caller wants a single short timeout +// instead of layering its own context. +func (c *Client) MustWaitFor(ctx context.Context, d time.Duration) error { + if c == nil || c.JS == nil { + return errors.New("nats client not initialized") + } + cctx, cancel := context.WithTimeout(ctx, d) + defer cancel() + _, err := c.JS.AccountInfo(cctx) + return err +} diff --git a/internal/nats/dlq.go b/internal/nats/dlq.go new file mode 100644 index 0000000..f25b115 --- /dev/null +++ b/internal/nats/dlq.go @@ -0,0 +1,106 @@ +package nats + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +// dlqSubjectFor derives the DLQ publish subject from the configured DLQ +// subject filter. We trim the trailing wildcard (`.>` or `.*`) and append +// the original subject so DLQ entries keep their full source hierarchy and +// filter by original signal/event name. Falls back to the package-level +// DLQSubject (dimo.dlq.*) when no DLQ subject is configured. +func (c *Client) dlqSubjectFor(original string) string { + prefix := strings.TrimSuffix(c.cfg.DLQSubject, ".>") + prefix = strings.TrimSuffix(prefix, ".*") + if prefix == "" || prefix == c.cfg.DLQSubject { + // No wildcard suffix or empty config — fall back to package default + // to avoid publishing to a subject the DLQ stream doesn't capture. + return DLQSubject(original) + } + return prefix + "." + original +} + +// publishDLQ writes a poison message to the DLQ stream, preserving the +// original subject hierarchy under the configured DLQ prefix and stamping +// headers with triage context: subject, source name, vehicle DID, failure +// reason, deliver count, original stream, and the timestamp the failure was +// recorded. +// +// Best-effort: returns an error if the publish fails so the caller (PullLoop) +// can fall back to nak instead of Term. +// +// We intentionally do NOT include a developer-license header because one +// inbound message may match webhooks across multiple developers; tagging +// "the" developer would be misleading. Triage tools look up impacted +// developers by joining the asset DID against the trigger registry. +func (c *Client) publishDLQ(m jetstream.Msg, handlerErr error) error { + meta, _ := m.Metadata() + headers := nats.Header{} + for k, vs := range m.Headers() { + headers[k] = vs + } + headers.Set("X-Original-Subject", m.Subject()) + headers.Set("X-Failure-Reason", handlerErr.Error()) + headers.Set("X-Recorded-At", time.Now().UTC().Format(time.RFC3339Nano)) + if name := sourceNameFromSubject(m.Subject()); name != "" { + headers.Set("X-Source-Name", name) + } + if did := extractAssetDID(m.Data()); did != "" { + headers.Set("X-Asset-DID", did) + } + if meta != nil { + headers.Set("X-Original-Stream", meta.Stream) + headers.Set("X-Delivered-Count", fmt.Sprintf("%d", meta.NumDelivered)) + } + dlq := &nats.Msg{ + Subject: c.dlqSubjectFor(m.Subject()), + Data: m.Data(), + Header: headers, + } + // Bounded context, detached from the handler's: the handler ctx is + // cancelled at shutdown, but the DLQ publish must still run (the caller + // falls back to nak if it fails, so the message isn't Term'd without a + // landing spot). The timeout stops it hanging forever if the connection + // is draining. + pubCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if _, err := c.JS.PublishMsg(pubCtx, dlq); err != nil { + MetricsPublish(c.cfg.DLQStream, "error") + return fmt.Errorf("dlq publish %q: %w", dlq.Subject, err) + } + MetricsPublish(c.cfg.DLQStream, "ok") + return nil +} + +// sourceNameFromSubject returns the trailing token after the dimo.signals. +// or dimo.events. prefix. Used to populate the X-Source-Name header on DLQ +// records so ops can group by signal/event name without parsing the subject. +func sourceNameFromSubject(subject string) string { + for _, prefix := range []string{SignalSubjectPrefix + ".", EventSubjectPrefix + "."} { + if strings.HasPrefix(subject, prefix) { + return subject[len(prefix):] + } + } + return "" +} + +// extractAssetDID does a best-effort parse of the payload to lift the +// CloudEvent subject (== ERC721 DID) out for the DLQ header. We use a +// minimal struct so we don't pay the full vss decode cost for messages that +// are already known broken. +func extractAssetDID(body []byte) string { + var envelope struct { + Subject string `json:"subject"` + } + if err := json.Unmarshal(body, &envelope); err != nil { + return "" + } + return envelope.Subject +} diff --git a/internal/nats/kv.go b/internal/nats/kv.go new file mode 100644 index 0000000..035a593 --- /dev/null +++ b/internal/nats/kv.go @@ -0,0 +1,42 @@ +package nats + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// SignalHistory returns the signal-history KV bucket (per-vehicle per-metric +// last fire snapshot). +func (c *Client) SignalHistory(ctx context.Context) (jetstream.KeyValue, error) { + return c.kv(ctx, c.cfg.SignalHistoryBucket) +} + +// TriggerState returns the trigger-state KV bucket. +func (c *Client) TriggerState(ctx context.Context) (jetstream.KeyValue, error) { + return c.kv(ctx, c.cfg.TriggerStateBucket) +} + +// RateLimit returns the cluster-shared rate-limit KV bucket used by the +// dispatcher's clusterLimiter. Returns ErrBucketNotFound when cluster +// limiting isn't provisioned; callers must handle that by falling back to +// the per-pod limiter. +func (c *Client) RateLimit(ctx context.Context) (jetstream.KeyValue, error) { + if c.cfg.RateLimitBucket == "" { + return nil, fmt.Errorf("rate limit bucket not configured") + } + return c.kv(ctx, c.cfg.RateLimitBucket) +} + +func (c *Client) kv(ctx context.Context, bucket string) (jetstream.KeyValue, error) { + kv, err := c.JS.KeyValue(ctx, bucket) + if err != nil { + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, fmt.Errorf("kv bucket %q not found (did provisioning run?): %w", bucket, err) + } + return nil, fmt.Errorf("kv bucket %q: %w", bucket, err) + } + return kv, nil +} diff --git a/internal/nats/metrics.go b/internal/nats/metrics.go new file mode 100644 index 0000000..d0f631d --- /dev/null +++ b/internal/nats/metrics.go @@ -0,0 +1,59 @@ +package nats + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Prometheus instrumentation for JetStream publish + consume health and +// end-to-end evaluation latency. Exposed via the service's existing /metrics +// endpoint (server-garage's monserver registers the default registry). +var ( + publishTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "nats", + Name: "publish_total", + Help: "Number of JetStream publishes, labeled by stream and outcome.", + }, []string{"stream", "outcome"}) + + consumeTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "nats", + Name: "consume_total", + Help: "Number of JetStream messages processed, labeled by stream and outcome (ack|nak|dlq).", + }, []string{"stream", "outcome"}) + + // evalLatency is the wall-clock time from when JetStream timestamped the + // message (i.e. message arrival in the stream) to when our handler + // returns. It includes our parse + CEL eval + webhook dispatch. SLO + // surface lives here. + evalLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "vehicle_triggers", + Subsystem: "nats", + Name: "eval_latency_seconds", + Help: "Time from JetStream message timestamp to handler return.", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }, []string{"stream", "outcome"}) +) + +// MetricsPublish records a publish outcome. +func MetricsPublish(stream, outcome string) { + publishTotal.WithLabelValues(stream, outcome).Inc() +} + +// MetricsConsume records a consume outcome. +func MetricsConsume(stream, outcome string) { + consumeTotal.WithLabelValues(stream, outcome).Inc() +} + +// MetricsEvalLatency records the end-to-end handler latency for a message. +// arrived is the JetStream message timestamp (when the broker received the +// message); pass time.Time{} when unknown and the observation is skipped. +func MetricsEvalLatency(stream, outcome string, arrived time.Time) { + if arrived.IsZero() { + return + } + evalLatency.WithLabelValues(stream, outcome).Observe(time.Since(arrived).Seconds()) +} diff --git a/internal/nats/provision.go b/internal/nats/provision.go new file mode 100644 index 0000000..ee6f5c5 --- /dev/null +++ b/internal/nats/provision.go @@ -0,0 +1,102 @@ +package nats + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// EnsureStreams creates-or-updates the five streams the service uses: +// DIMO_SIGNALS, DIMO_EVENTS, DIMO_TRIGGER_AUDIT, DIMO_TRIGGER_DLQ, +// DIMO_CONFIG_AUDIT. Idempotent. +func (c *Client) EnsureStreams(ctx context.Context) error { + streams := []jetstream.StreamConfig{ + { + Name: c.cfg.SignalsStream, + Subjects: []string{c.cfg.SignalsSubject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: c.cfg.SignalsMaxAge, + MaxBytes: c.cfg.SignalsMaxBytes, + Replicas: c.cfg.StreamReplicas, + Description: "DIMO vehicle signal telemetry", + }, + { + Name: c.cfg.EventsStream, + Subjects: []string{c.cfg.EventsSubject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: c.cfg.EventsMaxAge, + MaxBytes: c.cfg.EventsMaxBytes, + Replicas: c.cfg.StreamReplicas, + Description: "DIMO vehicle events", + }, + { + Name: c.cfg.AuditStream, + Subjects: []string{c.cfg.AuditSubject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: c.cfg.AuditMaxAge, + MaxBytes: c.cfg.AuditMaxBytes, + Replicas: c.cfg.StreamReplicas, + Description: "Trigger fire audit log for billing", + }, + { + Name: c.cfg.DLQStream, + Subjects: []string{c.cfg.DLQSubject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: c.cfg.DLQMaxAge, + MaxBytes: c.cfg.DLQMaxBytes, + Replicas: c.cfg.StreamReplicas, + Description: "Poison messages that exceeded MaxDeliver retries", + }, + { + Name: c.cfg.ConfigAuditStream, + Subjects: []string{c.cfg.ConfigAuditSubject}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: c.cfg.ConfigAuditMaxAge, + MaxBytes: c.cfg.ConfigAuditMaxBytes, + Replicas: c.cfg.StreamReplicas, + Description: "Immutable webhook config change history (CRUD audit trail)", + }, + } + for _, s := range streams { + if _, err := c.JS.CreateOrUpdateStream(ctx, s); err != nil { + return fmt.Errorf("ensure stream %s: %w", s.Name, err) + } + c.log.Info().Str("stream", s.Name).Msg("nats stream ready") + } + return nil +} + +// EnsureBuckets creates-or-updates the two KV buckets the service uses: +// trigger_state (per-trigger fire records driving cooldown and previousValue) +// and signal_history (per-vehicle per-metric fire records driving +// cross-trigger previousValue). Idempotent. +func (c *Client) EnsureBuckets(ctx context.Context) error { + buckets := []jetstream.KeyValueConfig{ + {Bucket: c.cfg.TriggerStateBucket, History: 1, Replicas: c.cfg.StreamReplicas, TTL: c.cfg.TriggerStateTTL, Description: "per-trigger per-vehicle fire record"}, + {Bucket: c.cfg.SignalHistoryBucket, History: 1, Replicas: c.cfg.StreamReplicas, TTL: c.cfg.SignalHistoryTTL, Description: "per-vehicle per-metric last fire snapshot"}, + } + if c.cfg.RateLimitBucket != "" { + buckets = append(buckets, jetstream.KeyValueConfig{ + Bucket: c.cfg.RateLimitBucket, History: 1, Replicas: c.cfg.StreamReplicas, TTL: c.cfg.RateLimitTTL, + Description: "cluster-shared per-host token bucket state", + }) + } + for _, b := range buckets { + if _, err := c.JS.CreateOrUpdateKeyValue(ctx, b); err != nil { + return fmt.Errorf("ensure kv %s: %w", b.Bucket, err) + } + c.log.Info().Str("bucket", b.Bucket).Msg("nats kv bucket ready") + } + return nil +} diff --git a/internal/nats/publisher.go b/internal/nats/publisher.go new file mode 100644 index 0000000..71ba4de --- /dev/null +++ b/internal/nats/publisher.go @@ -0,0 +1,35 @@ +package nats + +import ( + "context" + "fmt" + + "github.com/nats-io/nats.go/jetstream" +) + +// Publish synchronously publishes a payload to the given subject via JetStream. +// Returns the stream sequence assigned by the server, useful as a stable ID. +func (c *Client) Publish(ctx context.Context, subject string, payload []byte, opts ...jetstream.PublishOpt) (uint64, error) { + ack, err := c.JS.Publish(ctx, subject, payload, opts...) + if err != nil { + return 0, fmt.Errorf("js publish %q: %w", subject, err) + } + return ack.Sequence, nil +} + +// PublishAsync publishes without waiting for server ack. Returns the future so +// callers that want delivery confirmation can select on it. Used by the audit +// publisher where we trade a small loss risk for throughput. +func (c *Client) PublishAsync(subject string, payload []byte, opts ...jetstream.PublishOpt) (jetstream.PubAckFuture, error) { + f, err := c.JS.PublishAsync(subject, payload, opts...) + if err != nil { + return nil, fmt.Errorf("js publish async %q: %w", subject, err) + } + return f, nil +} + +// PublishAsyncComplete returns a channel closed when all in-flight async +// publishes have been acked. Useful on shutdown. +func (c *Client) PublishAsyncComplete() <-chan struct{} { + return c.JS.PublishAsyncComplete() +} diff --git a/internal/nats/subjects.go b/internal/nats/subjects.go new file mode 100644 index 0000000..a8d76c0 --- /dev/null +++ b/internal/nats/subjects.go @@ -0,0 +1,94 @@ +package nats + +import ( + "fmt" + "strings" +) + +// Subject design notes +// +// Stream subject cardinality scales with the number of signal/event NAMES, not +// the number of vehicles. With ~30 signals and a handful of event types this +// keeps total stream-subject space in the low hundreds even with millions of +// vehicles. The vehicle identity travels in the CloudEvent payload +// (`signalCE.Subject` is the ERC721 DID), so consumers parse it from the body +// rather than from the subject. This is the deliberate trade: pay one JSON +// decode per message in exchange for keeping JetStream's per-subject state +// tiny. +// +// The audit subject keys on developer license so per-developer billing +// aggregation can use subject-level consumer filters; trigger ID lives in the +// payload. + +const ( + SignalSubjectPrefix = "dimo.signals" + EventSubjectPrefix = "dimo.events" + AuditSubjectPrefix = "dimo.trigger.fired" + DLQSubjectPrefix = "dimo.dlq" +) + +// DLQSubject returns the dead-letter subject for an original subject. We +// preserve the rest of the hierarchy so DLQ entries can be filtered by their +// original signal/event name. +func DLQSubject(original string) string { + return DLQSubjectPrefix + "." + original +} + +// AllDLQFilter returns the wildcard subject matching every DLQ entry. +func AllDLQFilter() string { return DLQSubjectPrefix + ".>" } + +// SignalSubject builds the publish subject for a signal. +// Shape: dimo.signals. +func SignalSubject(signalName string) string { + return fmt.Sprintf("%s.%s", SignalSubjectPrefix, sanitize(signalName)) +} + +// EventSubject builds the publish subject for an event. +// Shape: dimo.events. +func EventSubject(eventName string) string { + return fmt.Sprintf("%s.%s", EventSubjectPrefix, sanitize(eventName)) +} + +// AuditSubject builds the publish subject for a trigger-fired audit event. +// Shape: dimo.trigger.fired. +func AuditSubject(developerLicense string) string { + return fmt.Sprintf("%s.%s", AuditSubjectPrefix, sanitize(developerLicense)) +} + +// SignalFilter returns the consumer filter subject for one signal name. +// Shape: dimo.signals. +func SignalFilter(signalName string) string { + return SignalSubject(signalName) +} + +// EventFilter returns the consumer filter subject for one event name. +// Shape: dimo.events. +func EventFilter(eventName string) string { + return EventSubject(eventName) +} + +// AllSignalsFilter returns the wildcard filter matching every signal subject. +func AllSignalsFilter() string { return SignalSubjectPrefix + ".>" } + +// AllEventsFilter returns the wildcard filter matching every event subject. +func AllEventsFilter() string { return EventSubjectPrefix + ".>" } + +// sanitize replaces characters illegal in NATS subject tokens with underscores +// and substitutes a placeholder for empty input so we never emit adjacent-dot +// subjects like "dimo.signals..speed". +// NATS tokens disallow spaces, dots (except as separators), *, >, and control chars. +func sanitize(s string) string { + if s == "" { + return "_" + } + r := strings.NewReplacer( + " ", "_", + ".", "_", + "*", "_", + ">", "_", + "\t", "_", + "\n", "_", + "\r", "_", + ) + return r.Replace(s) +} diff --git a/internal/nats/subjects_test.go b/internal/nats/subjects_test.go new file mode 100644 index 0000000..6b29b88 --- /dev/null +++ b/internal/nats/subjects_test.go @@ -0,0 +1,71 @@ +package nats + +import "testing" + +func TestSignalSubject(t *testing.T) { + cases := []struct { + name string + signal string + expected string + }{ + {"simple", "speed", "dimo.signals.speed"}, + {"dotted signal", "powertrain.fuelLevel", "dimo.signals.powertrain_fuelLevel"}, + {"wildcard stripped", "speed>", "dimo.signals.speed_"}, + {"empty signal becomes placeholder", "", "dimo.signals._"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := SignalSubject(tc.signal); got != tc.expected { + t.Fatalf("SignalSubject(%q) = %q, want %q", tc.signal, got, tc.expected) + } + }) + } +} + +func TestEventSubject(t *testing.T) { + if got := EventSubject("harshBraking"); got != "dimo.events.harshBraking" { + t.Fatalf("got %q", got) + } +} + +func TestAuditSubject(t *testing.T) { + if got := AuditSubject("0xDEADBEEF"); got != "dimo.trigger.fired.0xDEADBEEF" { + t.Fatalf("got %q", got) + } +} + +func TestSignalFilter(t *testing.T) { + if got := SignalFilter("speed"); got != "dimo.signals.speed" { + t.Fatalf("got %q", got) + } +} + +func TestEventFilter(t *testing.T) { + if got := EventFilter("ignitionOn"); got != "dimo.events.ignitionOn" { + t.Fatalf("got %q", got) + } +} + +func TestAllSignalsFilter(t *testing.T) { + if got := AllSignalsFilter(); got != "dimo.signals.>" { + t.Fatalf("got %q", got) + } +} + +func TestSanitizeReplacesIllegal(t *testing.T) { + cases := map[string]string{ + "hello world": "hello_world", + "a.b.c": "a_b_c", + "wild*": "wild_", + "term>": "term_", + "tab\there": "tab_here", + "line\nbreak": "line_break", + "carriage\rret": "carriage_ret", + "": "_", + } + for in, want := range cases { + if got := sanitize(in); got != want { + t.Errorf("sanitize(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/safetransport/safetransport.go b/internal/safetransport/safetransport.go new file mode 100644 index 0000000..85a6501 --- /dev/null +++ b/internal/safetransport/safetransport.go @@ -0,0 +1,77 @@ +// Package safetransport provides SSRF-hardened HTTP plumbing for outbound +// calls to developer-supplied webhook URLs. A developer can register an +// arbitrary target URL, so both the registration verification probe and the +// production webhook sender must refuse to dial internal network addresses +// (RFC-1918, loopback, link-local incl. the cloud metadata endpoint, etc.). +// +// The guard runs as a net.Dialer.Control hook, which fires with the +// post-DNS-resolution IP on every dial attempt. That placement also defends +// against DNS rebinding: even if a hostname resolves to a public IP at +// registration time and a private IP at delivery time, the private dial is +// refused at connect time rather than at validation time. +package safetransport + +import ( + "fmt" + "net" + "net/http" + "syscall" + "time" +) + +// cgnat is the RFC-6598 carrier-grade NAT range (100.64.0.0/10). net.IP's +// IsPrivate does not cover it, but it is routable only inside provider +// networks, so we treat it as blocked. +var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0), Mask: net.CIDRMask(10, 32)} + +// blocked reports whether ip is in a range we refuse to dial. +func blocked(ip net.IP) bool { + return ip.IsLoopback() || + ip.IsLinkLocalUnicast() || // 169.254.0.0/16 (incl. 169.254.169.254), fe80::/10 + ip.IsLinkLocalMulticast() || + ip.IsMulticast() || + ip.IsUnspecified() || + ip.IsPrivate() || // 10/8, 172.16/12, 192.168/16, fc00::/7 + cgnat.Contains(ip) +} + +// control is the net.Dialer.Control hook enforcing the blocklist. +func control(_, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("safetransport: bad dial address %q: %w", address, err) + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("safetransport: cannot parse dial IP %q", host) + } + if blocked(ip) { + return fmt.Errorf("safetransport: refusing to dial private/loopback address %s", ip) + } + return nil +} + +func dialer() *net.Dialer { + return &net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + Control: control, + } +} + +// Guard installs the SSRF dial guard onto t and returns it. t must not be +// nil. The guard replaces t.DialContext, so callers that need a custom dialer +// should compose around Guard rather than overwrite DialContext afterwards. +func Guard(t *http.Transport) *http.Transport { + t.DialContext = dialer().DialContext + return t +} + +// Client returns an http.Client with an SSRF-guarded transport and the given +// timeout. Used by the registration verification probe. +func Client(timeout time.Duration) *http.Client { + return &http.Client{ + Timeout: timeout, + Transport: Guard(http.DefaultTransport.(*http.Transport).Clone()), + } +} diff --git a/internal/safetransport/safetransport_test.go b/internal/safetransport/safetransport_test.go new file mode 100644 index 0000000..15a9fd3 --- /dev/null +++ b/internal/safetransport/safetransport_test.go @@ -0,0 +1,65 @@ +package safetransport + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestBlockedRanges(t *testing.T) { + t.Parallel() + cases := []struct { + ip string + blocked bool + }{ + {"127.0.0.1", true}, // loopback + {"::1", true}, // loopback v6 + {"169.254.169.254", true}, // cloud metadata (link-local) + {"10.0.0.5", true}, // RFC1918 + {"172.16.0.1", true}, // RFC1918 + {"192.168.1.1", true}, // RFC1918 + {"100.64.0.1", true}, // CGNAT + {"0.0.0.0", true}, // unspecified + {"224.0.0.1", true}, // multicast + {"fc00::1", true}, // ULA + {"8.8.8.8", false}, // public + {"1.1.1.1", false}, // public + } + for _, c := range cases { + ip := net.ParseIP(c.ip) + if ip == nil { + t.Fatalf("bad test IP %q", c.ip) + } + if got := blocked(ip); got != c.blocked { + t.Errorf("blocked(%s) = %v, want %v", c.ip, got, c.blocked) + } + } +} + +// TestGuardRefusesLoopback proves the guarded client refuses to dial a +// loopback server end-to-end (the SSRF control path), not just the unit +// predicate. +func TestGuardRefusesLoopback(t *testing.T) { + t.Parallel() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + client := Client(5 * time.Second) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, srv.URL, nil) + if err != nil { + t.Fatal(err) + } + _, err = client.Do(req) + if err == nil { + t.Fatal("expected guarded client to refuse loopback dial, got nil error") + } + if !strings.Contains(err.Error(), "refusing to dial") { + t.Fatalf("expected SSRF refusal, got: %v", err) + } +} diff --git a/internal/services/auditqueue/auditqueue.go b/internal/services/auditqueue/auditqueue.go new file mode 100644 index 0000000..b6f3bd7 --- /dev/null +++ b/internal/services/auditqueue/auditqueue.go @@ -0,0 +1,152 @@ +// Package auditqueue is a bounded-buffer fire-and-forget queue in front of +// the trigger-fired audit stream. The dispatcher spawns one goroutine per +// fire to call PublishAsync; at 30k fires/sec with a 5s detached context +// that's tens of thousands of goroutines wedged on a slow audit broker. +// This package collapses those goroutines into a single fixed-size queue +// with a tiny drainer pool, trading "audit always sent" for "service never +// stalls when audit is slow." Audit loss is observable via dropped_total. +package auditqueue + +import ( + "context" + "sync" + "time" + + "github.com/rs/zerolog" +) + +// Publisher is the minimal contract the queue needs from the NATS client. +type Publisher interface { + PublishTriggerFired(ctx context.Context, devLicense string, record []byte) error +} + +// Entry is one queued audit publish. The dispatcher hands us already- +// marshaled bodies because re-marshaling under a queue lock would amplify +// the back-pressure risk. +type Entry struct { + DevLicense string + Record []byte +} + +// Config tunes the queue. +type Config struct { + // Workers is the number of drainer goroutines reading from the buffer. + // 2-4 is typically plenty since each publish is a sub-ms async ack. + Workers int + // Buffer is the channel size. Set comfortably above expected steady- + // state fire rate * publish RTT. + Buffer int + // PublishTimeout caps each individual PublishTriggerFired call. + PublishTimeout time.Duration +} + +// Queue is a bounded buffer of audit publishes drained by a small pool. +type Queue struct { + cfg Config + publisher Publisher + log zerolog.Logger + ch chan Entry + wg sync.WaitGroup + stop chan struct{} + once sync.Once +} + +// New builds a queue. Caller must invoke Run to start the drainer pool. +func New(cfg Config, p Publisher, log zerolog.Logger) *Queue { + if cfg.Workers < 1 { + cfg.Workers = 2 + } + if cfg.Buffer < 1 { + cfg.Buffer = 1024 + } + if cfg.PublishTimeout <= 0 { + cfg.PublishTimeout = 5 * time.Second + } + return &Queue{ + cfg: cfg, + publisher: p, + log: log, + ch: make(chan Entry, cfg.Buffer), + stop: make(chan struct{}), + } +} + +// Submit queues an entry. Non-blocking: if the buffer is full the entry is +// dropped and dropped_total ticks. Returns true on accept, false on drop. +func (q *Queue) Submit(e Entry) bool { + select { + case q.ch <- e: + queueDepth.Set(float64(len(q.ch))) + return true + default: + droppedTotal.Inc() + return false + } +} + +// PublishTriggerFired adapts the queue to the webhookdispatcher.AuditPublisher +// interface. Returns nil even on drop: a full audit queue is a known, +// metric-tracked failure mode, not an error the dispatcher should propagate +// back to the JetStream handler. +func (q *Queue) PublishTriggerFired(_ context.Context, devLicense string, record []byte) error { + q.Submit(Entry{DevLicense: devLicense, Record: record}) + return nil +} + +// Run starts the drainer pool. Returns when ctx cancels and the buffer +// drains (best-effort within a 5s grace window). +func (q *Queue) Run(ctx context.Context) error { + for i := 0; i < q.cfg.Workers; i++ { + q.wg.Add(1) + go q.drain(i) + } + <-ctx.Done() + q.once.Do(func() { close(q.stop) }) + q.wg.Wait() + return nil +} + +func (q *Queue) drain(id int) { + defer q.wg.Done() + log := q.log.With().Int("worker", id).Logger() + for { + select { + case <-q.stop: + // Graceful shutdown: flush buffered entries before exiting so + // audit records already enqueued aren't dropped. The dispatcher + // drains before us (see the main shutdown sequence), so its final + // audit Submits land in this buffer in time to be flushed here. + for { + select { + case e, ok := <-q.ch: + if !ok { + return + } + q.publishEntry(&log, e) + default: + return + } + } + case e, ok := <-q.ch: + if !ok { + return + } + q.publishEntry(&log, e) + } + } +} + +// publishEntry performs one bounded audit publish and records its metrics. +func (q *Queue) publishEntry(log *zerolog.Logger, e Entry) { + queueDepth.Set(float64(len(q.ch))) + pubCtx, cancel := context.WithTimeout(context.Background(), q.cfg.PublishTimeout) + defer cancel() + started := time.Now() + if err := q.publisher.PublishTriggerFired(pubCtx, e.DevLicense, e.Record); err != nil { + errorTotal.Inc() + log.Warn().Err(err).Str("devLicense", e.DevLicense).Msg("audit publish failed") + } else { + publishedTotal.Inc() + } + publishBlocked.Observe(time.Since(started).Seconds()) +} diff --git a/internal/services/auditqueue/auditqueue_test.go b/internal/services/auditqueue/auditqueue_test.go new file mode 100644 index 0000000..09c1b70 --- /dev/null +++ b/internal/services/auditqueue/auditqueue_test.go @@ -0,0 +1,114 @@ +package auditqueue + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +type fakePublisher struct { + calls atomic.Uint64 + delay time.Duration + err error +} + +func (f *fakePublisher) PublishTriggerFired(ctx context.Context, _ string, _ []byte) error { + f.calls.Add(1) + if f.delay > 0 { + select { + case <-time.After(f.delay): + case <-ctx.Done(): + return ctx.Err() + } + } + return f.err +} + +func TestQueueDrains(t *testing.T) { + t.Parallel() + p := &fakePublisher{} + q := New(Config{Workers: 2, Buffer: 64, PublishTimeout: time.Second}, p, zerolog.Nop()) + ctx, cancel := context.WithCancel(t.Context()) + go func() { _ = q.Run(ctx) }() + + for i := 0; i < 32; i++ { + require.True(t, q.Submit(Entry{DevLicense: "0xdead", Record: []byte("{}")})) + } + require.Eventually(t, func() bool { return p.calls.Load() == 32 }, 2*time.Second, 10*time.Millisecond) + cancel() +} + +// TestQueueDrainsBufferOnShutdown asserts that cancelling ctx flushes entries +// still sitting in the buffer rather than dropping them. The dispatcher emits +// its final audit records during its own drain, so the audit queue must flush +// what's buffered when it is stopped (after the dispatcher) or those records +// are lost on every deploy. +func TestQueueDrainsBufferOnShutdown(t *testing.T) { + t.Parallel() + // Slow publisher so a backlog builds in the buffer before we cancel. + p := &fakePublisher{delay: 10 * time.Millisecond} + const N = 32 + q := New(Config{Workers: 2, Buffer: N, PublishTimeout: time.Second}, p, zerolog.Nop()) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- q.Run(ctx) }() + + for range N { + require.True(t, q.Submit(Entry{DevLicense: "0xdead", Record: []byte("{}")})) + } + // Cancel while most entries are still buffered; Run must drain them all + // before returning. + require.Eventually(t, func() bool { return p.calls.Load() >= 1 }, time.Second, time.Millisecond) + cancel() + + require.NoError(t, <-done) + require.EqualValues(t, N, p.calls.Load(), "graceful shutdown must flush every buffered audit record") +} + +func TestQueueDropsWhenFull(t *testing.T) { + t.Parallel() + p := &fakePublisher{delay: 200 * time.Millisecond} + q := New(Config{Workers: 1, Buffer: 1, PublishTimeout: time.Second}, p, zerolog.Nop()) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go func() { _ = q.Run(ctx) }() + + // Fill the worker slot + 1 buffer slot, then expect drop. + require.True(t, q.Submit(Entry{})) + require.Eventually(t, func() bool { return p.calls.Load() == 1 }, time.Second, 5*time.Millisecond) + require.True(t, q.Submit(Entry{})) + require.False(t, q.Submit(Entry{}), "should drop on full buffer") +} + +func TestQueuePublishTriggerFiredAdapter(t *testing.T) { + t.Parallel() + p := &fakePublisher{} + q := New(Config{Workers: 1, Buffer: 4, PublishTimeout: time.Second}, p, zerolog.Nop()) + ctx, cancel := context.WithCancel(t.Context()) + go func() { _ = q.Run(ctx) }() + + // AuditPublisher adapter must return nil even on full queue, because + // dispatcher cannot do anything with the error. + require.NoError(t, q.PublishTriggerFired(t.Context(), "0xa", []byte("{}"))) + require.Eventually(t, func() bool { return p.calls.Load() == 1 }, time.Second, 5*time.Millisecond) + cancel() +} + +func TestQueuePublishErrorObservable(t *testing.T) { + t.Parallel() + p := &fakePublisher{err: errors.New("boom")} + q := New(Config{Workers: 1, Buffer: 4, PublishTimeout: time.Second}, p, zerolog.Nop()) + ctx, cancel := context.WithCancel(t.Context()) + go func() { _ = q.Run(ctx) }() + + require.True(t, q.Submit(Entry{})) + require.Eventually(t, func() bool { return p.calls.Load() == 1 }, time.Second, 5*time.Millisecond) + cancel() + // Errors don't surface in Submit return; ops sees them via + // vehicle_triggers_audit_publish_errors_total. +} diff --git a/internal/services/auditqueue/metrics.go b/internal/services/auditqueue/metrics.go new file mode 100644 index 0000000..2812caa --- /dev/null +++ b/internal/services/auditqueue/metrics.go @@ -0,0 +1,48 @@ +package auditqueue + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Audit queue observability. dropped_total is the key SLO indicator - +// dropped audit records are billing miscounts. A non-zero rate means the +// queue is undersized relative to fire volume or the audit publisher is +// stalled. +var ( + queueDepth = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "vehicle_triggers", + Subsystem: "audit", + Name: "queue_depth", + Help: "Current number of audit records waiting in the queue.", + }) + + droppedTotal = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "audit", + Name: "dropped_total", + Help: "Number of audit records dropped because the queue was full. Each drop is a billing miscount; alarm on this.", + }) + + publishedTotal = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "audit", + Name: "published_total", + Help: "Number of audit records successfully published to the audit stream.", + }) + + errorTotal = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "audit", + Name: "publish_errors_total", + Help: "Number of audit publish attempts that returned an error from PublishTriggerFired.", + }) + + publishBlocked = promauto.NewHistogram(prometheus.HistogramOpts{ + Namespace: "vehicle_triggers", + Subsystem: "audit", + Name: "publish_blocked_seconds", + Help: "Time the audit publisher's call to PublishTriggerFired spent blocked. High values mean JetStream-side back-pressure is now upstream of our queue (max_pending exhausted).", + Buckets: []float64{0.001, 0.005, 0.01, 0.025, 0.1, 0.5, 1, 5}, + }) +) diff --git a/internal/services/cachebroadcast/cachebroadcast.go b/internal/services/cachebroadcast/cachebroadcast.go new file mode 100644 index 0000000..df12619 --- /dev/null +++ b/internal/services/cachebroadcast/cachebroadcast.go @@ -0,0 +1,129 @@ +// Package cachebroadcast carries webhook-config change notifications between +// vehicle-triggers-api replicas over NATS so the in-memory webhook cache can +// invalidate within milliseconds instead of waiting for the 5-minute poll. +// +// Topology: every CRUD endpoint in the webhook controller calls Notifier. +// Notify(webhookID, op) after a successful DB write. The notifier publishes +// a small JSON record to a single plain NATS subject (no JetStream - this +// is a fan-out cache invalidation, not durable audit). Every replica +// subscribes with no queue group, so every replica receives every event and +// calls webhookCache.ScheduleRefresh. Debouncing inside the cache collapses +// bursty CRUD into one rebuild. +// +// A missed notification (transient NATS outage) is bounded by the existing +// periodic poll, which now runs much less frequently (default 5 min). The +// poll is the reconciliation safety net. +package cachebroadcast + +import ( + "context" + "encoding/json" + "fmt" + "time" + + nc "github.com/nats-io/nats.go" + "github.com/rs/zerolog" +) + +// Subject is the NATS subject used for cache-change notifications. Single +// subject because we want every replica to receive every event - no need to +// shard by webhook ID. +const Subject = "dimo.cache.webhook.changed" + +// Op describes the kind of change so subscribers could filter if needed; the +// current implementation triggers the same full rebuild regardless. +type Op string + +const ( + OpCreate Op = "create" + OpUpdate Op = "update" + OpDelete Op = "delete" +) + +// Notification is the payload published on every change. +type Notification struct { + WebhookID string `json:"webhookId"` + Op Op `json:"op"` + At time.Time `json:"at"` +} + +// Notifier publishes change events. Implementations are safe for concurrent +// use from API handlers. The Op argument is a string to keep the interface +// tiny - callers pass an Op typed constant, the implementation just emits. +type Notifier interface { + Notify(ctx context.Context, webhookID string, op string) error +} + +// NoopNotifier is used when no NATS connection is configured; CRUD handlers +// can call it unconditionally without checking nil. +type NoopNotifier struct{} + +func (NoopNotifier) Notify(context.Context, string, string) error { return nil } + +// NATSNotifier publishes to NATS. Best-effort: a publish failure is logged +// and discarded - subscribers will eventually reconcile via the periodic +// poll on the receiving replicas. +type NATSNotifier struct { + conn *nc.Conn + log zerolog.Logger +} + +// NewNATSNotifier wraps an existing connection. +func NewNATSNotifier(conn *nc.Conn, log zerolog.Logger) *NATSNotifier { + return &NATSNotifier{conn: conn, log: log} +} + +// Notify publishes the change event. Returns the underlying NATS error so +// the caller can surface it in tests; production callers should log + ignore. +func (n *NATSNotifier) Notify(_ context.Context, webhookID string, op string) error { + body, err := json.Marshal(Notification{ + WebhookID: webhookID, + Op: Op(op), + At: time.Now().UTC(), + }) + if err != nil { + return fmt.Errorf("cachebroadcast marshal: %w", err) + } + if err := n.conn.Publish(Subject, body); err != nil { + n.log.Warn().Err(err).Str("webhookId", webhookID).Msg("cache invalidate publish failed") + return fmt.Errorf("cachebroadcast publish: %w", err) + } + return nil +} + +// Refresher is the callback Subscriber invokes on each received notification. +// InvalidateTrigger drops the compiled program for one specific trigger so +// the next refresh recompiles it; ScheduleRefreshSilent kicks the refresh +// without re-broadcasting. Together they implement diff-rebuild on the +// receive side: a CRUD on one trigger touches only that trigger's compiled +// program in every replica's cache, not every program. +type Refresher interface { + InvalidateTrigger(triggerID string) + ScheduleRefreshSilent(ctx context.Context) +} + +// Subscribe registers a long-lived subscription on Subject. The returned +// *nc.Subscription must be unsubscribed during shutdown. Each received +// notification with a non-empty WebhookID invalidates just that trigger's +// compiled program before scheduling the rebuild. Notifications with an +// empty WebhookID (legacy "refresh all") fall back to a full rebuild. +// Decode errors are logged and the message is dropped - schema mismatches +// shouldn't break the cache. +func Subscribe(conn *nc.Conn, ctx context.Context, r Refresher, log zerolog.Logger) (*nc.Subscription, error) { + sub, err := conn.Subscribe(Subject, func(msg *nc.Msg) { + var n Notification + if err := json.Unmarshal(msg.Data, &n); err != nil { + log.Warn().Err(err).Msg("cache invalidate: decode failed") + return + } + log.Debug().Str("webhookId", n.WebhookID).Str("op", string(n.Op)).Msg("cache invalidate received") + if n.WebhookID != "" { + r.InvalidateTrigger(n.WebhookID) + } + r.ScheduleRefreshSilent(ctx) + }) + if err != nil { + return nil, fmt.Errorf("cachebroadcast subscribe: %w", err) + } + return sub, nil +} diff --git a/internal/services/configaudit/configaudit.go b/internal/services/configaudit/configaudit.go new file mode 100644 index 0000000..378beb5 --- /dev/null +++ b/internal/services/configaudit/configaudit.go @@ -0,0 +1,111 @@ +// Package configaudit publishes an immutable record of every webhook-config +// CRUD to JetStream. The audit stream (DIMO_CONFIG_AUDIT) lives separately +// from the trigger-fired audit so config changes don't share retention or +// throughput characteristics with hot-path fires. Receivers (compliance, +// ops dashboards, change-management tooling) consume the stream however +// they like; the service only emits. +// +// Publishes are best-effort: a failure is logged and discarded. The DB row +// is the source of truth; the audit stream is a historical record. If NATS +// is unavailable, the API request still succeeds. +package configaudit + +import ( + "context" + "encoding/json" + "fmt" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" +) + +// Subject is the JetStream subject root. Keyed on webhookID so consumers +// can filter to a single webhook's history without server-side replay. +const SubjectPrefix = "dimo.config.changed" + +// Op is the lifecycle event type. +type Op string + +const ( + OpWebhookCreate Op = "webhook.create" + OpWebhookUpdate Op = "webhook.update" + OpWebhookDelete Op = "webhook.delete" + OpSubscribeVehicle Op = "subscription.create" + OpUnsubscribeVehicle Op = "subscription.delete" +) + +// Event is the JSON payload. +type Event struct { + Op Op `json:"op"` + At time.Time `json:"at"` + WebhookID string `json:"webhookId,omitempty"` + DevLicense string `json:"devLicense,omitempty"` + AssetDID string `json:"assetDid,omitempty"` + Snapshot map[string]any `json:"snapshot,omitempty"` +} + +// Publisher is the interface API handlers depend on. Noop is used when NATS +// is not configured. +type Publisher interface { + Publish(ctx context.Context, e Event) error +} + +// Noop discards events; safe to use as a default so handlers don't need +// nil-checks. +type Noop struct{} + +func (Noop) Publish(context.Context, Event) error { return nil } + +// NATSPublisher writes events to the audit JetStream stream. +type NATSPublisher struct { + client *nats.Client +} + +// New wraps a NATS client; nil client returns a Noop. +func New(client *nats.Client) Publisher { + if client == nil { + return Noop{} + } + return &NATSPublisher{client: client} +} + +// Publish emits the event. The subject embeds the webhook ID when present +// so consumers can filter to a specific webhook history; events without a +// webhook ID (e.g. global config changes, not currently emitted) land on a +// catch-all subject. +func (p *NATSPublisher) Publish(ctx context.Context, e Event) error { + if e.At.IsZero() { + e.At = time.Now().UTC() + } + body, err := json.Marshal(e) + if err != nil { + return fmt.Errorf("configaudit marshal: %w", err) + } + subject := SubjectPrefix + "._unknown" + if e.WebhookID != "" { + subject = SubjectPrefix + "." + sanitize(e.WebhookID) + } + if _, err := p.client.Publish(ctx, subject, body); err != nil { + return fmt.Errorf("configaudit publish %q: %w", subject, err) + } + return nil +} + +// sanitize matches the NATS subject sanitization elsewhere in the service. +// We don't import internal/nats's helper because it's package-private; the +// rules are tiny enough to duplicate. +func sanitize(s string) string { + out := make([]byte, 0, len(s)) + for _, c := range []byte(s) { + switch c { + case ' ', '.', '*', '>', '\t', '\n', '\r': + out = append(out, '_') + default: + out = append(out, c) + } + } + if len(out) == 0 { + return "_" + } + return string(out) +} diff --git a/internal/services/configaudit/configaudit_test.go b/internal/services/configaudit/configaudit_test.go new file mode 100644 index 0000000..6a82004 --- /dev/null +++ b/internal/services/configaudit/configaudit_test.go @@ -0,0 +1,35 @@ +package configaudit + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNoopPublishSucceeds(t *testing.T) { + t.Parallel() + require.NoError(t, Noop{}.Publish(context.Background(), Event{Op: OpWebhookCreate})) +} + +func TestSanitize(t *testing.T) { + t.Parallel() + cases := map[string]string{ + "abc-123": "abc-123", + "a.b": "a_b", + "with*": "with_", + "angle>": "angle_", + "line\nend": "line_end", + "": "_", + } + for in, want := range cases { + require.Equal(t, want, sanitize(in), "sanitize(%q)", in) + } +} + +func TestNewReturnsNoopOnNilClient(t *testing.T) { + t.Parallel() + p := New(nil) + _, ok := p.(Noop) + require.True(t, ok, "nil client should produce a Noop publisher") +} diff --git a/internal/services/secrets/secrets.go b/internal/services/secrets/secrets.go new file mode 100644 index 0000000..31933c8 --- /dev/null +++ b/internal/services/secrets/secrets.go @@ -0,0 +1,100 @@ +// Package secrets is a tiny encryption seam for the per-trigger HMAC +// signing secret. It defines a Cipher interface so the repository can write +// ciphertext to Postgres instead of plaintext, and an implementation backed +// by AES-256-GCM with a key supplied by the operator via env / KMS. +// +// We do not depend on AWS KMS directly because: +// - The service is cloud-portable; tying to KMS would force operators on +// other clouds to swap the package out. +// - A single AES-GCM key from env works for the deploy size we ship. +// Operators who need rotation can layer in their own KMS data-key +// wrapping on top by implementing this interface in a custom package. +// +// The default Cipher when no key is configured is Plaintext, which writes +// and reads the raw secret. That preserves backwards compatibility with the +// existing schema while letting deployments opt in to encryption by setting +// SIGNING_SECRET_KEY_HEX to a 64-char hex string (32 bytes). +package secrets + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" +) + +// Cipher encrypts/decrypts the per-trigger signing secret at the storage +// boundary. Encrypt returns a value safe to persist; Decrypt returns the +// original. Both must be idempotent over the round-trip: +// Decrypt(Encrypt(x)) == x. +type Cipher interface { + Encrypt(plaintext string) (string, error) + Decrypt(stored string) (string, error) +} + +// Plaintext is the no-op cipher used when SIGNING_SECRET_KEY_HEX is unset. +// Existing rows continue to round-trip unchanged. +type Plaintext struct{} + +func (Plaintext) Encrypt(s string) (string, error) { return s, nil } +func (Plaintext) Decrypt(s string) (string, error) { return s, nil } + +// AESGCM encrypts each secret with a fresh 96-bit nonce, prefixed to the +// ciphertext: storage layout is base16(nonce || ciphertext). The output is +// distinguishable from plaintext by length (32 hex prefix for nonce) and by +// the leading byte being valid hex, so we don't need a magic prefix. +type AESGCM struct { + aead cipher.AEAD +} + +// NewAESGCM builds an AES-256-GCM cipher from a 32-byte key. Pass the key +// as raw bytes - the caller is responsible for fetching it from env/KMS and +// hex-decoding if needed. +func NewAESGCM(key []byte) (*AESGCM, error) { + if len(key) != 32 { + return nil, fmt.Errorf("secrets: AES-256-GCM requires a 32-byte key, got %d", len(key)) + } + block, err := aes.NewCipher(key) + if err != nil { + return nil, fmt.Errorf("secrets: aes.NewCipher: %w", err) + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("secrets: cipher.NewGCM: %w", err) + } + return &AESGCM{aead: aead}, nil +} + +// Encrypt produces a hex-encoded nonce||ciphertext blob. +func (c *AESGCM) Encrypt(plaintext string) (string, error) { + nonce := make([]byte, c.aead.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return "", fmt.Errorf("secrets: nonce: %w", err) + } + out := c.aead.Seal(nonce, nonce, []byte(plaintext), nil) + return hex.EncodeToString(out), nil +} + +// Decrypt accepts either a hex-encoded encrypted blob or, for backwards +// compatibility with rows persisted before encryption was enabled, a raw +// plaintext secret. Heuristic: if the input hex-decodes AND the decoded +// length is at least nonce + 16 (auth tag), treat as encrypted; otherwise +// return as-is. +func (c *AESGCM) Decrypt(stored string) (string, error) { + raw, err := hex.DecodeString(stored) + if err != nil || len(raw) < c.aead.NonceSize()+16 { + // Doesn't look like our encrypted format. Treat as legacy plaintext. + return stored, nil + } + nonce, ct := raw[:c.aead.NonceSize()], raw[c.aead.NonceSize():] + pt, err := c.aead.Open(nil, nonce, ct, nil) + if err != nil { + // Looked like ciphertext but auth failed. Could be legacy plaintext + // that happens to hex-decode; surface a clear error rather than + // silently mishandling. + return "", errors.New("secrets: decrypt failed (auth tag mismatch); is SIGNING_SECRET_KEY_HEX wrong?") + } + return string(pt), nil +} diff --git a/internal/services/secrets/secrets_test.go b/internal/services/secrets/secrets_test.go new file mode 100644 index 0000000..b3dbdfb --- /dev/null +++ b/internal/services/secrets/secrets_test.go @@ -0,0 +1,73 @@ +package secrets + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPlaintextRoundtrip(t *testing.T) { + t.Parallel() + c := Plaintext{} + ct, err := c.Encrypt("hello") + require.NoError(t, err) + require.Equal(t, "hello", ct) + pt, err := c.Decrypt(ct) + require.NoError(t, err) + require.Equal(t, "hello", pt) +} + +func TestAESGCMRequiresCorrectKeySize(t *testing.T) { + t.Parallel() + _, err := NewAESGCM(make([]byte, 16)) + require.Error(t, err) +} + +func TestAESGCMRoundtrip(t *testing.T) { + t.Parallel() + key := make([]byte, 32) + _, err := rand.Read(key) + require.NoError(t, err) + c, err := NewAESGCM(key) + require.NoError(t, err) + + secret := "the rain in spain" + ct, err := c.Encrypt(secret) + require.NoError(t, err) + require.NotEqual(t, secret, ct, "ciphertext must differ from plaintext") + + pt, err := c.Decrypt(ct) + require.NoError(t, err) + require.Equal(t, secret, pt) +} + +func TestAESGCMDecryptLegacyPlaintext(t *testing.T) { + t.Parallel() + key := make([]byte, 32) + _, err := rand.Read(key) + require.NoError(t, err) + c, err := NewAESGCM(key) + require.NoError(t, err) + + // A raw plaintext secret from before encryption was enabled doesn't + // look like our nonce||ciphertext format. We pass it through unchanged. + pt, err := c.Decrypt("plain-old-secret") + require.NoError(t, err) + require.Equal(t, "plain-old-secret", pt) +} + +func TestAESGCMDecryptWrongKey(t *testing.T) { + t.Parallel() + k1 := make([]byte, 32) + k2 := make([]byte, 32) + _, _ = rand.Read(k1) + _, _ = rand.Read(k2) + c1, _ := NewAESGCM(k1) + c2, _ := NewAESGCM(k2) + + ct, err := c1.Encrypt("secret") + require.NoError(t, err) + _, err = c2.Decrypt(ct) + require.ErrorContains(t, err, "auth tag mismatch") +} diff --git a/internal/services/triggerevaluator/trigger_evaluator.go b/internal/services/triggerevaluator/trigger_evaluator.go index d2cb7c1..5ae300d 100644 --- a/internal/services/triggerevaluator/trigger_evaluator.go +++ b/internal/services/triggerevaluator/trigger_evaluator.go @@ -2,9 +2,7 @@ package triggerevaluator import ( "context" - "database/sql" "encoding/json" - "errors" "net/http" "time" @@ -13,16 +11,13 @@ import ( "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/celcondition" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" "github.com/ethereum/go-ethereum/common" "github.com/google/cel-go/cel" + "github.com/rs/zerolog" ) -type TriggerRepo interface { - GetLastLogValue(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (*models.TriggerLog, error) - GetLastLogForMetric(ctx context.Context, assetDid cloudevent.ERC721DID, metricName string) (*models.TriggerLog, error) -} - // SignalEvaluationData is a struct that contains the data needed to evaluate a signal trigger. type SignalEvaluationData struct { Signal vss.Signal @@ -38,10 +33,21 @@ type EventEvaluationData struct { RawData json.RawMessage } +// StateStore is the evaluator's read-only view of the NATS KV state buckets. +// LastFire drives the cooldown check and the per-trigger previousEvent input; +// LastMetric drives the cross-trigger previousValue input on the signal path. +// A nil store, a miss, or a transport error all degrade to "no prior data" +// so evaluation never blocks on NATS - the cost is one redundant fire when +// state is unavailable, which is acceptable given at-least-once delivery. +type StateStore interface { + LastFire(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (triggerstate.Record, bool, error) + LastMetric(ctx context.Context, vehicleDID cloudevent.ERC721DID, metricName string) (triggerstate.MetricRecord, bool, error) +} + // TriggerEvaluator handles trigger condition evaluation and related logic type TriggerEvaluator struct { - repo TriggerRepo tokenClient TokenExchangeClient + state StateStore } type TriggerEvaluationResult struct { @@ -56,14 +62,23 @@ type TokenExchangeClient interface { HasVehiclePermissions(ctx context.Context, vehicleDID cloudevent.ERC721DID, developerLicense common.Address, permissions []string) (bool, error) } -// NewTriggerEvaluator creates a new TriggerEvaluator -func NewTriggerEvaluator(r TriggerRepo, t TokenExchangeClient) *TriggerEvaluator { - return &TriggerEvaluator{repo: r, tokenClient: t} +// NewTriggerEvaluator creates a new TriggerEvaluator. State is required for +// production wiring; tests can pass nil to exercise the zero-previous-value +// path. The DB-backed previousValue / cooldown fallback was removed - all +// state lives in NATS now (see internal/services/triggerstate). +func NewTriggerEvaluator(t TokenExchangeClient) *TriggerEvaluator { + return &TriggerEvaluator{tokenClient: t} +} + +// WithStateStore returns a copy of the evaluator wired to a state store. +func (t *TriggerEvaluator) WithStateStore(s StateStore) *TriggerEvaluator { + cp := *t + cp.state = s + return &cp } -// EvaluateSignalTrigger evaluates a signal trigger and returns whether it should fire return true if it should fire, false if not. +// EvaluateSignalTrigger evaluates a signal trigger and returns whether it should fire. func (t *TriggerEvaluator) EvaluateSignalTrigger(ctx context.Context, trigger *models.Trigger, program cel.Program, signal *SignalEvaluationData) (*TriggerEvaluationResult, error) { - // Check permissions first hasPerm, err := t.tokenClient.HasVehiclePermissions(ctx, signal.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signal.Def.Permissions) if err != nil { return nil, richerrors.Error{ @@ -79,18 +94,8 @@ func (t *TriggerEvaluator) EvaluateSignalTrigger(ctx context.Context, trigger *m }, nil } - // Get last trigger log for cooldown (per-trigger) - lastTrigger, err := t.getLastLogValue(ctx, trigger.ID, signal.VehicleDID) - if err != nil { - return nil, richerrors.Error{ - Code: http.StatusInternalServerError, - Err: err, - ExternalMsg: "failed to retrieve trigger logs for signal trigger", - } - } - - // Check cooldown - cooldownPassed, err := t.checkCooldown(trigger, lastTrigger.LastTriggeredAt) + lastFired := t.lookupLastFireTime(ctx, trigger.ID, signal.VehicleDID) + cooldownPassed, err := t.checkCooldown(trigger, lastFired) if err != nil { return nil, richerrors.Error{ Code: http.StatusInternalServerError, @@ -105,26 +110,11 @@ func (t *TriggerEvaluator) EvaluateSignalTrigger(ctx context.Context, trigger *m }, nil } - // Previous signal for condition: use last log for this (vehicle, metric) so transition-to-zero - // conditions see the prior value (e.g. 1) from when another trigger fired, not only this trigger's log. - lastLogForMetric, err := t.repo.GetLastLogForMetric(ctx, signal.VehicleDID, trigger.MetricName) - if err != nil { - return nil, richerrors.Error{ - Code: http.StatusInternalServerError, - Err: err, - ExternalMsg: "failed to retrieve last signal for metric", - } - } - var previousSignal vss.Signal - if lastLogForMetric != nil { - if err := json.Unmarshal(lastLogForMetric.SnapshotData, &previousSignal); err != nil { - return nil, richerrors.Error{ - Code: http.StatusInternalServerError, - Err: err, - ExternalMsg: "failed to unmarshal previous signal", - } - } - } + // Previous signal for CEL transition conditions: the snapshot from the + // most recent fire of any trigger on this (vehicle, metric). Comes from + // the signal_history KV bucket. Miss = zero-valued previousSignal, same + // as the first-time-fire case. + previousSignal := t.lookupPreviousSignal(ctx, signal.VehicleDID, trigger.MetricName) conditionMet, err := celcondition.EvaluateSignalCondition(program, &signal.Signal, &previousSignal, signal.Def.ValueType) if err != nil { @@ -146,10 +136,8 @@ func (t *TriggerEvaluator) EvaluateSignalTrigger(ctx context.Context, trigger *m }, nil } -// EvaluateEventTrigger evaluates an event trigger and returns whether it should fire -// Returns: shouldFire, permissionDenied, cooldownActive, error +// EvaluateEventTrigger evaluates an event trigger and returns whether it should fire. func (t *TriggerEvaluator) EvaluateEventTrigger(ctx context.Context, trigger *models.Trigger, program cel.Program, ev *EventEvaluationData) (*TriggerEvaluationResult, error) { - // Check permissions for events (use standard permissions) hasPerm, err := t.tokenClient.HasVehiclePermissions(ctx, ev.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signals.DefaultPermissions) if err != nil { return nil, richerrors.Error{ @@ -165,18 +153,11 @@ func (t *TriggerEvaluator) EvaluateEventTrigger(ctx context.Context, trigger *mo }, nil } - // Get last trigger log for cooldown and condition evaluation - lastTrigger, err := t.getLastLogValue(ctx, trigger.ID, ev.VehicleDID) - if err != nil { - return nil, richerrors.Error{ - Code: http.StatusInternalServerError, - Err: err, - ExternalMsg: "failed to retrieve trigger logs for event trigger", - } - } + // trigger_state stores both cooldown timestamp and the prior fire's + // payload, so one KV read covers both event-side needs. + lastFired, previousEvent := t.lookupPreviousEvent(ctx, trigger.ID, ev.VehicleDID) - // Check cooldown - cooldownPassed, err := t.checkCooldown(trigger, lastTrigger.LastTriggeredAt) + cooldownPassed, err := t.checkCooldown(trigger, lastFired) if err != nil { return nil, richerrors.Error{ Code: http.StatusInternalServerError, @@ -191,16 +172,6 @@ func (t *TriggerEvaluator) EvaluateEventTrigger(ctx context.Context, trigger *mo }, nil } - // Evaluate condition - var previousEvent vss.Event - if err := json.Unmarshal(lastTrigger.SnapshotData, &previousEvent); err != nil { - return nil, richerrors.Error{ - Code: http.StatusInternalServerError, - Err: err, - ExternalMsg: "failed to unmarshal previous event", - } - } - conditionMet, err := celcondition.EvaluateEventCondition(program, &ev.Event, &previousEvent) if err != nil { return nil, richerrors.Error{ @@ -221,7 +192,7 @@ func (t *TriggerEvaluator) EvaluateEventTrigger(ctx context.Context, trigger *mo }, nil } -// checkCooldown checks if the cooldown period has passed since the last trigger +// checkCooldown checks if the cooldown period has passed since the last trigger. func (e *TriggerEvaluator) checkCooldown(t *models.Trigger, lastTriggeredAt time.Time) (bool, error) { if lastTriggeredAt.IsZero() { return true, nil @@ -230,19 +201,61 @@ func (e *TriggerEvaluator) checkCooldown(t *models.Trigger, lastTriggeredAt time return time.Since(lastTriggeredAt) >= cooldown, nil } -// getLastLogValue retrieves the last trigger log for a given trigger and vehicle -func (t *TriggerEvaluator) getLastLogValue(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (*models.TriggerLog, error) { - lastTrigger, err := t.repo.GetLastLogValue(ctx, triggerID, assetDid) - if err != nil { - // If no previous log exists, create a default one - if errors.Is(err, sql.ErrNoRows) { - return &models.TriggerLog{ - SnapshotData: []byte("{}"), - AssetDid: assetDid.String(), - TriggerID: triggerID, - }, nil +// lookupLastFireTime returns the last fire timestamp from the trigger_state +// KV bucket. Errors and misses both return zero time so evaluation proceeds +// (matching first-time behavior). +func (t *TriggerEvaluator) lookupLastFireTime(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) time.Time { + if t.state == nil { + return time.Time{} + } + rec, ok, _ := t.state.LastFire(ctx, triggerID, vehicleDID) + if !ok { + return time.Time{} + } + return rec.LastFiredAt +} + +// lookupPreviousSignal returns the most recent signal payload across any +// trigger for this (vehicle, metric). Used as the previousValue input for +// transition CEL conditions like `valueNumber != previousValue`. Errors and +// misses return a zero-valued Signal; decode errors bump a metric so silent +// corruption is observable. +func (t *TriggerEvaluator) lookupPreviousSignal(ctx context.Context, vehicleDID cloudevent.ERC721DID, metricName string) vss.Signal { + if t.state == nil { + return vss.Signal{} + } + rec, ok, _ := t.state.LastMetric(ctx, vehicleDID, metricName) + if !ok || len(rec.LastSnapshot) == 0 { + return vss.Signal{} + } + var sig vss.Signal + if err := json.Unmarshal(rec.LastSnapshot, &sig); err != nil { + if preview := triggerstate.MetricsDecodeErrorWithSample("signal_history", rec.LastSnapshot, 100); preview != "" { + zerolog.Ctx(ctx).Warn().Err(err).Str("bucket", "signal_history").Str("preview", preview).Msg("kv decode error sample") + } + return vss.Signal{} + } + return sig +} + +// lookupPreviousEvent returns both the cooldown timestamp and the snapshot of +// the most recent fire of this trigger on this vehicle in one KV round-trip. +// Errors and misses return zero values; decode errors bump a metric. +func (t *TriggerEvaluator) lookupPreviousEvent(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (time.Time, vss.Event) { + if t.state == nil { + return time.Time{}, vss.Event{} + } + rec, ok, _ := t.state.LastFire(ctx, triggerID, vehicleDID) + if !ok { + return time.Time{}, vss.Event{} + } + var ev vss.Event + if len(rec.LastSnapshot) > 0 { + if err := json.Unmarshal(rec.LastSnapshot, &ev); err != nil { + if preview := triggerstate.MetricsDecodeErrorWithSample("trigger_state", rec.LastSnapshot, 100); preview != "" { + zerolog.Ctx(ctx).Warn().Err(err).Str("bucket", "trigger_state").Str("preview", preview).Msg("kv decode error sample") + } } - return nil, err } - return lastTrigger, nil + return rec.LastFiredAt, ev } diff --git a/internal/services/triggerevaluator/trigger_evaluator_mock_test.go b/internal/services/triggerevaluator/trigger_evaluator_mock_test.go index 7a4b42b..21a7494 100644 --- a/internal/services/triggerevaluator/trigger_evaluator_mock_test.go +++ b/internal/services/triggerevaluator/trigger_evaluator_mock_test.go @@ -14,63 +14,65 @@ import ( reflect "reflect" cloudevent "github.com/DIMO-Network/cloudevent" - models "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + triggerstate "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" common "github.com/ethereum/go-ethereum/common" gomock "go.uber.org/mock/gomock" ) -// MockTriggerRepo is a mock of TriggerRepo interface. -type MockTriggerRepo struct { +// MockStateStore is a mock of StateStore interface. +type MockStateStore struct { ctrl *gomock.Controller - recorder *MockTriggerRepoMockRecorder + recorder *MockStateStoreMockRecorder isgomock struct{} } -// MockTriggerRepoMockRecorder is the mock recorder for MockTriggerRepo. -type MockTriggerRepoMockRecorder struct { - mock *MockTriggerRepo +// MockStateStoreMockRecorder is the mock recorder for MockStateStore. +type MockStateStoreMockRecorder struct { + mock *MockStateStore } -// NewMockTriggerRepo creates a new mock instance. -func NewMockTriggerRepo(ctrl *gomock.Controller) *MockTriggerRepo { - mock := &MockTriggerRepo{ctrl: ctrl} - mock.recorder = &MockTriggerRepoMockRecorder{mock} +// NewMockStateStore creates a new mock instance. +func NewMockStateStore(ctrl *gomock.Controller) *MockStateStore { + mock := &MockStateStore{ctrl: ctrl} + mock.recorder = &MockStateStoreMockRecorder{mock} return mock } // EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockTriggerRepo) EXPECT() *MockTriggerRepoMockRecorder { +func (m *MockStateStore) EXPECT() *MockStateStoreMockRecorder { return m.recorder } -// GetLastLogForMetric mocks base method. -func (m *MockTriggerRepo) GetLastLogForMetric(ctx context.Context, assetDid cloudevent.ERC721DID, metricName string) (*models.TriggerLog, error) { +// LastFire mocks base method. +func (m *MockStateStore) LastFire(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (triggerstate.Record, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLastLogForMetric", ctx, assetDid, metricName) - ret0, _ := ret[0].(*models.TriggerLog) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "LastFire", ctx, triggerID, vehicleDID) + ret0, _ := ret[0].(triggerstate.Record) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 } -// GetLastLogForMetric indicates an expected call of GetLastLogForMetric. -func (mr *MockTriggerRepoMockRecorder) GetLastLogForMetric(ctx, assetDid, metricName any) *gomock.Call { +// LastFire indicates an expected call of LastFire. +func (mr *MockStateStoreMockRecorder) LastFire(ctx, triggerID, vehicleDID any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastLogForMetric", reflect.TypeOf((*MockTriggerRepo)(nil).GetLastLogForMetric), ctx, assetDid, metricName) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LastFire", reflect.TypeOf((*MockStateStore)(nil).LastFire), ctx, triggerID, vehicleDID) } -// GetLastLogValue mocks base method. -func (m *MockTriggerRepo) GetLastLogValue(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (*models.TriggerLog, error) { +// LastMetric mocks base method. +func (m *MockStateStore) LastMetric(ctx context.Context, vehicleDID cloudevent.ERC721DID, metricName string) (triggerstate.MetricRecord, bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetLastLogValue", ctx, triggerID, assetDid) - ret0, _ := ret[0].(*models.TriggerLog) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "LastMetric", ctx, vehicleDID, metricName) + ret0, _ := ret[0].(triggerstate.MetricRecord) + ret1, _ := ret[1].(bool) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 } -// GetLastLogValue indicates an expected call of GetLastLogValue. -func (mr *MockTriggerRepoMockRecorder) GetLastLogValue(ctx, triggerID, assetDid any) *gomock.Call { +// LastMetric indicates an expected call of LastMetric. +func (mr *MockStateStoreMockRecorder) LastMetric(ctx, vehicleDID, metricName any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetLastLogValue", reflect.TypeOf((*MockTriggerRepo)(nil).GetLastLogValue), ctx, triggerID, assetDid) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LastMetric", reflect.TypeOf((*MockStateStore)(nil).LastMetric), ctx, vehicleDID, metricName) } // MockTokenExchangeClient is a mock of TokenExchangeClient interface. diff --git a/internal/services/triggerevaluator/trigger_evaluator_test.go b/internal/services/triggerevaluator/trigger_evaluator_test.go index 514824c..1fda4ee 100644 --- a/internal/services/triggerevaluator/trigger_evaluator_test.go +++ b/internal/services/triggerevaluator/trigger_evaluator_test.go @@ -4,7 +4,6 @@ package triggerevaluator import ( "context" - "database/sql" "encoding/json" "errors" "math/big" @@ -17,6 +16,7 @@ import ( "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/celcondition" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" "github.com/DIMO-Network/vehicle-triggers-api/internal/signals" "github.com/ethereum/go-ethereum/common" "github.com/stretchr/testify/assert" @@ -24,21 +24,26 @@ import ( "go.uber.org/mock/gomock" ) -func TestNewTriggerEvaluator(t *testing.T) { - t.Parallel() - - t.Run("creates evaluator with dependencies", func(t *testing.T) { - ctrl := gomock.NewController(t) +// noState satisfies StateStore with no records - used by tests that exercise +// permission paths where state should not be consulted. +type noState struct{} - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) +func (noState) LastFire(context.Context, string, cloudevent.ERC721DID) (triggerstate.Record, bool, error) { + return triggerstate.Record{}, false, nil +} - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) +func (noState) LastMetric(context.Context, cloudevent.ERC721DID, string) (triggerstate.MetricRecord, bool, error) { + return triggerstate.MetricRecord{}, false, nil +} - require.NotNil(t, evaluator) - assert.Equal(t, mockRepo, evaluator.repo) - assert.Equal(t, mockTokenClient, evaluator.tokenClient) - }) +func TestNewTriggerEvaluator(t *testing.T) { + t.Parallel() + ctrl := gomock.NewController(t) + mockTokenClient := NewMockTokenExchangeClient(ctrl) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(noState{}) + require.NotNil(t, evaluator) + assert.Equal(t, mockTokenClient, evaluator.tokenClient) + assert.NotNil(t, evaluator.state) } func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { @@ -46,10 +51,9 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { t.Run("successful evaluation - should fire", func(t *testing.T) { ctrl := gomock.NewController(t) - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() @@ -57,35 +61,19 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - success mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromSignal(t, vss.Signal{ - Data: vss.SignalData{ - Timestamp: signalData.Signal.Data.Timestamp.Add(-time.Hour), // previous value 1 hour ago - ValueNumber: 59, - }, - }), - AssetDid: signalData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-time.Hour), // Cooldown passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(lastLog, nil). - Times(1) - mockRepo.EXPECT(). - GetLastLogForMetric(ctx, signalData.VehicleDID, trigger.MetricName). - Return(lastLog, nil). - Times(1) + Return(true, nil) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, signalData.VehicleDID). + Return(triggerstate.Record{LastFiredAt: time.Now().Add(-time.Hour)}, true, nil) + // previousValue source - any trigger fired previously with value 59 + prev := snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: 59}}) + mockState.EXPECT(). + LastMetric(ctx, signalData.VehicleDID, trigger.MetricName). + Return(triggerstate.MetricRecord{LastSnapshot: prev}, true, nil) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) require.NotNil(t, result) assert.True(t, result.ShouldFire) @@ -97,10 +85,8 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { t.Run("permission denied", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(noState{}) ctx := context.Background() trigger := createTestTrigger() @@ -108,29 +94,22 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - denied mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(false, nil). - Times(1) + Return(false, nil) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) require.NotNil(t, result) assert.False(t, result.ShouldFire) assert.True(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) }) t.Run("permission check error", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(noState{}) ctx := context.Background() trigger := createTestTrigger() @@ -138,14 +117,11 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - error mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(false, errors.New("permission service error")). - Times(1) + Return(false, errors.New("permission service error")) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.Error(t, err) assert.Nil(t, result) richErr, ok := richerrors.AsRichError(err) @@ -157,58 +133,38 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { t.Run("cooldown not met", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() - trigger.CooldownPeriod = int(time.Hour.Seconds()) // 1 hour cooldown + trigger.CooldownPeriod = int(time.Hour.Seconds()) signalData := createTestSignalData() program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - success mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - recent trigger (cooldown not passed; GetLastLogForMetric not called) - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromSignal(t, vss.Signal{ - Data: vss.SignalData{ - Timestamp: signalData.Signal.Data.Timestamp.Add(-time.Hour), // previous value 1 hour ago - ValueNumber: 59, - }, - }), - AssetDid: signalData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-30 * time.Minute), // Cooldown not passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(lastLog, nil). - Times(1) + Return(true, nil) + // 30 minutes ago - cooldown of 1 hour not met + mockState.EXPECT(). + LastFire(ctx, trigger.ID, signalData.VehicleDID). + Return(triggerstate.Record{LastFiredAt: time.Now().Add(-30 * time.Minute)}, true, nil) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) require.NotNil(t, result) assert.False(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) assert.True(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) }) t.Run("condition not met", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() @@ -216,50 +172,30 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - success mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromSignal(t, vss.Signal{ - Data: vss.SignalData{ - Timestamp: signalData.Signal.Data.Timestamp.Add(-time.Hour), - ValueNumber: 60, // value the same as current - }, - }), - AssetDid: signalData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-time.Hour), // Cooldown passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(lastLog, nil). - Times(1) - mockRepo.EXPECT(). - GetLastLogForMetric(ctx, signalData.VehicleDID, trigger.MetricName). - Return(lastLog, nil). - Times(1) + Return(true, nil) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, signalData.VehicleDID). + Return(triggerstate.Record{LastFiredAt: time.Now().Add(-time.Hour)}, true, nil) + prev := snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: 60}}) + mockState.EXPECT(). + LastMetric(ctx, signalData.VehicleDID, trigger.MetricName). + Return(triggerstate.MetricRecord{LastSnapshot: prev}, true, nil) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) require.NotNil(t, result) assert.False(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) assert.True(t, result.ConditionNotMet) }) - t.Run("no previous log - first trigger", func(t *testing.T) { + t.Run("no previous state - first trigger", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() @@ -267,115 +203,26 @@ func TestTriggerEvaluator_EvaluateSignalTrigger(t *testing.T) { program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) require.NoError(t, err) - // Mock permission check - success mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - no rows (first trigger) - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(nil, sql.ErrNoRows). - Times(1) - mockRepo.EXPECT(). - GetLastLogForMetric(ctx, signalData.VehicleDID, trigger.MetricName). - Return(nil, nil). - Times(1) + Return(true, nil) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, signalData.VehicleDID). + Return(triggerstate.Record{}, false, nil) + mockState.EXPECT(). + LastMetric(ctx, signalData.VehicleDID, trigger.MetricName). + Return(triggerstate.MetricRecord{}, false, nil) result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) require.NotNil(t, result) + // value=60 > previousValue=0 (zero-valued) -> fires assert.True(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) - }) - - t.Run("database error", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - ctx := context.Background() - trigger := createTestTrigger() - signalData := createTestSignalData() - program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) - require.NoError(t, err) - - // Mock permission check - success - mockTokenClient.EXPECT(). - HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - database error - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(nil, errors.New("database connection error")). - Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - - require.Error(t, err) - assert.Nil(t, result) - richErr, ok := richerrors.AsRichError(err) - require.True(t, ok) - assert.Equal(t, http.StatusInternalServerError, richErr.Code) - }) - - t.Run("invalid previous signal JSON", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - ctx := context.Background() - trigger := createTestTrigger() - signalData := createTestSignalData() - program, err := celcondition.PrepareSignalCondition("value > previousValue", signalData.Def.ValueType) - require.NoError(t, err) - - // Mock permission check - success - mockTokenClient.EXPECT(). - HasVehiclePermissions(ctx, signalData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), signalData.Def.Permissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - cooldown passed; previous from GetLastLogForMetric has invalid JSON - lastLog := &models.TriggerLog{ - SnapshotData: []byte(`invalid json`), - AssetDid: signalData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-time.Hour), - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, signalData.VehicleDID). - Return(lastLog, nil). - Times(1) - mockRepo.EXPECT(). - GetLastLogForMetric(ctx, signalData.VehicleDID, trigger.MetricName). - Return(lastLog, nil). - Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - - require.Error(t, err) - assert.Nil(t, result) - richErr, ok := richerrors.AsRichError(err) - require.True(t, ok) - assert.Equal(t, http.StatusInternalServerError, richErr.Code) }) } // TestTransitionConditions verifies isIgnitionOn (1 and 0) and obdIsPluggedIn (0) transition -// conditions fire correctly with and without an existing previous value (fix for transition-to-zero). +// conditions fire correctly with and without an existing previous value. func TestTransitionConditions(t *testing.T) { t.Parallel() @@ -383,208 +230,76 @@ func TestTransitionConditions(t *testing.T) { perm := []string{"privilege:GetNonLocationHistory"} numberDef := signals.SignalDefinition{Name: "ignition", ValueType: signals.NumberType, Permissions: perm} - t.Run("isIgnitionOn valueNumber==1 with existing value (previous 0) - should fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-ignition-on", Service: "signals", MetricName: "vss.isIgnitionOn", - Condition: "valueNumber == 1 && valueNumber != previousValueNumber", - CooldownPeriod: 600, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 1}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 1}`), - } - signalData.Def.Name = "isIgnitionOn" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(&models.TriggerLog{LastTriggeredAt: time.Now().Add(-2 * time.Hour)}, nil).Times(1) - prevLog := &models.TriggerLog{SnapshotData: snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: 0}})} - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.isIgnitionOn").Return(prevLog, nil).Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.True(t, result.ShouldFire, "isIgnitionOn==1 with previous 0 should fire") - }) - - t.Run("isIgnitionOn valueNumber==1 with no existing value - should fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-ignition-on", Service: "signals", MetricName: "vss.isIgnitionOn", - Condition: "valueNumber == 1 && valueNumber != previousValueNumber", - CooldownPeriod: 600, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 1}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 1}`), - } - signalData.Def.Name = "isIgnitionOn" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(nil, sql.ErrNoRows).Times(1) - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.isIgnitionOn").Return(nil, nil).Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.True(t, result.ShouldFire, "isIgnitionOn==1 with no previous (0) should fire: 1 != 0") - }) - - t.Run("isIgnitionOn valueNumber==0 with existing value (previous 1) - should fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-ignition-off", Service: "signals", MetricName: "vss.isIgnitionOn", - Condition: "valueNumber == 0 && valueNumber != previousValueNumber", - CooldownPeriod: 600, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 0}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 0}`), - } - signalData.Def.Name = "isIgnitionOn" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(&models.TriggerLog{LastTriggeredAt: time.Now().Add(-2 * time.Hour)}, nil).Times(1) - prevLog := &models.TriggerLog{SnapshotData: snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: 1}})} - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.isIgnitionOn").Return(prevLog, nil).Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.True(t, result.ShouldFire, "isIgnitionOn==0 with previous 1 should fire (transition to off)") - }) - - t.Run("isIgnitionOn valueNumber==0 with no existing value - should not fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-ignition-off", Service: "signals", MetricName: "vss.isIgnitionOn", - Condition: "valueNumber == 0 && valueNumber != previousValueNumber", - CooldownPeriod: 600, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 0}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 0}`), - } - signalData.Def.Name = "isIgnitionOn" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(nil, sql.ErrNoRows).Times(1) - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.isIgnitionOn").Return(nil, nil).Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.False(t, result.ShouldFire, "isIgnitionOn==0 with no previous (0) should not fire: 0 == 0") - assert.True(t, result.ConditionNotMet) - }) - - t.Run("obdIsPluggedIn valueNumber==0 with existing value (previous 1) - should fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-obd-unplugged", Service: "signals", MetricName: "vss.obdisPluggedin", - Condition: "valueNumber == 0 && valueNumber != previousValueNumber", - CooldownPeriod: 60, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 0}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 0}`), - } - signalData.Def.Name = "obdisPluggedin" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(&models.TriggerLog{LastTriggeredAt: time.Now().Add(-2 * time.Minute)}, nil).Times(1) - prevLog := &models.TriggerLog{SnapshotData: snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: 1}})} - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.obdisPluggedin").Return(prevLog, nil).Times(1) - - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.True(t, result.ShouldFire, "obdIsPluggedIn==0 with previous 1 should fire (transition to unplugged)") - }) - - t.Run("obdIsPluggedIn valueNumber==0 with no existing value - should not fire", func(t *testing.T) { - ctrl := gomock.NewController(t) - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - trigger := &models.Trigger{ - ID: "trigger-obd-unplugged", Service: "signals", MetricName: "vss.obdisPluggedin", - Condition: "valueNumber == 0 && valueNumber != previousValueNumber", - CooldownPeriod: 60, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), - } - signalData := &SignalEvaluationData{ - Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: 0}}, - VehicleDID: vehicleDID, - Def: numberDef, - RawData: json.RawMessage(`{"valueNumber": 0}`), - } - signalData.Def.Name = "obdisPluggedin" - - program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) - require.NoError(t, err) - - ctx := context.Background() - mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil).Times(1) - mockRepo.EXPECT().GetLastLogValue(ctx, trigger.ID, vehicleDID).Return(nil, sql.ErrNoRows).Times(1) - mockRepo.EXPECT().GetLastLogForMetric(ctx, vehicleDID, "vss.obdisPluggedin").Return(nil, nil).Times(1) + type ignitionCase struct { + name string + condition string + signalValue float64 + // previous: nil = no previous, &v = previous fire with that value + previous *float64 + shouldFire bool + } + pf := func(v float64) *float64 { return &v } + cases := []ignitionCase{ + { + name: "isIgnitionOn==1 with previous 0", condition: "valueNumber == 1 && valueNumber != previousValueNumber", + signalValue: 1, previous: pf(0), shouldFire: true, + }, + { + name: "isIgnitionOn==1 with no previous", condition: "valueNumber == 1 && valueNumber != previousValueNumber", + signalValue: 1, previous: nil, shouldFire: true, // 1 != 0 + }, + { + name: "isIgnitionOn==0 with previous 1", condition: "valueNumber == 0 && valueNumber != previousValueNumber", + signalValue: 0, previous: pf(1), shouldFire: true, + }, + { + name: "isIgnitionOn==0 with no previous", condition: "valueNumber == 0 && valueNumber != previousValueNumber", + signalValue: 0, previous: nil, shouldFire: false, // 0 == 0 + }, + } - result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) - require.NoError(t, err) - require.NotNil(t, result) - assert.False(t, result.ShouldFire, "obdIsPluggedIn==0 with no previous (0) should not fire") - assert.True(t, result.ConditionNotMet) - }) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + mockState := NewMockStateStore(ctrl) + mockTokenClient := NewMockTokenExchangeClient(ctrl) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) + + trigger := &models.Trigger{ + ID: "trigger-x", Service: "signals", MetricName: "vss.isIgnitionOn", + Condition: c.condition, CooldownPeriod: 600, + DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), + } + signalData := &SignalEvaluationData{ + Signal: vss.Signal{Data: vss.SignalData{Timestamp: time.Now().UTC(), ValueNumber: c.signalValue}}, + VehicleDID: vehicleDID, + Def: numberDef, + RawData: json.RawMessage(`{}`), + } + signalData.Def.Name = "isIgnitionOn" + program, err := celcondition.PrepareSignalCondition(trigger.Condition, signals.NumberType) + require.NoError(t, err) + + ctx := context.Background() + mockTokenClient.EXPECT().HasVehiclePermissions(ctx, vehicleDID, gomock.Any(), perm).Return(true, nil) + // Past cooldown + mockState.EXPECT().LastFire(ctx, trigger.ID, vehicleDID). + Return(triggerstate.Record{LastFiredAt: time.Now().Add(-2 * time.Hour)}, true, nil) + if c.previous != nil { + prev := snapShotFromSignal(t, vss.Signal{Data: vss.SignalData{ValueNumber: *c.previous}}) + mockState.EXPECT().LastMetric(ctx, vehicleDID, "vss.isIgnitionOn"). + Return(triggerstate.MetricRecord{LastSnapshot: prev}, true, nil) + } else { + mockState.EXPECT().LastMetric(ctx, vehicleDID, "vss.isIgnitionOn"). + Return(triggerstate.MetricRecord{}, false, nil) + } + + result, err := evaluator.EvaluateSignalTrigger(ctx, trigger, program, signalData) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, c.shouldFire, result.ShouldFire, c.name) + }) + } } func TestTriggerEvaluator_EvaluateEventTrigger(t *testing.T) { @@ -593,10 +308,9 @@ func TestTriggerEvaluator_EvaluateEventTrigger(t *testing.T) { t.Run("successful evaluation - should fire", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() @@ -604,51 +318,33 @@ func TestTriggerEvaluator_EvaluateEventTrigger(t *testing.T) { program, err := celcondition.PrepareEventCondition("durationNs > previousDurationNs") require.NoError(t, err) - // Mock permission check - success expectedPermissions := []string{ "privilege:GetNonLocationHistory", "privilege:GetLocationHistory", } mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, eventData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), expectedPermissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromEvent(t, vss.Event{ - Data: vss.EventData{ - Timestamp: eventData.Event.Data.Timestamp.Add(-time.Hour), - DurationNs: 5, - Name: "ignition", - }, - }), - AssetDid: eventData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-time.Hour), // Cooldown passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, eventData.VehicleDID). - Return(lastLog, nil). - Times(1) + Return(true, nil) - result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) + prev := snapShotFromEvent(t, vss.Event{Data: vss.EventData{DurationNs: 5, Name: "ignition"}}) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, eventData.VehicleDID). + Return(triggerstate.Record{ + LastFiredAt: time.Now().Add(-time.Hour), + LastSnapshot: prev, + }, true, nil) + result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) require.NoError(t, err) require.NotNil(t, result) assert.True(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) }) t.Run("permission denied", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(noState{}) ctx := context.Background() trigger := createTestTrigger() @@ -656,138 +352,57 @@ func TestTriggerEvaluator_EvaluateEventTrigger(t *testing.T) { program, err := celcondition.PrepareEventCondition("durationNs > previousDurationNs") require.NoError(t, err) - // Mock permission check - denied expectedPermissions := []string{ "privilege:GetNonLocationHistory", "privilege:GetLocationHistory", } mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, eventData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), expectedPermissions). - Return(false, nil). - Times(1) + Return(false, nil) result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) - require.NoError(t, err) require.NotNil(t, result) - assert.False(t, result.ShouldFire) assert.True(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) }) t.Run("cooldown not met", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() - trigger.CooldownPeriod = 3600 // 1 hour cooldown + trigger.CooldownPeriod = 3600 eventData := createTestEventData() program, err := celcondition.PrepareEventCondition("durationNs > previousDurationNs") require.NoError(t, err) - // Mock permission check - success expectedPermissions := []string{ "privilege:GetNonLocationHistory", "privilege:GetLocationHistory", } mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, eventData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), expectedPermissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - recent trigger - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromEvent(t, vss.Event{ - Data: vss.EventData{ - Timestamp: eventData.Event.Data.Timestamp.Add(-time.Hour), - DurationNs: 5, - Name: "HarshBraking", - }, - }), - AssetDid: eventData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-30 * time.Minute), // Cooldown not passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, eventData.VehicleDID). - Return(lastLog, nil). - Times(1) + Return(true, nil) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, eventData.VehicleDID). + Return(triggerstate.Record{LastFiredAt: time.Now().Add(-30 * time.Minute)}, true, nil) result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) - require.NoError(t, err) require.NotNil(t, result) - assert.False(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) assert.True(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) - }) - - t.Run("condition not met", func(t *testing.T) { - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) - mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) - - ctx := context.Background() - trigger := createTestTrigger() - eventData := createTestEventData() - program, err := celcondition.PrepareEventCondition("durationNs > previousDurationNs") - require.NoError(t, err) - - // Mock permission check - success - expectedPermissions := []string{ - "privilege:GetNonLocationHistory", - "privilege:GetLocationHistory", - } - mockTokenClient.EXPECT(). - HasVehiclePermissions(ctx, eventData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), expectedPermissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - same event type - lastLog := &models.TriggerLog{ - SnapshotData: snapShotFromEvent(t, vss.Event{ - Data: vss.EventData{ - Timestamp: eventData.Event.Data.Timestamp.Add(-time.Hour), - DurationNs: 15, - Name: "HarshBraking", - }, - }), // Same as current - AssetDid: eventData.VehicleDID.String(), - TriggerID: trigger.ID, - LastTriggeredAt: time.Now().Add(-time.Hour), // Cooldown passed - } - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, eventData.VehicleDID). - Return(lastLog, nil). - Times(1) - - result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) - - require.NoError(t, err) - require.NotNil(t, result) - assert.False(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.True(t, result.ConditionNotMet) }) t.Run("no previous log - first trigger", func(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - - mockRepo := NewMockTriggerRepo(ctrl) + mockState := NewMockStateStore(ctrl) mockTokenClient := NewMockTokenExchangeClient(ctrl) - evaluator := NewTriggerEvaluator(mockRepo, mockTokenClient) + evaluator := NewTriggerEvaluator(mockTokenClient).WithStateStore(mockState) ctx := context.Background() trigger := createTestTrigger() @@ -795,30 +410,22 @@ func TestTriggerEvaluator_EvaluateEventTrigger(t *testing.T) { program, err := celcondition.PrepareEventCondition("durationNs > previousDurationNs") require.NoError(t, err) - // Mock permission check - success expectedPermissions := []string{ "privilege:GetNonLocationHistory", "privilege:GetLocationHistory", } mockTokenClient.EXPECT(). HasVehiclePermissions(ctx, eventData.VehicleDID, common.BytesToAddress(trigger.DeveloperLicenseAddress), expectedPermissions). - Return(true, nil). - Times(1) - - // Mock last log retrieval - no rows (first trigger) - mockRepo.EXPECT(). - GetLastLogValue(ctx, trigger.ID, eventData.VehicleDID). - Return(nil, sql.ErrNoRows). - Times(1) + Return(true, nil) + mockState.EXPECT(). + LastFire(ctx, trigger.ID, eventData.VehicleDID). + Return(triggerstate.Record{}, false, nil) result, err := evaluator.EvaluateEventTrigger(ctx, trigger, program, eventData) - require.NoError(t, err) require.NotNil(t, result) + // durationNs=10 > previousDurationNs=0 -> fires assert.True(t, result.ShouldFire) - assert.False(t, result.PermissionDenied) - assert.False(t, result.CoolDownNotMet) - assert.False(t, result.ConditionNotMet) }) } @@ -832,7 +439,7 @@ func createTestTrigger() *models.Trigger { Condition: "valueNumber > 55", TargetURI: "https://example.com/webhook", Status: "enabled", - CooldownPeriod: 300, // 5 minutes + CooldownPeriod: 300, DeveloperLicenseAddress: common.HexToAddress("0x1234567890abcdef").Bytes(), } } @@ -846,15 +453,15 @@ func createTestAssetDID() cloudevent.ERC721DID { } func snapShotFromSignal(t *testing.T, signal vss.Signal) []byte { - json, err := json.Marshal(signal) + b, err := json.Marshal(signal) require.NoError(t, err) - return json + return b } func snapShotFromEvent(t *testing.T, event vss.Event) []byte { - json, err := json.Marshal(event) + b, err := json.Marshal(event) require.NoError(t, err) - return json + return b } func createTestSignalData() *SignalEvaluationData { @@ -862,7 +469,7 @@ func createTestSignalData() *SignalEvaluationData { Signal: vss.Signal{ Data: vss.SignalData{ Timestamp: time.Now().UTC(), - ValueNumber: 60.0, // Higher than threshold in test condition + ValueNumber: 60.0, }, }, VehicleDID: createTestAssetDID(), diff --git a/internal/services/triggersrepo/failures.go b/internal/services/triggersrepo/failures.go new file mode 100644 index 0000000..601979a --- /dev/null +++ b/internal/services/triggersrepo/failures.go @@ -0,0 +1,84 @@ +package triggersrepo + +import ( + "context" + "fmt" + "net/http" + + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/ethereum/go-ethereum/common" + "github.com/rs/zerolog" +) + +// ResetTriggerFailureCount zeroes the circuit-breaker counter and re-enables +// the trigger if it was previously knocked into StatusFailed. Called by the +// dispatcher on every successful delivery; cheap when there's nothing to +// reset (single SELECT, then no UPDATE). +func (r *Repository) ResetTriggerFailureCount(ctx context.Context, trigger *models.Trigger) error { + updatedTrigger, tx, err := r.GetTriggerByIDAndDeveloperLicenseForUpdate(ctx, trigger.ID, common.BytesToAddress(trigger.DeveloperLicenseAddress)) + if err != nil { + return fmt.Errorf("failed to fetch trigger for success reset: %w", err) + } + defer RollbackTx(ctx, tx) + + if updatedTrigger.FailureCount < 1 { + return nil + } + + updatedTrigger.FailureCount = 0 + if updatedTrigger.Status == StatusFailed { + updatedTrigger.Status = StatusEnabled + } + + if err := r.UpdateTriggerWithTx(ctx, tx, updatedTrigger); err != nil { + return fmt.Errorf("failed to reset failure count: %w", err) + } + + if err := tx.Commit(); err != nil { + return richerrors.Error{ + ExternalMsg: "Failed to commit Update.", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return nil +} + +// IncrementTriggerFailureCount bumps the per-trigger failure counter and +// flips the trigger to StatusFailed when it hits maxFailureCount. The +// circuit-breaker check on the listener / dispatcher side reads Status + +// FailureCount; once failed, the trigger is dropped from the cache on the +// next refresh. +func (r *Repository) IncrementTriggerFailureCount(ctx context.Context, trigger *models.Trigger, failureReason error, maxFailureCount int) error { + updatedTrigger, tx, err := r.GetTriggerByIDAndDeveloperLicenseForUpdate(ctx, trigger.ID, common.BytesToAddress(trigger.DeveloperLicenseAddress)) + if err != nil { + return fmt.Errorf("failed to fetch trigger for failure handling: %w", err) + } + defer RollbackTx(ctx, tx) + + updatedTrigger.FailureCount++ + + if updatedTrigger.FailureCount >= maxFailureCount { + updatedTrigger.Status = StatusFailed + zerolog.Ctx(ctx).Warn(). + Str("triggerId", trigger.ID). + Int("maxFailures", maxFailureCount). + Msg("webhook disabled due to excessive failures") + } + + if err := r.UpdateTriggerWithTx(ctx, tx, updatedTrigger); err != nil { + return fmt.Errorf("failed to update failure count: %w", err) + } + + if err := tx.Commit(); err != nil { + return richerrors.Error{ + ExternalMsg: "Failed to commit Update.", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return nil +} diff --git a/internal/services/triggersrepo/internal.go b/internal/services/triggersrepo/internal.go new file mode 100644 index 0000000..464e596 --- /dev/null +++ b/internal/services/triggersrepo/internal.go @@ -0,0 +1,65 @@ +package triggersrepo + +import ( + "context" + "fmt" + "net/http" + + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/migrations" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/aarondl/sqlboiler/v4/queries/qm" +) + +// InternalGetAllVehicleSubscriptions returns every active subscription joined +// against its trigger. Used by the cache rebuild; never call from a request +// handler - it has no developer-license scoping and the result set is the +// full configured fleet. +func (r *Repository) InternalGetAllVehicleSubscriptions(ctx context.Context) ([]*models.VehicleSubscription, error) { + subs, err := models.VehicleSubscriptions( + qm.InnerJoin(fmt.Sprintf("%s.%s on %s = %s", + migrations.SchemaName, + models.TableNames.Triggers, + models.TriggerTableColumns.ID, + models.VehicleSubscriptionTableColumns.TriggerID, + )), + qm.Where(fmt.Sprintf("%s != ?", models.TriggerTableColumns.Status), StatusDeleted), + ).All(ctx, r.db) + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Failed to get all vehicle subscriptions", + Err: fmt.Errorf("failed to get all vehicle subscriptions: %w", err), + Code: http.StatusInternalServerError, + } + } + return subs, nil +} + +// InternalGetTriggerByID retrieves a specific trigger by ID with no developer +// scoping. Used by the dispatcher and cache to rehydrate trigger rows after +// a CRUD-broadcast invalidation. Never call from a request handler; use +// GetTriggerByIDAndDeveloperLicense for owner-scoped reads. +func (r *Repository) InternalGetTriggerByID(ctx context.Context, triggerID string) (*models.Trigger, error) { + if triggerID == "" { + return nil, richerrors.Error{ + ExternalMsg: "Webhook id is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + + trigger, err := models.Triggers( + models.TriggerWhere.ID.EQ(triggerID), + models.TriggerWhere.Status.NEQ(StatusDeleted), + ).One(ctx, r.db) + + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Error getting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return trigger, nil +} diff --git a/internal/services/triggersrepo/repository.go b/internal/services/triggersrepo/repository.go new file mode 100644 index 0000000..fb8ebb1 --- /dev/null +++ b/internal/services/triggersrepo/repository.go @@ -0,0 +1,126 @@ +// Package triggersrepo is the SQL persistence layer for triggers and the +// vehicle subscriptions that bind them to assets. Files are split by concern: +// +// repository.go - type, ctor, status/service constants, crypto helpers, tx helpers +// triggers.go - trigger CRUD (Create/Get/Update/Delete) +// subscriptions.go - vehicle subscription CRUD +// failures.go - circuit-breaker failure-count accounting +// secrets.go - per-trigger HMAC signing-secret rotation +// internal.go - reads used by the cache rebuild and dispatcher; +// no auth scoping, not for handler use. +package triggersrepo + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "errors" + "fmt" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/secrets" + "github.com/rs/zerolog" +) + +const ( + // StatusEnabled is the status of a trigger that is enabled. + StatusEnabled = "enabled" + // StatusDisabled is the status of a trigger that is disabled. + StatusDisabled = "disabled" + // StatusFailed is the status of a trigger that has failed. + StatusFailed = "failed" + // StatusDeleted is the status of a trigger that has been deleted. + StatusDeleted = "deleted" +) + +const ( + // ServiceSignal is the service name for signal webhooks. MetricName includes the schema prefix (e.g. "vss.speed", "vss.currentLocationCoordinates"). + ServiceSignal = "signals" + // ServiceEvent is the service name for event webhooks. MetricName is the full event name (e.g. "behavior.harshBraking", "security.isEngineBlocked"). + ServiceEvent = "events" +) + +// IsSignalService returns true if service is a signal service. +func IsSignalService(service string) bool { + return service == ServiceSignal +} + +// IsEventService returns true if service is an event service. +func IsEventService(service string) bool { + return service == ServiceEvent +} + +// Repository is the SQL-backed trigger store. Construct with NewRepository +// and wire an at-rest cipher via SetCipher when SIGNING_SECRET_KEY_HEX is set. +type Repository struct { + db *sql.DB + cipher secrets.Cipher +} + +func NewRepository(db *sql.DB) *Repository { + return &Repository{db: db, cipher: secrets.Plaintext{}} +} + +// Ping verifies the database connection is alive. Used by /health so K8s +// readiness probes don't keep a pod in service routing webhooks when the +// dispatcher's eventual repo writes are about to fail. Wraps the standard +// sql.DB.PingContext with the caller's deadline. +func (r *Repository) Ping(ctx context.Context) error { + if r.db == nil { + return fmt.Errorf("triggersrepo: nil db") + } + return r.db.PingContext(ctx) +} + +// SetCipher wires the at-rest encryption cipher used for per-trigger +// signing secrets. Defaults to Plaintext (no encryption) so existing +// deployments behave unchanged until they configure SIGNING_SECRET_KEY_HEX. +func (r *Repository) SetCipher(c secrets.Cipher) { + if c == nil { + c = secrets.Plaintext{} + } + r.cipher = c +} + +// encryptSecret applies the wired cipher to a freshly generated secret +// before it lands in the trigger row. Used by CreateTrigger and +// RotateSigningSecret. +func (r *Repository) encryptSecret(plaintext string) (string, error) { + if r.cipher == nil { + return plaintext, nil + } + return r.cipher.Encrypt(plaintext) +} + +// DecryptSigningSecret is the reverse for read paths (audit, send). When no +// cipher is wired or the stored value looks like legacy plaintext, returns +// the input unchanged. +func (r *Repository) DecryptSigningSecret(stored string) (string, error) { + if r.cipher == nil { + return stored, nil + } + return r.cipher.Decrypt(stored) +} + +// randomHex returns 2*n hex characters of cryptographic randomness, used for +// per-trigger HMAC signing secrets. Failure here is fatal for the create +// path - we never want to fall back to a weak secret. +func randomHex(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} + +// RollbackTx rolls back tx unless it has already been committed. Logs +// non-sql.ErrTxDone failures so a leaked tx surfaces in logs instead of +// silently piling onto the connection pool. +func RollbackTx(ctx context.Context, tx *sql.Tx) { + if tx == nil { + return + } + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { + zerolog.Ctx(ctx).Error().Err(err).Msg("failed to rollback transaction") + } +} diff --git a/internal/services/triggersrepo/secrets.go b/internal/services/triggersrepo/secrets.go new file mode 100644 index 0000000..e72aff0 --- /dev/null +++ b/internal/services/triggersrepo/secrets.go @@ -0,0 +1,44 @@ +package triggersrepo + +import ( + "context" + "net/http" + + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/aarondl/null/v8" + "github.com/ethereum/go-ethereum/common" +) + +// RotateSigningSecret generates a fresh per-trigger HMAC signing secret, +// writes it under the developer's ownership check, and returns the new +// value. Returns the secret only via the function result - it must be +// surfaced to the API caller exactly once. Subsequent reads via GetTrigger* +// expose only the stored value, which is the same secret until the next +// rotation. +func (r *Repository) RotateSigningSecret(ctx context.Context, triggerID string, devLicense common.Address) (string, error) { + trigger, err := r.GetTriggerByIDAndDeveloperLicense(ctx, triggerID, devLicense) + if err != nil { + return "", err + } + secret, err := randomHex(32) + if err != nil { + return "", richerrors.Error{ + ExternalMsg: "Failed to generate signing secret", + Err: err, + Code: http.StatusInternalServerError, + } + } + stored, err := r.encryptSecret(secret) + if err != nil { + return "", richerrors.Error{ + ExternalMsg: "Failed to encrypt signing secret", + Err: err, + Code: http.StatusInternalServerError, + } + } + trigger.SigningSecret = null.StringFrom(stored) + if err := r.UpdateTrigger(ctx, trigger); err != nil { + return "", err + } + return secret, nil +} diff --git a/internal/services/triggersrepo/subscriptions.go b/internal/services/triggersrepo/subscriptions.go new file mode 100644 index 0000000..8546e82 --- /dev/null +++ b/internal/services/triggersrepo/subscriptions.go @@ -0,0 +1,181 @@ +package triggersrepo + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/migrations" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/aarondl/sqlboiler/v4/boil" + "github.com/aarondl/sqlboiler/v4/queries/qm" + "github.com/ethereum/go-ethereum/common" + "github.com/lib/pq" +) + +// CreateVehicleSubscription creates a new vehicle subscription. +func (r *Repository) CreateVehicleSubscription(ctx context.Context, assetDid cloudevent.ERC721DID, triggerID string) (*models.VehicleSubscription, error) { + if assetDid == (cloudevent.ERC721DID{}) { + return nil, richerrors.Error{ + ExternalMsg: "Asset DID is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + if triggerID == "" { + return nil, richerrors.Error{ + ExternalMsg: "Webhook id is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + + subscription := &models.VehicleSubscription{ + AssetDid: assetDid.String(), + TriggerID: triggerID, + CreatedAt: time.Now().UTC(), + UpdatedAt: time.Now().UTC(), + } + + if err := subscription.Insert(ctx, r.db, boil.Infer()); err != nil { + var pqErr *pq.Error + if errors.As(err, &pqErr) { + if pqErr.Code == ForeignKeyViolation { + return nil, richerrors.Error{ + ExternalMsg: "Webhook not found", + Err: err, + Code: http.StatusNotFound, + } + } + if pqErr.Code == DuplicateKeyError { + return nil, richerrors.Error{ + ExternalMsg: "Already subscribed", + Err: err, + Code: http.StatusBadRequest, + } + } + } + return nil, richerrors.Error{ + ExternalMsg: "Failed to create vehicle subscription", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return subscription, nil +} + +// GetVehicleSubscriptionsByTriggerID retrieves all vehicle subscriptions for a trigger. +func (r *Repository) GetVehicleSubscriptionsByTriggerID(ctx context.Context, triggerID string) ([]*models.VehicleSubscription, error) { + subscriptions, err := models.VehicleSubscriptions( + models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), + ).All(ctx, r.db) + + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Failed to get vehicle subscriptions", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return subscriptions, nil +} + +// GetVehicleSubscriptionsByVehicleAndDeveloperLicense retrieves all +// subscriptions for trigger IDs owned by developerLicenseAddress. +func (r *Repository) GetVehicleSubscriptionsByVehicleAndDeveloperLicense(ctx context.Context, assetDid cloudevent.ERC721DID, developerLicenseAddress common.Address) ([]*models.VehicleSubscription, error) { + if developerLicenseAddress == (common.Address{}) { + return nil, richerrors.Error{ + ExternalMsg: "Developer license address is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + if assetDid == (cloudevent.ERC721DID{}) { + return nil, richerrors.Error{ + ExternalMsg: "Asset DID is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + subscriptions, err := models.VehicleSubscriptions( + models.VehicleSubscriptionWhere.AssetDid.EQ(assetDid.String()), + qm.InnerJoin(fmt.Sprintf("%s.%s on %s = %s", + migrations.SchemaName, + models.TableNames.Triggers, + models.TriggerTableColumns.ID, + models.VehicleSubscriptionTableColumns.TriggerID, + )), + qm.Where(fmt.Sprintf("%s = ?", + models.TriggerTableColumns.DeveloperLicenseAddress, + ), developerLicenseAddress.Bytes()), + qm.Where(fmt.Sprintf("%s != ?", models.TriggerTableColumns.Status), StatusDeleted), + ).All(ctx, r.db) + + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Failed to get vehicle subscriptions", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return subscriptions, nil +} + +// DeleteVehicleSubscription deletes a specific vehicle subscription. +func (r *Repository) DeleteVehicleSubscription(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (int64, error) { + if triggerID == "" { + return 0, richerrors.Error{ + ExternalMsg: "Trigger id is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + if assetDid == (cloudevent.ERC721DID{}) { + return 0, richerrors.Error{ + ExternalMsg: "Asset DID is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + deleteCount, err := models.VehicleSubscriptions( + models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), + models.VehicleSubscriptionWhere.AssetDid.EQ(assetDid.String()), + ).DeleteAll(ctx, r.db) + if err != nil { + return 0, richerrors.Error{ + ExternalMsg: "Failed to delete vehicle subscription", + Err: err, + Code: http.StatusInternalServerError, + } + } + return deleteCount, nil +} + +// DeleteAllVehicleSubscriptionsForTrigger deletes all vehicle subscriptions for a trigger. +func (r *Repository) DeleteAllVehicleSubscriptionsForTrigger(ctx context.Context, triggerID string) (int64, error) { + if triggerID == "" { + return 0, richerrors.Error{ + ExternalMsg: "Trigger id is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + deleteCount, err := models.VehicleSubscriptions( + models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), + ).DeleteAll(ctx, r.db) + if err != nil { + return 0, richerrors.Error{ + ExternalMsg: "Failed to delete vehicle subscriptions", + Err: err, + Code: http.StatusInternalServerError, + } + } + return deleteCount, nil +} diff --git a/internal/services/triggersrepo/triggers.go b/internal/services/triggersrepo/triggers.go new file mode 100644 index 0000000..4467a5b --- /dev/null +++ b/internal/services/triggersrepo/triggers.go @@ -0,0 +1,364 @@ +package triggersrepo + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "time" + + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/aarondl/null/v8" + "github.com/aarondl/sqlboiler/v4/boil" + "github.com/aarondl/sqlboiler/v4/queries/qm" + "github.com/ethereum/go-ethereum/common" + "github.com/google/uuid" +) + +// CreateTriggerRequest represents the data needed to create a new trigger. +type CreateTriggerRequest struct { + DisplayName string + Service string + MetricName string + Condition string + TargetURI string + Status string + Description string + CooldownPeriod int + DeveloperLicenseAddress common.Address +} + +func (req CreateTriggerRequest) Validate() error { + if req.MetricName == "" { + return fmt.Errorf("%w metricName is required", ValidationError) + } + if req.DeveloperLicenseAddress == (common.Address{}) { + return fmt.Errorf("%w developerLicenseAddress is required", ValidationError) + } + if req.Service == "" { + return fmt.Errorf("%w service is required", ValidationError) + } + if req.Condition == "" { + return fmt.Errorf("%w condition is required", ValidationError) + } + if req.TargetURI == "" { + return fmt.Errorf("%w target_uri is required", ValidationError) + } + if req.Status == "" { + return fmt.Errorf("%w status is required", ValidationError) + } + if req.CooldownPeriod < 0 { + return fmt.Errorf("%w cooldownPeriod cannot be negative", ValidationError) + } + return nil +} + +// CreateTrigger creates a new trigger/webhook. +func (r *Repository) CreateTrigger(ctx context.Context, req CreateTriggerRequest) (*models.Trigger, error) { + if err := req.Validate(); err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Invalid request: " + err.Error(), + Err: err, + Code: http.StatusBadRequest, + } + } + id := uuid.New().String() + displayName := req.DisplayName + if displayName == "" { + displayName = id + } + currTime := time.Now().UTC() + + // Per-trigger HMAC signing secret. 32 bytes (256 bits) is enough margin + // for collision-resistant HMAC-SHA256. The plaintext value is returned + // in the registration response exactly once; the column stores the + // cipher-encrypted form so a DB compromise doesn't leak signing keys. + plaintextSecret, err := randomHex(32) + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Failed to generate signing secret", + Err: err, + Code: http.StatusInternalServerError, + } + } + storedSecret, err := r.encryptSecret(plaintextSecret) + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Failed to encrypt signing secret", + Err: err, + Code: http.StatusInternalServerError, + } + } + + trigger := &models.Trigger{ + ID: id, + DisplayName: displayName, + Service: req.Service, + MetricName: req.MetricName, + Condition: req.Condition, + Description: null.StringFrom(req.Description), + TargetURI: req.TargetURI, + CooldownPeriod: req.CooldownPeriod, + DeveloperLicenseAddress: req.DeveloperLicenseAddress.Bytes(), + Status: req.Status, + SigningSecret: null.StringFrom(storedSecret), + CreatedAt: currTime, + UpdatedAt: currTime, + } + + if err := trigger.Insert(ctx, r.db, boil.Infer()); err != nil { + if isDuplicateDisplayNameError(err) { + return nil, richerrors.Error{ + ExternalMsg: "Display name must be unique", + Err: err, + Code: http.StatusBadRequest, + } + } + return nil, richerrors.Error{ + ExternalMsg: "Error during creation", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return trigger, nil +} + +// GetTriggersByDeveloperLicense retrieves all triggers for a developer license. +func (r *Repository) GetTriggersByDeveloperLicense(ctx context.Context, developerLicenseAddress common.Address) ([]*models.Trigger, error) { + triggers, err := models.Triggers( + models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), + models.TriggerWhere.Status.NEQ(StatusDeleted), + qm.OrderBy("id"), + ).All(ctx, r.db) + + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, richerrors.Error{ + ExternalMsg: "No triggers found", + Err: err, + Code: http.StatusNotFound, + } + } + return nil, richerrors.Error{ + ExternalMsg: "Error getting triggers", + Err: err, + Code: http.StatusInternalServerError, + } + } + + if triggers == nil { + triggers = make([]*models.Trigger, 0) + } + + return triggers, nil +} + +// GetTriggerByIDAndDeveloperLicense retrieves a specific trigger by ID and developer license. +func (r *Repository) GetTriggerByIDAndDeveloperLicense(ctx context.Context, triggerID string, developerLicenseAddress common.Address) (*models.Trigger, error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Error starting transaction", + Err: err, + Code: http.StatusInternalServerError, + } + } + defer RollbackTx(ctx, tx) + trigger, err := r.getTriggerByIDAndDeveloperLicense(tx, ctx, triggerID, developerLicenseAddress, false) + if err != nil { + return nil, err + } + err = tx.Commit() + if err != nil { + return nil, richerrors.Error{ + ExternalMsg: "Error committing transaction", + Err: err, + Code: http.StatusInternalServerError, + } + } + return trigger, nil +} + +// GetTriggerByIDAndDeveloperLicenseForUpdate retrieves a specific trigger by ID and developer license in the given transaction for update. +func (r *Repository) GetTriggerByIDAndDeveloperLicenseForUpdate(ctx context.Context, triggerID string, developerLicenseAddress common.Address) (*models.Trigger, *sql.Tx, error) { + tx, err := r.db.BeginTx(ctx, &sql.TxOptions{ + Isolation: sql.LevelReadCommitted, + }) + if err != nil { + return nil, nil, richerrors.Error{ + ExternalMsg: "Error starting transaction", + Err: err, + Code: http.StatusInternalServerError, + } + } + trigger, err := r.getTriggerByIDAndDeveloperLicense(tx, ctx, triggerID, developerLicenseAddress, true) + if err != nil { + return nil, nil, err + } + return trigger, tx, nil +} + +func (r *Repository) getTriggerByIDAndDeveloperLicense(tx *sql.Tx, ctx context.Context, triggerID string, developerLicenseAddress common.Address, forUpdate bool) (*models.Trigger, error) { + if triggerID == "" { + return nil, richerrors.Error{ + ExternalMsg: "Webhook id is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + if developerLicenseAddress == (common.Address{}) { + return nil, richerrors.Error{ + ExternalMsg: "Developer license address is required", + Err: ValidationError, + Code: http.StatusBadRequest, + } + } + mods := []qm.QueryMod{ + models.TriggerWhere.ID.EQ(triggerID), + models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), + models.TriggerWhere.Status.NEQ(StatusDeleted), + } + if forUpdate { + mods = append(mods, qm.For("UPDATE")) + } + trigger, err := models.Triggers(mods...).One(ctx, tx) + + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, richerrors.Error{ + ExternalMsg: "Webhook not found", + Err: err, + Code: http.StatusNotFound, + } + } + return nil, richerrors.Error{ + ExternalMsg: "Error getting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + + return trigger, nil +} + +func (r *Repository) UpdateTrigger(ctx context.Context, trigger *models.Trigger) error { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return richerrors.Error{ + ExternalMsg: "Error starting transaction", + Err: err, + Code: http.StatusInternalServerError, + } + } + defer RollbackTx(ctx, tx) + err = r.updateTrigger(tx, ctx, trigger) + if err != nil { + return err + } + err = tx.Commit() + if err != nil { + return richerrors.Error{ + ExternalMsg: "Failed to commit Update.", + Err: err, + Code: http.StatusInternalServerError, + } + } + return nil +} + +func (r *Repository) UpdateTriggerWithTx(ctx context.Context, tx *sql.Tx, trigger *models.Trigger) error { + return r.updateTrigger(tx, ctx, trigger) +} + +func (r *Repository) updateTrigger(tx *sql.Tx, ctx context.Context, trigger *models.Trigger) error { + trigger.UpdatedAt = time.Now().UTC() + ret, err := trigger.Update(ctx, tx, boil.Blacklist(models.TriggerColumns.ID, + models.TriggerColumns.ID, + models.TriggerColumns.DeveloperLicenseAddress, + models.TriggerColumns.Service, + models.TriggerColumns.CreatedAt, + )) + if err != nil { + if isDuplicateDisplayNameError(err) { + return richerrors.Error{ + ExternalMsg: "Display name must be unique", + Err: err, + Code: http.StatusBadRequest, + } + } + + return richerrors.Error{ + ExternalMsg: "Error updating trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + if ret == 0 { + return richerrors.Error{ + ExternalMsg: "Webhook not found", + Err: sql.ErrNoRows, + Code: http.StatusNotFound, + } + } + return nil +} + +// DeleteTrigger soft-deletes a trigger and removes all of its vehicle +// subscriptions in one transaction. The trigger row stays so audit / billing +// can still resolve historical fires. +func (r *Repository) DeleteTrigger(ctx context.Context, triggerID string, developerLicenseAddress common.Address) error { + trigger, err := models.Triggers( + models.TriggerWhere.ID.EQ(triggerID), + models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), + models.TriggerWhere.Status.NEQ(StatusDeleted), + ).One(ctx, r.db) + if err != nil { + return richerrors.Error{ + ExternalMsg: "Error deleting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return richerrors.Error{ + ExternalMsg: "Error deleting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + defer RollbackTx(ctx, tx) + + trigger.Status = StatusDeleted + trigger.UpdatedAt = time.Now().UTC() + if _, err := trigger.Update(ctx, tx, boil.Whitelist(models.TriggerColumns.Status, models.TriggerColumns.UpdatedAt)); err != nil { + return richerrors.Error{ + ExternalMsg: "Error deleting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + + if _, err := models.VehicleSubscriptions( + models.VehicleSubscriptionWhere.TriggerID.EQ(trigger.ID), + ).DeleteAll(ctx, tx); err != nil { + return richerrors.Error{ + ExternalMsg: "Error deleting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + + if err := tx.Commit(); err != nil { + return richerrors.Error{ + ExternalMsg: "Error deleting trigger", + Err: err, + Code: http.StatusInternalServerError, + } + } + return nil +} diff --git a/internal/services/triggersrepo/triggersrepo.go b/internal/services/triggersrepo/triggersrepo.go deleted file mode 100644 index 4657f30..0000000 --- a/internal/services/triggersrepo/triggersrepo.go +++ /dev/null @@ -1,767 +0,0 @@ -package triggersrepo - -import ( - "context" - "database/sql" - "errors" - "fmt" - "net/http" - "time" - - "github.com/DIMO-Network/cloudevent" - "github.com/DIMO-Network/server-garage/pkg/richerrors" - "github.com/DIMO-Network/vehicle-triggers-api/internal/db/migrations" - "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" - "github.com/aarondl/null/v8" - "github.com/aarondl/sqlboiler/v4/boil" - "github.com/aarondl/sqlboiler/v4/queries/qm" - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "github.com/lib/pq" - "github.com/rs/zerolog" -) - -const ( - // StatusEnabled is the status of a trigger that is enabled. - StatusEnabled = "enabled" - // StatusDisabled is the status of a trigger that is disabled. - StatusDisabled = "disabled" - // StatusFailed is the status of a trigger that has failed. - StatusFailed = "failed" - // StatusDeleted is the status of a trigger that has been deleted. - StatusDeleted = "deleted" -) - -const ( - // ServiceSignal is the service name for signal webhooks. MetricName includes the schema prefix (e.g. "vss.speed", "vss.currentLocationCoordinates"). - ServiceSignal = "signals" - // ServiceEvent is the service name for event webhooks. MetricName is the full event name (e.g. "behavior.harshBraking", "security.isEngineBlocked"). - ServiceEvent = "events" -) - -// IsSignalService returns true if service is a signal service. -func IsSignalService(service string) bool { - return service == ServiceSignal -} - -// IsEventService returns true if service is an event service. -func IsEventService(service string) bool { - return service == ServiceEvent -} - -type Repository struct { - db *sql.DB -} - -func NewRepository(db *sql.DB) *Repository { - return &Repository{db: db} -} - -// CreateTriggerRequest represents the data needed to create a new trigger. -type CreateTriggerRequest struct { - DisplayName string - Service string - MetricName string - Condition string - TargetURI string - Status string - Description string - CooldownPeriod int - DeveloperLicenseAddress common.Address -} - -func (req CreateTriggerRequest) Validate() error { - // Validate required fields - if req.MetricName == "" { - return fmt.Errorf("%w metricName is required", ValidationError) - } - if req.DeveloperLicenseAddress == (common.Address{}) { - return fmt.Errorf("%w developerLicenseAddress is required", ValidationError) - } - if req.Service == "" { - return fmt.Errorf("%w service is required", ValidationError) - } - if req.Condition == "" { - return fmt.Errorf("%w condition is required", ValidationError) - } - if req.TargetURI == "" { - return fmt.Errorf("%w target_uri is required", ValidationError) - } - if req.Status == "" { - return fmt.Errorf("%w status is required", ValidationError) - } - if req.CooldownPeriod < 0 { - return fmt.Errorf("%w cooldownPeriod cannot be negative", ValidationError) - } - return nil -} - -// CreateTrigger creates a new trigger/webhook. -func (r *Repository) CreateTrigger(ctx context.Context, req CreateTriggerRequest) (*models.Trigger, error) { - if err := req.Validate(); err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Invalid request: " + err.Error(), - Err: err, - Code: http.StatusBadRequest, - } - } - id := uuid.New().String() - displayName := req.DisplayName - if displayName == "" { - displayName = id - } - currTime := time.Now().UTC() - - trigger := &models.Trigger{ - ID: id, - DisplayName: displayName, - Service: req.Service, - MetricName: req.MetricName, - Condition: req.Condition, - Description: null.StringFrom(req.Description), - TargetURI: req.TargetURI, - CooldownPeriod: req.CooldownPeriod, - DeveloperLicenseAddress: req.DeveloperLicenseAddress.Bytes(), - Status: req.Status, - CreatedAt: currTime, - UpdatedAt: currTime, - } - - if err := trigger.Insert(ctx, r.db, boil.Infer()); err != nil { - if isDuplicateDisplayNameError(err) { - return nil, richerrors.Error{ - ExternalMsg: "Display name must be unique", - Err: err, - Code: http.StatusBadRequest, - } - } - return nil, richerrors.Error{ - ExternalMsg: "Error during creation", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return trigger, nil -} - -// GetTriggersByDeveloperLicense retrieves all triggers for a developer license. -func (r *Repository) GetTriggersByDeveloperLicense(ctx context.Context, developerLicenseAddress common.Address) ([]*models.Trigger, error) { - triggers, err := models.Triggers( - models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), - models.TriggerWhere.Status.NEQ(StatusDeleted), - qm.OrderBy("id"), - ).All(ctx, r.db) - - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, richerrors.Error{ - ExternalMsg: "No triggers found", - Err: err, - Code: http.StatusNotFound, - } - } - return nil, richerrors.Error{ - ExternalMsg: "Error getting triggers", - Err: err, - Code: http.StatusInternalServerError, - } - } - - if triggers == nil { - triggers = make([]*models.Trigger, 0) - } - - return triggers, nil -} - -// GetTriggerByIDAndDeveloperLicense retrieves a specific trigger by ID and developer license. -func (r *Repository) GetTriggerByIDAndDeveloperLicense(ctx context.Context, triggerID string, developerLicenseAddress common.Address) (*models.Trigger, error) { - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Error starting transaction", - Err: err, - Code: http.StatusInternalServerError, - } - } - defer RollbackTx(ctx, tx) - trigger, err := r.getTriggerByIDAndDeveloperLicense(tx, ctx, triggerID, developerLicenseAddress, false) - if err != nil { - return nil, err - } - err = tx.Commit() - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Error committing transaction", - Err: err, - Code: http.StatusInternalServerError, - } - } - return trigger, nil -} - -// GetTriggerByIDAndDeveloperLicenseForUpdate retrieves a specific trigger by ID and developer license in the given transaction for update. -func (r *Repository) GetTriggerByIDAndDeveloperLicenseForUpdate(ctx context.Context, triggerID string, developerLicenseAddress common.Address) (*models.Trigger, *sql.Tx, error) { - tx, err := r.db.BeginTx(ctx, &sql.TxOptions{ - Isolation: sql.LevelReadCommitted, - }) - if err != nil { - return nil, nil, richerrors.Error{ - ExternalMsg: "Error starting transaction", - Err: err, - Code: http.StatusInternalServerError, - } - } - trigger, err := r.getTriggerByIDAndDeveloperLicense(tx, ctx, triggerID, developerLicenseAddress, true) - if err != nil { - return nil, nil, err - } - return trigger, tx, nil -} - -func (r *Repository) getTriggerByIDAndDeveloperLicense(tx *sql.Tx, ctx context.Context, triggerID string, developerLicenseAddress common.Address, forUpdate bool) (*models.Trigger, error) { - if triggerID == "" { - return nil, richerrors.Error{ - ExternalMsg: "Webhook id is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if developerLicenseAddress == (common.Address{}) { - return nil, richerrors.Error{ - ExternalMsg: "Developer license address is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - mods := []qm.QueryMod{ - models.TriggerWhere.ID.EQ(triggerID), - models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), - models.TriggerWhere.Status.NEQ(StatusDeleted), - } - if forUpdate { - mods = append(mods, qm.For("UPDATE")) - } - trigger, err := models.Triggers(mods...).One(ctx, tx) - - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, richerrors.Error{ - ExternalMsg: "Webhook not found", - Err: err, - Code: http.StatusNotFound, - } - } - return nil, richerrors.Error{ - ExternalMsg: "Error getting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return trigger, nil -} - -func (r *Repository) UpdateTrigger(ctx context.Context, trigger *models.Trigger) error { - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return richerrors.Error{ - ExternalMsg: "Error starting transaction", - Err: err, - Code: http.StatusInternalServerError, - } - } - defer RollbackTx(ctx, tx) - err = r.updateTrigger(tx, ctx, trigger) - if err != nil { - return err - } - err = tx.Commit() - if err != nil { - return richerrors.Error{ - ExternalMsg: "Failed to commit Update.", - Err: err, - Code: http.StatusInternalServerError, - } - } - return nil -} - -func (r *Repository) UpdateTriggerWithTx(ctx context.Context, tx *sql.Tx, trigger *models.Trigger) error { - return r.updateTrigger(tx, ctx, trigger) -} - -func (r *Repository) updateTrigger(tx *sql.Tx, ctx context.Context, trigger *models.Trigger) error { - trigger.UpdatedAt = time.Now().UTC() - ret, err := trigger.Update(ctx, tx, boil.Blacklist(models.TriggerColumns.ID, - models.TriggerColumns.ID, - models.TriggerColumns.DeveloperLicenseAddress, - models.TriggerColumns.Service, - models.TriggerColumns.CreatedAt, - )) - if err != nil { - if isDuplicateDisplayNameError(err) { - return richerrors.Error{ - ExternalMsg: "Display name must be unique", - Err: err, - Code: http.StatusBadRequest, - } - } - - return richerrors.Error{ - ExternalMsg: "Error updating trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - if ret == 0 { - return richerrors.Error{ - ExternalMsg: "Webhook not found", - Err: sql.ErrNoRows, - Code: http.StatusNotFound, - } - } - return nil -} - -// DeleteTrigger deletes a trigger by ID and developer license -func (r *Repository) DeleteTrigger(ctx context.Context, triggerID string, developerLicenseAddress common.Address) error { - // Find the trigger owned by the developer and not already deleted - trigger, err := models.Triggers( - models.TriggerWhere.ID.EQ(triggerID), - models.TriggerWhere.DeveloperLicenseAddress.EQ(developerLicenseAddress.Bytes()), - models.TriggerWhere.Status.NEQ(StatusDeleted), - ).One(ctx, r.db) - if err != nil { - return richerrors.Error{ - ExternalMsg: "Error deleting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - - tx, err := r.db.BeginTx(ctx, nil) - if err != nil { - return richerrors.Error{ - ExternalMsg: "Error deleting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - defer RollbackTx(ctx, tx) - - // Soft-delete the trigger by setting status to Deleted - trigger.Status = StatusDeleted - trigger.UpdatedAt = time.Now().UTC() - if _, err := trigger.Update(ctx, tx, boil.Whitelist(models.TriggerColumns.Status, models.TriggerColumns.UpdatedAt)); err != nil { - return richerrors.Error{ - ExternalMsg: "Error deleting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - - // Delete all related vehicle subscriptions - if _, err := models.VehicleSubscriptions( - models.VehicleSubscriptionWhere.TriggerID.EQ(trigger.ID), - ).DeleteAll(ctx, tx); err != nil { - return richerrors.Error{ - ExternalMsg: "Error deleting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - - if err := tx.Commit(); err != nil { - return richerrors.Error{ - ExternalMsg: "Error deleting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - return nil -} - -// Vehicle Subscription operations - -// CreateVehicleSubscription creates a new vehicle subscription -func (r *Repository) CreateVehicleSubscription(ctx context.Context, assetDid cloudevent.ERC721DID, triggerID string) (*models.VehicleSubscription, error) { - if assetDid == (cloudevent.ERC721DID{}) { - return nil, richerrors.Error{ - ExternalMsg: "Asset DID is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if triggerID == "" { - return nil, richerrors.Error{ - ExternalMsg: "Webhook id is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - - subscription := &models.VehicleSubscription{ - AssetDid: assetDid.String(), - TriggerID: triggerID, - CreatedAt: time.Now().UTC(), - UpdatedAt: time.Now().UTC(), - } - - if err := subscription.Insert(ctx, r.db, boil.Infer()); err != nil { - var pqErr *pq.Error - if errors.As(err, &pqErr) { - if pqErr.Code == ForeignKeyViolation { - // This means the trigger does not exist - return nil, richerrors.Error{ - ExternalMsg: "Webhook not found", - Err: err, - Code: http.StatusNotFound, - } - } - if pqErr.Code == DuplicateKeyError { - return nil, richerrors.Error{ - ExternalMsg: "Already subscribed", - Err: err, - Code: http.StatusBadRequest, - } - } - } - return nil, richerrors.Error{ - ExternalMsg: "Failed to create vehicle subscription", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return subscription, nil -} - -// GetVehicleSubscriptionsByTriggerID retrieves all vehicle subscriptions for a trigger -func (r *Repository) GetVehicleSubscriptionsByTriggerID(ctx context.Context, triggerID string) ([]*models.VehicleSubscription, error) { - subscriptions, err := models.VehicleSubscriptions( - models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), - ).All(ctx, r.db) - - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Failed to get vehicle subscriptions", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return subscriptions, nil -} - -// GetVehicleSubscriptionsByVehicleAndDeveloperLicense retrieves all subscriptions for webhook IDs that the device_license created. -func (r *Repository) GetVehicleSubscriptionsByVehicleAndDeveloperLicense(ctx context.Context, assetDid cloudevent.ERC721DID, developerLicenseAddress common.Address) ([]*models.VehicleSubscription, error) { - if developerLicenseAddress == (common.Address{}) { - return nil, richerrors.Error{ - ExternalMsg: "Developer license address is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if assetDid == (cloudevent.ERC721DID{}) { - return nil, richerrors.Error{ - ExternalMsg: "Asset DID is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - subscriptions, err := models.VehicleSubscriptions( - models.VehicleSubscriptionWhere.AssetDid.EQ(assetDid.String()), - qm.InnerJoin(fmt.Sprintf("%s.%s on %s = %s", - migrations.SchemaName, - models.TableNames.Triggers, - models.TriggerTableColumns.ID, - models.VehicleSubscriptionTableColumns.TriggerID, - )), - qm.Where(fmt.Sprintf("%s = ?", - models.TriggerTableColumns.DeveloperLicenseAddress, - ), developerLicenseAddress.Bytes()), - qm.Where(fmt.Sprintf("%s != ?", models.TriggerTableColumns.Status), StatusDeleted), - ).All(ctx, r.db) - - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Failed to get vehicle subscriptions", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return subscriptions, nil -} - -// DeleteVehicleSubscription deletes a specific vehicle subscription. -func (r *Repository) DeleteVehicleSubscription(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (int64, error) { - if triggerID == "" { - return 0, richerrors.Error{ - ExternalMsg: "Trigger id is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if assetDid == (cloudevent.ERC721DID{}) { - return 0, richerrors.Error{ - ExternalMsg: "Asset DID is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - deleteCount, err := models.VehicleSubscriptions( - models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), - models.VehicleSubscriptionWhere.AssetDid.EQ(assetDid.String()), - ).DeleteAll(ctx, r.db) - if err != nil { - return 0, richerrors.Error{ - ExternalMsg: "Failed to delete vehicle subscription", - Err: err, - Code: http.StatusInternalServerError, - } - } - return deleteCount, nil -} - -// DeleteAllVehicleSubscriptionsForTrigger deletes all vehicle subscriptions for a trigger. -func (r *Repository) DeleteAllVehicleSubscriptionsForTrigger(ctx context.Context, triggerID string) (int64, error) { - if triggerID == "" { - return 0, richerrors.Error{ - ExternalMsg: "Trigger id is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - deleteCount, err := models.VehicleSubscriptions( - models.VehicleSubscriptionWhere.TriggerID.EQ(triggerID), - ).DeleteAll(ctx, r.db) - if err != nil { - return 0, richerrors.Error{ - ExternalMsg: "Failed to delete vehicle subscriptions", - Err: err, - Code: http.StatusInternalServerError, - } - } - return deleteCount, nil -} - -// InternalGetAllVehicleSubscriptions returns all vehicle subscriptions. -// This should not be used with handler calls. Instead use GetVehicleSubscriptionsByVehicleAndDeveloperLicense. -func (r *Repository) InternalGetAllVehicleSubscriptions(ctx context.Context) ([]*models.VehicleSubscription, error) { - subs, err := models.VehicleSubscriptions( - qm.InnerJoin(fmt.Sprintf("%s.%s on %s = %s", - migrations.SchemaName, - models.TableNames.Triggers, - models.TriggerTableColumns.ID, - models.VehicleSubscriptionTableColumns.TriggerID, - )), - qm.Where(fmt.Sprintf("%s != ?", models.TriggerTableColumns.Status), StatusDeleted), - ).All(ctx, r.db) - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Failed to get all vehicle subscriptions", - Err: fmt.Errorf("failed to get all vehicle subscriptions: %w", err), - Code: http.StatusInternalServerError, - } - } - return subs, nil -} - -// InternalGetTriggerByID retrieves a specific trigger by ID. -// This should not be used with handler calls. Instead use GetTriggerByIDAndDeveloperLicense. -func (r *Repository) InternalGetTriggerByID(ctx context.Context, triggerID string) (*models.Trigger, error) { - if triggerID == "" { - return nil, richerrors.Error{ - ExternalMsg: "Webhook id is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - - trigger, err := models.Triggers( - models.TriggerWhere.ID.EQ(triggerID), - models.TriggerWhere.Status.NEQ(StatusDeleted), - ).One(ctx, r.db) - - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Error getting trigger", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return trigger, nil -} - -// GetLastLogValue returns the last triggered at timestamp for a trigger and vehicle token ID. -func (r *Repository) GetLastLogValue(ctx context.Context, triggerID string, assetDid cloudevent.ERC721DID) (*models.TriggerLog, error) { - logs, err := models.TriggerLogs( - models.TriggerLogWhere.TriggerID.EQ(triggerID), - models.TriggerLogWhere.AssetDid.EQ(assetDid.String()), - qm.OrderBy("last_triggered_at DESC"), - ).One(ctx, r.db) - - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Failed to get last log value", - Err: err, - Code: http.StatusInternalServerError, - } - } - return logs, nil -} - -// GetLastLogForMetric returns the most recent trigger log for the given asset and metric (across any trigger with that metric). -// Used for condition evaluation so "previous" is the last seen value for the metric, not only when this trigger fired. -// Returns (nil, nil) when no log exists for this vehicle+metric. -func (r *Repository) GetLastLogForMetric(ctx context.Context, assetDid cloudevent.ERC721DID, metricName string) (*models.TriggerLog, error) { - triggers, err := models.Triggers(models.TriggerWhere.MetricName.EQ(metricName)).All(ctx, r.db) - if err != nil { - return nil, richerrors.Error{ - ExternalMsg: "Failed to get triggers for metric", - Err: err, - Code: http.StatusInternalServerError, - } - } - if len(triggers) == 0 { - return nil, nil - } - triggerIDs := make([]string, 0, len(triggers)) - for _, t := range triggers { - triggerIDs = append(triggerIDs, t.ID) - } - log, err := models.TriggerLogs( - models.TriggerLogWhere.AssetDid.EQ(assetDid.String()), - models.TriggerLogWhere.TriggerID.IN(triggerIDs), - qm.OrderBy(models.TriggerLogTableColumns.LastTriggeredAt+" DESC"), - ).One(ctx, r.db) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return nil, richerrors.Error{ - ExternalMsg: "Failed to get last log for metric", - Err: err, - Code: http.StatusInternalServerError, - } - } - return log, nil -} - -// CreateTriggerLog creates a new trigger log. -func (r *Repository) CreateTriggerLog(ctx context.Context, log *models.TriggerLog) error { - if log.AssetDid == "" { - return richerrors.Error{ - ExternalMsg: "Asset DID is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if log.TriggerID == "" { - return richerrors.Error{ - ExternalMsg: "Trigger ID is required", - Err: ValidationError, - Code: http.StatusBadRequest, - } - } - if log.ID == "" { - log.ID = uuid.New().String() - } - if log.CreatedAt.IsZero() { - log.CreatedAt = time.Now().UTC() - } - if err := log.Insert(ctx, r.db, boil.Infer()); err != nil { - return richerrors.Error{ - ExternalMsg: "Failed to create trigger log", - Err: err, - Code: http.StatusInternalServerError, - } - } - return nil -} - -// ResetTriggerFailureCount resets failure count on successful webhook delivery -func (r *Repository) ResetTriggerFailureCount(ctx context.Context, trigger *models.Trigger) error { - // Fetch latest trigger state - updatedTrigger, tx, err := r.GetTriggerByIDAndDeveloperLicenseForUpdate(ctx, trigger.ID, common.BytesToAddress(trigger.DeveloperLicenseAddress)) - if err != nil { - return fmt.Errorf("failed to fetch trigger for success reset: %w", err) - } - defer RollbackTx(ctx, tx) - - if updatedTrigger.FailureCount < 1 { - // do not update if don't have anything to reset - return nil - } - - // Reset failure count - updatedTrigger.FailureCount = 0 - - // If trigger was in failed state, re-enable it - if updatedTrigger.Status == StatusFailed { - updatedTrigger.Status = StatusEnabled - } - - if err := r.UpdateTriggerWithTx(ctx, tx, updatedTrigger); err != nil { - return fmt.Errorf("failed to reset failure count: %w", err) - } - - if err := tx.Commit(); err != nil { - return richerrors.Error{ - ExternalMsg: "Failed to commit Update.", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return nil -} - -// IncrementTriggerFailureCount increments failure count and disables webhook if threshold reached. -func (r *Repository) IncrementTriggerFailureCount(ctx context.Context, trigger *models.Trigger, failureReason error, maxFailureCount int) error { - // Fetch latest trigger state - updatedTrigger, tx, err := r.GetTriggerByIDAndDeveloperLicenseForUpdate(ctx, trigger.ID, common.BytesToAddress(trigger.DeveloperLicenseAddress)) - if err != nil { - return fmt.Errorf("failed to fetch trigger for failure handling: %w", err) - } - defer RollbackTx(ctx, tx) - - // Increment failure count - updatedTrigger.FailureCount++ - - // Disable webhook if failure threshold reached - if updatedTrigger.FailureCount >= maxFailureCount { - updatedTrigger.Status = StatusFailed - zerolog.Ctx(ctx).Warn(). - Str("triggerId", trigger.ID). - Int("maxFailures", maxFailureCount). - Msg("webhook disabled due to excessive failures") - } - - if err := r.UpdateTriggerWithTx(ctx, tx, updatedTrigger); err != nil { - return fmt.Errorf("failed to update failure count: %w", err) - } - - if err := tx.Commit(); err != nil { - return richerrors.Error{ - ExternalMsg: "Failed to commit Update.", - Err: err, - Code: http.StatusInternalServerError, - } - } - - return nil -} - -func RollbackTx(ctx context.Context, tx *sql.Tx) { - if tx == nil { - return - } - if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { - zerolog.Ctx(ctx).Error().Err(err).Msg("failed to rollback transaction") - } -} diff --git a/internal/services/triggersrepo/triggersrepo_test.go b/internal/services/triggersrepo/triggersrepo_test.go index a848fca..5d929f8 100644 --- a/internal/services/triggersrepo/triggersrepo_test.go +++ b/internal/services/triggersrepo/triggersrepo_test.go @@ -5,19 +5,16 @@ import ( "context" "crypto/rand" "database/sql" - "encoding/json" "errors" "math/big" "net/http" "slices" "testing" - "time" "github.com/DIMO-Network/cloudevent" "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" "github.com/DIMO-Network/vehicle-triggers-api/tests" - "github.com/aarondl/null/v8" "github.com/ethereum/go-ethereum/common" "github.com/google/uuid" "github.com/stretchr/testify/assert" @@ -1717,205 +1714,6 @@ func createTestTriggerWithFailures(t *testing.T, repo *Repository, ctx context.C return trigger } -func TestCreateTriggerLog(t *testing.T) { - t.Parallel() - tc := tests.SetupTestContainer(t) - - repo := NewRepository(tc.DB) - ctx := context.Background() - - // Create a test trigger first - baseReq := CreateTriggerRequest{ - Service: ServiceSignal, - MetricName: "vss.speed", - Condition: "valueNumber > 20", - TargetURI: "https://example.com/webhook", - Status: StatusEnabled, - Description: "Speed alert", - CooldownPeriod: 10, - DeveloperLicenseAddress: tests.RandomAddr(t), - } - - trigger, err := repo.CreateTrigger(ctx, baseReq) - require.NoError(t, err) - require.NotNil(t, trigger) - - assetDid := randAssetDID(t) - - t.Run("successful creation", func(t *testing.T) { - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 25, "timestamp": "2023-10-01T12:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err := repo.CreateTriggerLog(ctx, log) - require.NoError(t, err) - }) - - t.Run("create multiple logs for same trigger", func(t *testing.T) { - log1 := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 30, "timestamp": "2023-10-01T13:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - log2 := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 35, "timestamp": "2023-10-01T14:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err := repo.CreateTriggerLog(ctx, log1) - require.NoError(t, err) - - err = repo.CreateTriggerLog(ctx, log2) - require.NoError(t, err) - }) - - t.Run("create log with failure reason", func(t *testing.T) { - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 40, "timestamp": "2023-10-01T15:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - FailureReason: null.StringFrom("Webhook timeout after 30 seconds"), - } - - err := repo.CreateTriggerLog(ctx, log) - require.NoError(t, err) - }) - - t.Run("create log for non-existent trigger", func(t *testing.T) { - nonExistentTriggerID := uuid.New().String() - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: nonExistentTriggerID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 45, "timestamp": "2023-10-01T16:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err := repo.CreateTriggerLog(ctx, log) - require.Error(t, err) - var richErr richerrors.Error - require.ErrorAs(t, err, &richErr) - assert.Equal(t, http.StatusInternalServerError, richErr.Code) - }) - - t.Run("create log with missing required fields", func(t *testing.T) { - // Test with empty trigger ID - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: "", - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 50, "timestamp": "2023-10-01T17:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err := repo.CreateTriggerLog(ctx, log) - require.Error(t, err) - var richErr richerrors.Error - require.ErrorAs(t, err, &richErr) - assert.Equal(t, http.StatusBadRequest, richErr.Code) - }) - - t.Run("create log with empty asset DID", func(t *testing.T) { - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: "", - SnapshotData: []byte(`{"speed": 55, "timestamp": "2023-10-01T18:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err := repo.CreateTriggerLog(ctx, log) - require.Error(t, err) - var richErr richerrors.Error - require.ErrorAs(t, err, &richErr) - assert.Equal(t, http.StatusBadRequest, richErr.Code) - }) - - t.Run("create log with duplicate ID", func(t *testing.T) { - logID := uuid.New().String() - - log1 := &models.TriggerLog{ - ID: logID, - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 60, "timestamp": "2023-10-01T19:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - log2 := &models.TriggerLog{ - ID: logID, // Same ID - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: []byte(`{"speed": 65, "timestamp": "2023-10-01T20:00:00Z"}`), - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - // First log should succeed - err := repo.CreateTriggerLog(ctx, log1) - require.NoError(t, err) - - // Second log with duplicate ID should fail - err = repo.CreateTriggerLog(ctx, log2) - require.Error(t, err) - var richErr richerrors.Error - require.ErrorAs(t, err, &richErr) - assert.Equal(t, http.StatusInternalServerError, richErr.Code) - }) - - t.Run("create log with complex snapshot data", func(t *testing.T) { - complexSnapshot := map[string]interface{}{ - "speed": 25.5, - "temperature": 85.2, - "fuel_level": 0.75, - "location": map[string]float64{ - "latitude": 40.7128, - "longitude": -74.0060, - }, - "metadata": map[string]interface{}{ - "vehicle_id": "VIN123456789", - "trip_id": "TRIP789", - "event_time": "2023-10-01T21:00:00Z", - }, - } - - snapshotBytes, err := json.Marshal(complexSnapshot) - require.NoError(t, err) - - log := &models.TriggerLog{ - ID: uuid.New().String(), - TriggerID: trigger.ID, - AssetDid: assetDid.String(), - SnapshotData: snapshotBytes, - LastTriggeredAt: time.Now().UTC(), - CreatedAt: time.Now().UTC(), - } - - err = repo.CreateTriggerLog(ctx, log) - require.NoError(t, err) - }) -} - func randAssetDID(t *testing.T) cloudevent.ERC721DID { tokenID := make([]byte, 32) _, err := rand.Read(tokenID) diff --git a/internal/services/triggerstate/memory.go b/internal/services/triggerstate/memory.go new file mode 100644 index 0000000..c5dab3c --- /dev/null +++ b/internal/services/triggerstate/memory.go @@ -0,0 +1,66 @@ +package triggerstate + +import ( + "context" + "encoding/json" + "sync" + "time" + + "github.com/DIMO-Network/cloudevent" +) + +// InMemoryStore is a process-local fallback Store for setups without NATS. +// It satisfies cooldown and previousValue lookups within one replica but does +// NOT share state across replicas. Use only in single-process tests or in the +// legacy NATS_MODE=off deployment topology that's being phased out. +type InMemoryStore struct { + mu sync.RWMutex + triggers map[string]Record + metrics map[string]MetricRecord +} + +// NewInMemory returns an empty in-process store. +func NewInMemory() *InMemoryStore { + return &InMemoryStore{ + triggers: make(map[string]Record), + metrics: make(map[string]MetricRecord), + } +} + +// LastFire returns the last fire record for (trigger, vehicle). +func (s *InMemoryStore) LastFire(_ context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (Record, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.triggers[TriggerKey(triggerID, vehicleDID)] + return r, ok, nil +} + +// LastMetric returns the last fire record for (vehicle, metric). +func (s *InMemoryStore) LastMetric(_ context.Context, vehicleDID cloudevent.ERC721DID, metricName string) (MetricRecord, bool, error) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.metrics[MetricKey(vehicleDID, metricName)] + return r, ok, nil +} + +// RecordFire writes both records under one lock so a concurrent LastFire + +// LastMetric pair sees a consistent view. +func (s *InMemoryStore) RecordFire(_ context.Context, triggerID, metricName string, vehicleDID cloudevent.ERC721DID, at time.Time, snapshot json.RawMessage) error { + s.mu.Lock() + defer s.mu.Unlock() + s.triggers[TriggerKey(triggerID, vehicleDID)] = Record{ + LastFiredAt: at.UTC(), + TriggerID: triggerID, + AssetDID: vehicleDID.String(), + LastSnapshot: snapshot, + } + if metricName != "" { + s.metrics[MetricKey(vehicleDID, metricName)] = MetricRecord{ + LastFiredAt: at.UTC(), + AssetDID: vehicleDID.String(), + MetricName: metricName, + LastSnapshot: snapshot, + } + } + return nil +} diff --git a/internal/services/triggerstate/metrics.go b/internal/services/triggerstate/metrics.go new file mode 100644 index 0000000..5828466 --- /dev/null +++ b/internal/services/triggerstate/metrics.go @@ -0,0 +1,73 @@ +package triggerstate + +import ( + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Prometheus instrumentation for the evaluator state path. Counters surface +// the rate of concurrent-writer races so we can see how often the +// at-least-once delivery contract is actually producing duplicate fires in +// production. Decode-error counters surface silent data corruption that +// would otherwise just degrade the previousValue lookup to a zero default. +// sampledErrors is a parallel atomic counter to the prom decodeErrors metric. +// Prom counters don't expose their value publicly; we keep our own so the +// "sample every Nth" decision in MetricsDecodeErrorWithSample is cheap and +// deterministic without scraping our own metrics. +var sampledErrors atomic.Uint64 + +var ( + casConflicts = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "state", + Name: "cas_conflicts_total", + Help: "Number of CAS conflicts observed on the per-trigger state bucket. A non-zero rate means two replicas raced on the same (trigger, vehicle); see PROD_HARDENING.md for the receiver-dedup contract.", + }, []string{"bucket", "outcome"}) + + decodeErrors = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "state", + Name: "decode_errors_total", + Help: "Number of KV records the service failed to JSON-decode. A non-zero rate means writers and readers disagree on the schema or the bucket has corruption.", + }, []string{"bucket"}) +) + +// metricsCASConflict records the outcome of an attempted CAS write. +// outcome ∈ {retry, fallback}. retry = first conflict that we resolved by +// re-fetching and retrying. fallback = persistent conflict; we wrote +// unconditionally and the receiver must dedup. +func metricsCASConflict(bucket, outcome string) { + casConflicts.WithLabelValues(bucket, outcome).Inc() +} + +// MetricsDecodeError records a single decode failure on the named bucket. +// Exported so the evaluator can call it from its read path. +func MetricsDecodeError(bucket string) { + decodeErrors.WithLabelValues(bucket).Inc() +} + +// MetricsDecodeErrorWithSample bumps the counter and, every sampleEvery +// errors, returns a non-empty preview of the bad payload (truncated). The +// caller is responsible for logging that preview - we return rather than +// log here so we don't take a dependency on a logger. Returns "" when we +// shouldn't sample this occurrence. +// +// The intent is to surface enough of a real malformed payload that ops can +// reproduce locally without flooding logs at high error rates. +func MetricsDecodeErrorWithSample(bucket string, payload []byte, sampleEvery int) string { + MetricsDecodeError(bucket) + if sampleEvery < 1 { + sampleEvery = 100 + } + n := sampledErrors.Add(1) + if int(n%uint64(sampleEvery)) != 0 { + return "" + } + const max = 200 + if len(payload) > max { + return string(payload[:max]) + "...(truncated)" + } + return string(payload) +} diff --git a/internal/services/triggerstate/triggerstate.go b/internal/services/triggerstate/triggerstate.go new file mode 100644 index 0000000..49def22 --- /dev/null +++ b/internal/services/triggerstate/triggerstate.go @@ -0,0 +1,254 @@ +// Package triggerstate stores evaluator state in NATS JetStream KV buckets so +// signal/event evaluation has zero Postgres reads on the hot path. +// +// Two buckets are used: +// +// - trigger_state (per trigger + vehicle): last fire timestamp and the +// payload snapshot from that fire. Drives the cooldown check and the +// "previous value for THIS trigger" CEL input on the event path. TTL is +// configurable (NATS_TRIGGER_STATE_TTL, default 7d) and bounds storage at +// the longest reasonable cooldown window. +// +// - signal_history (per vehicle + metric): payload snapshot from the most +// recent fire of ANY trigger on this metric for this vehicle. Drives the +// "previous value across triggers" CEL input on the signal path. TTL +// bounded similarly. +// +// Writes happen synchronously after a successful webhook delivery so any +// replica reading next sees the new state. Reads are best-effort: a KV miss +// or error returns "no prior data" so evaluation uses zero-valued previous +// data, matching the first-time-fire case. That trades a single redundant +// fire on a hard NATS outage for never hanging the eval loop on the DB. +package triggerstate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/nats-io/nats.go/jetstream" +) + +// Store reads and writes evaluator state. The interface is small so it can be +// swapped or mocked in tests. +type Store interface { + LastFire(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (Record, bool, error) + LastMetric(ctx context.Context, vehicleDID cloudevent.ERC721DID, metricName string) (MetricRecord, bool, error) + RecordFire(ctx context.Context, triggerID, metricName string, vehicleDID cloudevent.ERC721DID, at time.Time, snapshot json.RawMessage) error +} + +// Record is the JSON payload stored per (trigger, vehicle) key in the +// trigger_state bucket. +type Record struct { + LastFiredAt time.Time `json:"lastFiredAt"` + TriggerID string `json:"triggerId"` + AssetDID string `json:"assetDid"` + LastSnapshot json.RawMessage `json:"lastSnapshot,omitempty"` +} + +// MetricRecord is the JSON payload stored per (vehicle, metric) key in the +// signal_history bucket. +type MetricRecord struct { + LastFiredAt time.Time `json:"lastFiredAt"` + AssetDID string `json:"assetDid"` + MetricName string `json:"metricName"` + LastSnapshot json.RawMessage `json:"lastSnapshot,omitempty"` +} + +// KVStore is a Store backed by two NATS JetStream KV buckets - one per-trigger +// and one per-metric. Both buckets are opened by the caller; passing nil for +// either disables that half (used by tests that exercise only one path). +type KVStore struct { + state jetstream.KeyValue + history jetstream.KeyValue +} + +// New wraps existing KV bucket handles. state holds per-(trigger, vehicle) +// records, history holds per-(vehicle, metric) records. Either may be nil. +func New(state, history jetstream.KeyValue) *KVStore { + return &KVStore{state: state, history: history} +} + +// TriggerKey builds the trigger_state bucket key for (triggerID, vehicleDID). +// NATS KV keys allow alphanumeric plus -_=./ — we replace anything else with +// '_' so triggerIDs (UUIDs) and DIDs (did:erc721:...) round-trip cleanly. +func TriggerKey(triggerID string, vehicleDID cloudevent.ERC721DID) string { + return sanitize(triggerID) + "." + sanitize(vehicleDID.String()) +} + +// MetricKey builds the signal_history bucket key for (vehicleDID, metricName). +func MetricKey(vehicleDID cloudevent.ERC721DID, metricName string) string { + return sanitize(vehicleDID.String()) + "." + sanitize(metricName) +} + +// Key is kept as an alias for the trigger key for backwards compatibility +// with the CLI and earlier call sites. +func Key(triggerID string, vehicleDID cloudevent.ERC721DID) string { + return TriggerKey(triggerID, vehicleDID) +} + +func sanitize(s string) string { + r := strings.NewReplacer( + ":", "_", + " ", "_", + "*", "_", + ">", "_", + ) + return r.Replace(s) +} + +// LastFire returns the most recent fire record for this (trigger, vehicle). +// The second return value is false when the key is absent. +func (s *KVStore) LastFire(ctx context.Context, triggerID string, vehicleDID cloudevent.ERC721DID) (Record, bool, error) { + if s == nil || s.state == nil { + return Record{}, false, nil + } + entry, err := s.state.Get(ctx, TriggerKey(triggerID, vehicleDID)) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return Record{}, false, nil + } + return Record{}, false, fmt.Errorf("kv get trigger_state: %w", err) + } + var r Record + if err := json.Unmarshal(entry.Value(), &r); err != nil { + return Record{}, false, fmt.Errorf("kv decode trigger_state: %w", err) + } + return r, true, nil +} + +// LastMetric returns the most recent fire record for this (vehicle, metric) +// across any trigger. False when absent. +func (s *KVStore) LastMetric(ctx context.Context, vehicleDID cloudevent.ERC721DID, metricName string) (MetricRecord, bool, error) { + if s == nil || s.history == nil { + return MetricRecord{}, false, nil + } + entry, err := s.history.Get(ctx, MetricKey(vehicleDID, metricName)) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + return MetricRecord{}, false, nil + } + return MetricRecord{}, false, fmt.Errorf("kv get signal_history: %w", err) + } + var r MetricRecord + if err := json.Unmarshal(entry.Value(), &r); err != nil { + return MetricRecord{}, false, fmt.Errorf("kv decode signal_history: %w", err) + } + return r, true, nil +} + +// RecordFire writes both KV records in one logical call. +// +// trigger_state uses optimistic CAS so concurrent writers from two replicas +// on the same (trigger, vehicle) don't both blindly Put. Race semantics: +// +// 1. Read current revision via Get. Marshal new Record. +// 2. Try Update on the observed revision. On revision-mismatch, refresh and +// retry once - covers the common "another writer just landed" case. +// 3. On a second conflict, fall back to Put (last-writer-wins) and record +// the conflict on a counter. The other writer already sent its webhook; +// we already sent ours. Receivers must dedup via the deterministic +// webhook ID. See PROD_HARDENING.md for the contract. +// +// signal_history is per (vehicle, metric) across triggers - Put is correct +// because the most-recent value wins by definition. +// +// metricName may be empty when the caller does not have a per-metric concept +// (e.g. unit tests); in that case the signal_history write is skipped. +func (s *KVStore) RecordFire(ctx context.Context, triggerID, metricName string, vehicleDID cloudevent.ERC721DID, at time.Time, snapshot json.RawMessage) error { + now := at.UTC() + if s != nil && s.state != nil { + body, err := json.Marshal(Record{ + LastFiredAt: now, + TriggerID: triggerID, + AssetDID: vehicleDID.String(), + LastSnapshot: snapshot, + }) + if err != nil { + return fmt.Errorf("kv encode trigger_state: %w", err) + } + key := TriggerKey(triggerID, vehicleDID) + if err := writeWithCAS(ctx, s.state, key, body, "trigger_state"); err != nil { + return err + } + } + if metricName == "" || s == nil || s.history == nil { + return nil + } + body, err := json.Marshal(MetricRecord{ + LastFiredAt: now, + AssetDID: vehicleDID.String(), + MetricName: metricName, + LastSnapshot: snapshot, + }) + if err != nil { + return fmt.Errorf("kv encode signal_history: %w", err) + } + if _, err := s.history.Put(ctx, MetricKey(vehicleDID, metricName), body); err != nil { + return fmt.Errorf("kv put signal_history: %w", err) + } + return nil +} + +// writeWithCAS executes the CAS-with-retry-with-fallback policy described +// above on the supplied bucket + key. Errors that aren't conflict-shaped +// propagate immediately. bucketLabel is used only for the conflict metric. +func writeWithCAS(ctx context.Context, kv jetstream.KeyValue, key string, body []byte, bucketLabel string) error { + const maxRetries = 1 + + for attempt := 0; attempt <= maxRetries; attempt++ { + entry, getErr := kv.Get(ctx, key) + if getErr != nil && !errors.Is(getErr, jetstream.ErrKeyNotFound) { + return fmt.Errorf("kv get %s: %w", bucketLabel, getErr) + } + var rev uint64 + if getErr == nil { + rev = entry.Revision() + } + + var writeErr error + if rev == 0 { + _, writeErr = kv.Create(ctx, key, body) + } else { + _, writeErr = kv.Update(ctx, key, body, rev) + } + if writeErr == nil { + return nil + } + if !isConflict(writeErr) { + return fmt.Errorf("kv write %s: %w", bucketLabel, writeErr) + } + + if attempt < maxRetries { + metricsCASConflict(bucketLabel, "retry") + continue + } + // Persistent conflict: write unconditionally so state isn't lost, + // and surface the race via the conflict counter so ops can see it. + metricsCASConflict(bucketLabel, "fallback") + if _, err := kv.Put(ctx, key, body); err != nil { + return fmt.Errorf("kv put fallback %s: %w", bucketLabel, err) + } + return nil + } + return nil +} + +// isConflict matches the error shapes the KV API returns when an optimistic +// write loses the CAS race. Both Create (key already exists) and Update +// (wrong revision) surface as JSErrCodeStreamWrongLastSequence (10071) under +// the hood; ErrKeyExists is the sentinel for the Create form. +func isConflict(err error) bool { + if errors.Is(err, jetstream.ErrKeyExists) { + return true + } + var apiErr *jetstream.APIError + if errors.As(err, &apiErr) { + return apiErr.ErrorCode == jetstream.JSErrCodeStreamWrongLastSequence + } + return false +} diff --git a/internal/services/triggerstate/triggerstate_test.go b/internal/services/triggerstate/triggerstate_test.go new file mode 100644 index 0000000..0c903b9 --- /dev/null +++ b/internal/services/triggerstate/triggerstate_test.go @@ -0,0 +1,38 @@ +package triggerstate + +import ( + "math/big" + "testing" + + "github.com/DIMO-Network/cloudevent" + "github.com/ethereum/go-ethereum/common" +) + +func TestKey(t *testing.T) { + did := cloudevent.ERC721DID{ + ChainID: 137, + ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), + TokenID: big.NewInt(42), + } + got := Key("abc-123-uuid", did) + // triggerID kept as-is, DID's colons replaced. + want := "abc-123-uuid.did_erc721_137_0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF_42" + if got != want { + t.Fatalf("Key = %q, want %q", got, want) + } +} + +func TestSanitize(t *testing.T) { + cases := map[string]string{ + "plain": "plain", + "with:colons": "with_colons", + "spaces here": "spaces_here", + "wild*card": "wild_card", + "angle>brackets": "angle_brackets", + } + for in, want := range cases { + if got := sanitize(in); got != want { + t.Errorf("sanitize(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/services/webhookcache/webhook_cache.go b/internal/services/webhookcache/webhook_cache.go index 579eae9..f4045d4 100644 --- a/internal/services/webhookcache/webhook_cache.go +++ b/internal/services/webhookcache/webhook_cache.go @@ -33,34 +33,100 @@ type Webhook struct { type Repository interface { InternalGetAllVehicleSubscriptions(ctx context.Context) ([]*models.VehicleSubscription, error) InternalGetTriggerByID(ctx context.Context, triggerID string) (*models.Trigger, error) + DecryptSigningSecret(stored string) (string, error) } -// WebhookCache is an in-memory map: assetDID -> signal name -> []*models.Trigger. +// Notifier broadcasts a "config changed" event over NATS so other replicas +// can invalidate their own caches in milliseconds instead of waiting for +// the periodic poll. Set via SetNotifier when NATS is configured. +type Notifier interface { + Notify(ctx context.Context, webhookID string, op string) error +} + +type noopNotifier struct{} + +func (noopNotifier) Notify(context.Context, string, string) error { return nil } + +// WebhookCache is an in-memory map: assetDID -> signal name -> []*Webhook. +// +// Two levels of caching live here: +// - `webhooks`: the per-vehicle dispatch index hit on every signal. +// - `compiled`: a shared cache of {trigger row + compiled CEL program} +// keyed by trigger ID. Rebuilds reuse compiled programs for triggers +// whose config hasn't changed, so a 5-min periodic refresh on a +// stable config does ~0 CEL compiles. CRUD invalidations drop the +// affected trigger from `compiled` so the next rebuild recompiles it. +// +// This is the "diff-rebuild" half of the V2 review. Full tenant-scoped +// lazy loading (per-developer subscriptions fetched on first signal) +// is deliberately not implemented yet - the compiled-program cache is +// the easy 80% of the CPU win and avoids the cache-coherence complexity +// of partial scoping. type WebhookCache struct { - mu sync.RWMutex - webhooks map[string]map[string][]*Webhook - repo Repository - lastRefresh time.Time // last time the cache was refreshed - schedule atomic.Bool - debounce time.Duration - buildWorkers int + mu sync.RWMutex + webhooks map[string]map[string][]*Webhook + repo Repository + lastRefresh time.Time // last time the cache was refreshed + schedule atomic.Bool + debounce time.Duration + buildWorkers int + notifier Notifier + + // compiled is the shared {trigger row, CEL program} cache keyed by + // trigger ID. Owned by mu (read under RLock for build-time lookup, + // write under Lock for refresh + targeted invalidation). nil program + // entries are never stored - compile failures are skipped at fetch + // time. The dispatch path doesn't read this map; it only reads the + // `webhooks` map populated from it. + compiled map[string]*Webhook } func NewWebhookCache(repo Repository, settings *config.Settings) *WebhookCache { - debounce := settings.CacheDebounceTime + debounce := settings.Cache.DebounceTime if debounce == 0 { debounce = defaultCacheDebounceTime } - workers := settings.CacheBuildWorkers + workers := settings.Cache.BuildWorkers if workers < 1 { workers = defaultCacheBuildWorkers } return &WebhookCache{ webhooks: make(map[string]map[string][]*Webhook), + compiled: make(map[string]*Webhook), repo: repo, debounce: debounce, buildWorkers: workers, + notifier: noopNotifier{}, + } +} + +// InvalidateTrigger drops a single trigger from the compiled-program cache +// so the next rebuild re-fetches and recompiles it. Called by the +// cachebroadcast subscriber when a remote replica's CRUD broadcast carries +// a specific webhook ID. Targeted invalidation lets a CRUD on one trigger +// avoid recompiling every other trigger's CEL program on every replica. +// +// Does NOT trigger an immediate rebuild - the caller decides cadence via +// ScheduleRefreshSilent. Keep these methods orthogonal so a burst of +// CRUDs collapses into one rebuild after the debounce window. +func (wc *WebhookCache) InvalidateTrigger(triggerID string) { + if triggerID == "" { + return } + wc.mu.Lock() + delete(wc.compiled, triggerID) + wc.mu.Unlock() +} + +// SetNotifier wires the cache to publish a "config changed" event on +// ScheduleRefresh. Used by the API CRUD paths; the subscriber that receives +// remote notifications calls ScheduleRefreshSilent instead so the +// invalidation event doesn't echo back into another publish. +func (wc *WebhookCache) SetNotifier(n Notifier) { + if n == nil { + n = noopNotifier{} + } + wc.notifier = n } // PopulateCache builds the cache from the database @@ -104,16 +170,40 @@ func logMemStats(logger *zerolog.Logger, phase string) { Msg("memstats") } +// ScheduleRefresh debounces a rebuild of this replica's cache and publishes +// a cross-replica invalidation event. Called from CRUD handlers. func (wc *WebhookCache) ScheduleRefresh(ctx context.Context) { + wc.scheduleRefresh(ctx, true) +} + +// ScheduleRefreshSilent is the receiver side of cross-replica invalidation: +// rebuilds the local cache but does NOT re-broadcast, so notifications +// don't echo into infinite republishing across replicas. +func (wc *WebhookCache) ScheduleRefreshSilent(ctx context.Context) { + wc.scheduleRefresh(ctx, false) +} + +func (wc *WebhookCache) scheduleRefresh(ctx context.Context, broadcast bool) { + if broadcast && wc.notifier != nil { + if err := wc.notifier.Notify(ctx, "", "any"); err != nil { + zerolog.Ctx(ctx).Warn().Err(err).Msg("cache invalidate publish failed; relying on poll") + } + } if wc.schedule.CompareAndSwap(false, true) { + // Capture the logger from the caller's ctx, but detach the value + // chain from the request-bound ctx itself. Fiber pools fasthttp + // request contexts; holding one across the debounce + DB roundtrip + // races with the next request reusing the same memory. Use a fresh + // background ctx for the actual refresh and copy only the logger so + // log lines still get the right structured fields. + logger := zerolog.Ctx(ctx) go func() { time.Sleep(wc.debounce) if wc.schedule.Load() { - // if we waited and we still want to refresh, do it wc.schedule.Store(false) - err := wc.PopulateCache(ctx) - if err != nil { - zerolog.Ctx(ctx).Error().Err(err).Msg("failed to populate webhook cache") + bgCtx := logger.WithContext(context.Background()) + if err := wc.PopulateCache(bgCtx); err != nil { + logger.Error().Err(err).Msg("failed to populate webhook cache") } } }() @@ -142,6 +232,41 @@ func (wc *WebhookCache) Update(newData map[string]map[string][]*Webhook) { wc.lastRefresh = time.Now() } +// InvalidateVehicleTrigger removes the cached webhook entries for the given +// (assetDID, triggerID) pair without broadcasting a cross-replica refresh. +// Use this on the hot path - for example when eval discovers a revoked +// permission and unsubscribes the vehicle locally. The 5-minute periodic +// refresh is the reconciliation safety net for other replicas; we +// deliberately do NOT publish a cachebroadcast event here because +// permission-denied is a per-signal event and broadcasting it would +// stampede every replica thousands of times per second on a misconfigured +// developer. +func (wc *WebhookCache) InvalidateVehicleTrigger(assetDID, triggerID string) { + wc.mu.Lock() + defer wc.mu.Unlock() + + byVehicle, ok := wc.webhooks[assetDID] + if !ok { + return + } + for key, hooks := range byVehicle { + filtered := hooks[:0] + for _, h := range hooks { + if h.Trigger == nil || h.Trigger.ID != triggerID { + filtered = append(filtered, h) + } + } + if len(filtered) == 0 { + delete(byVehicle, key) + } else { + byVehicle[key] = filtered + } + } + if len(byVehicle) == 0 { + delete(wc.webhooks, assetDID) + } +} + func (wc *WebhookCache) fetchVehicleWebhooks(ctx context.Context) (map[string]map[string][]*Webhook, error) { logger := zerolog.Ctx(ctx) fetchStart := time.Now() @@ -161,12 +286,12 @@ func (wc *WebhookCache) fetchVehicleWebhooks(ctx context.Context) (map[string]ma uniqueTriggerIDs[sub.TriggerID] = struct{}{} } - // Fetch + CEL-compile each unique trigger in parallel. Each compile is - // CPU-bound (~5ms) and the loop is hot on startup (~10k triggers => 50s - // serial). Parallelising across GOMAXPROCS workers brings build_elapsed - // well under the kubelet liveness deadline. + // Diff-rebuild: split the set into already-compiled (cheap, reused) + // and new (must fetch + compile). Eviction of compiled-but-no-longer- + // subscribed entries is done at the end so we keep CPU bounded by + // "what actually changed since the last refresh." triggerFetchStart := time.Now() - uniqueTriggers := wc.compileTriggersParallel(ctx, uniqueTriggerIDs) + uniqueTriggers, reusedCount := wc.buildCompiledIndex(ctx, uniqueTriggerIDs) newData := make(map[string]map[string][]*Webhook) for _, sub := range subs { @@ -189,6 +314,7 @@ func (wc *WebhookCache) fetchVehicleWebhooks(ctx context.Context) (map[string]ma Int("sub_count", len(subs)). Int("unique_trigger_ids", len(uniqueTriggerIDs)). Int("unique_trigger_cache", len(uniqueTriggers)). + Int("reused_compiled", reusedCount). Int("asset_count", len(newData)). Dur("build_elapsed", time.Since(triggerFetchStart)). Msg("webhook cache build complete") @@ -204,6 +330,48 @@ func webhookKey(service, metricName string) string { return service + ":" + metricName } +// buildCompiledIndex returns the full {triggerID -> *Webhook} index for the +// requested IDs, reusing entries from wc.compiled when present and only +// fetching + CEL-compiling the new ones. Returns (index, reusedCount). +// +// Eviction: triggers that fall out of triggerIDs (no remaining subscribers) +// are dropped from wc.compiled at the end so the map can't grow unbounded +// as customers churn triggers. +func (wc *WebhookCache) buildCompiledIndex(ctx context.Context, triggerIDs map[string]struct{}) (map[string]*Webhook, int) { + wc.mu.RLock() + out := make(map[string]*Webhook, len(triggerIDs)) + missing := make(map[string]struct{}, len(triggerIDs)) + for id := range triggerIDs { + if w, ok := wc.compiled[id]; ok { + out[id] = w + continue + } + missing[id] = struct{}{} + } + wc.mu.RUnlock() + reused := len(out) + + if len(missing) > 0 { + fresh := wc.compileTriggersParallel(ctx, missing) + wc.mu.Lock() + for id, w := range fresh { + wc.compiled[id] = w + out[id] = w + } + // Evict compiled entries that no longer have any subscribers so + // the map stays bounded as triggers churn. Comparing against + // triggerIDs (the *current* subscription set) is the right + // invariant - anything not in there isn't being dispatched on. + for id := range wc.compiled { + if _, ok := triggerIDs[id]; !ok { + delete(wc.compiled, id) + } + } + wc.mu.Unlock() + } + return out, reused +} + // compileTriggersParallel fetches and CEL-compiles each unique trigger in // parallel. Worker count comes from CACHE_BUILD_WORKERS so prod can tune it // against its CPU limit and DB connection pool. Triggers that fail to fetch @@ -240,6 +408,18 @@ func (wc *WebhookCache) compileTriggersParallel(ctx context.Context, triggerIDs logger.Error().Err(err).Str("trigger_id", id).Msg("failed to get trigger by id for webhook cache") continue } + // Decrypt the signing secret here so downstream code paths + // (sender HMAC, audit) see plaintext. Failure to decrypt is + // not fatal - the trigger still functions for non-signing + // concerns; the sender just won't add a signature header. + if trigger.SigningSecret.Valid { + if pt, err := wc.repo.DecryptSigningSecret(trigger.SigningSecret.String); err == nil { + trigger.SigningSecret.String = pt + } else { + logger.Warn().Err(err).Str("trigger_id", id).Msg("failed to decrypt signing secret; webhook will be sent unsigned") + trigger.SigningSecret.Valid = false + } + } valueType := "" if triggersrepo.IsSignalService(trigger.Service) { valueType = signals.GetSignalDefinitionOrDefault(signals.BareSignalName(trigger.MetricName), signals.NumberType).ValueType diff --git a/internal/services/webhookcache/webhook_cache_mock_test.go b/internal/services/webhookcache/webhook_cache_mock_test.go index 2a00da2..fb9de99 100644 --- a/internal/services/webhookcache/webhook_cache_mock_test.go +++ b/internal/services/webhookcache/webhook_cache_mock_test.go @@ -41,6 +41,21 @@ func (m *MockRepository) EXPECT() *MockRepositoryMockRecorder { return m.recorder } +// DecryptSigningSecret mocks base method. +func (m *MockRepository) DecryptSigningSecret(stored string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DecryptSigningSecret", stored) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DecryptSigningSecret indicates an expected call of DecryptSigningSecret. +func (mr *MockRepositoryMockRecorder) DecryptSigningSecret(stored any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DecryptSigningSecret", reflect.TypeOf((*MockRepository)(nil).DecryptSigningSecret), stored) +} + // InternalGetAllVehicleSubscriptions mocks base method. func (m *MockRepository) InternalGetAllVehicleSubscriptions(ctx context.Context) ([]*models.VehicleSubscription, error) { m.ctrl.T.Helper() @@ -70,3 +85,41 @@ func (mr *MockRepositoryMockRecorder) InternalGetTriggerByID(ctx, triggerID any) mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InternalGetTriggerByID", reflect.TypeOf((*MockRepository)(nil).InternalGetTriggerByID), ctx, triggerID) } + +// MockNotifier is a mock of Notifier interface. +type MockNotifier struct { + ctrl *gomock.Controller + recorder *MockNotifierMockRecorder + isgomock struct{} +} + +// MockNotifierMockRecorder is the mock recorder for MockNotifier. +type MockNotifierMockRecorder struct { + mock *MockNotifier +} + +// NewMockNotifier creates a new mock instance. +func NewMockNotifier(ctrl *gomock.Controller) *MockNotifier { + mock := &MockNotifier{ctrl: ctrl} + mock.recorder = &MockNotifierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockNotifier) EXPECT() *MockNotifierMockRecorder { + return m.recorder +} + +// Notify mocks base method. +func (m *MockNotifier) Notify(ctx context.Context, webhookID, op string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Notify", ctx, webhookID, op) + ret0, _ := ret[0].(error) + return ret0 +} + +// Notify indicates an expected call of Notify. +func (mr *MockNotifierMockRecorder) Notify(ctx, webhookID, op any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Notify", reflect.TypeOf((*MockNotifier)(nil).Notify), ctx, webhookID, op) +} diff --git a/internal/services/webhookdispatcher/backoff_test.go b/internal/services/webhookdispatcher/backoff_test.go new file mode 100644 index 0000000..6ddfa86 --- /dev/null +++ b/internal/services/webhookdispatcher/backoff_test.go @@ -0,0 +1,99 @@ +package webhookdispatcher + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhooksender" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// timestampedSender records the exact moment each delivery attempt arrived +// so we can verify the backoff schedule matches what's documented in code +// comments and the PROD_HARDENING_V2 description for item F. +type timestampedSender struct { + mu sync.Mutex + times []time.Time + err error +} + +func (s *timestampedSender) SendWebhook(_ context.Context, _ *models.Trigger, _ *cloudevent.CloudEvent[webhook.WebhookPayload]) error { + s.mu.Lock() + s.times = append(s.times, time.Now()) + s.mu.Unlock() + return s.err +} + +// TestBackoffScheduleMatchesContract pins the exponential backoff schedule. +// If anyone tunes the multiplier without thinking, this test catches it +// and forces a doc update. +func TestBackoffScheduleMatchesContract(t *testing.T) { + t.Parallel() + sender := ×tampedSender{err: errors.New("transient")} + d := New(Config{ + Workers: 0, + QueueSize: 1, + MaxFailureCount: 5, + RetryAttempts: 2, + RetryInitialDelay: 100 * time.Millisecond, + }, sender, nil, nil, nil, zerolog.Nop()) + + require.NoError(t, d.Enqueue(context.Background(), Job{ + Trigger: &models.Trigger{ID: "t", Status: "enabled"}, + Payload: &cloudevent.CloudEvent[webhook.WebhookPayload]{}, + })) + + sender.mu.Lock() + defer sender.mu.Unlock() + require.Len(t, sender.times, 3, "first attempt + 2 retries") + + // Expected gaps: 100ms initial, 500ms second (×5 multiplier). + gap1 := sender.times[1].Sub(sender.times[0]) + gap2 := sender.times[2].Sub(sender.times[1]) + require.InDelta(t, float64(100*time.Millisecond), float64(gap1), float64(40*time.Millisecond), + "first retry should fire ~100ms after initial attempt, got %s", gap1) + require.InDelta(t, float64(500*time.Millisecond), float64(gap2), float64(80*time.Millisecond), + "second retry should fire ~500ms after first retry (5x multiplier), got %s", gap2) +} + +// TestErrQueueFullIsBackpressure pins the integration with the NATS +// PullLoop's backpressure classifier. Without this, an Enqueue rejection +// would be treated as poison and consume MaxDeliver budget. +func TestErrQueueFullIsBackpressure(t *testing.T) { + t.Parallel() + require.True(t, errors.Is(ErrQueueFull, vtnats.ErrBackpressure), + "ErrQueueFull must wrap nats.ErrBackpressure so PullLoop uses the long-delay nak path") +} + +// TestPermanentErrorSkipsRetries pins the 4xx classifier: a permanent +// receiver error (wrapped as webhooksender.ErrPermanent) must short-circuit +// the in-worker retry loop. Without this, we burn RetryAttempts + per-host +// rate-limit tokens on a receiver that will never accept the payload. +func TestPermanentErrorSkipsRetries(t *testing.T) { + t.Parallel() + sender := ×tampedSender{err: webhooksender.ErrPermanent} + d := New(Config{ + Workers: 0, + QueueSize: 1, + MaxFailureCount: 5, + RetryAttempts: 3, + RetryInitialDelay: 100 * time.Millisecond, + }, sender, nil, nil, nil, zerolog.Nop()) + + require.NoError(t, d.Enqueue(context.Background(), Job{ + Trigger: &models.Trigger{ID: "t", Status: "enabled"}, + Payload: &cloudevent.CloudEvent[webhook.WebhookPayload]{}, + })) + + sender.mu.Lock() + defer sender.mu.Unlock() + require.Len(t, sender.times, 1, "permanent error must NOT retry; got %d attempts", len(sender.times)) +} diff --git a/internal/services/webhookdispatcher/clusterlimit.go b/internal/services/webhookdispatcher/clusterlimit.go new file mode 100644 index 0000000..31f18f1 --- /dev/null +++ b/internal/services/webhookdispatcher/clusterlimit.go @@ -0,0 +1,204 @@ +package webhookdispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "hash/fnv" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +// clusterLimiter is the cluster-shared cousin of hostLimiter. It stores a +// per-host token-bucket state in a JetStream KV bucket, so N pods sharing +// the same configured RPS truly add up to RPS aggregate, not N*RPS. +// +// Each bucket entry is a small JSON record with the current token count +// and last-refill timestamp. Acquisition is a CAS loop: Get -> compute new +// tokens -> Update with the previous revision. CAS conflicts indicate +// contention; the caller retries after a small backoff. This is more +// expensive than a per-pod limiter (one KV round-trip per send vs. one +// in-memory CAS), so it only kicks in when explicitly enabled. +// +// Wire it up by passing a non-nil jetstream.KeyValue to newClusterLimiter. +// Falls back to the per-pod hostLimiter when the KV bucket is unavailable, +// so a JetStream outage degrades to "weaker but still bounded" rather +// than "unlimited." +type clusterLimiter struct { + kv jetstream.KeyValue + rps float64 + burst int + fallback *hostLimiter // used when KV operations fail + maxRetries int + pollDelay time.Duration + now func() time.Time + + // In-process backoff sleep cache so contention on a hot key doesn't + // pile every worker against KV at once. Each host keeps its own + // "next allowed" timestamp updated when the bucket was empty. + mu sync.Mutex + nextWake map[string]time.Time +} + +type bucketState struct { + Tokens float64 `json:"t"` + LastRefill time.Time `json:"r"` +} + +func newClusterLimiter(kv jetstream.KeyValue, rps float64, burst int, fallback *hostLimiter) *clusterLimiter { + if rps <= 0 || kv == nil { + return nil + } + if burst < 1 { + burst = 1 + } + return &clusterLimiter{ + kv: kv, + rps: rps, + burst: burst, + fallback: fallback, + maxRetries: 8, + pollDelay: 50 * time.Millisecond, + now: time.Now, + nextWake: make(map[string]time.Time), + } +} + +// Wait blocks until a token is available for `target`'s host, or ctx is +// done. Falls back to the per-pod limiter on persistent KV errors so a +// JetStream blip doesn't unbound outbound throughput. +func (c *clusterLimiter) Wait(ctx context.Context, target string) error { + if c == nil { + return nil + } + host := extractHost(target) + key := kvKey(host) + + // Fast path: respect any cached "wait until" so a hot empty bucket + // doesn't have every worker hammering KV in lockstep. + c.mu.Lock() + wake, ok := c.nextWake[host] + c.mu.Unlock() + if ok && wake.After(c.now()) { + select { + case <-time.After(time.Until(wake)): + case <-ctx.Done(): + return ctx.Err() + } + } + + for attempt := 0; attempt < c.maxRetries; attempt++ { + if err := ctx.Err(); err != nil { + return err + } + entry, rev, err := c.getOrInit(ctx, key) + if err != nil { + // KV unreachable - lean on the per-pod fallback so we stay + // bounded rather than unbounded. + if c.fallback != nil { + return c.fallback.Wait(ctx, target) + } + return err + } + now := c.now() + c.refill(&entry, now) + if entry.Tokens >= 1 { + entry.Tokens-- + if err := c.update(ctx, key, entry, rev); err != nil { + // CAS conflict - retry the loop. Brief backoff so the + // retry burst doesn't immediately re-collide. + select { + case <-time.After(time.Duration(attempt+1) * c.pollDelay / 4): + case <-ctx.Done(): + return ctx.Err() + } + continue + } + return nil + } + // Empty bucket: sleep until at least one token is expected, then + // retry. We don't try to be clever about token-debt scheduling - + // the limit is approximate by design, the goal is bounded send + // rate across replicas, not perfect fairness. + sleep := time.Duration(float64(time.Second) / c.rps) + c.mu.Lock() + c.nextWake[host] = now.Add(sleep) + c.mu.Unlock() + select { + case <-time.After(sleep): + case <-ctx.Done(): + return ctx.Err() + } + } + // Persistent CAS contention: drop to fallback so we don't block forever. + if c.fallback != nil { + return c.fallback.Wait(ctx, target) + } + return fmt.Errorf("clusterlimit: exhausted retries on %s", host) +} + +func (c *clusterLimiter) refill(b *bucketState, now time.Time) { + if b.LastRefill.IsZero() { + b.LastRefill = now + b.Tokens = float64(c.burst) + return + } + elapsed := now.Sub(b.LastRefill).Seconds() + if elapsed <= 0 { + return + } + b.Tokens += elapsed * c.rps + if b.Tokens > float64(c.burst) { + b.Tokens = float64(c.burst) + } + b.LastRefill = now +} + +func (c *clusterLimiter) getOrInit(ctx context.Context, key string) (bucketState, uint64, error) { + entry, err := c.kv.Get(ctx, key) + if err != nil { + if errors.Is(err, jetstream.ErrKeyNotFound) { + // Treat missing as a full bucket with revision 0 so the + // first Update either creates the key (success) or sees a + // race-created revision (retry). + return bucketState{Tokens: float64(c.burst), LastRefill: c.now()}, 0, nil + } + return bucketState{}, 0, fmt.Errorf("kv get %s: %w", key, err) + } + var st bucketState + if uerr := json.Unmarshal(entry.Value(), &st); uerr != nil { + // Corrupt entry: reset. + return bucketState{Tokens: float64(c.burst), LastRefill: c.now()}, entry.Revision(), nil + } + return st, entry.Revision(), nil +} + +func (c *clusterLimiter) update(ctx context.Context, key string, st bucketState, rev uint64) error { + body, err := json.Marshal(st) + if err != nil { + return err + } + if rev == 0 { + _, err = c.kv.Create(ctx, key, body) + } else { + _, err = c.kv.Update(ctx, key, body, rev) + } + if err != nil { + return fmt.Errorf("kv update %s rev=%d: %w", key, rev, err) + } + return nil +} + +// kvKey returns a NATS-subject-safe key derived from the host string. +// We hash to keep cardinality bounded (a misconfigured trigger with a +// novel host per signal would otherwise grow the bucket without bound). +// FNV-1a is fine: collisions just share a budget which is the right +// behaviour for "limit per receiver." +func kvKey(host string) string { + h := fnv.New64a() + _, _ = h.Write([]byte(host)) + return fmt.Sprintf("host_%016x", h.Sum64()) +} diff --git a/internal/services/webhookdispatcher/dispatcher.go b/internal/services/webhookdispatcher/dispatcher.go new file mode 100644 index 0000000..4954cc5 --- /dev/null +++ b/internal/services/webhookdispatcher/dispatcher.go @@ -0,0 +1,450 @@ +// Package webhookdispatcher decouples outbound webhook delivery from the +// JetStream message handler. The handler hands off a Job and returns; a +// worker pool owns the actual HTTP dispatch, state writes, audit publish, +// and the failure-count bookkeeping that drives the circuit breaker. +// +// The decoupling matters at scale. Sync delivery makes a slow receiver +// directly throttle our consumer because the JetStream message stays +// un-acked until the HTTP round-trip completes. With a worker pool, the +// consumer keeps pulling, MaxAckPending isn't consumed by a single slow +// destination, and per-pod connection pools stay warm independent of +// which replica evaluated which fire. +// +// Backpressure: when the queue is full, Enqueue returns ErrQueueFull. The +// caller (the JetStream handler) returns that error, the message is naked, +// and JetStream redelivers per its BackOff ladder. Consume_total{outcome=nak} +// climbs visibly, and once MaxDeliver is exhausted the message lands in the +// DLQ. Operators can tune workers / queue size from those signals. +// +// Pool size 0 = inline / synchronous mode. Used by single-replica deploys +// and tests that don't want the worker-pool machinery. +package webhookdispatcher + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/server-garage/pkg/richerrors" + "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/webhooksender" + "github.com/ethereum/go-ethereum/common" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" +) + +// ErrQueueFull is returned by Enqueue when no worker slot is available. The +// caller (a JetStream handler) should propagate this so the message is naked +// and redelivered. We wrap nats.ErrBackpressure so the PullLoop treats it +// as load-shed (long nak delay, no DLQ chain) rather than poison. +var ErrQueueFull = fmt.Errorf("webhook dispatcher queue full: %w", vtnats.ErrBackpressure) + +// Sender delivers a single webhook to its target URL. +type Sender interface { + SendWebhook(ctx context.Context, t *models.Trigger, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error +} + +// StateRecorder persists the fire to distributed state after successful +// delivery. RecordFire is called best-effort; errors are logged. +type StateRecorder interface { + RecordFire(ctx context.Context, triggerID, metricName string, vehicleDID cloudevent.ERC721DID, at time.Time, snapshot json.RawMessage) error +} + +// AuditPublisher emits a record to the trigger-fired audit stream. The +// production implementation (auditqueue.Queue) is a bounded fire-and-forget +// queue; the raw nats client implementation is OK for synchronous mode and +// tests. +type AuditPublisher interface { + PublishTriggerFired(ctx context.Context, devLicense string, record []byte) error +} + +// FailureRepo lets the dispatcher drive the circuit breaker. Reset on +// success, increment on delivery failure; no other DB writes happen here. +type FailureRepo interface { + ResetTriggerFailureCount(ctx context.Context, trigger *models.Trigger) error + IncrementTriggerFailureCount(ctx context.Context, trigger *models.Trigger, failureReason error, maxFailureCount int) error +} + +// Job is the unit of work flowing through the dispatcher. +type Job struct { + Trigger *models.Trigger + Payload *cloudevent.CloudEvent[webhook.WebhookPayload] + Snapshot json.RawMessage + MetricName string + VehicleDID cloudevent.ERC721DID +} + +// FailureFlushInterval is how often the failure-count coalescer drains its +// in-memory accumulator into the DB. Default 1s: short enough that the +// circuit breaker reacts within the same second as the receiver outage, +// long enough that 32 workers hammering a broken receiver collapse into +// one UPDATE per trigger per second instead of one per delivery. +const DefaultFailureFlushInterval = time.Second + +// Config tunes pool size and queue depth. +type Config struct { + // Workers is the number of goroutines pulling from the queue. 0 means + // synchronous mode: Enqueue does the work on the caller's goroutine. + Workers int + // QueueSize is the channel buffer. Set comfortably above Workers - if + // it's the same size, brief bursts immediately backpressure. + QueueSize int + // MaxFailureCount drives the circuit breaker; mirrors the listener's. + MaxFailureCount int + // JobTimeout is the per-attempt wall clock cap, protecting workers from + // a wedged receiver. Default 30s matches the http client timeout. + JobTimeout time.Duration + // AuditTimeout caps the detached audit publish. Default 5s. + AuditTimeout time.Duration + // RetryAttempts is the number of in-worker retries on a webhook + // delivery failure (in addition to the first attempt). Default 2 (so + // up to 3 total attempts per job). Each retry uses exponential backoff + // starting at RetryInitialDelay. The retry loop avoids the much heavier + // JetStream redelivery path which re-runs CEL eval + perms + state KV. + RetryAttempts int + // RetryInitialDelay is the backoff before the first retry. Default + // 100ms. Each subsequent retry multiplies by 5: 100ms, 500ms, 2.5s. + RetryInitialDelay time.Duration + // PerHostRPS is the per-pod per-receiver send rate ceiling. Multiple + // triggers on the same destination share the budget naturally. 0 + // disables limiting (default). Set per receiver-tolerance, e.g. 50. + PerHostRPS float64 + // PerHostBurst is the immediate token allowance per host. Defaults to + // PerHostRPS so a short spike doesn't queue. + PerHostBurst int + + // FailureFlushInterval is the failure-count coalescer drain cadence. + // 0 = DefaultFailureFlushInterval. Set to a negative value to disable + // coalescing entirely (writes go straight to the repo, legacy + // behaviour - only useful in tests that want immediate visibility). + FailureFlushInterval time.Duration +} + +// rateLimiter is the dispatch-side interface satisfied by hostLimiter +// (per-pod) and clusterLimiter (KV-shared). Both honour ctx cancellation +// and return nil when no limit applies. +type rateLimiter interface { + Wait(ctx context.Context, target string) error +} + +// Dispatcher owns the worker pool and the job channel. +type Dispatcher struct { + cfg Config + sender Sender + state StateRecorder + audit AuditPublisher + repo FailureRepo + limiter rateLimiter + coalescer *failureCoalescer + log zerolog.Logger + + queue chan Job + wg sync.WaitGroup + stop chan struct{} + once sync.Once +} + +// WithClusterLimiter swaps the per-pod token bucket for a cluster-shared one +// backed by the supplied JetStream KV. Returns the dispatcher for chaining. +// Falls back to the per-pod limiter (or no limit) for the same call when +// the KV is nil or PerHostRPS <= 0. +// +// Apply BEFORE Run; the limiter is consulted inline in the worker's send +// path and isn't safe to swap once jobs are flowing. +func (d *Dispatcher) WithClusterLimiter(kv jetstream.KeyValue) *Dispatcher { + cl := newClusterLimiter(kv, d.cfg.PerHostRPS, d.cfg.PerHostBurst, asHostLimiter(d.limiter)) + if cl != nil { + d.limiter = cl + } + return d +} + +// asHostLimiter is a tiny helper used by WithClusterLimiter to grab the +// existing per-pod limiter (if any) as the fallback. Returns nil when the +// current limiter is something else. +func asHostLimiter(l rateLimiter) *hostLimiter { + if hl, ok := l.(*hostLimiter); ok { + return hl + } + return nil +} + +// New builds a dispatcher. Call Run before Enqueue or jobs will block on a +// nil-ready queue. +func New(cfg Config, sender Sender, state StateRecorder, audit AuditPublisher, repo FailureRepo, log zerolog.Logger) *Dispatcher { + if cfg.QueueSize < 1 { + cfg.QueueSize = 1 + } + if cfg.JobTimeout <= 0 { + cfg.JobTimeout = 30 * time.Second + } + if cfg.AuditTimeout <= 0 { + cfg.AuditTimeout = 5 * time.Second + } + if cfg.MaxFailureCount < 1 { + cfg.MaxFailureCount = 1 + } + if cfg.RetryAttempts < 0 { + cfg.RetryAttempts = 0 + } + if cfg.RetryInitialDelay <= 0 { + cfg.RetryInitialDelay = 100 * time.Millisecond + } + if cfg.PerHostBurst < 1 { + cfg.PerHostBurst = int(cfg.PerHostRPS) + } + hl := newHostLimiter(cfg.PerHostRPS, cfg.PerHostBurst) + var lim rateLimiter + if hl != nil { + lim = hl + } + d := &Dispatcher{ + cfg: cfg, + sender: sender, + state: state, + audit: audit, + repo: repo, + limiter: lim, + log: log, + queue: make(chan Job, cfg.QueueSize), + stop: make(chan struct{}), + } + flush := cfg.FailureFlushInterval + if flush == 0 { + flush = DefaultFailureFlushInterval + } + if flush > 0 && repo != nil { + d.coalescer = newFailureCoalescer(repo, cfg.MaxFailureCount, flush, log) + } + return d +} + +// Run starts the worker pool and blocks until ctx cancels. On cancel the +// workers drain every job still buffered in the queue before exiting, so a +// graceful shutdown (SIGTERM during a deploy) never loses an acked-but-unsent +// webhook. The caller MUST stop feeding Enqueue before cancelling ctx - the +// main shutdown sequence cancels the JetStream pull loops first and waits for +// them to exit - otherwise a late Enqueue can race a drained worker. +// +// Shutdown ordering inside Run: +// 1. workers drain the queue and exit +// 2. THEN the failure coalescer flushes, so failure counts recorded while +// draining are folded into the final UPDATE instead of being dropped. +// +// Residual gap: a hard crash (SIGKILL / OOM / panic) still loses jobs sitting +// in the queue - at-least-once at the ingest boundary plus async dispatch +// can't survive that without persisting the queue. Graceful shutdown is +// lossless; an ungraceful exit is bounded by JetStream redelivery only for +// messages not yet acked. +func (d *Dispatcher) Run(ctx context.Context) error { + if d.cfg.Workers <= 0 { + return nil // synchronous mode: nothing to run + } + if d.coalescer != nil { + // Detached context: the coalescer must outlive ctx cancel so failures + // recorded during the worker drain still get flushed. We stop it + // explicitly via Close() after the workers have finished. + go d.coalescer.Run(context.Background()) + } + for i := 0; i < d.cfg.Workers; i++ { + d.wg.Add(1) + go d.worker(ctx, i) + } + <-ctx.Done() + d.once.Do(func() { close(d.stop) }) + d.wg.Wait() + if d.coalescer != nil { + d.coalescer.Close() + } + return nil +} + +// Enqueue submits a job. In synchronous mode the job runs inline. In async +// mode the job is queued; ErrQueueFull is returned when no slot is +// available so the caller can nak the JetStream message. +func (d *Dispatcher) Enqueue(ctx context.Context, j Job) error { + if d.cfg.Workers <= 0 { + d.process(ctx, j) + return nil + } + select { + case d.queue <- j: + queueDepth.Set(float64(len(d.queue))) + return nil + default: + queueFull.Inc() + return ErrQueueFull + } +} + +func (d *Dispatcher) worker(ctx context.Context, id int) { + defer d.wg.Done() + log := d.log.With().Int("worker", id).Logger() + log.Info().Msg("dispatcher worker started") + for { + select { + case <-d.stop: + // Graceful shutdown: drain every job already buffered before + // exiting so acked-but-unsent webhooks are still delivered. Pull + // loops have stopped by the time stop fires, so no new jobs + // arrive. process() detaches the parent context, so deliveries + // complete on their own JobTimeout even though ctx is cancelled. + for { + select { + case j, ok := <-d.queue: + if !ok { + return + } + queueDepth.Set(float64(len(d.queue))) + d.process(ctx, j) + default: + log.Info().Msg("dispatcher worker stopping") + return + } + } + case j, ok := <-d.queue: + if !ok { + return + } + queueDepth.Set(float64(len(d.queue))) + d.process(ctx, j) + } + } +} + +// process runs the full delivery + bookkeeping for a single job. Used by +// both the worker pool and the inline mode. +// +// Includes in-worker retry with exponential backoff so a transient receiver +// hiccup doesn't bounce us back to JetStream redelivery (which would re-run +// CEL eval, permission checks, KV lookups - all wasted work for what's +// already a known-good fire). After RetryAttempts retries we surface the +// last error to the caller which propagates back to PullLoop. +func (d *Dispatcher) process(parent context.Context, j Job) { + start := time.Now() + ctx, cancel := context.WithTimeout(detach(parent), d.cfg.JobTimeout) + defer cancel() + + delay := d.cfg.RetryInitialDelay + var lastErr error +retryLoop: + for attempt := 0; attempt <= d.cfg.RetryAttempts; attempt++ { + if attempt > 0 { + retryTotal.Inc() + select { + case <-time.After(delay): + case <-ctx.Done(): + lastErr = ctx.Err() + break retryLoop + } + delay *= 5 + } + // Per-host rate limit: blocks until the receiver's token bucket + // allows another send. Critically, this happens INSIDE the worker + // goroutine so other workers serving other receivers don't wait. + if d.limiter != nil { + if err := d.limiter.Wait(ctx, j.Trigger.TargetURI); err != nil { + lastErr = err + break + } + } + err := d.sender.SendWebhook(ctx, j.Trigger, j.Payload) + if err == nil { + d.onSuccess(ctx, j) + deliveryTotal.WithLabelValues("ok").Inc() + deliveryLatency.WithLabelValues("ok").Observe(time.Since(start).Seconds()) + return + } + lastErr = err + // Permanent errors (4xx other than 408/425/429) won't recover on + // retry. Skip remaining attempts so we don't burn per-host rate- + // limit tokens on a broken receiver and don't tarpit other jobs + // waiting for this worker. + if errors.Is(err, webhooksender.ErrPermanent) { + break retryLoop + } + } + d.onFailure(ctx, j, lastErr) + deliveryTotal.WithLabelValues("error").Inc() + deliveryLatency.WithLabelValues("error").Observe(time.Since(start).Seconds()) +} + +func (d *Dispatcher) onSuccess(ctx context.Context, j Job) { + if j.Trigger.FailureCount > 0 && d.repo != nil { + if err := d.repo.ResetTriggerFailureCount(ctx, j.Trigger); err != nil { + d.log.Error().Err(err).Str("triggerId", j.Trigger.ID).Msg("failed to reset failure count") + } + } + if d.state != nil { + if err := d.state.RecordFire(ctx, j.Trigger.ID, j.Trigger.MetricName, j.VehicleDID, time.Now().UTC(), j.Snapshot); err != nil { + d.log.Warn().Err(err).Str("triggerId", j.Trigger.ID).Msg("state recorder write failed") + } + } + d.publishAudit(j) +} + +func (d *Dispatcher) onFailure(ctx context.Context, j Job, err error) { + d.log.Warn().Err(err).Str("triggerId", j.Trigger.ID).Msg("webhook delivery failed") + if d.repo == nil { + return + } + if richError, ok := richerrors.AsRichError(err); ok && richError.Code == webhooksender.WebhookFailureCode { + // Route through the coalescer when wired so a receiver outage + // doesn't fan one UPDATE per worker per delivery at Postgres. + // Without it (tests, sync mode), call the repo directly. + if d.coalescer != nil { + d.coalescer.Record(j.Trigger, err) + return + } + if failErr := d.repo.IncrementTriggerFailureCount(ctx, j.Trigger, err, d.cfg.MaxFailureCount); failErr != nil { + d.log.Error().Err(failErr).Str("triggerId", j.Trigger.ID).Msg("failed to handle webhook failure") + } + } +} + +// publishAudit hands the audit record to the configured AuditPublisher. The +// production publisher is an auditqueue.Queue with a bounded buffer and a +// small drainer pool, so this returns immediately with no goroutine spawn. +// Tests and sync-mode setups can pass the raw NATS client; in that case the +// caller's goroutine takes the PublishAsync hit directly. +func (d *Dispatcher) publishAudit(j Job) { + if d.audit == nil { + return + } + devLicense := common.BytesToAddress(j.Trigger.DeveloperLicenseAddress).Hex() + record, err := json.Marshal(j.Payload) + if err != nil { + d.log.Warn().Err(err).Str("triggerId", j.Trigger.ID).Msg("audit: marshal failed") + return + } + // Bounded by the AuditTimeout so even sync-mode setups don't deadlock if + // the publisher wedges; queue-backed publishers reject on overflow + // internally, observable via vehicle_triggers_audit_dropped_total. + bgCtx, cancel := context.WithTimeout(context.Background(), d.cfg.AuditTimeout) + defer cancel() + if err := d.audit.PublishTriggerFired(bgCtx, devLicense, record); err != nil { + d.log.Warn().Err(err).Str("triggerId", j.Trigger.ID).Msg("audit publish failed") + } +} + +// detach returns a context that inherits ctx's values but not its cancel. +// Used so a JetStream handler's cancellation doesn't abort the dispatcher's +// in-progress delivery + state writes. +func detach(parent context.Context) context.Context { + return context.WithoutCancel(parent) +} + +// QueueDepth returns the current queue depth. Useful for tests. +func (d *Dispatcher) QueueDepth() int { + return len(d.queue) +} + +func (d *Dispatcher) String() string { + return fmt.Sprintf("Dispatcher(workers=%d queueDepth=%d/%d)", d.cfg.Workers, len(d.queue), cap(d.queue)) +} diff --git a/internal/services/webhookdispatcher/dispatcher_test.go b/internal/services/webhookdispatcher/dispatcher_test.go new file mode 100644 index 0000000..a14c271 --- /dev/null +++ b/internal/services/webhookdispatcher/dispatcher_test.go @@ -0,0 +1,220 @@ +package webhookdispatcher + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// fakeSender counts deliveries and can simulate latency or failure. +type fakeSender struct { + calls atomic.Uint64 + delay time.Duration + err error +} + +func (s *fakeSender) SendWebhook(ctx context.Context, _ *models.Trigger, _ *cloudevent.CloudEvent[webhook.WebhookPayload]) error { + s.calls.Add(1) + if s.delay > 0 { + select { + case <-time.After(s.delay): + case <-ctx.Done(): + return ctx.Err() + } + } + return s.err +} + +func sampleJob() Job { + return Job{ + Trigger: &models.Trigger{ID: "t-1", Status: "enabled"}, + Payload: &cloudevent.CloudEvent[webhook.WebhookPayload]{ + Data: webhook.WebhookPayload{ + WebhookID: "t-1", + AssetDID: cloudevent.ERC721DID{ + ChainID: 137, + }, + }, + }, + } +} + +func TestDispatcherSynchronous(t *testing.T) { + t.Parallel() + sender := &fakeSender{} + d := New(Config{Workers: 0, QueueSize: 1, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + require.EqualValues(t, 1, sender.calls.Load()) +} + +func TestDispatcherAsyncDrains(t *testing.T) { + t.Parallel() + sender := &fakeSender{} + d := New(Config{Workers: 4, QueueSize: 16, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- d.Run(ctx) }() + + const N = 10 + for i := 0; i < N; i++ { + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + } + + require.Eventually(t, func() bool { return sender.calls.Load() == N }, 2*time.Second, 10*time.Millisecond, "dispatcher must drain all jobs") + cancel() + require.NoError(t, <-done) +} + +func TestDispatcherQueueFullBackpressure(t *testing.T) { + t.Parallel() + // Slow sender so workers never drain. Tiny queue so Enqueue rejects. + sender := &fakeSender{delay: 500 * time.Millisecond} + d := New(Config{Workers: 1, QueueSize: 1, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + go func() { _ = d.Run(ctx) }() + + // Fill the worker slot + the buffer slot, then expect rejection. + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + // Wait until the first job is being processed so the queue can fill. + require.Eventually(t, func() bool { return sender.calls.Load() == 1 }, time.Second, 5*time.Millisecond) + + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) // sits in buffer + + err := d.Enqueue(t.Context(), sampleJob()) // must reject + require.ErrorIs(t, err, ErrQueueFull) +} + +func TestDispatcherShutdownDrainsInFlight(t *testing.T) { + t.Parallel() + sender := &fakeSender{delay: 50 * time.Millisecond} + d := New(Config{Workers: 2, QueueSize: 8, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + + ctx, cancel := context.WithCancel(t.Context()) + var runErr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + runErr = d.Run(ctx) + }() + + for i := 0; i < 4; i++ { + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + } + + // Wait for at least one worker to have dequeued before cancelling, + // otherwise on busy schedulers Run returns before any sender call lands + // and the test becomes flaky. + require.Eventually(t, func() bool { return sender.calls.Load() >= 1 }, 2*time.Second, 5*time.Millisecond) + + // Cancel and verify Run returns within a reasonable bound. + cancel() + gotEarly := make(chan struct{}) + go func() { wg.Wait(); close(gotEarly) }() + select { + case <-gotEarly: + case <-time.After(2 * time.Second): + t.Fatal("Run did not return after context cancel") + } + require.NoError(t, runErr) + // At least one worker dequeued before cancel; we don't pin a higher + // bound because scheduler timing makes it flaky in busy test suites. + require.GreaterOrEqual(t, sender.calls.Load(), uint64(1), "at least one job should have been delivered") +} + +// TestDispatcherShutdownDrainsAll asserts the graceful-shutdown contract: once +// no more jobs are enqueued, cancelling ctx must deliver EVERY job still +// buffered in the queue, not just the ones already dequeued. This is the fix +// for the ack-before-send loss window - a webhook is acked off JetStream at +// enqueue time, so dropping the queue on shutdown silently loses fires. +func TestDispatcherShutdownDrainsAll(t *testing.T) { + t.Parallel() + // Slow enough that several jobs are still buffered when we cancel, so the + // drain path (not just steady-state processing) is what delivers them. + sender := &fakeSender{delay: 20 * time.Millisecond} + const N = 32 + d := New(Config{Workers: 4, QueueSize: N, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + go func() { done <- d.Run(ctx) }() + + for range N { + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + } + + // Cancel while the bulk of the jobs are still buffered. The buffer should + // be drained by the workers' stop path before Run returns. + require.Eventually(t, func() bool { return sender.calls.Load() >= 1 }, time.Second, time.Millisecond) + cancel() + + require.NoError(t, <-done) + require.EqualValues(t, N, sender.calls.Load(), "graceful shutdown must drain every buffered job") +} + +func TestDispatcherDeliveryError(t *testing.T) { + t.Parallel() + sender := &fakeSender{err: errors.New("boom")} + d := New(Config{Workers: 0, QueueSize: 1, MaxFailureCount: 5}, sender, nil, nil, nil, zerolog.Nop()) + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) // sync mode swallows the error (metric only) + require.EqualValues(t, 1, sender.calls.Load()) +} + +// flakyThenOKSender fails the first N times then succeeds. Used to exercise +// the in-worker retry loop. +type flakyThenOKSender struct { + failuresLeft int32 + calls atomic.Uint64 +} + +func (s *flakyThenOKSender) SendWebhook(_ context.Context, _ *models.Trigger, _ *cloudevent.CloudEvent[webhook.WebhookPayload]) error { + s.calls.Add(1) + if atomic.AddInt32(&s.failuresLeft, -1) >= 0 { + return errors.New("transient") + } + return nil +} + +func TestDispatcherInWorkerRetry(t *testing.T) { + t.Parallel() + // Two transient failures then success - covered by RetryAttempts=2. + sender := &flakyThenOKSender{failuresLeft: 2} + d := New(Config{ + Workers: 0, + QueueSize: 1, + MaxFailureCount: 5, + RetryAttempts: 2, + RetryInitialDelay: time.Millisecond, // keep test fast + }, sender, nil, nil, nil, zerolog.Nop()) + + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + require.EqualValues(t, 3, sender.calls.Load(), "first attempt + 2 retries") +} + +func TestDispatcherRetryExhausted(t *testing.T) { + t.Parallel() + // Three transient failures with RetryAttempts=2 means we still error. + sender := &flakyThenOKSender{failuresLeft: 3} + d := New(Config{ + Workers: 0, + QueueSize: 1, + MaxFailureCount: 5, + RetryAttempts: 2, + RetryInitialDelay: time.Millisecond, + }, sender, nil, nil, nil, zerolog.Nop()) + + require.NoError(t, d.Enqueue(t.Context(), sampleJob())) + require.EqualValues(t, 3, sender.calls.Load(), "exhausted at 1+2 attempts") +} diff --git a/internal/services/webhookdispatcher/failurecoalescer.go b/internal/services/webhookdispatcher/failurecoalescer.go new file mode 100644 index 0000000..f7df138 --- /dev/null +++ b/internal/services/webhookdispatcher/failurecoalescer.go @@ -0,0 +1,138 @@ +package webhookdispatcher + +import ( + "context" + "sync" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + "github.com/rs/zerolog" +) + +// failureCoalescer batches IncrementTriggerFailureCount calls so a receiver +// outage doesn't fan one DB UPDATE per worker per failed delivery onto +// Postgres. At 32 workers + 30k fires/sec on a permanently-broken receiver +// that was 32k writes/sec hitting the same row family. +// +// Coalescing model: +// - Record(trigger, reason) just increments an in-memory counter keyed by +// trigger ID and remembers the most recent reason. No DB touch. +// - A single flusher goroutine wakes every flushInterval, snapshots the +// accumulated counts, zeros the map, and emits one UPDATE per +// distinct trigger with the accumulated count. +// +// Trade: a pod crash between flushes loses up to flushInterval worth of +// failure-count writes. Acceptable: the circuit-breaker recovers on the +// next failed delivery; transient under-counting is preferable to +// hammering the DB with a write storm during an outage. +type failureCoalescer struct { + repo FailureRepo + maxFailures int + flushInterval time.Duration + log zerolog.Logger + + mu sync.Mutex + pending map[string]*pendingFailure + + stop chan struct{} + done chan struct{} +} + +type pendingFailure struct { + trigger *models.Trigger + count int + reason error +} + +func newFailureCoalescer(repo FailureRepo, maxFailures int, flushInterval time.Duration, log zerolog.Logger) *failureCoalescer { + if flushInterval <= 0 { + flushInterval = time.Second + } + return &failureCoalescer{ + repo: repo, + maxFailures: maxFailures, + flushInterval: flushInterval, + log: log, + pending: make(map[string]*pendingFailure), + stop: make(chan struct{}), + done: make(chan struct{}), + } +} + +// Run starts the flusher goroutine; returns when ctx cancels or Close is +// called. Drains a final flush before returning so failures recorded in the +// last window aren't lost. +func (c *failureCoalescer) Run(ctx context.Context) { + defer close(c.done) + ticker := time.NewTicker(c.flushInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + c.flush(context.Background()) + return + case <-c.stop: + c.flush(context.Background()) + return + case <-ticker.C: + c.flush(ctx) + } + } +} + +// Close stops the flusher and waits for a final flush. +func (c *failureCoalescer) Close() { + select { + case <-c.stop: + // already closed + default: + close(c.stop) + } + <-c.done +} + +// Record accumulates a failure for the given trigger. The reason is +// retained as "most recent" for the eventual UPDATE; older reasons in the +// same window are discarded. This keeps the in-memory map tiny while still +// surfacing the latest failure cause when the circuit eventually opens. +func (c *failureCoalescer) Record(trigger *models.Trigger, reason error) { + if trigger == nil { + return + } + c.mu.Lock() + defer c.mu.Unlock() + p, ok := c.pending[trigger.ID] + if !ok { + p = &pendingFailure{trigger: trigger} + c.pending[trigger.ID] = p + } + p.count++ + p.reason = reason +} + +func (c *failureCoalescer) flush(ctx context.Context) { + c.mu.Lock() + if len(c.pending) == 0 { + c.mu.Unlock() + return + } + batch := c.pending + c.pending = make(map[string]*pendingFailure) + c.mu.Unlock() + + for _, p := range batch { + // Apply the accumulated count by calling Increment once per failure. + // IncrementTriggerFailureCount does a fetch-for-update + write per + // call; calling it N times here still beats calling it N times + // from N workers because we serialise to one writer per trigger + // and amortise the row-lock contention. A future optimisation + // could add a repo method that takes a delta and applies it in + // one statement; for now we keep the existing interface. + for i := 0; i < p.count; i++ { + if err := c.repo.IncrementTriggerFailureCount(ctx, p.trigger, p.reason, c.maxFailures); err != nil { + c.log.Error().Err(err).Str("triggerId", p.trigger.ID).Msg("failurecoalescer: increment failed") + break + } + } + } +} diff --git a/internal/services/webhookdispatcher/metrics.go b/internal/services/webhookdispatcher/metrics.go new file mode 100644 index 0000000..9d7931e --- /dev/null +++ b/internal/services/webhookdispatcher/metrics.go @@ -0,0 +1,48 @@ +package webhookdispatcher + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Dispatcher instrumentation. queueDepth is a gauge so dashboards can show +// in-flight pressure; queueFull tracks the rate at which Enqueue rejects +// (the backpressure signal that the caller surfaces by naking the message). +// deliveryTotal + deliveryLatency cover the actual HTTP dispatch outcome. +var ( + queueDepth = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "vehicle_triggers", + Subsystem: "dispatcher", + Name: "queue_depth", + Help: "Current number of jobs waiting in the dispatcher queue.", + }) + + queueFull = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "dispatcher", + Name: "queue_full_total", + Help: "Number of Enqueue rejections due to a full queue. Each rejection naks the inbound JetStream message.", + }) + + deliveryTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "dispatcher", + Name: "delivery_total", + Help: "Webhook delivery outcomes from the dispatcher path (ok|error).", + }, []string{"outcome"}) + + deliveryLatency = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Namespace: "vehicle_triggers", + Subsystem: "dispatcher", + Name: "delivery_latency_seconds", + Help: "Wall clock from dispatcher job start to receiver response.", + Buckets: []float64{0.005, 0.025, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30}, + }, []string{"outcome"}) + + retryTotal = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "vehicle_triggers", + Subsystem: "dispatcher", + Name: "retry_total", + Help: "Number of in-worker retry attempts (does not count the first attempt). High rates indicate flaky receivers.", + }) +) diff --git a/internal/services/webhookdispatcher/ratelimit.go b/internal/services/webhookdispatcher/ratelimit.go new file mode 100644 index 0000000..8f7d8bc --- /dev/null +++ b/internal/services/webhookdispatcher/ratelimit.go @@ -0,0 +1,70 @@ +package webhookdispatcher + +import ( + "context" + "net/url" + "sync" + + "golang.org/x/time/rate" +) + +// hostLimiter is a tiny per-host token bucket so one popular webhook +// receiver doesn't get hammered into rate-limiting our entire pod fleet. +// The limit is per-pod, not cluster-global: combined send rate at N pods is +// N * Rate. That's intentionally lenient - true cluster coordination would +// add a KV roundtrip per send. If we ever need cluster-global limits the +// hook is well-isolated. +// +// Limits per HOST (scheme+host of the trigger.TargetURI), not per trigger +// or per developer, so multiple webhooks on the same receiver share the +// budget naturally. +type hostLimiter struct { + mu sync.Mutex + limiters map[string]*rate.Limiter + rps rate.Limit + burst int +} + +// newHostLimiter builds a limiter that allows `rps` requests per second per +// host, with `burst` immediate tokens. rps <= 0 disables limiting entirely +// (returns a permissive limiter so callers don't need nil checks). +func newHostLimiter(rps float64, burst int) *hostLimiter { + if rps <= 0 { + return nil + } + if burst < 1 { + burst = 1 + } + return &hostLimiter{ + limiters: make(map[string]*rate.Limiter), + rps: rate.Limit(rps), + burst: burst, + } +} + +// Wait blocks until the per-host token bucket has a slot, or ctx is done. +// nil receiver = unlimited (production passthrough when config disabled it). +func (h *hostLimiter) Wait(ctx context.Context, target string) error { + if h == nil { + return nil + } + host := extractHost(target) + h.mu.Lock() + l, ok := h.limiters[host] + if !ok { + l = rate.NewLimiter(h.rps, h.burst) + h.limiters[host] = l + } + h.mu.Unlock() + return l.Wait(ctx) +} + +// extractHost returns scheme+host (no port stripping; receivers on different +// ports are usually different services). +func extractHost(target string) string { + u, err := url.Parse(target) + if err != nil || u.Host == "" { + return target + } + return u.Scheme + "://" + u.Host +} diff --git a/internal/services/webhooksender/webhook_sender.go b/internal/services/webhooksender/webhook_sender.go index 3d559ba..ce433e9 100644 --- a/internal/services/webhooksender/webhook_sender.go +++ b/internal/services/webhooksender/webhook_sender.go @@ -3,20 +3,45 @@ package webhooksender import ( "bytes" "context" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" + "strconv" "time" "github.com/DIMO-Network/cloudevent" "github.com/DIMO-Network/server-garage/pkg/richerrors" "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" "github.com/DIMO-Network/vehicle-triggers-api/internal/db/models" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/DIMO-Network/vehicle-triggers-api/internal/safetransport" ) +// signatureHeaders sign the request body using the trigger's signing_secret. +// Receivers verify with HMAC-SHA256(secret, timestamp + "." + body). The +// timestamp is included to give the receiver a replay window check (out-of- +// order delivery is fine; arbitrary-time replay is not). When the trigger's +// secret is null/empty (legacy rows pre-migration 00006), we skip signing - +// receivers should treat unsigned requests as an error. +func signatureHeaders(secret string, body []byte) (timestamp, signature string) { + if secret == "" { + return "", "" + } + timestamp = strconv.FormatInt(time.Now().Unix(), 10) + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(timestamp)) + mac.Write([]byte{'.'}) + mac.Write(body) + signature = hex.EncodeToString(mac.Sum(nil)) + return timestamp, signature +} + const ( // WebhookFailureCode is the code returned when a webhook caused an error WebhookFailureCode = -1 @@ -27,17 +52,47 @@ const ( maxResponseBodySize = 1024 ) +// ErrPermanent wraps receiver responses that retrying will never fix +// (most 4xx). The dispatcher checks for this with errors.Is and skips its +// in-worker retry loop, surfacing the failure immediately to onFailure so +// the FailureCount bookkeeping engages instead of burning the retry budget +// and per-host rate-limit tokens on a broken receiver. +var ErrPermanent = errors.New("webhook permanent failure") + +// isRetryableStatus returns false for 4xx that won't recover on retry. We +// keep 408 (Request Timeout), 425 (Too Early), and 429 (Too Many Requests) +// retryable because they signal "try again later," not "request is broken." +// 5xx is always retryable. +func isRetryableStatus(code int) bool { + if code < 400 { + return true + } + if code >= 500 { + return true + } + switch code { + case http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests: + return true + } + return false +} + // WebhookSender handles all webhook delivery operations type WebhookSender struct { client *http.Client } -// NewWebhookSender creates a new WebhookSender with proper HTTP client configuration +// NewWebhookSender creates a WebhookSender with an http.Client tuned for +// high-throughput webhook dispatch. The default transport is replaced so +// the keep-alive pool can actually sustain bursts to popular receivers - +// Go's MaxIdleConnsPerHost=2 default forces a new TCP+TLS handshake on the +// third concurrent request, which dominates per-fire latency at any real +// production rate. Pass a non-nil client to override (used by tests). func NewWebhookSender(client *http.Client) *WebhookSender { if client == nil { client = &http.Client{ - Timeout: defaultWebhookTimeout, - // TODO: Add transport configuration for connection pooling, TLS settings, etc. + Timeout: defaultWebhookTimeout, + Transport: defaultTransport(), } } return &WebhookSender{ @@ -45,6 +100,25 @@ func NewWebhookSender(client *http.Client) *WebhookSender { } } +// defaultTransport returns the production-tuned http.Transport. We keep a +// generous per-host pool because a single popular receiver may handle many +// triggers across many vehicles, all from the same pod. +func defaultTransport() *http.Transport { + t := http.DefaultTransport.(*http.Transport).Clone() + t.MaxIdleConns = 1024 + t.MaxIdleConnsPerHost = 64 + t.MaxConnsPerHost = 0 // 0 = unlimited; pool will reuse, not block + t.IdleConnTimeout = 90 * time.Second + t.TLSHandshakeTimeout = 10 * time.Second + t.ResponseHeaderTimeout = 20 * time.Second + t.ExpectContinueTimeout = 1 * time.Second + t.ForceAttemptHTTP2 = true + // SSRF guard: webhook targets are developer-supplied, so refuse to dial + // internal/loopback/metadata addresses (also defends against DNS + // rebinding since the check runs per-dial on the resolved IP). + return safetransport.Guard(t) +} + // SendWebhook sends a webhook notification to the specified trigger // Returns error for failures, nil for success func (w *WebhookSender) SendWebhook(ctx context.Context, t *models.Trigger, payload *cloudevent.CloudEvent[webhook.WebhookPayload]) error { @@ -71,7 +145,15 @@ func (w *WebhookSender) SendWebhook(ctx context.Context, t *models.Trigger, payl // Set headers req.Header.Set("Content-Type", "application/json") req.Header.Set("User-Agent", "DIMO-Webhook/1.0") - // TODO: Add webhook signature for security + if trace := vtnats.TraceFromContext(ctx); trace != "" { + req.Header.Set("X-DIMO-Trace-ID", trace) + } + if t.SigningSecret.Valid && t.SigningSecret.String != "" { + ts, sig := signatureHeaders(t.SigningSecret.String, body) + req.Header.Set("X-DIMO-Timestamp", ts) + req.Header.Set("X-DIMO-Signature", sig) + req.Header.Set("X-DIMO-Signature-Version", "v1") + } // Send request resp, err := w.client.Do(req) @@ -87,9 +169,17 @@ func (w *WebhookSender) SendWebhook(ctx context.Context, t *models.Trigger, payl if resp.StatusCode >= 400 { // Read response body for error details (limited size for security) respBody, _ := io.ReadAll(io.LimitReader(resp.Body, maxResponseBodySize)) + baseErr := fmt.Errorf("webhook returned status code %d: %s", resp.StatusCode, string(respBody)) + if !isRetryableStatus(resp.StatusCode) { + // Wrap ErrPermanent so the dispatcher recognises this with + // errors.Is and skips its retry loop. richerrors carries the + // existing failure code untouched so FailureCount accounting + // still engages. + baseErr = fmt.Errorf("%w: %w", ErrPermanent, baseErr) + } return richerrors.Error{ Code: WebhookFailureCode, - Err: fmt.Errorf("webhook returned status code %d: %s", resp.StatusCode, string(respBody)), + Err: baseErr, } } diff --git a/internal/services/webhooksender/webhook_sender_signing_test.go b/internal/services/webhooksender/webhook_sender_signing_test.go new file mode 100644 index 0000000..b4a9f86 --- /dev/null +++ b/internal/services/webhooksender/webhook_sender_signing_test.go @@ -0,0 +1,48 @@ +package webhooksender + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestSignatureHeaders(t *testing.T) { + t.Parallel() + + t.Run("empty secret returns empty", func(t *testing.T) { + ts, sig := signatureHeaders("", []byte("body")) + require.Empty(t, ts) + require.Empty(t, sig) + }) + + t.Run("signature is HMAC-SHA256 over timestamp + . + body", func(t *testing.T) { + secret := "test-secret" + body := []byte(`{"hello":"world"}`) + ts, sig := signatureHeaders(secret, body) + + // Timestamp parses as a unix epoch second within +/- 5s of now. + got, err := strconv.ParseInt(ts, 10, 64) + require.NoError(t, err) + require.InDelta(t, time.Now().Unix(), got, 5) + + // Recompute and compare. + mac := hmac.New(sha256.New, []byte(secret)) + mac.Write([]byte(ts)) + mac.Write([]byte{'.'}) + mac.Write(body) + expected := hex.EncodeToString(mac.Sum(nil)) + require.Equal(t, expected, sig) + }) + + t.Run("different secrets produce different sigs", func(t *testing.T) { + body := []byte("same body") + _, a := signatureHeaders("secret-a", body) + _, b := signatureHeaders("secret-b", body) + require.NotEqual(t, a, b) + }) +} diff --git a/internal/services/webhooksender/webhook_sender_test.go b/internal/services/webhooksender/webhook_sender_test.go index e21def0..8f1e9cf 100644 --- a/internal/services/webhooksender/webhook_sender_test.go +++ b/internal/services/webhooksender/webhook_sender_test.go @@ -39,14 +39,14 @@ func TestWebhookSender_SendWebhook(t *testing.T) { var payload cloudevent.CloudEvent[webhook.WebhookPayload] err = json.Unmarshal(body, &payload) require.NoError(t, err) - assert.Equal(t, "test-webhook-id", payload.Data.WebhookId) + assert.Equal(t, "test-webhook-id", payload.Data.WebhookID) w.WriteHeader(http.StatusOK) _, _ = fmt.Fprint(w, "success") })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -67,7 +67,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -92,7 +92,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -110,7 +110,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { }) t.Run("network connection failure", func(t *testing.T) { - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: "http://invalid.localhost:0", // Invalid endpoint @@ -128,7 +128,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { }) t.Run("invalid URL format", func(t *testing.T) { - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: "://invalid-url", // Invalid URL format @@ -181,7 +181,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -212,7 +212,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -226,8 +226,9 @@ func TestWebhookSender_SendWebhook(t *testing.T) { // Error message should contain truncated response (limited by maxResponseBodySize) assert.Contains(t, err.Error(), "webhook returned status code 400") - // The full large response should not be in the error (should be truncated) - assert.True(t, len(err.Error()) <= maxResponseBodySize+50, "Response should be truncated") // +50 for buffer + // The full large response should not be in the error (should be truncated). + // Buffer accommodates the "webhook permanent failure:" wrap added for 400. + assert.True(t, len(err.Error()) <= maxResponseBodySize+128, "Response should be truncated") }) t.Run("HTTPS server with custom client", func(t *testing.T) { @@ -264,7 +265,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -284,7 +285,7 @@ func TestWebhookSender_SendWebhook(t *testing.T) { })) defer testServer.Close() - sender := NewWebhookSender(nil) + sender := NewWebhookSender(unguardedClient()) trigger := &models.Trigger{ ID: "test-webhook-id", TargetURI: testServer.URL, @@ -325,6 +326,14 @@ func TestNewWebhookSender(t *testing.T) { }) } +// unguardedClient returns an http.Client without the production SSRF dial +// guard so these tests can reach httptest servers on loopback. The guard +// itself is covered by safetransport's own tests; here we exercise the +// send/retry/status logic, which is independent of the dial policy. +func unguardedClient() *http.Client { + return &http.Client{Timeout: defaultWebhookTimeout} +} + // Helper function to create test webhook payload func createTestPayload(webhookID string) *cloudevent.CloudEvent[webhook.WebhookPayload] { assetDID := cloudevent.ERC721DID{ @@ -347,7 +356,7 @@ func createTestPayload(webhookID string) *cloudevent.CloudEvent[webhook.WebhookP Data: webhook.WebhookPayload{ Service: "signals", MetricName: "vss.speed", - WebhookId: webhookID, + WebhookID: webhookID, WebhookName: "Test Webhook", AssetDID: assetDID, Condition: "valueNumber > 55", diff --git a/sample.env b/sample.env index 76bdf55..c5bba35 100644 --- a/sample.env +++ b/sample.env @@ -4,8 +4,40 @@ GRPC_PORT="50051" # Port number for the gRPC server LOG_LEVEL="info" # Logging level (e.g., debug, info, warn, error) SERVICE_NAME="vehicle-triggers-api" # Name of the service IDENTITY_API_URL="https://identity-api.dev.dimo.zone/query" # Identity API GraphQL endpoint -KAFKA_BROKERS="localhost:9092" -DEVICE_SIGNALS_TOPIC=topic.signals +JWK_KEY_SET_URL="https://auth.dev.dimo.zone/keys" # JWKS endpoint for JWT auth (required) +TOKEN_EXCHANGE_GRPC_ADDR="token-exchange-api-dev:8086" # Token-exchange gRPC for vehicle permission checks (required) + +# Distributed cooldown invariant: NATS_TRIGGER_STATE_TTL must be >= +# 2*MAX_ALLOWED_COOLDOWN_PERIOD when NATS is enabled, or the service refuses +# to start. The defaults below (63d TTL vs 30d ceiling) satisfy it; only set +# these if you change one of them. +# MAX_ALLOWED_COOLDOWN_PERIOD=2592000 # 30d (default) +# NATS_TRIGGER_STATE_TTL=1512h # 63d (default) + +# NATS JetStream ingest. Mode values: +# off - API only, no ingest pipeline. +# primary | exclusive - NATS owns evaluation + dispatch (the two are +# equivalent post-Kafka rip; "exclusive" is the canonical +# prod value, "primary" is kept for env-file compatibility). +NATS_MODE=exclusive +NATS_URL=nats://localhost:4222 +NATS_SIGNALS_STREAM=DIMO_SIGNALS +NATS_EVENTS_STREAM=DIMO_EVENTS +NATS_AUDIT_STREAM=DIMO_TRIGGER_AUDIT +NATS_DLQ_STREAM=DIMO_TRIGGER_DLQ +NATS_SIGNALS_SUBJECT=dimo.signals.> +NATS_EVENTS_SUBJECT=dimo.events.> +NATS_AUDIT_SUBJECT=dimo.trigger.fired.> +NATS_DLQ_SUBJECT=dimo.dlq.> +NATS_SIGNALS_DURABLE=triggers-signals +NATS_EVENTS_DURABLE=triggers-events +NATS_STREAM_REPLICAS=1 +NATS_ACK_WAIT=45s +NATS_MAX_DELIVER=5 +NATS_MAX_ACK_PENDING=5000 +NATS_FETCH_BATCH=100 +NATS_PUBLISH_ASYNC_MAX_PENDING=4000 +NATS_DLQ_MAX_AGE=168h # Worker count for the parallel webhook-cache compile loop. Each worker # takes one DB roundtrip + one CEL compile at a time, so this also caps DB diff --git a/tests/db.go b/tests/db.go index a69dffe..cef8b22 100644 --- a/tests/db.go +++ b/tests/db.go @@ -3,6 +3,7 @@ package tests import ( "context" "database/sql" + "os" "sync" "sync/atomic" "testing" @@ -14,6 +15,26 @@ import ( "github.com/testcontainers/testcontainers-go/modules/postgres" ) +// SkipIfNoDocker skips the test when neither DOCKER_HOST nor a known local +// Docker socket is reachable. Lets unit-style runs (CI without Docker, dev +// laptops without Docker Desktop running) avoid testcontainers panics. +func SkipIfNoDocker(t *testing.T) { + t.Helper() + if os.Getenv("DOCKER_HOST") != "" { + return + } + for _, sock := range []string{ + "/var/run/docker.sock", + os.ExpandEnv("$HOME/.docker/run/docker.sock"), + os.ExpandEnv("$HOME/.colima/default/docker.sock"), + } { + if _, err := os.Stat(sock); err == nil { + return + } + } + t.Skip("docker not available; set DOCKER_HOST or start Docker to run this test") +} + type TestContainer struct { container testcontainers.Container DB *sql.DB @@ -43,6 +64,7 @@ func (tc *TestContainer) Close() { } func SetupTestContainer(t *testing.T) *TestContainer { + SkipIfNoDocker(t) globalTestContainer.onceSetup.Do(func() { ctx := context.Background() var err error diff --git a/tests/e2e/cache_broadcast_test.go b/tests/e2e/cache_broadcast_test.go new file mode 100644 index 0000000..875334f --- /dev/null +++ b/tests/e2e/cache_broadcast_test.go @@ -0,0 +1,54 @@ +package e2e_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/cachebroadcast" + nc "github.com/nats-io/nats.go" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +type recordingRefresher struct { + count atomic.Uint64 +} + +func (r *recordingRefresher) InvalidateTrigger(string) {} + +func (r *recordingRefresher) ScheduleRefreshSilent(context.Context) { + r.count.Add(1) +} + +// TestCacheBroadcast verifies the publish->subscribe loop: a Notify call on +// one connection fires the Subscribe callback on a second connection, +// triggering the refresher. The receiver side calls ScheduleRefreshSilent, +// not ScheduleRefresh, so notifications don't echo into infinite republishes. +func TestCacheBroadcast(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + pubConn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(pubConn.Close) + subConn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(subConn.Close) + + notifier := cachebroadcast.NewNATSNotifier(pubConn, zerolog.Nop()) + refresher := &recordingRefresher{} + sub, err := cachebroadcast.Subscribe(subConn, t.Context(), refresher, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = sub.Unsubscribe() }) + // Flush both sides so the server registers the subscription before we + // publish; without this the publish can race ahead of subscription setup. + require.NoError(t, subConn.Flush()) + + require.NoError(t, notifier.Notify(t.Context(), "wh-1", "update")) + require.NoError(t, pubConn.Flush()) + + require.Eventually(t, func() bool { return refresher.count.Load() == 1 }, 2*time.Second, 10*time.Millisecond, "refresher should receive the notification") +} diff --git a/tests/e2e/kafka_server_test.go b/tests/e2e/kafka_server_test.go deleted file mode 100644 index 24a1a58..0000000 --- a/tests/e2e/kafka_server_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package e2e_test - -import ( - "context" - "encoding/json" - "fmt" - "sync" - "testing" - - "github.com/IBM/sarama" - "github.com/testcontainers/testcontainers-go/modules/kafka" -) - -type mockKafkaServer struct { - container *kafka.KafkaContainer - producer sarama.SyncProducer - topics map[string][]any - mu sync.RWMutex -} - -func setupMockKafkaServer(t *testing.T) *mockKafkaServer { - t.Helper() - - ctx := context.Background() - - // Start Kafka container using Testcontainers - kafkaContainer, err := kafka.Run(ctx, - "confluentinc/confluent-local:7.5.0", - kafka.WithClusterID("test-cluster"), - ) - if err != nil { - t.Fatalf("Failed to start Kafka container: %v", err) - } - - // Get broker addresses - brokers, err := kafkaContainer.Brokers(ctx) - if err != nil { - t.Fatalf("Failed to get Kafka brokers: %v", err) - } - - // Create a sync producer for sending messages - config := sarama.NewConfig() - config.Producer.Return.Successes = true - config.Producer.RequiredAcks = sarama.WaitForAll - config.Producer.Retry.Max = 5 - config.Version = sarama.V2_8_1_0 - - producer, err := sarama.NewSyncProducer(brokers, config) - if err != nil { - t.Fatalf("Failed to create producer: %v", err) - } - - mockServer := &mockKafkaServer{ - container: kafkaContainer, - producer: producer, - topics: make(map[string][]any), - } - - return mockServer -} - -// PushMessageToTopic sends a message to a specific topic -func (m *mockKafkaServer) PushMessageToTopic(topic string, payload []byte, headers map[string]string) error { - m.mu.Lock() - defer m.mu.Unlock() - - // Create producer message - msg := &sarama.ProducerMessage{ - Topic: topic, - Value: sarama.ByteEncoder(payload), - } - - for key, value := range headers { - msg.Headers = append(msg.Headers, sarama.RecordHeader{ - Key: []byte(key), - Value: []byte(value), - }) - } - - // Send message - _, _, err := m.producer.SendMessage(msg) - if err != nil { - return fmt.Errorf("failed to send message to topic %s: %w", topic, err) - } - - // Store in our internal topic storage for testing purposes - if m.topics[topic] == nil { - m.topics[topic] = make([]any, 0) - } - m.topics[topic] = append(m.topics[topic], string(payload)) - - return nil -} - -// PushJSONToTopic sends a JSON payload to a specific topic -func (m *mockKafkaServer) PushJSONToTopic(topic string, payload any) error { - jsonBytes, err := json.Marshal(payload) - if err != nil { - return fmt.Errorf("failed to marshal JSON: %w", err) - } - - return m.PushMessageToTopic(topic, jsonBytes, nil) -} - -// GetMessagesFromTopic returns all messages for a specific topic (from internal storage) -func (m *mockKafkaServer) GetMessagesFromTopic(topic string) []any { - m.mu.RLock() - defer m.mu.RUnlock() - - if messages, exists := m.topics[topic]; exists { - // Return a copy to avoid race conditions - result := make([]any, len(messages)) - copy(result, messages) - return result - } - return []any{} -} - -// ClearTopic removes all messages from a specific topic (from internal storage) -func (m *mockKafkaServer) ClearTopic(topic string) { - m.mu.Lock() - defer m.mu.Unlock() - delete(m.topics, topic) -} - -// GetBrokerAddress returns the first broker address as a string (for backward compatibility) -func (m *mockKafkaServer) GetBrokerAddress(t *testing.T) string { - brokers, err := m.container.Brokers(t.Context()) - if err != nil { - t.Fatalf("Failed to get Kafka brokers: %v", err) - } - if len(brokers) > 0 { - return brokers[0] - } - t.Fatalf("No brokers found") - return "" -} - -// Close closes the mock Kafka server and cleans up resources -func (m *mockKafkaServer) Close() error { - if m.producer != nil { - _ = m.producer.Close() - } - if m.container != nil { - return m.container.Terminate(context.Background()) - } - return nil -} diff --git a/tests/e2e/load_smoke_test.go b/tests/e2e/load_smoke_test.go new file mode 100644 index 0000000..ce5328e --- /dev/null +++ b/tests/e2e/load_smoke_test.go @@ -0,0 +1,127 @@ +//go:build load + +// Build-tagged smoke load test. Runs the production NATS publish + pull- +// consumer path at a modest rate against a testcontainer JetStream and +// asserts no publish drops + clean consumer drain. Wired into CI as a +// separate job (go test -tags=load ./tests/e2e/...) so it doesn't inflate +// the default unit-test budget but still catches wiring regressions before +// they hit prod. +// +// Knobs are deliberately small (5s, ~300 msg/s) so the run is bounded; the +// real bench tool (cmd/triggers-bench) is what we use for capacity planning. +package e2e_test + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +func TestLoadSmokeNATS(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + suffix := time.Now().Format("150405000") + conn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + require.NoError(t, err) + + streamName := "LOAD_SMOKE_" + suffix + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: streamName, + Subjects: []string{"load.smoke.>"}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: 5 * time.Minute, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = js.DeleteStream(context.Background(), streamName) }) + + cons, err := js.CreateOrUpdateConsumer(t.Context(), streamName, jetstream.ConsumerConfig{ + Durable: "load-cons-" + suffix, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + FilterSubject: "load.smoke.>", + MaxAckPending: 10_000, + }) + require.NoError(t, err) + + // Use the production Client + PullLoop so any wiring regression in those + // surfaces here. + client, err := vtnats.Connect(t.Context(), config.NATSSettings{ + Mode: "exclusive", + URL: server.URL(), + Name: "load-smoke-" + suffix, + FetchBatch: 100, + MaxDeliver: 5, + }, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + + var consumed, drops atomic.Uint64 + pullCtx, pullCancel := context.WithCancel(t.Context()) + t.Cleanup(pullCancel) + go func() { + _ = client.PullLoop(pullCtx, cons, 16, func(_ context.Context, _ []byte) error { + consumed.Add(1) + return nil + }) + }() + + const rate = 300 + const duration = 5 * time.Second + interval := time.Second / rate + + ticker := time.NewTicker(interval) + pubCtx, pubCancel := context.WithTimeout(t.Context(), duration) + defer pubCancel() + var published atomic.Uint64 + for { + stop := false + select { + case <-pubCtx.Done(): + ticker.Stop() + stop = true + case <-ticker.C: + _, err := client.Publish(pubCtx, "load.smoke.speed", []byte(`{"v":1}`)) + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + ticker.Stop() + stop = true + break + } + drops.Add(1) + continue + } + published.Add(1) + } + if stop { + break + } + } + + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if consumed.Load() >= published.Load() { + break + } + time.Sleep(50 * time.Millisecond) + } + + t.Logf("smoke load: published=%d consumed=%d drops=%d", published.Load(), consumed.Load(), drops.Load()) + require.Equal(t, uint64(0), drops.Load(), "publish drops are unacceptable at this rate") + require.GreaterOrEqual(t, consumed.Load(), published.Load(), "consumer must drain all published messages") +} diff --git a/tests/e2e/main_test.go b/tests/e2e/main_test.go index 8f1fcfe..4382145 100644 --- a/tests/e2e/main_test.go +++ b/tests/e2e/main_test.go @@ -23,7 +23,6 @@ var ( type TestServices struct { Identity *mockIdentityServer Auth *mockAuthServer - Kafka *mockKafkaServer Postgres *tests.TestContainer TokenExchange *mockTokenExchangeServer refs atomic.Int64 @@ -47,6 +46,7 @@ func TestMain(m *testing.M) { func GetTestServices(t *testing.T) *TestServices { t.Helper() + tests.SkipIfNoDocker(t) srvcLock.Lock() globalTestContainer.Do(func() { logger := zerolog.New(os.Stdout).Level(zerolog.WarnLevel) @@ -57,7 +57,7 @@ func GetTestServices(t *testing.T) *TestServices { JWKKeySetURL: "http://127.0.0.1:3003/keys", VehicleNFTAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), DIMORegistryChainID: 137, - CacheDebounceTime: -1, + Cache: config.CacheSettings{DebounceTime: -1}, } // Setup services @@ -75,13 +75,6 @@ func GetTestServices(t *testing.T) *TestServices { testServices.Auth = auth testServices.Settings.JWKKeySetURL = auth.URL() + "/keys" }) - waitForSetup(t, &wg, func(t *testing.T) { - kafka := setupMockKafkaServer(t) - testServices.Kafka = kafka - testServices.Settings.KafkaBrokers = kafka.GetBrokerAddress(t) - testServices.Settings.DeviceSignalsTopic = "test.default.signals.topic" - testServices.Settings.DeviceEventsTopic = "test.default.events.topic" - }) waitForSetup(t, &wg, func(t *testing.T) { db := tests.SetupTestContainer(t) testServices.Postgres = db @@ -109,9 +102,6 @@ func (tc *TestServices) TeardownIfLastTest(t *testing.T) { } tc.Identity.Close() tc.Auth.Close() - if err := tc.Kafka.Close(); err != nil { - t.Logf("Error closing Kafka: %v", err) - } tc.TokenExchange.Close() // reset the onceSetup to allow the next test to run if this one is closed globalTestContainer = sync.Once{} diff --git a/tests/e2e/nats_backpressure_test.go b/tests/e2e/nats_backpressure_test.go new file mode 100644 index 0000000..dd222d2 --- /dev/null +++ b/tests/e2e/nats_backpressure_test.go @@ -0,0 +1,212 @@ +package e2e_test + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// TestPullLoopBackpressureDoesNotDLQ verifies the D fix end-to-end: a +// handler that returns ErrBackpressure on every delivery gets long nak +// delays instead of burning through MaxDeliver and DLQing. Without the fix, +// MaxDeliver=3 attempts would land the message in the DLQ within seconds. +// With the fix, the message stays in the original stream undergoing slow +// retries until either the handler relents or operator scales out. +// +// We assert: the DLQ stream remains empty for at least 1s while the handler +// keeps returning ErrBackpressure, and consume metrics show +// nak_backpressure rather than dlq. +func TestPullLoopBackpressureDoesNotDLQ(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + conn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + require.NoError(t, err) + + suffix := time.Now().Format("150405000") + // Tight settings so the test runs fast: BackpressureNakDelay is the + // dominant timer (default 30s). Override it so the test finishes in + // reasonable time without weakening the assertion. + original := vtnats.BackpressureNakDelay + vtnats.BackpressureNakDelay = 500 * time.Millisecond + t.Cleanup(func() { vtnats.BackpressureNakDelay = original }) + + streamName := "BP_SIG_" + suffix + dlqName := "BP_DLQ_" + suffix + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: streamName, + Subjects: []string{"bp.signals.>"}, + Retention: jetstream.LimitsPolicy, + Storage: jetstream.FileStorage, + MaxAge: time.Minute, + }) + require.NoError(t, err) + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: dlqName, + Subjects: []string{"bp.dlq.>"}, + Retention: jetstream.LimitsPolicy, + Storage: jetstream.FileStorage, + MaxAge: time.Minute, + }) + require.NoError(t, err) + t.Cleanup(func() { + _ = js.DeleteStream(context.Background(), streamName) + _ = js.DeleteStream(context.Background(), dlqName) + }) + + client, err := vtnats.Connect(t.Context(), config.NATSSettings{ + Mode: "exclusive", + URL: server.URL(), + Name: "vt-bp-" + suffix, + SignalsStream: streamName, + SignalsSubject: "bp.signals.>", + DLQStream: dlqName, + DLQSubject: "bp.dlq.>", + MaxDeliver: 3, + MaxAckPending: 100, + AckWait: 2 * time.Second, + FetchBatch: 10, + }, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + + cons, err := client.EnsureConsumer(t.Context(), vtnats.ConsumerSpec{ + Stream: streamName, + Durable: "bp-cons-" + suffix, + FilterSubjects: []string{"bp.signals.>"}, + DeliverPolicy: jetstream.DeliverAllPolicy, + MaxDeliver: 3, + AckWait: 2 * time.Second, + MaxAckPending: 100, + }) + require.NoError(t, err) + + var deliveries atomic.Uint64 + bpHandler := func(_ context.Context, _ []byte) error { + deliveries.Add(1) + return fmt.Errorf("simulated dispatch saturation: %w", vtnats.ErrBackpressure) + } + + pullCtx, pullCancel := context.WithCancel(t.Context()) + t.Cleanup(pullCancel) + go func() { + _ = client.PullLoop(pullCtx, cons, 4, bpHandler) + }() + + _, err = client.Publish(t.Context(), "bp.signals.speed", []byte(`{"x":1}`)) + require.NoError(t, err) + + // Watch for redelivery. With MaxDeliver=3 and BackpressureNakDelay=500ms, + // we expect at most 3 attempts over ~1s+ wall time. Without the fix the + // same 3 attempts would happen back-to-back in <100ms and land DLQ. + require.Eventually(t, func() bool { return deliveries.Load() >= 2 }, 2*time.Second, 50*time.Millisecond, "at least 2 backpressure redeliveries") + + // Now assert the DLQ stream stays empty. If the backpressure path were + // not honoured, MaxDeliver=3 would have already landed the message in + // the DLQ during the eventually loop above. + dlqStream, err := js.Stream(t.Context(), dlqName) + require.NoError(t, err) + info, err := dlqStream.Info(t.Context()) + require.NoError(t, err) + require.Equal(t, uint64(0), info.State.Msgs, "DLQ must be empty for backpressure-only failures") +} + +// TestPullLoopRealErrorDLQs is the counter-assertion: a non-backpressure +// handler error DOES land in the DLQ after MaxDeliver. This pins the D fix +// to "only ErrBackpressure gets special treatment". +func TestPullLoopRealErrorDLQs(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + conn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + require.NoError(t, err) + + suffix := time.Now().Format("150405000") + streamName := "ERR_SIG_" + suffix + dlqName := "ERR_DLQ_" + suffix + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: streamName, + Subjects: []string{"err.signals.>"}, + Retention: jetstream.LimitsPolicy, + Storage: jetstream.FileStorage, + MaxAge: time.Minute, + }) + require.NoError(t, err) + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: dlqName, + Subjects: []string{"err.dlq.>"}, + Retention: jetstream.LimitsPolicy, + Storage: jetstream.FileStorage, + MaxAge: time.Minute, + }) + require.NoError(t, err) + t.Cleanup(func() { + _ = js.DeleteStream(context.Background(), streamName) + _ = js.DeleteStream(context.Background(), dlqName) + }) + + client, err := vtnats.Connect(t.Context(), config.NATSSettings{ + Mode: "exclusive", + URL: server.URL(), + Name: "vt-err-" + suffix, + SignalsStream: streamName, + SignalsSubject: "err.signals.>", + DLQStream: dlqName, + DLQSubject: "err.dlq.>", + MaxDeliver: 2, + MaxAckPending: 100, + AckWait: time.Second, + FetchBatch: 10, + }, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + + cons, err := client.EnsureConsumer(t.Context(), vtnats.ConsumerSpec{ + Stream: streamName, + Durable: "err-cons-" + suffix, + FilterSubjects: []string{"err.signals.>"}, + DeliverPolicy: jetstream.DeliverAllPolicy, + MaxDeliver: 2, + AckWait: time.Second, + MaxAckPending: 100, + BackOff: []time.Duration{200 * time.Millisecond}, + }) + require.NoError(t, err) + + pullCtx, pullCancel := context.WithCancel(t.Context()) + t.Cleanup(pullCancel) + go func() { + _ = client.PullLoop(pullCtx, cons, 4, func(_ context.Context, _ []byte) error { + return errors.New("permanent failure") + }) + }() + _, err = client.Publish(t.Context(), "err.signals.speed", []byte(`{"x":1}`)) + require.NoError(t, err) + + require.Eventually(t, func() bool { + dlqStream, err := js.Stream(t.Context(), dlqName) + if err != nil { + return false + } + info, err := dlqStream.Info(t.Context()) + return err == nil && info.State.Msgs > 0 + }, 15*time.Second, 200*time.Millisecond, "real handler error must land in DLQ after MaxDeliver") +} diff --git a/tests/e2e/nats_cluster_test.go b/tests/e2e/nats_cluster_test.go new file mode 100644 index 0000000..5575bf3 --- /dev/null +++ b/tests/e2e/nats_cluster_test.go @@ -0,0 +1,184 @@ +//go:build cluster + +// Build-tagged cluster e2e. Spins three NATS JetStream nodes on a shared +// testcontainers network and runs the production publish + consume path +// against replicas=3 streams. Catches bugs that single-node tests can't +// see - replica election, quorum loss, in-flight redelivery during +// leadership transfer. +// +// Tagged because cluster startup is multi-second and adds container churn +// to CI. Invoke in a dedicated job: `go test -tags=cluster ./tests/e2e/...`. +package e2e_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/DIMO-Network/vehicle-triggers-api/tests" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcnetwork "github.com/testcontainers/testcontainers-go/network" + "github.com/testcontainers/testcontainers-go/wait" +) + +const natsClusterConfTmpl = ` +server_name: %s +listen: 0.0.0.0:4222 +http: 0.0.0.0:8222 +jetstream { + store_dir: /data +} +cluster { + name: TRIGGERS_CLUSTER_TEST + listen: 0.0.0.0:6222 + routes = [ + nats-route://%s:6222 + nats-route://%s:6222 + nats-route://%s:6222 + ] +} +` + +type clusterNode struct { + name string + container testcontainers.Container + clientURL string +} + +// startCluster spins three nats containers on a shared network so they can +// form a cluster. Returns the first URL (any node works for clients) and a +// cleanup function. Skips the test gracefully if Docker isn't available. +func startCluster(t *testing.T) (string, func()) { + t.Helper() + tests.SkipIfNoDocker(t) + + ctx := context.Background() + net, err := tcnetwork.New(ctx) + require.NoError(t, err) + + names := []string{"nats-c1", "nats-c2", "nats-c3"} + nodes := make([]clusterNode, 0, len(names)) + cleanup := func() { + for _, n := range nodes { + _ = n.container.Terminate(context.Background()) + } + _ = net.Remove(context.Background()) + } + + for _, name := range names { + conf := fmt.Sprintf(natsClusterConfTmpl, name, names[0], names[1], names[2]) + req := testcontainers.ContainerRequest{ + Image: "nats:2.11-alpine", + Name: name, + Networks: []string{net.Name}, + ExposedPorts: []string{"4222/tcp"}, + Cmd: []string{"-c", "/etc/nats/nats.conf"}, + Files: []testcontainers.ContainerFile{{ + Reader: strings.NewReader(conf), + ContainerFilePath: "/etc/nats/nats.conf", + FileMode: 0o644, + }}, + WaitingFor: wait.ForListeningPort("4222/tcp").WithStartupTimeout(30 * time.Second), + } + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + if err != nil { + cleanup() + t.Fatalf("start %s: %v", name, err) + } + host, err := ctr.Host(ctx) + require.NoError(t, err) + port, err := ctr.MappedPort(ctx, "4222/tcp") + require.NoError(t, err) + nodes = append(nodes, clusterNode{ + name: name, + container: ctr, + clientURL: fmt.Sprintf("nats://%s:%s", host, port.Port()), + }) + } + return nodes[0].clientURL, cleanup +} + +// TestNATSCluster_PublishConsumeReplicas3 asserts that a replicas=3 stream +// accepts publishes from one client and that a pull consumer drains them. +// This is the smoke test for our cluster wiring. +func TestNATSCluster_PublishConsumeReplicas3(t *testing.T) { + t.Parallel() + url, cleanup := startCluster(t) + t.Cleanup(cleanup) + + // Cluster formation takes a couple of seconds beyond port listen. + time.Sleep(3 * time.Second) + + suffix := time.Now().Format("150405000") + conn, err := nc.Connect(url, nc.RetryOnFailedConnect(true), nc.MaxReconnects(-1), nc.ReconnectWait(time.Second), nc.Timeout(10*time.Second)) + require.NoError(t, err) + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + require.NoError(t, err) + + stream := "CLUSTER_T_" + suffix + _, err = js.CreateOrUpdateStream(t.Context(), jetstream.StreamConfig{ + Name: stream, + Subjects: []string{"cluster.t.>"}, + Retention: jetstream.LimitsPolicy, + Discard: jetstream.DiscardOld, + Storage: jetstream.FileStorage, + MaxAge: 5 * time.Minute, + Replicas: 3, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = js.DeleteStream(context.Background(), stream) }) + + cons, err := js.CreateOrUpdateConsumer(t.Context(), stream, jetstream.ConsumerConfig{ + Durable: "cluster-cons-" + suffix, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + FilterSubject: "cluster.t.>", + }) + require.NoError(t, err) + + client, err := vtnats.Connect(t.Context(), config.NATSSettings{ + Mode: "exclusive", + URL: url, + Name: "vt-cluster-" + suffix, + FetchBatch: 50, + MaxDeliver: 5, + MaxAckPending: 500, + StreamReplicas: 3, + }, zerolog.Nop()) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + + var consumed atomic.Uint64 + pullCtx, pullCancel := context.WithCancel(t.Context()) + t.Cleanup(pullCancel) + go func() { + _ = client.PullLoop(pullCtx, cons, 4, func(_ context.Context, _ []byte) error { + consumed.Add(1) + return nil + }) + }() + + const N = 50 + for i := 0; i < N; i++ { + _, err := client.Publish(t.Context(), "cluster.t.smoke", []byte(`{"v":1}`)) + require.NoError(t, err) + } + + require.Eventually(t, func() bool { return consumed.Load() == N }, 10*time.Second, 50*time.Millisecond, "consumer must drain all replicas=3 messages") +} + +var _ = errors.New // keep imports stable when adding follow-up tests diff --git a/tests/e2e/nats_dlq_test.go b/tests/e2e/nats_dlq_test.go new file mode 100644 index 0000000..c6651a3 --- /dev/null +++ b/tests/e2e/nats_dlq_test.go @@ -0,0 +1,126 @@ +package e2e_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/DIMO-Network/vehicle-triggers-api/internal/config" + vtnats "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// TestNATSDLQ verifies that a message whose handler keeps failing is parked in +// the DLQ stream (with diagnostic headers) after exceeding MaxDeliver, rather +// than redelivering forever or silently dropping. +func TestNATSDLQ(t *testing.T) { + t.Parallel() + natsServer := setupMockNATSServer(t) + t.Cleanup(func() { _ = natsServer.Close() }) + + suffix := time.Now().Format("150405000") + cfg := config.NATSSettings{ + Mode: "exclusive", + URL: natsServer.URL(), + Name: "vt-dlq-" + suffix, + SignalsStream: "DLQ_SIGNALS_" + suffix, + EventsStream: "DLQ_EVENTS_" + suffix, + AuditStream: "DLQ_AUDIT_" + suffix, + DLQStream: "DLQ_DLQ_" + suffix, + ConfigAuditStream: "DLQ_CFG_" + suffix, + SignalsSubject: "dimo.signals.>", + EventsSubject: "dimo.events.>", + AuditSubject: "dimo.trigger.fired.>", + DLQSubject: "dimo.dlq.>", + ConfigAuditSubject: "dimo.config.changed.>", + SignalsDurable: "dlq-sig-" + suffix, + EventsDurable: "dlq-evt-" + suffix, + TriggerStateBucket: "dlq_state_" + suffix, + SignalHistoryBucket: "dlq_hist_" + suffix, + StreamReplicas: 1, + SignalsMaxAge: time.Minute, + EventsMaxAge: time.Minute, + AuditMaxAge: time.Minute, + DLQMaxAge: time.Minute, + AckWait: time.Second, // short so redelivery is fast + MaxDeliver: 2, // first attempt + 1 retry, then DLQ + MaxAckPending: 100, + FetchBatch: 10, + TriggerStateTTL: time.Minute, + } + + client, err := vtnats.Connect(t.Context(), cfg, zerolog.New(zerolog.NewTestWriter(t))) + require.NoError(t, err) + t.Cleanup(func() { _ = client.Close() }) + require.NoError(t, client.EnsureStreams(t.Context())) + require.NoError(t, client.EnsureBuckets(t.Context())) + + cons, err := client.EnsureConsumer(t.Context(), vtnats.ConsumerSpec{ + Stream: cfg.SignalsStream, + Durable: cfg.SignalsDurable, + FilterSubjects: []string{vtnats.AllSignalsFilter()}, + DeliverPolicy: jetstream.DeliverAllPolicy, + AckWait: cfg.AckWait, + MaxDeliver: cfg.MaxDeliver, + MaxAckPending: cfg.MaxAckPending, + }) + require.NoError(t, err) + + // Handler that always fails: simulates an unparseable payload or a + // permanently broken downstream. + alwaysFail := func(_ context.Context, _ []byte) error { + return errors.New("synthetic permanent failure") + } + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + go func() { + if err := client.PullLoop(ctx, cons, 4, alwaysFail); err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("pull loop: %v", err) + } + }() + + // Publish one signal. It will fail, retry once, then be DLQ'd. + _, err = client.Publish(t.Context(), vtnats.SignalSubject("speed"), []byte(`{"garbage":true}`)) + require.NoError(t, err) + + // Tap the DLQ stream and assert the message lands with diagnostic headers. + conn, err := nc.Connect(natsServer.URL()) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + js, err := jetstream.New(conn) + require.NoError(t, err) + + dlqCons, err := js.CreateOrUpdateConsumer(t.Context(), cfg.DLQStream, jetstream.ConsumerConfig{ + Durable: "dlq-tap-" + suffix, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + FilterSubject: cfg.DLQSubject, + }) + require.NoError(t, err) + + msgs, err := dlqCons.Fetch(1, jetstream.FetchMaxWait(15*time.Second)) + require.NoError(t, err) + var got jetstream.Msg + for m := range msgs.Messages() { + got = m + require.NoError(t, m.Ack()) + break + } + require.NoError(t, msgs.Error()) + require.NotNil(t, got, "expected a message in the DLQ stream") + + require.Equal(t, "dimo.dlq.dimo.signals.speed", got.Subject()) + require.Equal(t, "dimo.signals.speed", got.Headers().Get("X-Original-Subject")) + require.Contains(t, got.Headers().Get("X-Failure-Reason"), "synthetic permanent failure") + require.Equal(t, `{"garbage":true}`, string(got.Data())) + require.Equal(t, "speed", got.Headers().Get("X-Source-Name")) + require.NotEmpty(t, got.Headers().Get("X-Recorded-At")) + // Payload here has no "subject" - X-Asset-DID should be empty rather + // than crash; downstream tooling treats empty as "unknown vehicle". + require.Empty(t, got.Headers().Get("X-Asset-DID")) +} diff --git a/tests/e2e/nats_duplicate_fire_test.go b/tests/e2e/nats_duplicate_fire_test.go new file mode 100644 index 0000000..81d7950 --- /dev/null +++ b/tests/e2e/nats_duplicate_fire_test.go @@ -0,0 +1,91 @@ +package e2e_test + +import ( + "context" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/require" +) + +// TestDuplicateFireProducesIdenticalIDs documents and enforces the receiver- +// dedup contract from NATS_CONTRACT.md. Two parallel writers acting on the +// same (trigger, vehicle, source) state produce records the receiver-side +// dedup is expected to collapse on - they share LastFiredAt and an asset DID +// derived from the same payload, and any webhook we send would carry the +// same deterministic CloudEvent ID. This is a regression guard: if anyone +// changes the state record shape or the deterministic ID derivation in a +// way that breaks receiver dedup, this test fails. +func TestDuplicateFireProducesIdenticalIDs(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + conn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + require.NoError(t, err) + + suffix := time.Now().Format("150405000") + stateKV, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: "dup_state_" + suffix, + TTL: time.Minute, + }) + require.NoError(t, err) + historyKV, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: "dup_hist_" + suffix, + TTL: time.Minute, + }) + require.NoError(t, err) + + store := triggerstate.New(stateKV, historyKV) + + did := cloudevent.ERC721DID{ + ChainID: 137, + ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), + TokenID: big.NewInt(42), + } + triggerID := "dedup-trigger" + metric := "vss.speed" + + // Force two writers to race on the same key. The CAS fallback metric + // ticks; both records persist on the KV stream. From a receiver's POV + // they look like two deliveries of the same logical fire. + const N = 8 + var wg sync.WaitGroup + var attempts atomic.Uint64 + wg.Add(N) + for i := 0; i < N; i++ { + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _ = store.RecordFire(ctx, triggerID, metric, did, time.Now(), []byte(`{"sourceId":"signal-abc"}`)) + attempts.Add(1) + }() + } + wg.Wait() + require.EqualValues(t, N, attempts.Load()) + + // The trigger_state record converges to one entry per (trigger, vehicle). + rec, ok, err := store.LastFire(t.Context(), triggerID, did) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, triggerID, rec.TriggerID) + require.Equal(t, did.String(), rec.AssetDID) + // LastSnapshot carries the source-identifying body; the receiver-side + // deterministic webhook ID derivation hashes (triggerID, sourceID) - all + // N writers used the same sourceID, so all N (had we delivered them) + // would yield the same data.webhookId. That is the receiver-dedup + // guarantee documented in NATS_CONTRACT.md. + require.Contains(t, string(rec.LastSnapshot), `"sourceId":"signal-abc"`) +} diff --git a/tests/e2e/nats_exclusive_flow_test.go b/tests/e2e/nats_exclusive_flow_test.go new file mode 100644 index 0000000..ec85253 --- /dev/null +++ b/tests/e2e/nats_exclusive_flow_test.go @@ -0,0 +1,209 @@ +package e2e_test + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/big" + "net/http" + "os" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/app" + "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" + "github.com/DIMO-Network/vehicle-triggers-api/tests" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/rs/zerolog" + "github.com/stretchr/testify/require" +) + +// TestNATSExclusiveFlow verifies the Kafka-free target state: NATS_MODE=exclusive +// means the service starts without Kafka consumers and runs evaluation purely +// from JetStream. We also assert that an audit record lands on the audit +// stream so billing downstream gets the fire. +func TestNATSExclusiveFlow(t *testing.T) { + t.Parallel() + tc := GetTestServices(t) + + natsServer := setupMockNATSServer(t) + t.Cleanup(func() { _ = natsServer.Close() }) + + devAddress := tests.RandomAddr(t) + settingsCopy := tc.Settings + + settingsCopy.NATS.Mode = "exclusive" + settingsCopy.NATS.URL = natsServer.URL() + suffix := devAddress.Hex()[2:10] + settingsCopy.NATS.SignalsStream = "EX_SIGNALS_" + suffix + settingsCopy.NATS.EventsStream = "EX_EVENTS_" + suffix + settingsCopy.NATS.AuditStream = "EX_AUDIT_" + suffix + settingsCopy.NATS.DLQStream = "EX_DLQ_" + suffix + settingsCopy.NATS.ConfigAuditStream = "EX_CFG_" + suffix + settingsCopy.NATS.SignalsSubject = "dimo.signals.>" + settingsCopy.NATS.EventsSubject = "dimo.events.>" + settingsCopy.NATS.AuditSubject = "dimo.trigger.fired.>" + settingsCopy.NATS.DLQSubject = "dimo.dlq.>" + settingsCopy.NATS.ConfigAuditSubject = "dimo.config.changed.>" + settingsCopy.NATS.SignalsDurable = "ex-sig-" + suffix + settingsCopy.NATS.EventsDurable = "ex-evt-" + suffix + settingsCopy.NATS.TriggerStateBucket = "ex_state_" + suffix + settingsCopy.NATS.SignalHistoryBucket = "ex_hist_" + suffix + settingsCopy.NATS.Name = "vt-ex-" + suffix + settingsCopy.NATS.StreamReplicas = 1 + settingsCopy.NATS.SignalsMaxAge = time.Minute + settingsCopy.NATS.EventsMaxAge = time.Minute + settingsCopy.NATS.AuditMaxAge = time.Minute + settingsCopy.NATS.DLQMaxAge = time.Minute + settingsCopy.NATS.AckWait = 5 * time.Second + settingsCopy.NATS.MaxDeliver = 10 + settingsCopy.NATS.MaxAckPending = 1000 + settingsCopy.NATS.FetchBatch = 50 + settingsCopy.NATS.TriggerStateTTL = time.Minute + + servers, err := app.CreateServers(t.Context(), &settingsCopy, zerolog.New(os.Stdout)) + require.NoError(t, err) + require.NotNil(t, servers.NATSClient) + require.NotNil(t, servers.NATSSignalConsumer) + + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + go func() { + if err := servers.NATSClient.PullLoop(ctx, servers.NATSSignalConsumer, settingsCopy.MaxInFlight, servers.NATSListener.HandleSignalPayload); err != nil && !errors.Is(err, context.Canceled) { + t.Errorf("nats pull loop: %v", err) + } + }() + if servers.Dispatcher != nil { + go func() { _ = servers.Dispatcher.Run(ctx) }() + } + if servers.AuditQueue != nil { + go func() { _ = servers.AuditQueue.Run(ctx) }() + } + t.Cleanup(func() { + shutdownCtx, c := context.WithTimeout(context.Background(), 5*time.Second) + defer c() + _ = servers.NATSClient.Shutdown(shutdownCtx) + }) + + receiver := NewWebhookReceiver() + t.Cleanup(receiver.Close) + + authToken, err := tc.Auth.CreateToken(t, devAddress) + require.NoError(t, err) + require.NoError(t, tc.Identity.SetRequestResponse( + fmt.Sprintf(`{"query":"\n\tquery($clientId: Address){\n\t\tdeveloperLicense(by: { clientId: $clientId }) {\n\t\t\tclientId\n\t\t}\n\t}","variables":{"clientId":"%s"}}`, devAddress.String()), + map[string]any{"data": map[string]any{"developerLicense": map[string]any{"clientId": devAddress.String()}}}, + )) + + regBody, _ := json.Marshal(webhook.RegisterWebhookRequest{ + Service: triggersrepo.ServiceSignal, + MetricName: "vss.speed", + Condition: "valueNumber > 10", + CoolDownPeriod: 0, + Description: "exclusive e2e: speed > 10", + TargetURL: receiver.URL(), + Status: triggersrepo.StatusEnabled, + VerificationToken: "test-verification-token", + }) + req, _ := http.NewRequestWithContext(t.Context(), "POST", "/v1/webhooks", bytes.NewBuffer(regBody)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+authToken) + resp, err := servers.Application.Test(req, -1) + require.NoError(t, err) + body, _ := io.ReadAll(resp.Body) + require.Equal(t, http.StatusCreated, resp.StatusCode, string(body)) + var regResp map[string]any + _ = json.Unmarshal(body, ®Resp) + webhookID := regResp["id"].(string) + + assetDid := cloudevent.ERC721DID{ + ChainID: 137, + ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), + TokenID: big.NewInt(99002), + } + tc.TokenExchange.SetAccessCheckReturn(devAddress.String(), true) + subReq, _ := http.NewRequestWithContext(t.Context(), "POST", fmt.Sprintf("/v1/webhooks/%s/subscribe/%s", webhookID, assetDid.String()), nil) + subReq.Header.Set("Authorization", "Bearer "+authToken) + subResp, err := servers.Application.Test(subReq, -1) + require.NoError(t, err) + require.Equal(t, http.StatusCreated, subResp.StatusCode) + time.Sleep(1 * time.Second) + + // Subscribe to audit stream first so the publish lands while we're listening. + conn, err := nc.Connect(natsServer.URL()) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + js, err := jetstream.New(conn) + require.NoError(t, err) + + auditCons, err := js.CreateOrUpdateConsumer(t.Context(), settingsCopy.NATS.AuditStream, jetstream.ConsumerConfig{ + Durable: "audit-tap-" + suffix, + AckPolicy: jetstream.AckExplicitPolicy, + DeliverPolicy: jetstream.DeliverAllPolicy, + FilterSubject: "dimo.trigger.fired.>", + }) + require.NoError(t, err) + + publishSignal(t, js, assetDid, "speed", 42.0) + + require.True(t, receiver.WaitForCall(10*time.Second), "webhook not called in exclusive mode") + calls := receiver.GetReceivedCalls() + require.Len(t, calls, 1) + + // Verify audit stream caught the fire. + auditMsgs, err := auditCons.Fetch(1, jetstream.FetchMaxWait(5*time.Second)) + require.NoError(t, err) + var auditPayload map[string]any + for m := range auditMsgs.Messages() { + require.NoError(t, json.Unmarshal(m.Data(), &auditPayload)) + require.NoError(t, m.Ack()) + break + } + require.NoError(t, auditMsgs.Error()) + require.NotEmpty(t, auditPayload, "audit stream did not capture the fire") + data := auditPayload["data"].(map[string]any) + require.Equal(t, "vss.speed", data["metricName"].(string)) + + // Verify trigger state KV captured the fire so other replicas would honor + // the cooldown without reading Postgres. + require.Eventually(t, func() bool { + kv, err := js.KeyValue(t.Context(), settingsCopy.NATS.TriggerStateBucket) + if err != nil { + return false + } + entry, err := kv.Get(t.Context(), triggerstate.TriggerKey(webhookID, assetDid)) + if err != nil { + return false + } + var rec triggerstate.Record + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return false + } + return !rec.LastFiredAt.IsZero() && rec.TriggerID == webhookID && rec.AssetDID == assetDid.String() && len(rec.LastSnapshot) > 0 + }, 5*time.Second, 100*time.Millisecond, "trigger state KV missing fire record") + + // Verify signal_history KV captured the per-metric snapshot. + require.Eventually(t, func() bool { + kv, err := js.KeyValue(t.Context(), settingsCopy.NATS.SignalHistoryBucket) + if err != nil { + return false + } + entry, err := kv.Get(t.Context(), triggerstate.MetricKey(assetDid, "vss.speed")) + if err != nil { + return false + } + var rec triggerstate.MetricRecord + if err := json.Unmarshal(entry.Value(), &rec); err != nil { + return false + } + return rec.MetricName == "vss.speed" && len(rec.LastSnapshot) > 0 + }, 5*time.Second, 100*time.Millisecond, "signal_history KV missing metric record") +} diff --git a/tests/e2e/nats_publish_helpers_test.go b/tests/e2e/nats_publish_helpers_test.go new file mode 100644 index 0000000..ae9dd5f --- /dev/null +++ b/tests/e2e/nats_publish_helpers_test.go @@ -0,0 +1,48 @@ +package e2e_test + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/model-garage/pkg/vss" + "github.com/DIMO-Network/vehicle-triggers-api/internal/nats" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/require" +) + +// publishSignal wraps a single signal into a CloudEvent and publishes one +// message on the per-signal subject the production consumers filter on. Used +// by the exclusive-mode e2e tests to feed the NATS pipeline directly, +// bypassing the deprecated Kafka entry. +func publishSignal(t *testing.T, js jetstream.JetStream, assetDID cloudevent.ERC721DID, signalName string, value float64) { + t.Helper() + ce := vss.PackSignals(cloudevent.CloudEventHeader{ + Subject: assetDID.String(), + Source: "test-source", + Producer: "test-producer", + ID: "test-event-" + signalName, + }, []vss.Signal{ + { + CloudEventHeader: cloudevent.CloudEventHeader{ + Subject: assetDID.String(), + Source: "test-source", + Producer: "test-producer", + }, + Data: vss.SignalData{ + Timestamp: time.Now(), + Name: signalName, + ValueNumber: value, + CloudEventID: "test-event-" + signalName, + }, + }, + }) + body, err := json.Marshal(ce) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + _, err = js.Publish(ctx, nats.SignalSubject(signalName), body) + require.NoError(t, err) +} diff --git a/tests/e2e/nats_server_test.go b/tests/e2e/nats_server_test.go new file mode 100644 index 0000000..ea2afb2 --- /dev/null +++ b/tests/e2e/nats_server_test.go @@ -0,0 +1,47 @@ +package e2e_test + +import ( + "context" + "testing" + + "github.com/DIMO-Network/vehicle-triggers-api/tests" + "github.com/testcontainers/testcontainers-go/modules/nats" +) + +type mockNATSServer struct { + container *nats.NATSContainer + url string +} + +// setupMockNATSServer launches a JetStream-enabled NATS broker in a container +// and returns its client URL. Skips the test cleanly when Docker is not +// available so unit-style runs stay green. +func setupMockNATSServer(t *testing.T) *mockNATSServer { + t.Helper() + tests.SkipIfNoDocker(t) + + ctx := context.Background() + // The default Run command already includes "-js" so we don't need to pass + // extra args. Passing them via WithArgument/CmdArgs replaces the default + // command and we'd lose JetStream. + container, err := nats.Run(ctx, "nats:2.11-alpine") + if err != nil { + t.Fatalf("Failed to start NATS container: %v", err) + } + + url, err := container.ConnectionString(ctx) + if err != nil { + t.Fatalf("Failed to read NATS connection string: %v", err) + } + + return &mockNATSServer{container: container, url: url} +} + +func (m *mockNATSServer) URL() string { return m.url } + +func (m *mockNATSServer) Close() error { + if m == nil || m.container == nil { + return nil + } + return m.container.Terminate(context.Background()) +} diff --git a/tests/e2e/nats_state_cas_test.go b/tests/e2e/nats_state_cas_test.go new file mode 100644 index 0000000..7b95ff4 --- /dev/null +++ b/tests/e2e/nats_state_cas_test.go @@ -0,0 +1,85 @@ +package e2e_test + +import ( + "context" + "encoding/json" + "math/big" + "sync" + "testing" + "time" + + "github.com/DIMO-Network/cloudevent" + "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggerstate" + "github.com/ethereum/go-ethereum/common" + nc "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/stretchr/testify/require" +) + +// TestTriggerStateCAS asserts that two goroutines racing RecordFire on the +// same key both eventually succeed (KV state is preserved) and the conflict +// counter records the race - the receiver-dedup story relies on this counter +// being observable to operators. +func TestTriggerStateCAS(t *testing.T) { + t.Parallel() + server := setupMockNATSServer(t) + t.Cleanup(func() { _ = server.Close() }) + + conn, err := nc.Connect(server.URL()) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + js, err := jetstream.New(conn) + require.NoError(t, err) + + suffix := time.Now().Format("150405000") + stateKV, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: "cas_state_" + suffix, + TTL: time.Minute, + }) + require.NoError(t, err) + historyKV, err := js.CreateOrUpdateKeyValue(t.Context(), jetstream.KeyValueConfig{ + Bucket: "cas_hist_" + suffix, + TTL: time.Minute, + }) + require.NoError(t, err) + + store := triggerstate.New(stateKV, historyKV) + + did := cloudevent.ERC721DID{ + ChainID: 137, + ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), + TokenID: big.NewInt(7), + } + triggerID := "race-trigger" + metric := "vss.speed" + + // First write - must be Create. + require.NoError(t, store.RecordFire(t.Context(), triggerID, metric, did, time.Now(), json.RawMessage(`{"v":1}`))) + + // Now race N writers. The KV value after all complete should be one of + // the snapshots (last writer wins on fallback), and the bucket must hold + // a valid Record. + const N = 8 + var wg sync.WaitGroup + wg.Add(N) + for i := 0; i < N; i++ { + go func(i int) { + defer wg.Done() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + snap, _ := json.Marshal(map[string]int{"i": i + 100}) + // Errors are not asserted here - the CAS-with-fallback contract + // means concurrent writers don't surface errors; instead the + // vehicle_triggers_state_cas_conflicts_total counter ticks. + _ = store.RecordFire(ctx, triggerID, metric, did, time.Now(), snap) + }(i) + } + wg.Wait() + + rec, ok, err := store.LastFire(t.Context(), triggerID, did) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, triggerID, rec.TriggerID) + require.Equal(t, did.String(), rec.AssetDID) + require.NotEmpty(t, rec.LastSnapshot, "snapshot from winning writer must be preserved") +} diff --git a/tests/e2e/webhook_flow_test.go b/tests/e2e/webhook_flow_test.go deleted file mode 100644 index 3f91954..0000000 --- a/tests/e2e/webhook_flow_test.go +++ /dev/null @@ -1,696 +0,0 @@ -package e2e_test - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "math/big" - "net/http" - "os" - "testing" - "time" - - "github.com/DIMO-Network/cloudevent" - "github.com/DIMO-Network/model-garage/pkg/vss" - "github.com/DIMO-Network/vehicle-triggers-api/internal/app" - "github.com/DIMO-Network/vehicle-triggers-api/internal/controllers/webhook" - "github.com/DIMO-Network/vehicle-triggers-api/internal/services/triggersrepo" - "github.com/DIMO-Network/vehicle-triggers-api/tests" - "github.com/ethereum/go-ethereum/common" - "github.com/rs/zerolog" - "github.com/stretchr/testify/require" -) - -func TestSignalWebhookFlow(t *testing.T) { - t.Parallel() - tc := GetTestServices(t) - - // Create a developer address for testing - devAddress := tests.RandomAddr(t) - settingsCopy := tc.Settings - settingsCopy.DeviceEventsTopic = "test-event-topic" + devAddress.String() - settingsCopy.DeviceSignalsTopic = "test-signal-topic" + devAddress.String() - - // Create the main application - servers, err := app.CreateServers(t.Context(), &settingsCopy, zerolog.New(os.Stdout)) - go func() { - if err := servers.SignalConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start signal consumer: %v", err) - } - } - }() - go func() { - if err := servers.EventConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start event consumer: %v", err) - } - } - }() - t.Cleanup(func() { - _ = servers.SignalConsumer.Stop(t.Context()) - _ = servers.EventConsumer.Stop(t.Context()) - }) - require.NoError(t, err) - - // Create a webhook receiver - webhookReceiver := NewWebhookReceiver() - t.Cleanup(webhookReceiver.Close) - - // Step 1: Create a webhook - webhookPayload := webhook.RegisterWebhookRequest{ - Service: triggersrepo.ServiceSignal, - MetricName: "vss.speed", - Condition: "valueNumber > 20 && valueNumber != previousValue", - CoolDownPeriod: 0, - Description: "Alert when vehicle speed exceeds 20 kph", - TargetURL: webhookReceiver.URL(), - Status: triggersrepo.StatusEnabled, - VerificationToken: "test-verification-token", - } - - webhookBody, err := json.Marshal(webhookPayload) - require.NoError(t, err) - - // Create auth token for the request - authToken, err := tc.Auth.CreateToken(t, devAddress) - require.NoError(t, err) - - // add dev license to identity api - err = tc.Identity.SetRequestResponse( - fmt.Sprintf(`{"query":"\n\tquery($clientId: Address){\n\t\tdeveloperLicense(by: { clientId: $clientId }) {\n\t\t\tclientId\n\t\t}\n\t}","variables":{"clientId":"%s"}}`, devAddress.String()), - map[string]any{ - "data": map[string]any{ - "developerLicense": map[string]any{ - "clientId": devAddress.String(), - }, - }, - }) - require.NoError(t, err) - - // Make the webhook creation request - req, err := http.NewRequestWithContext( - t.Context(), - "POST", - "/v1/webhooks", - bytes.NewBuffer(webhookBody), - ) - require.NoError(t, err) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err := servers.Application.Test(req, -1) - require.NoError(t, err) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - bodyStr := string(body) - require.Equal(t, http.StatusCreated, resp.StatusCode, bodyStr) - - // Parse the response to get the webhook ID - webhookResponse := map[string]any{} - err = json.Unmarshal(body, &webhookResponse) - require.NoError(t, err) - - webhookID, ok := webhookResponse["id"].(string) - require.True(t, ok, "Expected webhook ID in response") - require.NotEmpty(t, webhookID) - - // Step 2: Subscribe a vehicle to the webhook - - // Use a test vehicle token ID - assetDid := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - - // Set up token exchange API mock to return permissions for this vehicle - tc.TokenExchange.SetAccessCheckReturn(devAddress.String(), true) - - // Make the subscription request - subscribeURL := fmt.Sprintf("/v1/webhooks/%s/subscribe/%s", webhookID, assetDid.String()) - req, err = http.NewRequestWithContext( - t.Context(), - "POST", - subscribeURL, - nil, - ) - require.NoError(t, err) - - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err = servers.Application.Test(req, -1) - require.NoError(t, err) - require.Equal(t, http.StatusCreated, resp.StatusCode) - - // wait for webhook to be updated - time.Sleep(1 * time.Second) - - // Step 3: Send a signal to Kafka to trigger the webhook - signalPayload := vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Signal{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now(), - Name: "speed", - ValueNumber: 25.0, // Above the 20 threshold to trigger the webhook - CloudEventID: "test-event-id", - }, - }, - }) - - err = tc.Kafka.PushJSONToTopic(settingsCopy.DeviceSignalsTopic, signalPayload) - require.NoError(t, err) - - // Step 4: Verify the webhook was called - - // Wait for the webhook to be called (with timeout) - received := webhookReceiver.WaitForCall(10 * time.Second) - require.True(t, received, "Webhook was not called within timeout") - - // Get the received calls - calls := webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 1, "Expected exactly one webhook call") - - call := calls[0] - - // Verify the webhook call details - require.Equal(t, "POST", call.Method) - require.Equal(t, "/", call.URL) // The webhook receiver URL path - - // Parse the webhook body to verify it contains the signal data - var webhookBodyData map[string]any - err = json.Unmarshal([]byte(call.Body), &webhookBodyData) - require.NoError(t, err) - - // Verify the webhook contains the expected signal data - require.Contains(t, webhookBodyData, "data") - data := webhookBodyData["data"].(map[string]any) - require.Contains(t, data, "signal") - signal := data["signal"].(map[string]any) - require.Equal(t, float64(25), signal["value"].(float64)) - require.Equal(t, "speed", signal["name"].(string)) - webhookReceiver.ClearReceivedCalls() - - // Step 5: Send a second signal to Kafka to trigger the webhook - signalPayload = vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Signal{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now(), - Name: "speed", - ValueNumber: 25.0, // Above the 20 threshold to trigger the webhook - CloudEventID: "test-event-id", - }, - }, - }) - if err := tc.Kafka.PushJSONToTopic(settingsCopy.DeviceSignalsTopic, signalPayload); err != nil { - t.Errorf("failed to push signal to Kafka: %v", err) - } - - // Step 6: Verify the webhook was not called - received = webhookReceiver.WaitForCall(2 * time.Second) - require.False(t, received, "Webhook was unexpectedly called within timeout") - calls = webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 0, "Expected exactly one webhook call") - signalPayload = vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Signal{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now(), - Name: "speed", - ValueNumber: 24.0, // Below the 20 threshold to trigger the webhook - CloudEventID: "test-event-id", - }, - }, - }) - if err := tc.Kafka.PushJSONToTopic(settingsCopy.DeviceSignalsTopic, signalPayload); err != nil { - t.Errorf("failed to push signal to Kafka: %v", err) - } - - // Step 7: Verify the webhook was called - received = webhookReceiver.WaitForCall(10 * time.Second) - require.True(t, received, "Webhook was not called within timeout") - calls = webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 1, "Expected exactly one webhook call") - - call = calls[0] - - // Verify the webhook call details - require.Equal(t, "POST", call.Method) - require.Equal(t, "/", call.URL) // The webhook receiver URL path - - // Parse the webhook body to verify it contains the signal data - webhookBodyData = make(map[string]any) - err = json.Unmarshal([]byte(call.Body), &webhookBodyData) - require.NoError(t, err) - - // Verify the webhook contains the expected signal data - require.Contains(t, webhookBodyData, "data") - data = webhookBodyData["data"].(map[string]any) - require.Contains(t, data, "signal") - signal = data["signal"].(map[string]any) - require.Equal(t, float64(24), signal["value"].(float64)) - require.Equal(t, "speed", signal["name"].(string)) -} - -func TestSignalWebhookFlowLocation(t *testing.T) { - t.Parallel() - tc := GetTestServices(t) - - // Create a developer address for testing - devAddress := tests.RandomAddr(t) - settingsCopy := tc.Settings - settingsCopy.DeviceEventsTopic = "test-event-topic" + devAddress.String() - settingsCopy.DeviceSignalsTopic = "test-signal-topic" + devAddress.String() - - // Create the main application - servers, err := app.CreateServers(t.Context(), &settingsCopy, zerolog.New(os.Stdout)) - go func() { - if err := servers.SignalConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start signal consumer: %v", err) - } - } - }() - go func() { - if err := servers.EventConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start event consumer: %v", err) - } - } - }() - t.Cleanup(func() { - _ = servers.SignalConsumer.Stop(t.Context()) - _ = servers.EventConsumer.Stop(t.Context()) - }) - require.NoError(t, err) - - // Create a webhook receiver - webhookReceiver := NewWebhookReceiver() - t.Cleanup(webhookReceiver.Close) - - // Step 1: Create a webhook for location coordinates - webhookPayload := webhook.RegisterWebhookRequest{ - Service: triggersrepo.ServiceSignal, - MetricName: "vss.currentLocationCoordinates", - Condition: "geoDistance(value.latitude, value.longitude, 54.71061320000001, 25.239925999999997) < 0.7138406571965812", - CoolDownPeriod: 0, - Description: "Alert when vehicle is within 0.7km of target location", - TargetURL: webhookReceiver.URL(), - Status: triggersrepo.StatusEnabled, - VerificationToken: "test-verification-token", - } - - webhookBody, err := json.Marshal(webhookPayload) - require.NoError(t, err) - - // Create auth token for the request - authToken, err := tc.Auth.CreateToken(t, devAddress) - require.NoError(t, err) - - // add dev license to identity api - err = tc.Identity.SetRequestResponse( - fmt.Sprintf(`{"query":"\n\tquery($clientId: Address){\n\t\tdeveloperLicense(by: { clientId: $clientId }) {\n\t\t\tclientId\n\t\t}\n\t}","variables":{"clientId":"%s"}}`, devAddress.String()), - map[string]any{ - "data": map[string]any{ - "developerLicense": map[string]any{ - "clientId": devAddress.String(), - }, - }, - }) - require.NoError(t, err) - - // Make the webhook creation request - req, err := http.NewRequestWithContext( - t.Context(), - "POST", - "/v1/webhooks", - bytes.NewBuffer(webhookBody), - ) - require.NoError(t, err) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err := servers.Application.Test(req, -1) - require.NoError(t, err) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - bodyStr := string(body) - require.Equal(t, http.StatusCreated, resp.StatusCode, bodyStr) - - // Parse the response to get the webhook ID - webhookResponse := map[string]any{} - err = json.Unmarshal(body, &webhookResponse) - require.NoError(t, err) - - webhookID, ok := webhookResponse["id"].(string) - require.True(t, ok, "Expected webhook ID in response") - require.NotEmpty(t, webhookID) - - // Step 2: Subscribe a vehicle to the webhook - - // Use a test vehicle token ID - assetDid := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - - // Set up token exchange API mock to return permissions for this vehicle - tc.TokenExchange.SetAccessCheckReturn(devAddress.String(), true) - - // Make the subscription request - subscribeURL := fmt.Sprintf("/v1/webhooks/%s/subscribe/%s", webhookID, assetDid.String()) - req, err = http.NewRequestWithContext( - t.Context(), - "POST", - subscribeURL, - nil, - ) - require.NoError(t, err) - - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err = servers.Application.Test(req, -1) - require.NoError(t, err) - require.Equal(t, http.StatusCreated, resp.StatusCode) - - // wait for webhook to be updated - time.Sleep(1 * time.Second) - - // Step 3: Send a location signal to Kafka to trigger the webhook - // Using coordinates that are within the 0.7km radius of the target location (54.71061320000001, 25.239925999999997) - signalPayload := vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Signal{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now(), - Name: "currentLocationCoordinates", - ValueLocation: vss.Location{ - Latitude: 54.71061320000001, // Same as target latitude - Longitude: 25.239925999999997, // Same as target longitude - HDOP: 1.0, // Horizontal Dilution of Precision - }, - CloudEventID: "test-event-id", - }, - }, - }) - - err = tc.Kafka.PushJSONToTopic(settingsCopy.DeviceSignalsTopic, signalPayload) - require.NoError(t, err) - - // Step 4: Verify the webhook was called - - // Wait for the webhook to be called (with timeout) - received := webhookReceiver.WaitForCall(10 * time.Second) - require.True(t, received, "Webhook was not called within timeout") - - // Get the received calls - calls := webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 1, "Expected exactly one webhook call") - - call := calls[0] - - // Verify the webhook call details - require.Equal(t, "POST", call.Method) - require.Equal(t, "/", call.URL) // The webhook receiver URL path - - // Parse the webhook body to verify it contains the signal data - var webhookBodyData map[string]any - err = json.Unmarshal([]byte(call.Body), &webhookBodyData) - require.NoError(t, err) - - // Verify the webhook contains the expected signal data - require.Contains(t, webhookBodyData, "data") - data := webhookBodyData["data"].(map[string]any) - require.Contains(t, data, "signal") - signal := data["signal"].(map[string]any) - require.Equal(t, "currentLocationCoordinates", signal["name"].(string)) - require.Equal(t, "vss.Location", signal["valueType"].(string)) - // Verify the location value is a map with latitude, longitude, and HDOP - locationValue := signal["value"].(map[string]any) - require.Equal(t, 54.71061320000001, locationValue["latitude"].(float64)) - require.Equal(t, 25.239925999999997, locationValue["longitude"].(float64)) - require.Equal(t, 1.0, locationValue["hdop"].(float64)) - webhookReceiver.ClearReceivedCalls() - - // Step 5: Send a location signal with coordinates outside the radius - signalPayload = vss.PackSignals(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Signal{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.SignalData{ - Timestamp: time.Now(), - Name: "currentLocationCoordinates", - ValueLocation: vss.Location{ - Latitude: 55.0, // Far from target location (outside 0.7km radius) - Longitude: 26.0, - HDOP: 1.0, - }, - CloudEventID: "test-event-id", - }, - }, - }) - if err := tc.Kafka.PushJSONToTopic(settingsCopy.DeviceSignalsTopic, signalPayload); err != nil { - t.Errorf("failed to push signal to Kafka: %v", err) - } - - // Step 6: Verify the webhook was not called (coordinates outside radius) - received = webhookReceiver.WaitForCall(2 * time.Second) - require.False(t, received, "Webhook was unexpectedly called for coordinates outside radius") - calls = webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 0, "Expected no webhook calls for coordinates outside radius") -} -func TestEventWebhookFlow(t *testing.T) { - t.Parallel() - tc := GetTestServices(t) - - // Create a developer address for testing - devAddress := tests.RandomAddr(t) - settingsCopy := tc.Settings - settingsCopy.DeviceEventsTopic = "test-event-topic" + devAddress.String() - settingsCopy.DeviceSignalsTopic = "test-signal-topic" + devAddress.String() - - // Create the main application - servers, err := app.CreateServers(t.Context(), &settingsCopy, zerolog.New(os.Stdout)) - go func() { - if err := servers.SignalConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start signal consumer: %v", err) - } - } - }() - go func() { - if err := servers.EventConsumer.Start(t.Context()); err != nil { - if !errors.Is(err, context.Canceled) { - t.Errorf("failed to start event consumer: %v", err) - } - } - }() - t.Cleanup(func() { - _ = servers.SignalConsumer.Stop(t.Context()) - _ = servers.EventConsumer.Stop(t.Context()) - }) - require.NoError(t, err) - - // Create a webhook receiver - webhookReceiver := NewWebhookReceiver() - t.Cleanup(webhookReceiver.Close) - - // Step 1: Create a webhook - webhookPayload := webhook.RegisterWebhookRequest{ - Service: triggersrepo.ServiceEvent, - MetricName: "behavior.harshBraking", - Condition: "true", - CoolDownPeriod: 0, - Description: "Alert when vehicle harsh braking occurs", - TargetURL: webhookReceiver.URL(), - Status: triggersrepo.StatusEnabled, - VerificationToken: "test-verification-token", - } - - webhookBody, err := json.Marshal(webhookPayload) - require.NoError(t, err) - - // Create auth token for the request - authToken, err := tc.Auth.CreateToken(t, devAddress) - require.NoError(t, err) - - // add dev license to identity api - err = tc.Identity.SetRequestResponse( - fmt.Sprintf(`{"query":"\n\tquery($clientId: Address){\n\t\tdeveloperLicense(by: { clientId: $clientId }) {\n\t\t\tclientId\n\t\t}\n\t}","variables":{"clientId":"%s"}}`, devAddress.String()), - map[string]any{ - "data": map[string]any{ - "developerLicense": map[string]any{ - "clientId": devAddress.String(), - }, - }, - }) - require.NoError(t, err) - - // Make the webhook creation request - req, err := http.NewRequestWithContext( - t.Context(), - "POST", - "/v1/webhooks", - bytes.NewBuffer(webhookBody), - ) - require.NoError(t, err) - - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err := servers.Application.Test(req, -1) - require.NoError(t, err) - body, err := io.ReadAll(resp.Body) - require.NoError(t, err) - bodyStr := string(body) - require.Equal(t, http.StatusCreated, resp.StatusCode, bodyStr) - - // Parse the response to get the webhook ID - webhookResponse := map[string]any{} - err = json.Unmarshal(body, &webhookResponse) - require.NoError(t, err) - - webhookID, ok := webhookResponse["id"].(string) - require.True(t, ok, "Expected webhook ID in response") - require.NotEmpty(t, webhookID) - - // Step 2: Subscribe a vehicle to the webhook - - // Use a test vehicle token ID - assetDid := cloudevent.ERC721DID{ - ChainID: 137, - ContractAddress: common.HexToAddress("0xbA5738a18d83D41847dfFbDC6101d37C69c9B0cF"), - TokenID: big.NewInt(12345), - } - - // Set up token exchange API mock to return permissions for this vehicle - tc.TokenExchange.SetAccessCheckReturn(devAddress.String(), true) - - // Make the subscription request - subscribeURL := fmt.Sprintf("/v1/webhooks/%s/subscribe/%s", webhookID, assetDid.String()) - req, err = http.NewRequestWithContext( - t.Context(), - "POST", - subscribeURL, - nil, - ) - require.NoError(t, err) - - req.Header.Set("Authorization", "Bearer "+authToken) - - resp, err = servers.Application.Test(req, -1) - require.NoError(t, err) - require.Equal(t, http.StatusCreated, resp.StatusCode) - - // wait for webhook to be updated - time.Sleep(1 * time.Second) - - // Step 3: Send a signal to Kafka to trigger the webhook - eventCE := vss.PackEvents(cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - ID: "test-event-id", - }, []vss.Event{ - { - CloudEventHeader: cloudevent.CloudEventHeader{ - Subject: assetDid.String(), - Source: "test-source", - Producer: "test-producer", - }, - Data: vss.EventData{ - Timestamp: time.Now(), - Name: "behavior.harshBraking", - DurationNs: 1000000000, - Metadata: `{"counter": 1}`, - }, - }, - }) - - err = tc.Kafka.PushJSONToTopic(settingsCopy.DeviceEventsTopic, eventCE) - require.NoError(t, err) - - // Step 4: Verify the webhook was called - // Wait for the webhook to be called (with timeout) - received := webhookReceiver.WaitForCall(10 * time.Second) - require.True(t, received, "Webhook was not called within timeout") - - // Get the received calls - calls := webhookReceiver.GetReceivedCalls() - require.Len(t, calls, 1, "Expected exactly one webhook call") - - call := calls[0] - - // Verify the webhook call details - require.Equal(t, "POST", call.Method) - require.Equal(t, "/", call.URL) // The webhook receiver URL path - - // Parse the webhook body to verify it contains the signal data - var webhookBodyData map[string]any - err = json.Unmarshal([]byte(call.Body), &webhookBodyData) - require.NoError(t, err) - - // Verify the webhook contains the expected signal data - require.Contains(t, webhookBodyData, "data") - data := webhookBodyData["data"].(map[string]any) - require.Contains(t, data, "event") - event := data["event"].(map[string]any) - require.Equal(t, "behavior.harshBraking", event["name"].(string)) - require.Equal(t, "test-source", event["source"].(string)) - require.Equal(t, "test-producer", event["producer"].(string)) - require.Equal(t, float64(1000000000), event["durationNs"].(float64)) - require.Equal(t, `{"counter": 1}`, event["metadata"].(string)) - -}