diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 81ca0fe4..2be8a3c9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @jajeffries @leoparente @manrodrigues @mfiedorowicz @MicahParks @grant-nbl @paulstuart @marc-barry +* @jajeffries @leoparente @manrodrigues @mfiedorowicz @MicahParks @grant-nbl @paulstuart @marc-barry @davidlanouette diff --git a/.github/workflows/container-scan.yaml b/.github/workflows/container-scan.yaml index 29bb4efe..7a76fb6b 100644 --- a/.github/workflows/container-scan.yaml +++ b/.github/workflows/container-scan.yaml @@ -15,7 +15,7 @@ concurrency: cancel-in-progress: true env: - GO_VERSION: '1.25' + GO_VERSION: '1.26' jobs: build-and-scan: @@ -61,6 +61,7 @@ jobs: cache-to: type=gha,scope=${{ matrix.app }},mode=max build-args: | SVC=${{ matrix.app }} + GO_VERSION=${{ env.GO_VERSION }} tags: diode-${{ matrix.app }}:scan - name: Scan image for vulnerabilities diff --git a/diode-server/auth/manager_hydra.go b/diode-server/auth/manager_hydra.go index 3e9583c9..1c093648 100644 --- a/diode-server/auth/manager_hydra.go +++ b/diode-server/auth/manager_hydra.go @@ -9,7 +9,7 @@ import ( "strings" "time" - hydra "github.com/ory/hydra-client-go/v25" + hydra "github.com/ory/hydra-client-go/v26" ) const ( diff --git a/diode-server/dbstore/postgres/migrations/20260522000001_add_deviation_list_indexes.sql b/diode-server/dbstore/postgres/migrations/20260522000001_add_deviation_list_indexes.sql new file mode 100644 index 00000000..70d5d4db --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260522000001_add_deviation_list_indexes.sql @@ -0,0 +1,21 @@ +-- +goose Up + +-- Two B-tree indexes on ingestion_logs to support fast ORDER BY on the +-- ingestion_ts and updated_at columns. Without these, queries listing +-- ingestion_logs ordered by either column fall back to a parallel +-- sequential scan + top-N heapsort over the full table; on a tenant +-- with several million rows that runs ~700ms-1.2s hot and times out +-- the 5s client budget under concurrent traffic. Plain (non-partial) +-- B-tree because list-style consumers may filter by any subset of +-- state/object_type/branch in addition to ordering by these columns. +-- +-- B-tree indexes are bidirectional, so a single index per column covers +-- both ASC and DESC orderings without a second index. + +CREATE INDEX IF NOT EXISTS idx_ingestion_logs_ingestion_ts ON ingestion_logs (ingestion_ts); +CREATE INDEX IF NOT EXISTS idx_ingestion_logs_updated_at ON ingestion_logs (updated_at); + +-- +goose Down + +DROP INDEX IF EXISTS idx_ingestion_logs_ingestion_ts; +DROP INDEX IF EXISTS idx_ingestion_logs_updated_at; diff --git a/diode-server/dbstore/postgres/migrations/20260525000001_add_ingestion_log_state_object_type_counts.sql b/diode-server/dbstore/postgres/migrations/20260525000001_add_ingestion_log_state_object_type_counts.sql new file mode 100644 index 00000000..dcc4ed1d --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260525000001_add_ingestion_log_state_object_type_counts.sql @@ -0,0 +1,161 @@ +-- +goose Up +-- +goose StatementBegin + +-- Counter table for fast SUM-based aggregation over (state, object_type). +-- Read path: list queries that filter by any subset of {state, object_type} +-- can answer their total via a single sub-millisecond index lookup instead +-- of a parallel sequential scan over the whole ingestion_logs table. +-- For tenants with several million rows that drops the count cost from +-- ~600ms per call to <1ms. +CREATE TABLE IF NOT EXISTS ingestion_log_state_object_type_counts ( + state INT4 NOT NULL, + object_type TEXT NOT NULL, + n BIGINT NOT NULL, + PRIMARY KEY (state, object_type) +); + +-- Initial backfill from the current data. Runs once at migration time; after +-- this point the counter is maintained transactionally by the triggers below. +INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) +SELECT state, COALESCE(object_type, ''), COUNT(*) +FROM ingestion_logs +WHERE state IS NOT NULL +GROUP BY state, COALESCE(object_type, '') +ON CONFLICT (state, object_type) DO UPDATE SET n = EXCLUDED.n; + +-- +goose StatementEnd + +-- +goose StatementBegin +-- Statement-level triggers with transition tables. Each fires once per +-- INSERT / UPDATE / DELETE statement against ingestion_logs regardless of +-- how many rows it touches, so a 100-row bulk transition costs one trigger +-- invocation, not a hundred. The trigger body collapses the affected rows +-- into a small aggregated UPSERT. +-- +-- Rows with NULL state are skipped: state is only NULL transiently during +-- creation; the existing list queries filter `state = ANY(...)` which also +-- excludes nulls, so counter semantics match. + +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_insert() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_update() +RETURNS TRIGGER AS $$ +BEGIN + -- Aggregate the net delta per (state, object_type) cell. A row that + -- changes state from A to B contributes -1 to (A, *) and +1 to (B, *); + -- a row whose (state, object_type) is unchanged nets to zero and is + -- filtered out by the HAVING clause. + WITH deltas AS ( + SELECT state, COALESCE(object_type, '') AS object_type, -COUNT(*) AS delta + FROM old_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + UNION ALL + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ), + net AS ( + SELECT state, object_type, SUM(delta) AS delta + FROM deltas + GROUP BY state, object_type + HAVING SUM(delta) <> 0 + ) + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, object_type, delta FROM net + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_delete() +RETURNS TRIGGER AS $$ +BEGIN + WITH deltas AS ( + SELECT state, COALESCE(object_type, '') AS object_type, COUNT(*) AS n + FROM old_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ) + UPDATE ingestion_log_state_object_type_counts c + SET n = c.n - d.n + FROM deltas d + WHERE c.state = d.state AND c.object_type = d.object_type; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE TRIGGER ingestion_log_state_object_type_counts_after_insert +AFTER INSERT ON ingestion_logs +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT +EXECUTE FUNCTION ingestion_log_state_object_type_counts_on_insert(); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE TRIGGER ingestion_log_state_object_type_counts_after_update +AFTER UPDATE ON ingestion_logs +REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows +FOR EACH STATEMENT +EXECUTE FUNCTION ingestion_log_state_object_type_counts_on_update(); +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE TRIGGER ingestion_log_state_object_type_counts_after_delete +AFTER DELETE ON ingestion_logs +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT +EXECUTE FUNCTION ingestion_log_state_object_type_counts_on_delete(); +-- +goose StatementEnd + +-- +goose StatementBegin +-- Reconciliation function: replace the counter contents with the truth as +-- of NOW(). Called at reconciler startup as a defense-in-depth check against +-- drift (admin SQL that bypassed the trigger, replication anomalies, etc.). +-- Bounded to a single transaction so concurrent readers either see the old +-- value or the new value, not partial state. +CREATE OR REPLACE FUNCTION rebuild_ingestion_log_state_object_type_counts() +RETURNS VOID AS $$ +BEGIN + LOCK TABLE ingestion_log_state_object_type_counts IN EXCLUSIVE MODE; + DELETE FROM ingestion_log_state_object_type_counts; + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM ingestion_logs + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, ''); +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TRIGGER IF EXISTS ingestion_log_state_object_type_counts_after_delete ON ingestion_logs; +DROP TRIGGER IF EXISTS ingestion_log_state_object_type_counts_after_update ON ingestion_logs; +DROP TRIGGER IF EXISTS ingestion_log_state_object_type_counts_after_insert ON ingestion_logs; +DROP FUNCTION IF EXISTS rebuild_ingestion_log_state_object_type_counts(); +DROP FUNCTION IF EXISTS ingestion_log_state_object_type_counts_on_delete(); +DROP FUNCTION IF EXISTS ingestion_log_state_object_type_counts_on_update(); +DROP FUNCTION IF EXISTS ingestion_log_state_object_type_counts_on_insert(); +DROP TABLE IF EXISTS ingestion_log_state_object_type_counts; +-- +goose StatementEnd diff --git a/diode-server/dbstore/postgres/migrations/20260525000002_add_state_time_composite_indexes.sql b/diode-server/dbstore/postgres/migrations/20260525000002_add_state_time_composite_indexes.sql new file mode 100644 index 00000000..e1d987e9 --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260525000002_add_state_time_composite_indexes.sql @@ -0,0 +1,17 @@ +-- +goose Up + +-- Composite indexes to support the common deviation-list COUNT shape: +-- "WHERE state = ANY(...) AND (ingestion_ts | updated_at) BETWEEN ? AND ?". +-- Without these, Postgres has to pick between the single-column state +-- index and the single-column time index and filter by the other axis, +-- which scans many irrelevant rows when the time window is selective. +-- The composite lets it seek to each matching state value and range-scan +-- the time column within. + +CREATE INDEX IF NOT EXISTS idx_ingestion_logs_state_ingestion_ts ON ingestion_logs (state, ingestion_ts); +CREATE INDEX IF NOT EXISTS idx_ingestion_logs_state_updated_at ON ingestion_logs (state, updated_at); + +-- +goose Down + +DROP INDEX IF EXISTS idx_ingestion_logs_state_ingestion_ts; +DROP INDEX IF EXISTS idx_ingestion_logs_state_updated_at; diff --git a/diode-server/dbstore/postgres/migrations/20260526000001_add_ingestion_log_updated_at_desc_index.sql b/diode-server/dbstore/postgres/migrations/20260526000001_add_ingestion_log_updated_at_desc_index.sql new file mode 100644 index 00000000..236b1ddb --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260526000001_add_ingestion_log_updated_at_desc_index.sql @@ -0,0 +1,26 @@ +-- +goose Up + +-- Composite descending index to support the deviation-list LIST shape: +-- "ORDER BY updated_at DESC, id DESC LIMIT N". Without this index Postgres +-- falls back to Parallel Seq Scan + top-N heapsort over the full +-- ingestion_logs table because updated_at carries many duplicate values +-- from bulk operations, so the existing single-column +-- idx_ingestion_logs_updated_at cannot satisfy the multi-column sort +-- via Incremental Sort. With the composite, the planner does an +-- Index Scan over (updated_at DESC, id DESC) and stops at LIMIT N, +-- bringing top-N list queries from seconds to sub-millisecond on +-- multi-million-row tables. +-- +-- Complements idx_ingestion_logs_state_updated_at from migration +-- 20260525000002, which helps state-filtered COUNTs but does not satisfy +-- the DESC sort shape: state = ANY(...) requires merging multiple +-- ascending index streams, which the planner estimates as costlier than +-- a seq scan + sort. The composite below lets the planner satisfy the +-- DESC sort directly and apply state as a cheap post-scan filter. + +CREATE INDEX IF NOT EXISTS idx_ingestion_logs_updated_at_id_desc + ON ingestion_logs (updated_at DESC, id DESC); + +-- +goose Down + +DROP INDEX IF EXISTS idx_ingestion_logs_updated_at_id_desc; diff --git a/diode-server/dbstore/postgres/migrations/20260528000001_fix_ingestion_log_state_counts_trigger_deadlock.sql b/diode-server/dbstore/postgres/migrations/20260528000001_fix_ingestion_log_state_counts_trigger_deadlock.sql new file mode 100644 index 00000000..b42848b3 --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260528000001_fix_ingestion_log_state_counts_trigger_deadlock.sql @@ -0,0 +1,120 @@ +-- +goose Up + +-- The ingestion_log_state_object_type_counts triggers introduced in +-- 20260525000001 use multi-row `INSERT ... ON CONFLICT DO UPDATE` to +-- maintain the (state, object_type) counter cells. Their SELECT had no +-- ORDER BY, so Postgres acquired row locks on the counter table in the +-- non-deterministic order returned by the hash aggregation. Concurrent +-- statement-level trigger invocations (typically two ClaimQueuedForAutoApply +-- transitions, both 1 -> 8 across overlapping object_types) could then form +-- a lock cycle on the same (state, object_type) cells and deadlock with +-- SQLSTATE 40P01. +-- +-- The fix is the documented Postgres mitigation for this anti-pattern: +-- ORDER BY the conflict target so every concurrent invocation acquires +-- counter row locks in the same global order. Output rows are unchanged; +-- only the lock-acquisition order is constrained. +-- +-- CREATE OR REPLACE FUNCTION swaps the function body in place; existing +-- triggers pick up the new definition on the next statement, so no +-- ALTER TRIGGER or table change is needed. + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_insert() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, COALESCE(object_type, '') AS object_type, COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ORDER BY state, object_type + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_update() +RETURNS TRIGGER AS $$ +BEGIN + WITH deltas AS ( + SELECT state, COALESCE(object_type, '') AS object_type, -COUNT(*) AS delta + FROM old_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + UNION ALL + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ), + net AS ( + SELECT state, object_type, SUM(delta) AS delta + FROM deltas + GROUP BY state, object_type + HAVING SUM(delta) <> 0 + ) + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, object_type, delta FROM net + ORDER BY state, object_type + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose Down + +-- Restore the original (pre-fix) function bodies from 20260525000001. +-- The Down path reintroduces the deadlock risk; it exists only to make +-- the migration reversible during local testing. + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_insert() +RETURNS TRIGGER AS $$ +BEGIN + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd + +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION ingestion_log_state_object_type_counts_on_update() +RETURNS TRIGGER AS $$ +BEGIN + WITH deltas AS ( + SELECT state, COALESCE(object_type, '') AS object_type, -COUNT(*) AS delta + FROM old_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + UNION ALL + SELECT state, COALESCE(object_type, ''), COUNT(*) + FROM new_rows + WHERE state IS NOT NULL + GROUP BY state, COALESCE(object_type, '') + ), + net AS ( + SELECT state, object_type, SUM(delta) AS delta + FROM deltas + GROUP BY state, object_type + HAVING SUM(delta) <> 0 + ) + INSERT INTO ingestion_log_state_object_type_counts (state, object_type, n) + SELECT state, object_type, delta FROM net + ON CONFLICT (state, object_type) + DO UPDATE SET n = ingestion_log_state_object_type_counts.n + EXCLUDED.n; + RETURN NULL; +END; +$$ LANGUAGE plpgsql; +-- +goose StatementEnd diff --git a/diode-server/dbstore/postgres/migrations/20260707102832_add_requeued_from_state.sql b/diode-server/dbstore/postgres/migrations/20260707102832_add_requeued_from_state.sql new file mode 100644 index 00000000..10fcf3b4 --- /dev/null +++ b/diode-server/dbstore/postgres/migrations/20260707102832_add_requeued_from_state.sql @@ -0,0 +1,14 @@ +-- +goose Up +-- +goose StatementBegin +-- requeued_from_state records the terminal state an ingestion log was in when +-- a duplicate observation requeued it for re-plan (see BulkMarkDuplicates). +-- Consumed at persist time: a log requeued from APPLIED spawns a new +-- deviation on drift and is restored to APPLIED, preserving its history. +-- Cleared on every final state write. +ALTER TABLE ingestion_logs ADD COLUMN requeued_from_state INTEGER; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE ingestion_logs DROP COLUMN requeued_from_state; +-- +goose StatementEnd diff --git a/diode-server/dbstore/postgres/queries/ingestion_logs.sql b/diode-server/dbstore/postgres/queries/ingestion_logs.sql index 3aa7588e..1d129b36 100644 --- a/diode-server/dbstore/postgres/queries/ingestion_logs.sql +++ b/diode-server/dbstore/postgres/queries/ingestion_logs.sql @@ -7,7 +7,8 @@ RETURNING *; -- name: UpdateIngestionLogStateWithError :exec UPDATE ingestion_logs SET state = $2, - error = $3 + error = $3, + requeued_from_state = NULL WHERE id = $1 RETURNING *; @@ -58,12 +59,6 @@ WHERE il.entity_hash = sqlc.arg('entity_hash') ORDER BY il.created_at DESC LIMIT 1; --- name: IncrementDuplicateCount :exec -UPDATE ingestion_logs -SET duplicate_count = duplicate_count + 1, - last_seen = CURRENT_TIMESTAMP -WHERE id = $1; - -- name: FindPriorIngestionLogsByEntityHashes :many SELECT il.* FROM unnest(@entity_hashes::text[]) AS h(entity_hash) @@ -91,16 +86,57 @@ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14); -- name: FindIngestionLogIDsByExternalIDs :many SELECT id, external_id FROM ingestion_logs WHERE external_id = ANY(@external_ids::text[]); --- name: BulkIncrementDuplicateCounts :exec -UPDATE ingestion_logs -SET duplicate_count = duplicate_count + 1, - last_seen = CURRENT_TIMESTAMP -WHERE id = ANY(@ids::int4[]); +-- name: BulkMarkDuplicates :many +-- Increment duplicate_count/last_seen for prior ingestion logs of +-- deduplicated entities and atomically requeue rows whose NetBox state may +-- have drifted: APPLIED (3), FAILED (4), NO_CHANGES (5) -> QUEUED (1) so the +-- pollers re-plan (and re-apply under auto-apply). QUEUED (1) is already +-- pending, OPEN (2) may be claimed or awaiting review, IGNORED (6) is a user +-- opt-out, APPLYING (8) is in flight -- those keep their state and only get +-- the duplicate bookkeeping. +-- +-- The inner SELECT ... ORDER BY id FOR UPDATE acquires row locks in the same +-- global order as ClaimQueuedIngestionLogs/ClaimQueuedForAutoApply to avoid +-- deadlocks with concurrent claim/persist updates. The state guard is +-- evaluated on the locked row version, so a concurrent claim (1->2 / 1->8) +-- committed before we acquire the lock is correctly skipped. +UPDATE ingestion_logs il +SET duplicate_count = il.duplicate_count + 1, + last_seen = CURRENT_TIMESTAMP, + state = CASE WHEN locked.prev_state IN (3, 4, 5) THEN 1 ELSE il.state END, + error = CASE WHEN locked.prev_state IN (3, 4, 5) THEN NULL ELSE il.error END, + requeued_from_state = CASE WHEN locked.prev_state IN (3, 4, 5) THEN locked.prev_state ELSE il.requeued_from_state END +FROM ( + SELECT id, state AS prev_state + FROM ingestion_logs + WHERE id = ANY(@ids::int4[]) + ORDER BY id + FOR UPDATE +) locked +WHERE il.id = locked.id +RETURNING il.id, (locked.prev_state IN (3, 4, 5))::bool AS requeued; + +-- name: CloneIngestionLogForDrift :one +-- Clone a prior ingestion log into a new row representing a freshly detected +-- deviation (NetBox state drifted since the prior was applied). Entity data, +-- hash and source fields are copied; the new row gets its own external ID, +-- terminal state and ingestion timestamp. Dedup lookups pick the newest row +-- per entity hash, so subsequent duplicates track the new deviation. +INSERT INTO ingestion_logs (external_id, object_type, state, request_id, ingestion_ts, source_ts, + producer_app_name, producer_app_version, sdk_name, sdk_version, + entity, source_metadata, entity_hash) +SELECT @new_external_id, object_type, @new_state::int4, request_id, @ingestion_ts::bigint, source_ts, + producer_app_name, producer_app_version, sdk_name, sdk_version, + entity, source_metadata, entity_hash +FROM ingestion_logs AS prior +WHERE prior.id = @prior_id +RETURNING ingestion_logs.id; -- name: BulkUpdateIngestionLogStates :exec UPDATE ingestion_logs il SET state = bulk.new_state, - error = NULL + error = NULL, + requeued_from_state = NULL FROM ( SELECT unnest(@ids::int4[]) AS id, unnest(@states::int4[]) AS new_state diff --git a/diode-server/dbstore/postgres/repository.go b/diode-server/dbstore/postgres/repository.go index 22fabb4b..d2f59dbd 100644 --- a/diode-server/dbstore/postgres/repository.go +++ b/diode-server/dbstore/postgres/repository.go @@ -481,9 +481,19 @@ func (r *Repository) FindPriorIngestionLogByEntityHash(ctx context.Context, enti return &dbLog.ID, log, nil } -// IncrementDuplicateCount increments the duplicate count for an ingestion log -func (r *Repository) IncrementDuplicateCount(ctx context.Context, id int32) error { - return r.queries.IncrementDuplicateCount(ctx, id) +// BulkMarkDuplicates increments duplicate bookkeeping for the given prior +// ingestion logs and requeues drift-eligible ones (APPLIED/FAILED/NO_CHANGES +// -> QUEUED). Returns requeued flag by ingestion log ID. +func (r *Repository) BulkMarkDuplicates(ctx context.Context, ids []int32) (map[int32]bool, error) { + rows, err := r.queries.BulkMarkDuplicates(ctx, ids) + if err != nil { + return nil, err + } + requeued := make(map[int32]bool, len(rows)) + for _, row := range rows { + requeued[row.ID] = row.Requeued + } + return requeued, nil } // TruncateChangeSets truncates change sets for an ingestion log to the given limit (keeps latest n) @@ -590,11 +600,6 @@ func (r *Repository) allocateIngestionLogIDs(ctx context.Context, n int) ([]int3 return ids, rows.Err() } -// BulkIncrementDuplicateCounts increments the duplicate count for multiple ingestion logs. -func (r *Repository) BulkIncrementDuplicateCounts(ctx context.Context, ids []int32) error { - return r.queries.BulkIncrementDuplicateCounts(ctx, ids) -} - // BulkPersistChangeSets persists multiple changesets and their changes in a // single transaction, then bulk-updates ingestion log states. func (r *Repository) BulkPersistChangeSets(ctx context.Context, items []ops.BulkPersistItem, maxChangeSetsPerLog int32) ([]ops.BulkPersistResult, error) { @@ -630,51 +635,13 @@ func (r *Repository) BulkPersistChangeSets(ctx context.Context, items []ops.Bulk csID := csIDs[j] results[idx].ChangeSetID = &csID - p := postgres.BulkCreateChangeSetsParams{ - ID: csID, - ExternalID: item.ChangeSet.ID, - IngestionLogID: item.IngestionLogID, - } - if item.ChangeSet.BranchID != nil { - p.BranchID = pgtype.Text{String: *item.ChangeSet.BranchID, Valid: true} - } - if item.ChangeSet.DeviationName != nil { - p.DeviationName = pgtype.Text{String: *item.ChangeSet.DeviationName, Valid: true} - } - csParams = append(csParams, p) - - for seq, change := range item.ChangeSet.Changes { - beforeJSON, err := json.Marshal(change.Before) - if err != nil { - return nil, fmt.Errorf("failed to marshal before state: %w", err) - } - afterJSON, err := json.Marshal(change.After) - if err != nil { - return nil, fmt.Errorf("failed to marshal after state: %w", err) - } + csParams = append(csParams, buildChangeSetParams(csID, item.IngestionLogID, item.ChangeSet)) - cp := postgres.BulkCreateChangesParams{ - ExternalID: change.ID, - ChangeSetID: csID, - ChangeType: change.ChangeType, - ObjectType: change.ObjectType, - ObjectPrimaryValue: change.ObjectPrimaryValue, - Before: beforeJSON, - After: afterJSON, - NewRefs: change.NewRefs, - SequenceNumber: pgtype.Int4{Int32: int32(seq), Valid: true}, - } - if change.ObjectID != nil { - cp.ObjectID = pgtype.Int4{Int32: int32(*change.ObjectID), Valid: true} - } - if change.ObjectVersion != nil { - cp.ObjectVersion = pgtype.Int4{Int32: int32(*change.ObjectVersion), Valid: true} - } - if change.RefID != nil { - cp.RefID = pgtype.Text{String: *change.RefID, Valid: true} - } - allChangeParams = append(allChangeParams, cp) + changeParams, err := buildChangeParams(csID, item.ChangeSet) + if err != nil { + return nil, err } + allChangeParams = append(allChangeParams, changeParams...) } if _, err := qtx.BulkCreateChangeSets(ctx, csParams); err != nil { @@ -718,6 +685,135 @@ func (r *Repository) BulkPersistChangeSets(ctx context.Context, items []ops.Bulk return results, nil } +func buildChangeSetParams(csID int32, ingestionLogID int32, cs changeset.ChangeSet) postgres.BulkCreateChangeSetsParams { + p := postgres.BulkCreateChangeSetsParams{ + ID: csID, + ExternalID: cs.ID, + IngestionLogID: ingestionLogID, + } + if cs.BranchID != nil { + p.BranchID = pgtype.Text{String: *cs.BranchID, Valid: true} + } + if cs.DeviationName != nil { + p.DeviationName = pgtype.Text{String: *cs.DeviationName, Valid: true} + } + return p +} + +func buildChangeParams(csID int32, cs changeset.ChangeSet) ([]postgres.BulkCreateChangesParams, error) { + params := make([]postgres.BulkCreateChangesParams, 0, len(cs.Changes)) + for seq, change := range cs.Changes { + beforeJSON, err := json.Marshal(change.Before) + if err != nil { + return nil, fmt.Errorf("failed to marshal before state: %w", err) + } + afterJSON, err := json.Marshal(change.After) + if err != nil { + return nil, fmt.Errorf("failed to marshal after state: %w", err) + } + + cp := postgres.BulkCreateChangesParams{ + ExternalID: change.ID, + ChangeSetID: csID, + ChangeType: change.ChangeType, + ObjectType: change.ObjectType, + ObjectPrimaryValue: change.ObjectPrimaryValue, + Before: beforeJSON, + After: afterJSON, + NewRefs: change.NewRefs, + SequenceNumber: pgtype.Int4{Int32: int32(seq), Valid: true}, + } + if change.ObjectID != nil { + cp.ObjectID = pgtype.Int4{Int32: int32(*change.ObjectID), Valid: true} + } + if change.ObjectVersion != nil { + cp.ObjectVersion = pgtype.Int4{Int32: int32(*change.ObjectVersion), Valid: true} + } + if change.RefID != nil { + cp.RefID = pgtype.Text{String: *change.RefID, Valid: true} + } + params = append(params, cp) + } + return params, nil +} + +// BulkCreateDriftDeviations creates, in one transaction, a new ingestion log +// per item (clone of the prior with a fresh external ID) carrying the drift +// change set, and restores each prior log to APPLIED so its history stays +// intact. +func (r *Repository) BulkCreateDriftDeviations(ctx context.Context, items []ops.DriftDeviationItem) ([]ops.DriftDeviationResult, error) { + results := make([]ops.DriftDeviationResult, len(items)) + + csIDs, err := r.allocateChangeSetIDs(ctx, len(items)) + if err != nil { + return nil, fmt.Errorf("failed to allocate change set IDs: %w", err) + } + + tx, err := r.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("failed to start transaction: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + qtx := r.queries.WithTx(tx) + ingestionTs := time.Now().UnixNano() + + csParams := make([]postgres.BulkCreateChangeSetsParams, 0, len(items)) + var allChangeParams []postgres.BulkCreateChangesParams + priorIDs := make([]int32, len(items)) + priorStates := make([]int32, len(items)) + + for i, item := range items { + newLogID, err := qtx.CloneIngestionLogForDrift(ctx, postgres.CloneIngestionLogForDriftParams{ + NewExternalID: item.NewExternalID, + NewState: int32(item.NewState), + IngestionTs: ingestionTs, + PriorID: item.PriorIngestionLogID, + }) + if err != nil { + return nil, fmt.Errorf("failed to clone ingestion log %d: %w", item.PriorIngestionLogID, err) + } + + csID := csIDs[i] + results[i] = ops.DriftDeviationResult{ + PriorIngestionLogID: item.PriorIngestionLogID, + NewIngestionLogID: newLogID, + ChangeSetID: &csID, + } + + csParams = append(csParams, buildChangeSetParams(csID, newLogID, item.ChangeSet)) + changeParams, err := buildChangeParams(csID, item.ChangeSet) + if err != nil { + return nil, err + } + allChangeParams = append(allChangeParams, changeParams...) + + priorIDs[i] = item.PriorIngestionLogID + priorStates[i] = int32(reconcilerpb.State_APPLIED) + } + + if _, err := qtx.BulkCreateChangeSets(ctx, csParams); err != nil { + return nil, fmt.Errorf("failed to bulk create change sets: %w", err) + } + if _, err := qtx.BulkCreateChanges(ctx, allChangeParams); err != nil { + return nil, fmt.Errorf("failed to bulk create changes: %w", err) + } + + // Restore priors to APPLIED; also clears error and requeued_from_state. + if err := qtx.BulkUpdateIngestionLogStates(ctx, postgres.BulkUpdateIngestionLogStatesParams{ + Ids: priorIDs, + States: priorStates, + }); err != nil { + return nil, fmt.Errorf("failed to restore prior ingestion log states: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("failed to commit transaction: %w", err) + } + + return results, nil +} + func (r *Repository) allocateChangeSetIDs(ctx context.Context, n int) ([]int32, error) { rows, err := r.pool.Query(ctx, "SELECT nextval('change_sets_id_seq')::int4 FROM generate_series(1, $1)", n) if err != nil { @@ -750,8 +846,9 @@ func (r *Repository) ClaimQueuedIngestionLogs(ctx context.Context, batchSize int return nil, fmt.Errorf("failed to convert to proto: %w", err) } result = append(result, ops.QueuedIngestionLog{ - ID: dbLog.ID, - IngestionLog: log, + ID: dbLog.ID, + IngestionLog: log, + RequeuedFromState: reconcilerpb.State(dbLog.RequeuedFromState.Int32), }) } return result, nil @@ -774,8 +871,9 @@ func (r *Repository) ClaimQueuedForAutoApply(ctx context.Context, batchSize int3 return nil, fmt.Errorf("failed to convert ingestion log %d to proto: %w", dbLog.ID, err) } result = append(result, ops.QueuedIngestionLog{ - ID: dbLog.ID, - IngestionLog: log, + ID: dbLog.ID, + IngestionLog: log, + RequeuedFromState: reconcilerpb.State(dbLog.RequeuedFromState.Int32), }) } return result, nil diff --git a/diode-server/docker/Dockerfile-build b/diode-server/docker/Dockerfile-build index 4419c655..fab13afc 100644 --- a/diode-server/docker/Dockerfile-build +++ b/diode-server/docker/Dockerfile-build @@ -1,4 +1,4 @@ -ARG GO_VERSION=1.25 +ARG GO_VERSION=1.26 FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build diff --git a/diode-server/docker/Dockerfile-build.auth b/diode-server/docker/Dockerfile-build.auth index 5ba2e6a2..aed68db4 100644 --- a/diode-server/docker/Dockerfile-build.auth +++ b/diode-server/docker/Dockerfile-build.auth @@ -20,7 +20,7 @@ RUN --mount=target=. \ FROM alpine:3.22 -RUN apk add --no-cache jq +RUN apk upgrade --no-cache && apk add --no-cache jq RUN addgroup -S appgroup && adduser -S appuser -G appgroup diff --git a/diode-server/docker/Dockerfile.auth b/diode-server/docker/Dockerfile.auth index a0ba797b..92e2e76c 100644 --- a/diode-server/docker/Dockerfile.auth +++ b/diode-server/docker/Dockerfile.auth @@ -1,6 +1,6 @@ FROM alpine:3.22 -RUN apk add --no-cache jq +RUN apk upgrade --no-cache && apk add --no-cache jq RUN addgroup -S appgroup && adduser -S appuser -G appgroup diff --git a/diode-server/gen/dbstore/postgres/ingestion_logs.sql.go b/diode-server/gen/dbstore/postgres/ingestion_logs.sql.go index 3130d9da..83cf046d 100644 --- a/diode-server/gen/dbstore/postgres/ingestion_logs.sql.go +++ b/diode-server/gen/dbstore/postgres/ingestion_logs.sql.go @@ -29,22 +29,67 @@ type BulkCreateIngestionLogsParams struct { EntityHash pgtype.Text `json:"entity_hash"` } -const bulkIncrementDuplicateCounts = `-- name: BulkIncrementDuplicateCounts :exec -UPDATE ingestion_logs -SET duplicate_count = duplicate_count + 1, - last_seen = CURRENT_TIMESTAMP -WHERE id = ANY($1::int4[]) +const bulkMarkDuplicates = `-- name: BulkMarkDuplicates :many +UPDATE ingestion_logs il +SET duplicate_count = il.duplicate_count + 1, + last_seen = CURRENT_TIMESTAMP, + state = CASE WHEN locked.prev_state IN (3, 4, 5) THEN 1 ELSE il.state END, + error = CASE WHEN locked.prev_state IN (3, 4, 5) THEN NULL ELSE il.error END, + requeued_from_state = CASE WHEN locked.prev_state IN (3, 4, 5) THEN locked.prev_state ELSE il.requeued_from_state END +FROM ( + SELECT id, state AS prev_state + FROM ingestion_logs + WHERE id = ANY($1::int4[]) + ORDER BY id + FOR UPDATE +) locked +WHERE il.id = locked.id +RETURNING il.id, (locked.prev_state IN (3, 4, 5))::bool AS requeued ` -func (q *Queries) BulkIncrementDuplicateCounts(ctx context.Context, ids []int32) error { - _, err := q.db.Exec(ctx, bulkIncrementDuplicateCounts, ids) - return err +type BulkMarkDuplicatesRow struct { + ID int32 `json:"id"` + Requeued bool `json:"requeued"` +} + +// Increment duplicate_count/last_seen for prior ingestion logs of +// deduplicated entities and atomically requeue rows whose NetBox state may +// have drifted: APPLIED (3), FAILED (4), NO_CHANGES (5) -> QUEUED (1) so the +// pollers re-plan (and re-apply under auto-apply). QUEUED (1) is already +// pending, OPEN (2) may be claimed or awaiting review, IGNORED (6) is a user +// opt-out, APPLYING (8) is in flight -- those keep their state and only get +// the duplicate bookkeeping. +// +// The inner SELECT ... ORDER BY id FOR UPDATE acquires row locks in the same +// global order as ClaimQueuedIngestionLogs/ClaimQueuedForAutoApply to avoid +// deadlocks with concurrent claim/persist updates. The state guard is +// evaluated on the locked row version, so a concurrent claim (1->2 / 1->8) +// committed before we acquire the lock is correctly skipped. +func (q *Queries) BulkMarkDuplicates(ctx context.Context, ids []int32) ([]BulkMarkDuplicatesRow, error) { + rows, err := q.db.Query(ctx, bulkMarkDuplicates, ids) + if err != nil { + return nil, err + } + defer rows.Close() + var items []BulkMarkDuplicatesRow + for rows.Next() { + var i BulkMarkDuplicatesRow + if err := rows.Scan(&i.ID, &i.Requeued); err != nil { + return nil, err + } + items = append(items, i) + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil } const bulkUpdateIngestionLogStates = `-- name: BulkUpdateIngestionLogStates :exec UPDATE ingestion_logs il SET state = bulk.new_state, - error = NULL + error = NULL, + requeued_from_state = NULL FROM ( SELECT unnest($1::int4[]) AS id, unnest($2::int4[]) AS new_state @@ -72,7 +117,7 @@ WHERE id IN ( LIMIT $1 FOR UPDATE SKIP LOCKED ) -RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state ` // Claim a batch of QUEUED ingestion logs for the AutoApplyProcessor (combined @@ -108,6 +153,7 @@ func (q *Queries) ClaimQueuedForAutoApply(ctx context.Context, batchSize int32) &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ); err != nil { return nil, err } @@ -129,7 +175,7 @@ WHERE id IN ( LIMIT $1 FOR UPDATE SKIP LOCKED ) -RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state ` func (q *Queries) ClaimQueuedIngestionLogs(ctx context.Context, batchSize int32) ([]IngestionLog, error) { @@ -161,6 +207,7 @@ func (q *Queries) ClaimQueuedIngestionLogs(ctx context.Context, batchSize int32) &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ); err != nil { return nil, err } @@ -172,6 +219,42 @@ func (q *Queries) ClaimQueuedIngestionLogs(ctx context.Context, batchSize int32) return items, nil } +const cloneIngestionLogForDrift = `-- name: CloneIngestionLogForDrift :one +INSERT INTO ingestion_logs (external_id, object_type, state, request_id, ingestion_ts, source_ts, + producer_app_name, producer_app_version, sdk_name, sdk_version, + entity, source_metadata, entity_hash) +SELECT $1, object_type, $2::int4, request_id, $3::bigint, source_ts, + producer_app_name, producer_app_version, sdk_name, sdk_version, + entity, source_metadata, entity_hash +FROM ingestion_logs AS prior +WHERE prior.id = $4 +RETURNING ingestion_logs.id +` + +type CloneIngestionLogForDriftParams struct { + NewExternalID string `json:"new_external_id"` + NewState int32 `json:"new_state"` + IngestionTs int64 `json:"ingestion_ts"` + PriorID int32 `json:"prior_id"` +} + +// Clone a prior ingestion log into a new row representing a freshly detected +// deviation (NetBox state drifted since the prior was applied). Entity data, +// hash and source fields are copied; the new row gets its own external ID, +// terminal state and ingestion timestamp. Dedup lookups pick the newest row +// per entity hash, so subsequent duplicates track the new deviation. +func (q *Queries) CloneIngestionLogForDrift(ctx context.Context, arg CloneIngestionLogForDriftParams) (int32, error) { + row := q.db.QueryRow(ctx, cloneIngestionLogForDrift, + arg.NewExternalID, + arg.NewState, + arg.IngestionTs, + arg.PriorID, + ) + var id int32 + err := row.Scan(&id) + return id, err +} + const countIngestionLogsPerState = `-- name: CountIngestionLogsPerState :many SELECT state, COUNT(*) AS count FROM ingestion_logs @@ -207,7 +290,7 @@ const createIngestionLog = `-- name: CreateIngestionLog :one INSERT INTO ingestion_logs (external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, source_metadata, entity_hash) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13) -RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state ` type CreateIngestionLogParams struct { @@ -263,6 +346,7 @@ func (q *Queries) CreateIngestionLog(ctx context.Context, arg CreateIngestionLog &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ) return i, err } @@ -297,7 +381,7 @@ func (q *Queries) FindIngestionLogIDsByExternalIDs(ctx context.Context, external } const findPriorIngestionLogByEntityHash = `-- name: FindPriorIngestionLogByEntityHash :one -SELECT il.id, il.external_id, il.object_type, il.state, il.request_id, il.ingestion_ts, il.source_ts, il.producer_app_name, il.producer_app_version, il.sdk_name, il.sdk_version, il.entity, il.error, il.source_metadata, il.created_at, il.updated_at, il.entity_hash, il.last_seen, il.duplicate_count +SELECT il.id, il.external_id, il.object_type, il.state, il.request_id, il.ingestion_ts, il.source_ts, il.producer_app_name, il.producer_app_version, il.sdk_name, il.sdk_version, il.entity, il.error, il.source_metadata, il.created_at, il.updated_at, il.entity_hash, il.last_seen, il.duplicate_count, il.requeued_from_state FROM ingestion_logs il LEFT JOIN LATERAL ( SELECT branch_id @@ -340,15 +424,16 @@ func (q *Queries) FindPriorIngestionLogByEntityHash(ctx context.Context, arg Fin &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ) return i, err } const findPriorIngestionLogsByEntityHashes = `-- name: FindPriorIngestionLogsByEntityHashes :many -SELECT il.id, il.external_id, il.object_type, il.state, il.request_id, il.ingestion_ts, il.source_ts, il.producer_app_name, il.producer_app_version, il.sdk_name, il.sdk_version, il.entity, il.error, il.source_metadata, il.created_at, il.updated_at, il.entity_hash, il.last_seen, il.duplicate_count +SELECT il.id, il.external_id, il.object_type, il.state, il.request_id, il.ingestion_ts, il.source_ts, il.producer_app_name, il.producer_app_version, il.sdk_name, il.sdk_version, il.entity, il.error, il.source_metadata, il.created_at, il.updated_at, il.entity_hash, il.last_seen, il.duplicate_count, il.requeued_from_state FROM unnest($1::text[]) AS h(entity_hash) CROSS JOIN LATERAL ( - SELECT il2.id, il2.external_id, il2.object_type, il2.state, il2.request_id, il2.ingestion_ts, il2.source_ts, il2.producer_app_name, il2.producer_app_version, il2.sdk_name, il2.sdk_version, il2.entity, il2.error, il2.source_metadata, il2.created_at, il2.updated_at, il2.entity_hash, il2.last_seen, il2.duplicate_count + SELECT il2.id, il2.external_id, il2.object_type, il2.state, il2.request_id, il2.ingestion_ts, il2.source_ts, il2.producer_app_name, il2.producer_app_version, il2.sdk_name, il2.sdk_version, il2.entity, il2.error, il2.source_metadata, il2.created_at, il2.updated_at, il2.entity_hash, il2.last_seen, il2.duplicate_count, il2.requeued_from_state FROM ingestion_logs il2 WHERE il2.entity_hash = h.entity_hash AND ( @@ -397,6 +482,7 @@ func (q *Queries) FindPriorIngestionLogsByEntityHashes(ctx context.Context, arg &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ); err != nil { return nil, err } @@ -408,18 +494,6 @@ func (q *Queries) FindPriorIngestionLogsByEntityHashes(ctx context.Context, arg return items, nil } -const incrementDuplicateCount = `-- name: IncrementDuplicateCount :exec -UPDATE ingestion_logs -SET duplicate_count = duplicate_count + 1, - last_seen = CURRENT_TIMESTAMP -WHERE id = $1 -` - -func (q *Queries) IncrementDuplicateCount(ctx context.Context, id int32) error { - _, err := q.db.Exec(ctx, incrementDuplicateCount, id) - return err -} - const resetApplyingIngestionLogs = `-- name: ResetApplyingIngestionLogs :exec UPDATE ingestion_logs SET state = 1 @@ -434,7 +508,7 @@ func (q *Queries) ResetApplyingIngestionLogs(ctx context.Context) error { } const retrieveIngestionLogByExternalID = `-- name: RetrieveIngestionLogByExternalID :one -SELECT id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +SELECT id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state FROM ingestion_logs WHERE external_id = $1 ` @@ -462,12 +536,13 @@ func (q *Queries) RetrieveIngestionLogByExternalID(ctx context.Context, external &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ) return i, err } const retrieveIngestionLogs = `-- name: RetrieveIngestionLogs :many -SELECT id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +SELECT id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state FROM ingestion_logs WHERE (state = $1 OR $1 IS NULL) AND (object_type = $2 OR $2 IS NULL) @@ -522,6 +597,7 @@ func (q *Queries) RetrieveIngestionLogs(ctx context.Context, arg RetrieveIngesti &i.EntityHash, &i.LastSeen, &i.DuplicateCount, + &i.RequeuedFromState, ); err != nil { return nil, err } @@ -604,9 +680,10 @@ func (q *Queries) RetrieveIngestionLogsWithChangeSets(ctx context.Context, arg R const updateIngestionLogStateWithError = `-- name: UpdateIngestionLogStateWithError :exec UPDATE ingestion_logs SET state = $2, - error = $3 + error = $3, + requeued_from_state = NULL WHERE id = $1 -RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count +RETURNING id, external_id, object_type, state, request_id, ingestion_ts, source_ts, producer_app_name, producer_app_version, sdk_name, sdk_version, entity, error, source_metadata, created_at, updated_at, entity_hash, last_seen, duplicate_count, requeued_from_state ` type UpdateIngestionLogStateWithErrorParams struct { diff --git a/diode-server/gen/dbstore/postgres/types.go b/diode-server/gen/dbstore/postgres/types.go index bc02935d..5d0e7eed 100644 --- a/diode-server/gen/dbstore/postgres/types.go +++ b/diode-server/gen/dbstore/postgres/types.go @@ -109,6 +109,13 @@ type IngestionLog struct { EntityHash pgtype.Text `json:"entity_hash"` LastSeen pgtype.Timestamptz `json:"last_seen"` DuplicateCount int32 `json:"duplicate_count"` + RequeuedFromState pgtype.Int4 `json:"requeued_from_state"` +} + +type IngestionLogStateObjectTypeCount struct { + State int32 `json:"state"` + ObjectType string `json:"object_type"` + N int64 `json:"n"` } type VDeviation struct { diff --git a/diode-server/go.mod b/diode-server/go.mod index ec393846..2b3209e8 100644 --- a/diode-server/go.mod +++ b/diode-server/go.mod @@ -1,6 +1,6 @@ module github.com/netboxlabs/diode/diode-server -go 1.26.1 +go 1.26.4 require ( github.com/MicahParks/keyfunc/v3 v3.3.11 @@ -16,12 +16,12 @@ require ( github.com/jackc/pgx/v5 v5.9.2 github.com/kelseyhightower/envconfig v1.4.0 github.com/oklog/run v1.1.0 - github.com/ory/hydra-client-go/v25 v25.4.0 + github.com/ory/hydra-client-go/v26 v26.2.0 github.com/pressly/goose/v3 v3.24.3 github.com/prometheus/client_golang v1.22.0 github.com/redis/go-redis/v9 v9.8.0 github.com/stretchr/testify v1.11.1 - github.com/testcontainers/testcontainers-go v0.37.0 + github.com/testcontainers/testcontainers-go v0.42.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 go.opentelemetry.io/otel v1.43.0 @@ -33,8 +33,9 @@ require ( go.opentelemetry.io/otel/metric v1.43.0 go.opentelemetry.io/otel/sdk v1.43.0 go.opentelemetry.io/otel/sdk/metric v1.43.0 - golang.org/x/oauth2 v0.34.0 - golang.org/x/text v0.32.0 + go.opentelemetry.io/otel/trace v1.43.0 + golang.org/x/oauth2 v0.36.0 + golang.org/x/text v0.37.0 golang.org/x/time v0.11.0 google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.10 @@ -52,47 +53,46 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cenkalti/backoff/v5 v5.0.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/platforms v0.2.1 // indirect github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/distribution/reference v0.6.0 // indirect - github.com/docker/docker v28.1.1+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/ebitengine/purego v0.8.4 // indirect + github.com/ebitengine/purego v0.10.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.5 // indirect github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect github.com/magiconair/properties v1.8.10 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mfridman/interpolate v0.0.2 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect - github.com/moby/go-archive v0.1.0 // indirect - github.com/moby/patternmatcher v0.6.0 // indirect - github.com/moby/sys/atomicwriter v0.1.0 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.54.1 // indirect + github.com/moby/moby/client v0.4.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect github.com/moby/sys/sequential v0.6.0 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/moby/sys/userns v0.1.0 // indirect github.com/moby/term v0.5.2 // indirect - github.com/morikuni/aec v1.0.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/prometheus/client_model v0.6.2 // indirect @@ -100,23 +100,22 @@ require ( github.com/prometheus/procfs v0.16.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sethvargo/go-retry v0.3.0 // indirect - github.com/shirou/gopsutil/v4 v4.25.4 // indirect - github.com/sirupsen/logrus v1.9.3 // indirect - github.com/stretchr/objx v0.5.2 // indirect - github.com/tklauser/go-sysconf v0.3.15 // indirect - github.com/tklauser/numcpus v0.10.0 // indirect + github.com/shirou/gopsutil/v4 v4.26.3 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/stretchr/objx v0.5.3 // indirect + github.com/tklauser/go-sysconf v0.3.16 // indirect + github.com/tklauser/numcpus v0.11.0 // indirect github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect - go.opentelemetry.io/otel/trace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.6.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.46.0 // indirect + golang.org/x/crypto v0.51.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.42.0 // indirect + golang.org/x/net v0.55.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.45.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect modernc.org/libc v1.65.8 // indirect diff --git a/diode-server/go.sum b/diode-server/go.sum index 403474a1..37730c84 100644 --- a/diode-server/go.sum +++ b/diode-server/go.sum @@ -30,14 +30,18 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/backoff v0.0.0-20240920015135-e46b80a3a7d0 h1:pRcxfaAlK0vR6nOeQs7eAEvjJzdGXl8+KaBlcvpQTyQ= github.com/cloudflare/backoff v0.0.0-20240920015135-e46b80a3a7d0/go.mod h1:rzgs2ZOiguV6/NpiDgADjRLPNyZlApIWxKpkT+X8SdY= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -45,16 +49,14 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v28.1.1+incompatible h1:49M11BFLsVO1gxY9UX9p/zwkE/rswggs8AdFmXQw51I= -github.com/docker/docker v28.1.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw= -github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/ebitengine/purego v0.10.0 h1:QIw4xfpWT6GWTzaW5XEKy3HXoqrJGx1ijYHzTF0/ISU= +github.com/ebitengine/purego v0.10.0/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= @@ -73,8 +75,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -107,10 +107,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.5 h1:/h1gH5Ce+VWNLSWqPzOVn6XBO+vJbCNGvjoaGBFW2IE= +github.com/klauspost/compress v1.18.5/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -129,12 +127,14 @@ github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6B github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= -github.com/moby/go-archive v0.1.0 h1:Kk/5rdW/g+H8NHdJW2gsXyZ7UnzvJNOy6VKJqueWdcQ= -github.com/moby/go-archive v0.1.0/go.mod h1:G9B+YoujNohJmrIYFBpSd54GTUB4lt9S+xVQvsJyFuo= -github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= -github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= -github.com/moby/sys/atomicwriter v0.1.0 h1:kw5D/EqkBwsBFi0ss9v1VG3wIkVhzGvLklJ+w3A14Sw= -github.com/moby/sys/atomicwriter v0.1.0/go.mod h1:Ul8oqv2ZMNHOceF643P6FKPXeCmYtlQMvpizfsSoaWs= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.54.1 h1:TqVzuJkOLsgLDDwNLmYqACUuTehOHRGKiPhvH8V3Nn4= +github.com/moby/moby/api v1.54.1/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.4.0 h1:S+2XegzHQrrvTCvF6s5HFzcrywWQmuVnhOXe2kiWjIw= +github.com/moby/moby/client v0.4.0/go.mod h1:QWPbvWchQbxBNdaLSpoKpCdf5E+WxFAgNHogCWDoa7g= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= @@ -143,8 +143,6 @@ github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= -github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= -github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= @@ -155,8 +153,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/ory/hydra-client-go/v25 v25.4.0 h1:ydJ9yZj3oSH/UgpR10JJJogYwlBA+im435WwBRiYDRg= -github.com/ory/hydra-client-go/v25 v25.4.0/go.mod h1:jxa+7mEZLkWgCu+Njr/rsPGgQKSIjCZPQJaV/jBa3i4= +github.com/ory/hydra-client-go/v26 v26.2.0 h1:el+M6dV5CK3Am2V4isorOoZ8pJdDaJm0irQzqnxZHqU= +github.com/ory/hydra-client-go/v26 v26.2.0/go.mod h1:IkTtqWmwQoFGo6dclIP2n51uror2sGNHC4Fjs/QlFqM= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -183,27 +181,25 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sethvargo/go-retry v0.3.0 h1:EEt31A35QhrcRZtrYFDTBg91cqZVnFL2navjDrah2SE= github.com/sethvargo/go-retry v0.3.0/go.mod h1:mNX17F0C/HguQMyMyJxcnU471gOZGxCLyYaFyAZraas= -github.com/shirou/gopsutil/v4 v4.25.4 h1:cdtFO363VEOOFrUCjZRh4XVJkb548lyF0q0uTeMqYPw= -github.com/shirou/gopsutil/v4 v4.25.4/go.mod h1:xbuxyoZj+UsgnZrENu3lQivsngRR5BdjbJwf2fv4szA= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= +github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= -github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg= -github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM= -github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= -github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= -github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= -github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= +github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= +github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= +github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= +github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= +github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= @@ -222,8 +218,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0 h1:JgtbA0xkWHnTmYk7YusopJFX6uleBmAuZ8n05NEh8nQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.36.0/go.mod h1:179AK5aar5R3eS9FucPy6rggvU0g52cvKId8pv4+v0c= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/prometheus v0.58.0 h1:CJAxWKFIqdBennqxJyOgnt5LqkeFRT+Mz3Yjz3hL+h8= go.opentelemetry.io/otel/exporters/prometheus v0.58.0/go.mod h1:7qo/4CLI+zYSNbv0GMNquzuss2FVZo3OYrGh96n4HNc= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= @@ -244,59 +238,33 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU= -golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0= +golang.org/x/crypto v0.51.0 h1:IBPXwPfKxY7cWQZ38ZCIRPI50YLeevDLlLnyC5wRGTI= +golang.org/x/crypto v0.51.0/go.mod h1:8AdwkbraGNABw2kOX6YFPs3WM22XqI4EXEd8g+x7Oc8= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.30.0 h1:fDEXFVZ/fmCKProc/yAXXUijritrDzahmwwefnjoPFk= -golang.org/x/mod v0.30.0/go.mod h1:lAsf5O2EvJeSFMiBxXDki7sCgAxEUcZHXoXMKT4GJKc= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= +golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= +golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= +golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q= -golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.43.0 h1:S4RLU2sB31O/NCl+zFN9Aru9A/Cq2aqKpTZJ6B+DwT4= +golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= +golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= +golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.39.0 h1:ik4ho21kwuQln40uelmciQPp9SipgNDdrafrYA4TmQQ= -golang.org/x/tools v0.39.0/go.mod h1:JnefbkDPyD8UU2kI5fuf8ZX4/yUeh9W877ZeBONxUqQ= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= +golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= @@ -339,3 +307,5 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/diode-server/netboxdiodeplugin/client.go b/diode-server/netboxdiodeplugin/client.go index b0f9b711..1ffb357a 100644 --- a/diode-server/netboxdiodeplugin/client.go +++ b/diode-server/netboxdiodeplugin/client.go @@ -28,6 +28,7 @@ import ( "github.com/netboxlabs/diode/diode-server/authutil" diodeErrors "github.com/netboxlabs/diode/diode-server/errors" "github.com/netboxlabs/diode/diode-server/reconciler/changeset" + "github.com/netboxlabs/diode/diode-server/telemetry" ) const ( @@ -254,6 +255,12 @@ func httpTimeout() (time.Duration, error) { return time.Duration(timeout) * time.Second, nil } +func (c *Client) waitForRateLimit(ctx context.Context) (err error) { + _, span := telemetry.StartSpan(ctx, telemetry.SpanRateLimiterWait) + defer func() { telemetry.End(span, err) }() + return c.limiter.Wait(ctx) +} + // NetBoxAPI is the interface for the NetBox Diode plugin API type NetBoxAPI interface { // BulkPlan generates diffs for multiple entities in a single request @@ -325,7 +332,7 @@ func (c *Client) BulkPlan(ctx context.Context, payload BulkPlanRequest) (*BulkPl return nil, err } - if err := c.limiter.Wait(ctx); err != nil { + if err := c.waitForRateLimit(ctx); err != nil { return nil, err } @@ -419,7 +426,7 @@ func (c *Client) BulkPlanApply(ctx context.Context, payload BulkPlanApplyRequest return nil, err } - if err := c.limiter.Wait(ctx); err != nil { + if err := c.waitForRateLimit(ctx); err != nil { return nil, err } @@ -474,7 +481,7 @@ func (c *Client) GetDefaultBranch(ctx context.Context) (*Branch, error) { return nil, err } - if err := c.limiter.Wait(ctx); err != nil { + if err := c.waitForRateLimit(ctx); err != nil { return nil, err } diff --git a/diode-server/reconciler/differ/differ.go b/diode-server/reconciler/differ/differ.go index 238c222e..e50812ee 100644 --- a/diode-server/reconciler/differ/differ.go +++ b/diode-server/reconciler/differ/differ.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "google.golang.org/protobuf/proto" + diodeErrors "github.com/netboxlabs/diode/diode-server/errors" "github.com/netboxlabs/diode/diode-server/gen/diode/v1/diodepb" "github.com/netboxlabs/diode/diode-server/gen/netbox" "github.com/netboxlabs/diode/diode-server/netboxdiodeplugin" @@ -24,7 +25,7 @@ type IngestEntity struct { // Returns nil changeset if the result contains errors. func ConvertBulkPlanResult(result netboxdiodeplugin.BulkPlanResult, objectType string) (*changeset.ChangeSet, error) { if len(result.Errors) > 0 && string(result.Errors) != "null" { - return nil, fmt.Errorf("bulk plan error for entity %s: %s", result.ID, string(result.Errors)) + return nil, changeset.NewError("failed to determine deviation", diodeErrors.ErrCodeOpsGenerateDiff, result.Errors) } if result.ChangeSet == nil { @@ -66,10 +67,10 @@ func ConvertBulkPlanApplyResult(result netboxdiodeplugin.BulkPlanApplyResult, ob var planErr, applyErr error if result.Errors != nil { if len(result.Errors.Plan) > 0 && string(result.Errors.Plan) != "null" { - planErr = fmt.Errorf("plan error for entity %s: %s", result.ID, string(result.Errors.Plan)) + planErr = changeset.NewError("failed to determine deviation", diodeErrors.ErrCodeOpsGenerateDiff, result.Errors.Plan) } if len(result.Errors.Apply) > 0 && string(result.Errors.Apply) != "null" { - applyErr = fmt.Errorf("apply error for entity %s: %s", result.ID, string(result.Errors.Apply)) + applyErr = changeset.NewError("failed to apply deviation to NetBox", diodeErrors.ErrCodeOpsApplyChangeSet, result.Errors.Apply) } } diff --git a/diode-server/reconciler/ingestion_log_processor.go b/diode-server/reconciler/ingestion_log_processor.go index 806be37b..2b787bf6 100644 --- a/diode-server/reconciler/ingestion_log_processor.go +++ b/diode-server/reconciler/ingestion_log_processor.go @@ -8,10 +8,33 @@ import ( "go.opentelemetry.io/otel/attribute" + "github.com/netboxlabs/diode/diode-server/gen/diode/v1/reconcilerpb" + "github.com/netboxlabs/diode/diode-server/reconciler/changeset" "github.com/netboxlabs/diode/diode-server/reconciler/ops" "github.com/netboxlabs/diode/diode-server/telemetry" ) +// AutoApplyPolicy decides per-record whether an already-planned change set +// should bypass the OPEN review queue and be applied immediately. Returns +// true to apply, false to leave the row in OPEN for manual review. +type AutoApplyPolicy func(log *reconcilerpb.IngestionLog, cs *changeset.ChangeSet) bool + +// IngestionLogProcessorOption is a functional option for IngestionLogProcessor. +type IngestionLogProcessorOption func(*IngestionLogProcessor) + +// WithAutoApplyPolicy attaches a per-record auto-apply policy. When set, +// matching records are applied immediately rather than left in OPEN. +// +// Layers on top of the cfg.AutoApplyChangesets default: it only widens +// auto-apply for matching records, never restricts. The boot-time decision +// between IngestionLogProcessor (plan-only) and AutoApplyProcessor (plan + +// apply) is unchanged. +func WithAutoApplyPolicy(policy AutoApplyPolicy) IngestionLogProcessorOption { + return func(p *IngestionLogProcessor) { + p.autoApplyPolicy = policy + } +} + const ( defaultIngestionLogPollInterval = 100 * time.Millisecond defaultIngestionLogIdleInterval = time.Second @@ -55,6 +78,8 @@ type IngestionLogProcessor struct { metrics Metrics backpressure BackpressureFunc + autoApplyPolicy AutoApplyPolicy + // mx protects the lifecycle fields below: workCancel, pollCancel, done. // Set by Start, read by Stop/shutdown/watchParent. mx sync.Mutex @@ -66,12 +91,12 @@ type IngestionLogProcessor struct { } // NewIngestionLogProcessor creates a new ingestion log processor. -func NewIngestionLogProcessor(logger *slog.Logger, cfg Config, repo Repository, ops IngestionProcessorOps, metrics Metrics, backpressure BackpressureFunc) *IngestionLogProcessor { +func NewIngestionLogProcessor(logger *slog.Logger, cfg Config, repo Repository, ops IngestionProcessorOps, metrics Metrics, backpressure BackpressureFunc, opts ...IngestionLogProcessorOption) *IngestionLogProcessor { batchSize := cfg.IngestionLogProcessorBatchSize if batchSize <= 0 { batchSize = defaultIngestionLogBatchSize } - return &IngestionLogProcessor{ + p := &IngestionLogProcessor{ config: cfg, logger: logger, ops: ops, @@ -80,6 +105,10 @@ func NewIngestionLogProcessor(logger *slog.Logger, cfg Config, repo Repository, backpressure: backpressure, batchSize: batchSize, } + for _, opt := range opts { + opt(p) + } + return p } // Name returns the name of the component. @@ -241,14 +270,10 @@ func (p *IngestionLogProcessor) processBatch(ctx context.Context, batch []ops.Qu results := p.ops.BulkPlan(ctx, batch, branchID) + var matched []ops.QueuedIngestionLog for i, result := range results { item := batch[i] - - attrs := []attribute.KeyValue{ - attribute.String(telemetry.AttributeSDKName, item.IngestionLog.GetSdkName()), - attribute.String(telemetry.AttributeProducerAppName, item.IngestionLog.GetProducerAppName()), - } - metricsCtx := telemetry.ContextWithMetricAttributes(ctx, attrs...) + metricsCtx := p.metricsContext(ctx, item) if result.Err != nil { p.logger.Error("error generating changeset", "error", result.Err, "ingestionLogID", item.ID) @@ -258,5 +283,49 @@ func (p *IngestionLogProcessor) processBatch(ctx context.Context, batch []ops.Qu if result.ChangeSet != nil { p.metrics.RecordChangeSetCreate(metricsCtx, true, int64(len(result.ChangeSet.Changes))) } + + if p.autoApplyPolicy != nil && result.ChangeSet != nil && + p.autoApplyPolicy(item.IngestionLog, result.ChangeSet) { + // Re-drive the log the change set was persisted on: for a drift + // re-check that is the freshly spawned deviation, not the restored + // prior (RequeuedFromState deliberately left zero so BulkPlanApply + // treats it as a regular log). + matched = append(matched, ops.QueuedIngestionLog{ID: result.IngestionLogID, IngestionLog: item.IngestionLog}) + } + } + + if len(matched) == 0 { + return } + + // Interim limitation: the policy above was evaluated against the BulkPlan + // change sets, but BulkPlanApply re-plans server-side and applies the + // fresh sets in one call — a divergent re-plan can apply changes the + // policy never approved. Accepted until an apply-only endpoint (no + // re-plan) exists in the netbox plugin. + applyResults := p.ops.BulkPlanApply(ctx, matched, branchID) + for i, applyResult := range applyResults { + item := matched[i] + metricsCtx := p.metricsContext(ctx, item) + switch { + case applyResult.PlanErr != nil: + p.logger.Error("auto-apply policy matched but re-plan failed", + "error", applyResult.PlanErr, "ingestionLogID", item.ID) + p.metrics.RecordChangeSetApply(metricsCtx, false, 0) + case applyResult.ApplyErr != nil: + p.logger.Error("auto-apply policy matched but apply failed", + "error", applyResult.ApplyErr, "ingestionLogID", item.ID) + p.metrics.RecordChangeSetApply(metricsCtx, false, 0) + case applyResult.ChangeSet != nil && len(applyResult.ChangeSet.Changes) > 0: + p.metrics.RecordChangeSetApply(metricsCtx, true, int64(len(applyResult.ChangeSet.Changes))) + } + } +} + +// metricsContext returns ctx annotated with the per-record metric attributes. +func (p *IngestionLogProcessor) metricsContext(ctx context.Context, item ops.QueuedIngestionLog) context.Context { + return telemetry.ContextWithMetricAttributes(ctx, + attribute.String(telemetry.AttributeSDKName, item.IngestionLog.GetSdkName()), + attribute.String(telemetry.AttributeProducerAppName, item.IngestionLog.GetProducerAppName()), + ) } diff --git a/diode-server/reconciler/ingestion_log_processor_autoapply_test.go b/diode-server/reconciler/ingestion_log_processor_autoapply_test.go new file mode 100644 index 00000000..d26f6f03 --- /dev/null +++ b/diode-server/reconciler/ingestion_log_processor_autoapply_test.go @@ -0,0 +1,163 @@ +package reconciler_test + +import ( + "context" + "log/slog" + "os" + "testing" + "time" + + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/netboxlabs/diode/diode-server/gen/diode/v1/diodepb" + "github.com/netboxlabs/diode/diode-server/gen/diode/v1/reconcilerpb" + "github.com/netboxlabs/diode/diode-server/reconciler" + "github.com/netboxlabs/diode/diode-server/reconciler/changeset" + "github.com/netboxlabs/diode/diode-server/reconciler/mocks" + "github.com/netboxlabs/diode/diode-server/reconciler/ops" +) + +func newAutoApplyTestLog() *reconcilerpb.IngestionLog { + return &reconcilerpb.IngestionLog{ + DataType: "extras.customfield", + ObjectType: "extras.customfield", + State: reconcilerpb.State_QUEUED, + Entity: &diodepb.Entity{}, + SdkName: "test-sdk", + ProducerAppName: "orb-pro-credentials-bootstrap", + } +} + +func newAutoApplyTestLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug})) +} + +// TestIngestionLogProcessor_AutoApplyPolicyMatch_AppliesItem verifies that +// when WithAutoApplyPolicy is set and the policy returns true for a record, +// the processor calls BulkPlanApply for that item and records a successful +// apply metric — bypassing the OPEN review queue. +func TestIngestionLogProcessor_AutoApplyPolicyMatch_AppliesItem(t *testing.T) { + repo := mocks.NewRepository(t) + mockOps := mocks.NewIngestionProcessorOps(t) + mockMetrics := mocks.NewMetrics(t) + + log := newAutoApplyTestLog() + batch := []ops.QueuedIngestionLog{{ID: 1, IngestionLog: log}} + cs := &changeset.ChangeSet{ + Changes: []changeset.Change{{ChangeType: changeset.ChangeTypeCreate, ObjectType: "extras.customfield"}}, + } + + repo.On("ClaimQueuedIngestionLogs", mock.Anything, int32(100)). + Return(batch, nil).Once() + repo.On("ClaimQueuedIngestionLogs", mock.Anything, int32(100)). + Return([]ops.QueuedIngestionLog{}, nil).Maybe() + + mockOps.On("DefaultBranch", mock.Anything).Return(nil, nil).Maybe() + mockOps.On("BulkPlan", mock.Anything, batch, "").Return([]ops.BulkGenerateChangeSetResult{ + {IngestionLogID: 1, ChangeSetID: int32Ptr(10), ChangeSet: cs}, + }).Once() + mockOps.On("BulkPlanApply", mock.Anything, []ops.QueuedIngestionLog{batch[0]}, ""). + Return([]ops.BulkPlanApplyResult{{IngestionLogID: 1, ChangeSet: cs}}).Once() + + applied := make(chan struct{}) + mockMetrics.On("RecordChangeSetCreate", mock.Anything, true, int64(1)).Once() + mockMetrics.On("RecordChangeSetApply", mock.Anything, true, int64(1)).Once(). + Run(func(_ mock.Arguments) { close(applied) }) + + policy := reconciler.AutoApplyPolicy(func(_ *reconcilerpb.IngestionLog, _ *changeset.ChangeSet) bool { + return true + }) + + p := reconciler.NewIngestionLogProcessor( + newAutoApplyTestLogger(), + reconciler.Config{}, + repo, + mockOps, + mockMetrics, + nil, + reconciler.WithAutoApplyPolicy(policy), + ) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.Start(ctx) }() + + select { + case <-applied: + case <-time.After(5 * time.Second): + t.Fatal("apply metric was never recorded") + } + cancel() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("processor did not exit") + } +} + +// TestIngestionLogProcessor_AutoApplyPolicyNoMatch_DoesNotApply verifies that +// when the policy returns false for a record, the processor does NOT call +// BulkPlanApply — leaving the row in OPEN for review, as it would without +// any policy. +func TestIngestionLogProcessor_AutoApplyPolicyNoMatch_DoesNotApply(t *testing.T) { + repo := mocks.NewRepository(t) + mockOps := mocks.NewIngestionProcessorOps(t) + mockMetrics := mocks.NewMetrics(t) + + log := newAutoApplyTestLog() + batch := []ops.QueuedIngestionLog{{ID: 1, IngestionLog: log}} + cs := &changeset.ChangeSet{ + Changes: []changeset.Change{{ChangeType: changeset.ChangeTypeUpdate, ObjectType: "extras.customfield"}}, + } + + repo.On("ClaimQueuedIngestionLogs", mock.Anything, int32(100)). + Return(batch, nil).Once() + repo.On("ClaimQueuedIngestionLogs", mock.Anything, int32(100)). + Return([]ops.QueuedIngestionLog{}, nil).Maybe() + + mockOps.On("DefaultBranch", mock.Anything).Return(nil, nil).Maybe() + mockOps.On("BulkPlan", mock.Anything, batch, "").Return([]ops.BulkGenerateChangeSetResult{ + {IngestionLogID: 1, ChangeSetID: int32Ptr(10), ChangeSet: cs}, + }).Once() + + planned := make(chan struct{}) + mockMetrics.On("RecordChangeSetCreate", mock.Anything, true, int64(1)).Once(). + Run(func(_ mock.Arguments) { close(planned) }) + + policy := reconciler.AutoApplyPolicy(func(_ *reconcilerpb.IngestionLog, _ *changeset.ChangeSet) bool { + return false + }) + + p := reconciler.NewIngestionLogProcessor( + newAutoApplyTestLogger(), + reconciler.Config{}, + repo, + mockOps, + mockMetrics, + nil, + reconciler.WithAutoApplyPolicy(policy), + ) + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- p.Start(ctx) }() + + select { + case <-planned: + case <-time.After(5 * time.Second): + t.Fatal("create metric was never recorded") + } + cancel() + + select { + case err := <-done: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("processor did not exit") + } + + mockOps.AssertNotCalled(t, "BulkPlanApply") +} diff --git a/diode-server/reconciler/ingestion_processor.go b/diode-server/reconciler/ingestion_processor.go index d3df36a2..9d4ad285 100644 --- a/diode-server/reconciler/ingestion_processor.go +++ b/diode-server/reconciler/ingestion_processor.go @@ -209,7 +209,12 @@ func (p *IngestionProcessor) consumeIngestionStream(ctx context.Context, redisSt } } -func (p *IngestionProcessor) handleStreamMessage(ctx context.Context, msg redis.XMessage) error { +func (p *IngestionProcessor) handleStreamMessage(ctx context.Context, msg redis.XMessage) (err error) { + ctx, span := telemetry.StartSpan(ctx, telemetry.SpanIngestionHandleStreamMessage, + attribute.String(telemetry.AttributeHostname, p.hostname), + ) + defer func() { telemetry.End(span, err) }() + attrs := []attribute.KeyValue{ attribute.String(telemetry.AttributeHostname, p.hostname), } @@ -244,8 +249,16 @@ func (p *IngestionProcessor) handleStreamMessage(ctx context.Context, msg redis. ingestionTs, err := strconv.Atoi(msg.Values["ingestion_ts"].(string)) if err != nil { errs = append(errs, fmt.Errorf("failed to convert ingestion timestamp: %v", err)) + } else { + streamLag := int64(time.Since(time.Unix(0, int64(ingestionTs))).Seconds()) + span.SetAttributes(attribute.Int64(telemetry.AttributeStreamLag, streamLag)) } + span.SetAttributes( + attribute.String(telemetry.AttributeRequestID, ingestReq.GetId()), + attribute.Int(telemetry.AttributeEntityCount, len(ingestReq.GetEntities())), + ) + p.logger.Debug("handling ingest request", "request", ingestReq) createIngestionLogsErrs := p.CreateIngestionLogs(ctx, ingestReq, ingestionTs) @@ -278,8 +291,20 @@ func (p *IngestionProcessor) handleStreamMessage(ctx context.Context, msg redis. } // CreateIngestionLogs creates ingestion logs for an ingest request using bulk operations -func (p *IngestionProcessor) CreateIngestionLogs(ctx context.Context, ingestReq *diodepb.IngestRequest, ingestionTs int) []error { - errs := make([]error, 0) +func (p *IngestionProcessor) CreateIngestionLogs(ctx context.Context, ingestReq *diodepb.IngestRequest, ingestionTs int) (errs []error) { + ctx, span := telemetry.StartSpan(ctx, telemetry.SpanIngestionCreateIngestionLogs, + attribute.String(telemetry.AttributeRequestID, ingestReq.GetId()), + attribute.Int(telemetry.AttributeEntityCount, len(ingestReq.GetEntities())), + ) + defer func() { + if len(errs) > 0 { + telemetry.End(span, errors.Join(errs...)) + } else { + telemetry.End(span, nil) + } + }() + + errs = make([]error, 0) // Resolve the default branch via the 60s LRU cache. A short staleness // window on a value that changes rarely is acceptable, and avoids a @@ -378,6 +403,11 @@ func (p *IngestionProcessor) CreateIngestionLogs(ctx context.Context, ingestReq p.logger.Debug("ingested duplicate ingestion log", "id", id, "externalID", ingestionLog.GetId()) } + if result.Requeued { + p.logger.Debug("requeued duplicate ingestion log for re-plan", "id", id, "externalID", ingestionLog.GetId()) + p.metrics.RecordIngestionLogRequeue(ctx) + } + attrs := []attribute.KeyValue{ attribute.Bool(telemetry.AttributeDuplicate, result.WasDuplicate), } diff --git a/diode-server/reconciler/ingestion_processor_test.go b/diode-server/reconciler/ingestion_processor_test.go index 6c3b3db1..c9b8e7b3 100644 --- a/diode-server/reconciler/ingestion_processor_test.go +++ b/diode-server/reconciler/ingestion_processor_test.go @@ -341,6 +341,7 @@ func TestIngestionProcessor_DuplicateHandling(t *testing.T) { tests := []struct { name string existingLogState reconcilerpb.State + expectRequeued bool }{ { name: "duplicate of IGNORED", @@ -357,14 +358,17 @@ func TestIngestionProcessor_DuplicateHandling(t *testing.T) { { name: "duplicate of FAILED", existingLogState: reconcilerpb.State_FAILED, + expectRequeued: true, }, { name: "duplicate of APPLIED", existingLogState: reconcilerpb.State_APPLIED, + expectRequeued: true, }, { name: "duplicate of NO_CHANGES", existingLogState: reconcilerpb.State_NO_CHANGES, + expectRequeued: true, }, } @@ -411,10 +415,15 @@ func TestIngestionProcessor_DuplicateHandling(t *testing.T) { Entity: testEntity, } + if tt.expectRequeued { + existingLog.State = reconcilerpb.State_QUEUED + } + duplicateResult := &reconops.CreateIngestionLogResult{ ID: existingLogID, IngestionLog: existingLog, WasDuplicate: true, + Requeued: tt.expectRequeued, } mockOps.On("DefaultBranch", mock.Anything).Return((*netboxdiodeplugin.Branch)(nil), nil) @@ -422,6 +431,9 @@ func TestIngestionProcessor_DuplicateHandling(t *testing.T) { mockMetrics.On("RecordHandleMessage", mock.Anything, mock.Anything).Return() mockMetrics.On("RecordIngestionLogCreate", mock.Anything, mock.Anything).Return() + if tt.expectRequeued { + mockMetrics.On("RecordIngestionLogRequeue", mock.Anything).Return() + } processor, err := reconciler.NewIngestionProcessor( ctx, logger, cfg, redisClient, redisStreamClient, diff --git a/diode-server/reconciler/metrics.go b/diode-server/reconciler/metrics.go index c3ec971f..62c89f4d 100644 --- a/diode-server/reconciler/metrics.go +++ b/diode-server/reconciler/metrics.go @@ -26,6 +26,13 @@ var ReconcilerMetricDefinitions = map[string]otel.MetricDefinition{ Description: "Total number of ingestion logs created", Attributes: []string{"success"}, }, + "ingestionlog.requeue": { + Name: "ingestion_log_requeue_total", + Type: otel.Counter, + Unit: otel.Dimensionless, + Description: "Total number of duplicate ingestion logs requeued for re-plan due to possible NetBox state drift", + Attributes: []string{}, + }, "changeset.create": { Name: "change_set_create_total", Type: otel.Counter, @@ -74,6 +81,8 @@ type Metrics interface { RecordHandleMessage(ctx context.Context, success bool) // RecordIngestionLogCreate records the creation of an ingestion log RecordIngestionLogCreate(ctx context.Context, success bool) + // RecordIngestionLogRequeue records the requeue of a duplicate ingestion log for re-plan + RecordIngestionLogRequeue(ctx context.Context) // RecordChangeSetCreate records the creation of a change set RecordChangeSetCreate(ctx context.Context, success bool, changes int64) // RecordChangeSetApply records the application of a change set @@ -129,6 +138,13 @@ func (r *MetricRecorder) RecordIngestionLogCreate(ctx context.Context, success b r.mr.RecordCounter(ctx, "ingestionlog.create", 1, attrs...) } +// RecordIngestionLogRequeue records the requeue of a duplicate ingestion log for re-plan +func (r *MetricRecorder) RecordIngestionLogRequeue(ctx context.Context) { + attrs := telemetry.GetAttributesFromContext(ctx, r.contextAttributeKeys...) + + r.mr.RecordCounter(ctx, "ingestionlog.requeue", 1, attrs...) +} + // RecordChangeSetCreate records the creation of a change set func (r *MetricRecorder) RecordChangeSetCreate(ctx context.Context, success bool, changes int64) { attrs := append([]attribute.KeyValue{ diff --git a/diode-server/reconciler/mocks/metrics.go b/diode-server/reconciler/mocks/metrics.go index cbaf4abf..fb3897d8 100644 --- a/diode-server/reconciler/mocks/metrics.go +++ b/diode-server/reconciler/mocks/metrics.go @@ -195,6 +195,39 @@ func (_c *Metrics_RecordIngestionLogCreate_Call) RunAndReturn(run func(context.C return _c } +// RecordIngestionLogRequeue provides a mock function with given fields: ctx +func (_m *Metrics) RecordIngestionLogRequeue(ctx context.Context) { + _m.Called(ctx) +} + +// Metrics_RecordIngestionLogRequeue_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RecordIngestionLogRequeue' +type Metrics_RecordIngestionLogRequeue_Call struct { + *mock.Call +} + +// RecordIngestionLogRequeue is a helper method to define mock.On call +// - ctx context.Context +func (_e *Metrics_Expecter) RecordIngestionLogRequeue(ctx interface{}) *Metrics_RecordIngestionLogRequeue_Call { + return &Metrics_RecordIngestionLogRequeue_Call{Call: _e.mock.On("RecordIngestionLogRequeue", ctx)} +} + +func (_c *Metrics_RecordIngestionLogRequeue_Call) Run(run func(ctx context.Context)) *Metrics_RecordIngestionLogRequeue_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *Metrics_RecordIngestionLogRequeue_Call) Return() *Metrics_RecordIngestionLogRequeue_Call { + _c.Call.Return() + return _c +} + +func (_c *Metrics_RecordIngestionLogRequeue_Call) RunAndReturn(run func(context.Context)) *Metrics_RecordIngestionLogRequeue_Call { + _c.Run(run) + return _c +} + // RecordServiceStartupAttempt provides a mock function with given fields: ctx, success func (_m *Metrics) RecordServiceStartupAttempt(ctx context.Context, success bool) { _m.Called(ctx, success) diff --git a/diode-server/reconciler/mocks/repository.go b/diode-server/reconciler/mocks/repository.go index 7ac0a26c..85b93c85 100644 --- a/diode-server/reconciler/mocks/repository.go +++ b/diode-server/reconciler/mocks/repository.go @@ -27,6 +27,65 @@ func (_m *Repository) EXPECT() *Repository_Expecter { return &Repository_Expecter{mock: &_m.Mock} } +// BulkCreateDriftDeviations provides a mock function with given fields: ctx, items +func (_m *Repository) BulkCreateDriftDeviations(ctx context.Context, items []ops.DriftDeviationItem) ([]ops.DriftDeviationResult, error) { + ret := _m.Called(ctx, items) + + if len(ret) == 0 { + panic("no return value specified for BulkCreateDriftDeviations") + } + + var r0 []ops.DriftDeviationResult + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []ops.DriftDeviationItem) ([]ops.DriftDeviationResult, error)); ok { + return rf(ctx, items) + } + if rf, ok := ret.Get(0).(func(context.Context, []ops.DriftDeviationItem) []ops.DriftDeviationResult); ok { + r0 = rf(ctx, items) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]ops.DriftDeviationResult) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, []ops.DriftDeviationItem) error); ok { + r1 = rf(ctx, items) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Repository_BulkCreateDriftDeviations_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BulkCreateDriftDeviations' +type Repository_BulkCreateDriftDeviations_Call struct { + *mock.Call +} + +// BulkCreateDriftDeviations is a helper method to define mock.On call +// - ctx context.Context +// - items []ops.DriftDeviationItem +func (_e *Repository_Expecter) BulkCreateDriftDeviations(ctx interface{}, items interface{}) *Repository_BulkCreateDriftDeviations_Call { + return &Repository_BulkCreateDriftDeviations_Call{Call: _e.mock.On("BulkCreateDriftDeviations", ctx, items)} +} + +func (_c *Repository_BulkCreateDriftDeviations_Call) Run(run func(ctx context.Context, items []ops.DriftDeviationItem)) *Repository_BulkCreateDriftDeviations_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].([]ops.DriftDeviationItem)) + }) + return _c +} + +func (_c *Repository_BulkCreateDriftDeviations_Call) Return(_a0 []ops.DriftDeviationResult, _a1 error) *Repository_BulkCreateDriftDeviations_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *Repository_BulkCreateDriftDeviations_Call) RunAndReturn(run func(context.Context, []ops.DriftDeviationItem) ([]ops.DriftDeviationResult, error)) *Repository_BulkCreateDriftDeviations_Call { + _c.Call.Return(run) + return _c +} + // BulkCreateIngestionLogs provides a mock function with given fields: ctx, logs, sourceMetadata, entityHashes func (_m *Repository) BulkCreateIngestionLogs(ctx context.Context, logs []*reconcilerpb.IngestionLog, sourceMetadata [][]byte, entityHashes []string) (map[string]int32, error) { ret := _m.Called(ctx, logs, sourceMetadata, entityHashes) @@ -88,49 +147,61 @@ func (_c *Repository_BulkCreateIngestionLogs_Call) RunAndReturn(run func(context return _c } -// BulkIncrementDuplicateCounts provides a mock function with given fields: ctx, ids -func (_m *Repository) BulkIncrementDuplicateCounts(ctx context.Context, ids []int32) error { +// BulkMarkDuplicates provides a mock function with given fields: ctx, ids +func (_m *Repository) BulkMarkDuplicates(ctx context.Context, ids []int32) (map[int32]bool, error) { ret := _m.Called(ctx, ids) if len(ret) == 0 { - panic("no return value specified for BulkIncrementDuplicateCounts") + panic("no return value specified for BulkMarkDuplicates") } - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, []int32) error); ok { + var r0 map[int32]bool + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, []int32) (map[int32]bool, error)); ok { + return rf(ctx, ids) + } + if rf, ok := ret.Get(0).(func(context.Context, []int32) map[int32]bool); ok { r0 = rf(ctx, ids) } else { - r0 = ret.Error(0) + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[int32]bool) + } } - return r0 + if rf, ok := ret.Get(1).(func(context.Context, []int32) error); ok { + r1 = rf(ctx, ids) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } -// Repository_BulkIncrementDuplicateCounts_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BulkIncrementDuplicateCounts' -type Repository_BulkIncrementDuplicateCounts_Call struct { +// Repository_BulkMarkDuplicates_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BulkMarkDuplicates' +type Repository_BulkMarkDuplicates_Call struct { *mock.Call } -// BulkIncrementDuplicateCounts is a helper method to define mock.On call +// BulkMarkDuplicates is a helper method to define mock.On call // - ctx context.Context // - ids []int32 -func (_e *Repository_Expecter) BulkIncrementDuplicateCounts(ctx interface{}, ids interface{}) *Repository_BulkIncrementDuplicateCounts_Call { - return &Repository_BulkIncrementDuplicateCounts_Call{Call: _e.mock.On("BulkIncrementDuplicateCounts", ctx, ids)} +func (_e *Repository_Expecter) BulkMarkDuplicates(ctx interface{}, ids interface{}) *Repository_BulkMarkDuplicates_Call { + return &Repository_BulkMarkDuplicates_Call{Call: _e.mock.On("BulkMarkDuplicates", ctx, ids)} } -func (_c *Repository_BulkIncrementDuplicateCounts_Call) Run(run func(ctx context.Context, ids []int32)) *Repository_BulkIncrementDuplicateCounts_Call { +func (_c *Repository_BulkMarkDuplicates_Call) Run(run func(ctx context.Context, ids []int32)) *Repository_BulkMarkDuplicates_Call { _c.Call.Run(func(args mock.Arguments) { run(args[0].(context.Context), args[1].([]int32)) }) return _c } -func (_c *Repository_BulkIncrementDuplicateCounts_Call) Return(_a0 error) *Repository_BulkIncrementDuplicateCounts_Call { - _c.Call.Return(_a0) +func (_c *Repository_BulkMarkDuplicates_Call) Return(_a0 map[int32]bool, _a1 error) *Repository_BulkMarkDuplicates_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *Repository_BulkIncrementDuplicateCounts_Call) RunAndReturn(run func(context.Context, []int32) error) *Repository_BulkIncrementDuplicateCounts_Call { +func (_c *Repository_BulkMarkDuplicates_Call) RunAndReturn(run func(context.Context, []int32) (map[int32]bool, error)) *Repository_BulkMarkDuplicates_Call { _c.Call.Return(run) return _c } @@ -621,53 +692,6 @@ func (_c *Repository_FindPriorIngestionLogsByEntityHashes_Call) RunAndReturn(run return _c } -// IncrementDuplicateCount provides a mock function with given fields: ctx, id -func (_m *Repository) IncrementDuplicateCount(ctx context.Context, id int32) error { - ret := _m.Called(ctx, id) - - if len(ret) == 0 { - panic("no return value specified for IncrementDuplicateCount") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, int32) error); ok { - r0 = rf(ctx, id) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Repository_IncrementDuplicateCount_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'IncrementDuplicateCount' -type Repository_IncrementDuplicateCount_Call struct { - *mock.Call -} - -// IncrementDuplicateCount is a helper method to define mock.On call -// - ctx context.Context -// - id int32 -func (_e *Repository_Expecter) IncrementDuplicateCount(ctx interface{}, id interface{}) *Repository_IncrementDuplicateCount_Call { - return &Repository_IncrementDuplicateCount_Call{Call: _e.mock.On("IncrementDuplicateCount", ctx, id)} -} - -func (_c *Repository_IncrementDuplicateCount_Call) Run(run func(ctx context.Context, id int32)) *Repository_IncrementDuplicateCount_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(int32)) - }) - return _c -} - -func (_c *Repository_IncrementDuplicateCount_Call) Return(_a0 error) *Repository_IncrementDuplicateCount_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Repository_IncrementDuplicateCount_Call) RunAndReturn(run func(context.Context, int32) error) *Repository_IncrementDuplicateCount_Call { - _c.Call.Return(run) - return _c -} - // ResetApplyingIngestionLogs provides a mock function with given fields: ctx func (_m *Repository) ResetApplyingIngestionLogs(ctx context.Context) error { ret := _m.Called(ctx) diff --git a/diode-server/reconciler/ops.go b/diode-server/reconciler/ops.go index 0f9c167a..9c31627b 100644 --- a/diode-server/reconciler/ops.go +++ b/diode-server/reconciler/ops.go @@ -9,6 +9,8 @@ import ( "sync" "time" + "github.com/google/uuid" + "github.com/netboxlabs/diode/diode-server/entityhash" diodeErrors "github.com/netboxlabs/diode/diode-server/errors" "github.com/netboxlabs/diode/diode-server/gen/diode/v1/reconcilerpb" @@ -210,8 +212,10 @@ func (o *Ops) CreateIngestionLog(ctx context.Context, ingestionLog *reconcilerpb return result, nil } - // It was a duplicate, increment the duplicate count and return the prior ingestion log - if err := o.repository.IncrementDuplicateCount(ctx, *existingID); err != nil { + // It was a duplicate: increment the duplicate count, requeue the prior + // ingestion log if its NetBox state may have drifted, and return it + requeued, err := o.repository.BulkMarkDuplicates(ctx, []int32{*existingID}) + if err != nil { return nil, fmt.Errorf("failed to mark record as duplicate: %w", err) } @@ -219,8 +223,12 @@ func (o *Ops) CreateIngestionLog(ctx context.Context, ingestionLog *reconcilerpb ID: *existingID, IngestionLog: existingLog, WasDuplicate: true, + Requeued: requeued[*existingID], BranchID: branchIDForResult, } + if result.Requeued { + result.IngestionLog.State = reconcilerpb.State_QUEUED + } return result, nil } @@ -287,10 +295,20 @@ func (o *Ops) BulkCreateIngestionLogs(ctx context.Context, ingestionLogs []*reco } } - // Bulk increment duplicate counts + // Bulk mark duplicates: increment duplicate counts and requeue prior + // ingestion logs whose NetBox state may have drifted since last plan/apply if len(duplicateIDs) > 0 { - if err := o.repository.BulkIncrementDuplicateCounts(ctx, duplicateIDs); err != nil { - return nil, fmt.Errorf("failed to bulk increment duplicate counts: %w", err) + requeued, err := o.repository.BulkMarkDuplicates(ctx, duplicateIDs) + if err != nil { + return nil, fmt.Errorf("failed to bulk mark duplicates: %w", err) + } + // Results with the same entity hash share one prior *IngestionLog; + // writing the same QUEUED value into it repeatedly is idempotent. + for _, res := range results { + if res != nil && res.WasDuplicate && requeued[res.ID] { + res.Requeued = true + res.IngestionLog.State = reconcilerpb.State_QUEUED + } } } @@ -351,6 +369,8 @@ func (o *Ops) BulkPlan(ctx context.Context, items []ops.QueuedIngestionLog, bran persistItems := make([]ops.BulkPersistItem, 0, len(items)) persistIndex := make([]int, 0, len(items)) + var driftItems []ops.DriftDeviationItem + var driftIndex []int for i, item := range items { entityID := fmt.Sprintf("%d", item.ID) @@ -366,6 +386,22 @@ func (o *Ops) BulkPlan(ctx context.Context, items []ops.QueuedIngestionLog, bran continue } + stripNoopOnlyChanges(cs) + + // NetBox drifted since this entity was applied: the applied deviation + // stays APPLIED and the drift becomes a new deviation of its own. + if item.RequeuedFromState == reconcilerpb.State_APPLIED && len(cs.Changes) > 0 { + driftItems = append(driftItems, ops.DriftDeviationItem{ + PriorIngestionLogID: item.ID, + NewExternalID: uuid.NewString(), + NewState: reconcilerpb.State_OPEN, + ChangeSet: *cs, + }) + driftIndex = append(driftIndex, i) + results[i] = ops.BulkGenerateChangeSetResult{IngestionLogID: item.ID, ChangeSet: cs} + continue + } + persistItems, persistIndex = collectPersistItem(persistItems, persistIndex, results, i, item, cs) } @@ -375,7 +411,7 @@ func (o *Ops) BulkPlan(ctx context.Context, items []ops.QueuedIngestionLog, bran o.logger.Error("bulk persist failed, falling back to per-item persist", "error", err) for j, idx := range persistIndex { item := items[idx] - results[idx] = o.persistChangeSet(ctx, item.ID, item.IngestionLog, &persistItems[j].ChangeSet) + results[idx] = o.persistChangeSet(ctx, item.ID, item.IngestionLog, &persistItems[j].ChangeSet, persistItems[j].NewState) } } else { for j, pr := range persistResults { @@ -385,15 +421,47 @@ func (o *Ops) BulkPlan(ctx context.Context, items []ops.QueuedIngestionLog, bran } } + if len(driftItems) > 0 { + driftResults, err := o.repository.BulkCreateDriftDeviations(ctx, driftItems) + if err != nil { + o.logger.Error("failed to create drift deviations", "error", err) + for _, idx := range driftIndex { + // Restore the prior so it does not linger in the claimed + // state; the drift is re-detected on the next observation. + o.restoreRequeuedPrior(ctx, items[idx].ID) + results[idx].Err = err + } + } else { + for j, dr := range driftResults { + idx := driftIndex[j] + results[idx].IngestionLogID = dr.NewIngestionLogID + results[idx].ChangeSetID = dr.ChangeSetID + } + } + } + return results } -func collectPersistItem(persistItems []ops.BulkPersistItem, persistIndex []int, results []ops.BulkGenerateChangeSetResult, i int, item ops.QueuedIngestionLog, cs *changeset.ChangeSet) ([]ops.BulkPersistItem, []int) { - stripNoopOnlyChanges(cs) +// restoreRequeuedPrior returns a drift-recheck prior to APPLIED (clearing its +// requeue marker) after a failure that must not destroy its applied record. +func (o *Ops) restoreRequeuedPrior(ctx context.Context, ingestionLogID int32) { + if err := o.repository.UpdateIngestionLogStateWithError(ctx, ingestionLogID, reconcilerpb.State_APPLIED, nil); err != nil { + o.logger.Error("failed to restore requeued prior to APPLIED", "ingestionLogID", ingestionLogID, "error", err) + } +} +// collectPersistItem appends a persist item for cs, whose noop-only changes +// the caller has already stripped. +func collectPersistItem(persistItems []ops.BulkPersistItem, persistIndex []int, results []ops.BulkGenerateChangeSetResult, i int, item ops.QueuedIngestionLog, cs *changeset.ChangeSet) ([]ops.BulkPersistItem, []int) { newState := reconcilerpb.State_OPEN if len(cs.Changes) == 0 { newState = reconcilerpb.State_NO_CHANGES + if item.RequeuedFromState == reconcilerpb.State_APPLIED { + // Drift re-check of an applied entity found NetBox still in sync: + // keep the applied fact instead of degrading it to NO_CHANGES. + newState = reconcilerpb.State_APPLIED + } } persistItems = append(persistItems, ops.BulkPersistItem{ @@ -431,6 +499,14 @@ func (o *Ops) handleGenerateChangeSetFailure(ctx context.Context, item ops.Queue contextMap := map[string]any{"request_id": ingestEntity.RequestID, "object_type": ingestEntity.ObjectType} sentry.CaptureError(err, tags, "Ingest Entity", contextMap) + // A plan failure on a drift re-check must not destroy the prior APPLIED + // record: restore it and surface the error via Sentry/logs only. The + // check retries on the next duplicate observation. + if item.RequeuedFromState == reconcilerpb.State_APPLIED { + o.restoreRequeuedPrior(ctx, item.ID) + return ops.BulkGenerateChangeSetResult{IngestionLogID: item.ID, Err: err} + } + item.IngestionLog.State = reconcilerpb.State_FAILED changeSetErr := handleChangeSetError(err) if err2 := o.repository.UpdateIngestionLogStateWithError(ctx, item.ID, reconcilerpb.State_FAILED, changeSetErr); err2 != nil { @@ -447,9 +523,13 @@ func (o *Ops) handleGenerateChangeSetFailure(ctx context.Context, item ops.Queue return ops.BulkGenerateChangeSetResult{IngestionLogID: item.ID, ChangeSetID: id, ChangeSet: cs, Err: err} } -func (o *Ops) persistChangeSet(ctx context.Context, ingestionLogID int32, ingestionLog *reconcilerpb.IngestionLog, cs *changeset.ChangeSet) ops.BulkGenerateChangeSetResult { - if len(cs.Changes) == 0 && (ingestionLog.State == reconcilerpb.State_NO_CHANGES || ingestionLog.State == reconcilerpb.State_APPLIED) { - if err := o.repository.UpdateIngestionLogStateWithError(ctx, ingestionLogID, ingestionLog.State, nil); err != nil { +// persistChangeSet is the per-item fallback when bulk persist fails. newState +// is the state the bulk path already decided for this item; re-deriving it +// here would demote a drift re-check restored to APPLIED back to NO_CHANGES. +func (o *Ops) persistChangeSet(ctx context.Context, ingestionLogID int32, ingestionLog *reconcilerpb.IngestionLog, cs *changeset.ChangeSet, newState reconcilerpb.State) ops.BulkGenerateChangeSetResult { + if len(cs.Changes) == 0 { + ingestionLog.State = newState + if err := o.repository.UpdateIngestionLogStateWithError(ctx, ingestionLogID, newState, nil); err != nil { o.logger.Error("failed to update ingestion log state (error ignored)", "ingestionLogID", ingestionLogID, "error", err) } return ops.BulkGenerateChangeSetResult{IngestionLogID: ingestionLogID, ChangeSet: cs} @@ -467,12 +547,8 @@ func (o *Ops) persistChangeSet(ctx context.Context, ingestionLogID int32, ingest } } - if len(cs.Changes) > 0 { - ingestionLog.State = reconcilerpb.State_OPEN - } else { - ingestionLog.State = reconcilerpb.State_NO_CHANGES - } - if err := o.repository.UpdateIngestionLogStateWithError(ctx, ingestionLogID, ingestionLog.State, nil); err != nil { + ingestionLog.State = newState + if err := o.repository.UpdateIngestionLogStateWithError(ctx, ingestionLogID, newState, nil); err != nil { o.logger.Error("failed to update ingestion log state (error ignored)", "ingestionLogID", ingestionLogID, "error", err) } @@ -519,6 +595,8 @@ func (o *Ops) BulkPlanApply(ctx context.Context, items []ops.QueuedIngestionLog, var persistItems []ops.BulkPersistItem var persistIndex []int + var driftItems []ops.DriftDeviationItem + var driftIndex []int for i, item := range items { entityID := fmt.Sprintf("%d", item.ID) @@ -545,10 +623,34 @@ func (o *Ops) BulkPlanApply(ctx context.Context, items []ops.QueuedIngestionLog, // Plan succeeded — collect for bulk persist with the right terminal state. stripNoopOnlyChanges(cs) + + // NetBox drifted since this entity was applied: the applied deviation + // stays APPLIED and the drift becomes a new deviation of its own, + // APPLIED on success or FAILED when the apply phase reported errors. + if item.RequeuedFromState == reconcilerpb.State_APPLIED && len(cs.Changes) > 0 { + newState := reconcilerpb.State_APPLIED + if applyErr != nil { + newState = reconcilerpb.State_FAILED + } + driftItems = append(driftItems, ops.DriftDeviationItem{ + PriorIngestionLogID: item.ID, + NewExternalID: uuid.NewString(), + NewState: newState, + ChangeSet: *cs, + }) + driftIndex = append(driftIndex, i) + continue + } + newState := reconcilerpb.State_APPLIED switch { case len(cs.Changes) == 0: newState = reconcilerpb.State_NO_CHANGES + if item.RequeuedFromState == reconcilerpb.State_APPLIED { + // Drift re-check found NetBox still in sync: keep the applied + // fact instead of degrading it to NO_CHANGES. + newState = reconcilerpb.State_APPLIED + } case applyErr != nil: newState = reconcilerpb.State_FAILED } @@ -578,6 +680,35 @@ func (o *Ops) BulkPlanApply(ctx context.Context, items []ops.QueuedIngestionLog, } } + if len(driftItems) > 0 { + driftResults, err := o.repository.BulkCreateDriftDeviations(ctx, driftItems) + if err != nil { + o.logger.Error("failed to create drift deviations during bulk-plan-apply", "error", err) + for _, idx := range driftIndex { + // Restore the prior so it does not linger in APPLYING; the + // drift is re-detected on the next observation. + o.restoreRequeuedPrior(ctx, items[idx].ID) + if results[idx].ApplyErr == nil { + results[idx].ApplyErr = err + } + } + } else { + for j, dr := range driftResults { + idx := driftIndex[j] + results[idx].IngestionLogID = dr.NewIngestionLogID + results[idx].ChangeSetID = dr.ChangeSetID + // The new deviation was created in FAILED state when the apply + // phase reported errors; annotate it with the reason. + if results[idx].ApplyErr != nil { + changeSetErr := handleChangeSetError(results[idx].ApplyErr) + if err := o.repository.UpdateIngestionLogStateWithError(ctx, dr.NewIngestionLogID, reconcilerpb.State_FAILED, changeSetErr); err != nil { + o.logger.Warn("failed to annotate apply failure", "ingestionLogID", dr.NewIngestionLogID, "error", err) + } + } + } + } + } + // For entities whose apply phase failed, attach the apply error message to the // ingestion log row. BulkPersistChangeSets clears the error column when it sets // the state, so this second pass annotates the FAILED rows with their reason. @@ -610,6 +741,14 @@ func (o *Ops) persistPlanApplyFailurePlaceholder(ctx context.Context, item ops.Q contextMap := map[string]any{"request_id": ingestEntity.RequestID, "object_type": ingestEntity.ObjectType} sentry.CaptureError(planErr, tags, "BulkPlanApply", contextMap) + // A plan failure on a drift re-check must not destroy the prior APPLIED + // record: restore it and surface the error via Sentry/logs only. The + // check retries on the next duplicate observation. + if item.RequeuedFromState == reconcilerpb.State_APPLIED { + o.restoreRequeuedPrior(ctx, item.ID) + return ops.BulkPlanApplyResult{IngestionLogID: item.ID, PlanErr: planErr} + } + changeSetErr := handleChangeSetError(planErr) if err2 := o.repository.UpdateIngestionLogStateWithError(ctx, item.ID, reconcilerpb.State_FAILED, changeSetErr); err2 != nil { planErr = errors.Join(planErr, err2) diff --git a/diode-server/reconciler/ops/types.go b/diode-server/reconciler/ops/types.go index 6a76eeb8..8cdf0138 100644 --- a/diode-server/reconciler/ops/types.go +++ b/diode-server/reconciler/ops/types.go @@ -10,6 +10,7 @@ type CreateIngestionLogResult struct { ID int32 IngestionLog *reconcilerpb.IngestionLog WasDuplicate bool // true if the ingestion log was a duplicate, in this case the prior ingestion log is returned + Requeued bool // true if a duplicate's prior ingestion log was requeued to re-check NetBox state drift BranchID string // the branch ID used for this ingestion log (empty string means main branch) } @@ -23,6 +24,27 @@ type PriorIngestionLog struct { type QueuedIngestionLog struct { ID int32 IngestionLog *reconcilerpb.IngestionLog + // RequeuedFromState is the terminal state the log was in when a duplicate + // observation requeued it for re-plan (STATE_UNSPECIFIED when the log is a + // first-time plan). A log requeued from APPLIED spawns a new deviation on + // drift and is restored to APPLIED, preserving its history. + RequeuedFromState reconcilerpb.State +} + +// DriftDeviationItem describes a new deviation to create for an entity whose +// NetBox state drifted since its prior ingestion log was applied. +type DriftDeviationItem struct { + PriorIngestionLogID int32 + NewExternalID string + NewState reconcilerpb.State + ChangeSet changeset.ChangeSet +} + +// DriftDeviationResult holds the outcome of creating one drift deviation. +type DriftDeviationResult struct { + PriorIngestionLogID int32 + NewIngestionLogID int32 + ChangeSetID *int32 } // BulkGenerateChangeSetResult holds the result of generating a change set for a single item in a bulk operation. diff --git a/diode-server/reconciler/ops_test.go b/diode-server/reconciler/ops_test.go index 094be719..3e1f1aa1 100644 --- a/diode-server/reconciler/ops_test.go +++ b/diode-server/reconciler/ops_test.go @@ -19,6 +19,7 @@ import ( pluginmocks "github.com/netboxlabs/diode/diode-server/netboxdiodeplugin/mocks" "github.com/netboxlabs/diode/diode-server/reconciler" "github.com/netboxlabs/diode/diode-server/reconciler/mocks" + reconops "github.com/netboxlabs/diode/diode-server/reconciler/ops" ) func strPtr(s string) *string { @@ -64,9 +65,11 @@ func TestOpsCreateIngestionLog(t *testing.T) { mockFindPriorIngestionLogID *int32 mockFindPriorIngestionLog *pb.IngestionLog mockFindPriorIngestionLogError error + mockRequeued bool expectedError string expectWasDuplicate bool + expectRequeued bool }{ { name: "no duplicate found - successful creation", @@ -94,6 +97,26 @@ func TestOpsCreateIngestionLog(t *testing.T) { expectedError: "", expectWasDuplicate: true, }, + { + name: "duplicate found - prior requeued for re-plan", + ingestionLog: testIngestionLog, + sourceMetadata: testSourceMetadata, + + mockFindPriorIngestionLogID: int32Ptr(5678), + mockFindPriorIngestionLog: &pb.IngestionLog{ + Id: "8a8ae517-85b9-466e-890c-aadb0771cc9e", + ObjectType: netbox.SiteObjectType, + State: pb.State_APPLIED, + RequestId: "1abf059c-496f-4037-83c2-0e9b1d021e85", + Entity: testEntity, + }, + mockFindPriorIngestionLogError: nil, + mockRequeued: true, + + expectedError: "", + expectWasDuplicate: true, + expectRequeued: true, + }, { name: "create ingestion log fails", ingestionLog: testIngestionLog, @@ -139,7 +162,8 @@ func TestOpsCreateIngestionLog(t *testing.T) { } if tt.expectWasDuplicate { - mockRepository.EXPECT().IncrementDuplicateCount(mock.Anything, *tt.mockFindPriorIngestionLogID).Return(nil) + mockRepository.EXPECT().BulkMarkDuplicates(mock.Anything, []int32{*tt.mockFindPriorIngestionLogID}). + Return(map[int32]bool{*tt.mockFindPriorIngestionLogID: tt.mockRequeued}, nil) } result, err := opsInstance.CreateIngestionLog(ctx, tt.ingestionLog, tt.sourceMetadata) @@ -156,8 +180,16 @@ func TestOpsCreateIngestionLog(t *testing.T) { if tt.mockCreateIngestionLog != nil { require.Equal(t, *tt.mockCreateIngestionLog.id, result.ID) } - require.Equal(t, tt.ingestionLog, result.IngestionLog) + if tt.expectWasDuplicate { + require.Equal(t, tt.mockFindPriorIngestionLog, result.IngestionLog) + } else { + require.Equal(t, tt.ingestionLog, result.IngestionLog) + } require.Equal(t, tt.expectWasDuplicate, result.WasDuplicate) + require.Equal(t, tt.expectRequeued, result.Requeued) + if tt.expectRequeued { + require.Equal(t, pb.State_QUEUED, result.IngestionLog.State) + } }) } } @@ -179,6 +211,279 @@ func waitForBranch(t *testing.T, ops *reconciler.Ops, predicate func(*netboxdiod return b } +func TestOpsBulkCreateIngestionLogsDuplicateRequeue(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + + newLog := &pb.IngestionLog{Id: "new-log", ObjectType: netbox.SiteObjectType, State: pb.State_QUEUED, Entity: testEntity} + dupOfApplied := &pb.IngestionLog{Id: "dup-of-applied", ObjectType: netbox.SiteObjectType, State: pb.State_QUEUED, Entity: testEntity} + dupOfOpen := &pb.IngestionLog{Id: "dup-of-open", ObjectType: netbox.SiteObjectType, State: pb.State_QUEUED, Entity: testEntity} + + priorApplied := &reconops.PriorIngestionLog{ID: 10, IngestionLog: &pb.IngestionLog{Id: "prior-applied", State: pb.State_APPLIED, Entity: testEntity}} + priorOpen := &reconops.PriorIngestionLog{ID: 11, IngestionLog: &pb.IngestionLog{Id: "prior-open", State: pb.State_OPEN, Entity: testEntity}} + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + // Ops dedupes hashes via a map, so the argument order is nondeterministic + mockRepository.EXPECT().FindPriorIngestionLogsByEntityHashes(mock.Anything, mock.MatchedBy(func(hashes []string) bool { + want := map[string]struct{}{"hash-new": {}, "hash-applied": {}, "hash-open": {}} + if len(hashes) != len(want) { + return false + } + for _, h := range hashes { + if _, ok := want[h]; !ok { + return false + } + } + return true + }), (*string)(nil)). + Return(map[string]*reconops.PriorIngestionLog{"hash-applied": priorApplied, "hash-open": priorOpen}, nil) + mockRepository.EXPECT().BulkMarkDuplicates(mock.Anything, []int32{10, 11}). + Return(map[int32]bool{10: true, 11: false}, nil) + mockRepository.EXPECT().BulkCreateIngestionLogs(mock.Anything, []*pb.IngestionLog{newLog}, mock.Anything, []string{"hash-new"}). + Return(map[string]int32{"new-log": 1}, nil) + + results, err := opsInstance.BulkCreateIngestionLogs(ctx, + []*pb.IngestionLog{newLog, dupOfApplied, dupOfOpen}, + [][]byte{nil, nil, nil}, + []string{"hash-new", "hash-applied", "hash-open"}) + require.NoError(t, err) + require.Len(t, results, 3) + + require.False(t, results[0].WasDuplicate) + require.False(t, results[0].Requeued) + require.Equal(t, int32(1), results[0].ID) + + require.True(t, results[1].WasDuplicate) + require.True(t, results[1].Requeued) + require.Equal(t, int32(10), results[1].ID) + require.Equal(t, pb.State_QUEUED, results[1].IngestionLog.State) + + require.True(t, results[2].WasDuplicate) + require.False(t, results[2].Requeued) + require.Equal(t, int32(11), results[2].ID) + require.Equal(t, pb.State_OPEN, results[2].IngestionLog.State) +} + +func TestOpsBulkCreateIngestionLogsMarkDuplicatesError(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + dupLog := &pb.IngestionLog{Id: "dup-log", ObjectType: netbox.SiteObjectType, State: pb.State_QUEUED, Entity: testEntity} + prior := &reconops.PriorIngestionLog{ID: 10, IngestionLog: &pb.IngestionLog{Id: "prior", State: pb.State_APPLIED, Entity: testEntity}} + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + mockRepository.EXPECT().FindPriorIngestionLogsByEntityHashes(mock.Anything, []string{"hash-dup"}, (*string)(nil)). + Return(map[string]*reconops.PriorIngestionLog{"hash-dup": prior}, nil) + mockRepository.EXPECT().BulkMarkDuplicates(mock.Anything, []int32{10}). + Return(nil, fmt.Errorf("database error")) + + _, err := opsInstance.BulkCreateIngestionLogs(ctx, []*pb.IngestionLog{dupLog}, [][]byte{nil}, []string{"hash-dup"}) + require.Error(t, err) + require.Contains(t, err.Error(), "failed to bulk mark duplicates") +} + +func TestOpsBulkPlanDriftDeviation(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + + // Both logs were requeued from APPLIED by a duplicate observation. + // Log 1 re-plans with drift -> new deviation; log 2 re-plans clean -> + // restored to APPLIED, nothing persisted. + items := []reconops.QueuedIngestionLog{ + {ID: 1, IngestionLog: &pb.IngestionLog{Id: "log-1", ObjectType: netbox.SiteObjectType, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + {ID: 2, IngestionLog: &pb.IngestionLog{Id: "log-2", ObjectType: netbox.SiteObjectType, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + } + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + mockNetBoxClient.EXPECT().BulkPlan(mock.Anything, mock.Anything).Return(&netboxdiodeplugin.BulkPlanResponse{ + Results: []netboxdiodeplugin.BulkPlanResult{ + {ID: "1", ChangeSet: &netboxdiodeplugin.ChangeSet{ID: "cs-1", Changes: []netboxdiodeplugin.Change{ + {ID: "ch-1", ChangeType: "update", ObjectType: netbox.SiteObjectType, ObjectPrimaryValue: "test-site-1"}, + }}}, + {ID: "2", ChangeSet: &netboxdiodeplugin.ChangeSet{ID: "cs-2"}}, + }, + }, nil) + + newCSID := int32(201) + mockRepository.EXPECT().BulkCreateDriftDeviations(mock.Anything, mock.MatchedBy(func(driftItems []reconops.DriftDeviationItem) bool { + return len(driftItems) == 1 && + driftItems[0].PriorIngestionLogID == 1 && + driftItems[0].NewExternalID != "" && + driftItems[0].NewState == pb.State_OPEN && + len(driftItems[0].ChangeSet.Changes) == 1 + })).Return([]reconops.DriftDeviationResult{ + {PriorIngestionLogID: 1, NewIngestionLogID: 42, ChangeSetID: &newCSID}, + }, nil) + + mockRepository.EXPECT().BulkPersistChangeSets(mock.Anything, mock.MatchedBy(func(persistItems []reconops.BulkPersistItem) bool { + return len(persistItems) == 1 && + persistItems[0].IngestionLogID == 2 && + persistItems[0].NewState == pb.State_APPLIED && + len(persistItems[0].ChangeSet.Changes) == 0 + }), mock.Anything).Return([]reconops.BulkPersistResult{{IngestionLogID: 2}}, nil) + + results := opsInstance.BulkPlan(ctx, items, "") + require.Len(t, results, 2) + + require.NoError(t, results[0].Err) + require.Equal(t, int32(42), results[0].IngestionLogID) + require.Equal(t, &newCSID, results[0].ChangeSetID) + + require.NoError(t, results[1].Err) + require.Equal(t, int32(2), results[1].IngestionLogID) +} + +func TestOpsBulkPlanDriftPlanFailureRestoresPrior(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + items := []reconops.QueuedIngestionLog{ + {ID: 1, IngestionLog: &pb.IngestionLog{Id: "log-1", ObjectType: netbox.SiteObjectType, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + } + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + mockNetBoxClient.EXPECT().BulkPlan(mock.Anything, mock.Anything).Return(&netboxdiodeplugin.BulkPlanResponse{ + Results: []netboxdiodeplugin.BulkPlanResult{ + {ID: "1", Errors: []byte(`["boom"]`)}, + }, + }, nil) + + // The prior is restored to APPLIED; no FAILED state, no placeholder change set. + mockRepository.EXPECT().UpdateIngestionLogStateWithError(mock.Anything, int32(1), pb.State_APPLIED, nil).Return(nil) + + results := opsInstance.BulkPlan(ctx, items, "") + require.Len(t, results, 1) + require.Error(t, results[0].Err) + require.Equal(t, int32(1), results[0].IngestionLogID) +} + +func TestOpsBulkPlanFallbackPreservesAppliedState(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + // Requeued from APPLIED, re-plans clean; bulk persist fails, so the + // per-item fallback must restore APPLIED (not derive NO_CHANGES) and must + // not write a change set. + items := []reconops.QueuedIngestionLog{ + {ID: 1, IngestionLog: &pb.IngestionLog{Id: "log-1", ObjectType: netbox.SiteObjectType, State: pb.State_OPEN, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + } + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + mockNetBoxClient.EXPECT().BulkPlan(mock.Anything, mock.Anything).Return(&netboxdiodeplugin.BulkPlanResponse{ + Results: []netboxdiodeplugin.BulkPlanResult{ + {ID: "1", ChangeSet: &netboxdiodeplugin.ChangeSet{ID: "cs-1"}}, + }, + }, nil) + mockRepository.EXPECT().BulkPersistChangeSets(mock.Anything, mock.Anything, mock.Anything). + Return(nil, fmt.Errorf("database error")) + mockRepository.EXPECT().UpdateIngestionLogStateWithError(mock.Anything, int32(1), pb.State_APPLIED, nil).Return(nil) + + results := opsInstance.BulkPlan(ctx, items, "") + require.Len(t, results, 1) + require.NoError(t, results[0].Err) + require.Nil(t, results[0].ChangeSetID) +} + +func TestOpsBulkPlanApplyDriftDeviation(t *testing.T) { + ctx := context.Background() + logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) + + testEntity := &diodepb.Entity{ + Entity: &diodepb.Entity_Site{ + Site: &diodepb.Site{Name: "test-site-1"}, + }, + } + + // Both requeued from APPLIED with drift: log 1 applies cleanly -> new + // APPLIED deviation; log 2 fails apply -> new FAILED deviation annotated + // with the apply error. + items := []reconops.QueuedIngestionLog{ + {ID: 1, IngestionLog: &pb.IngestionLog{Id: "log-1", ObjectType: netbox.SiteObjectType, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + {ID: 2, IngestionLog: &pb.IngestionLog{Id: "log-2", ObjectType: netbox.SiteObjectType, Entity: testEntity}, RequeuedFromState: pb.State_APPLIED}, + } + + mockRepository := mocks.NewRepository(t) + mockNetBoxClient := pluginmocks.NewNetBoxAPI(t) + opsInstance := reconciler.NewOps(mockRepository, mockNetBoxClient, logger, nil) + + change := netboxdiodeplugin.Change{ID: "ch-1", ChangeType: "update", ObjectType: netbox.SiteObjectType, ObjectPrimaryValue: "test-site-1"} + mockNetBoxClient.EXPECT().BulkPlanApply(mock.Anything, mock.Anything).Return(&netboxdiodeplugin.BulkPlanApplyResponse{ + Results: []netboxdiodeplugin.BulkPlanApplyResult{ + {ID: "1", ChangeSet: &netboxdiodeplugin.ChangeSet{ID: "cs-1", Changes: []netboxdiodeplugin.Change{change}}}, + { + ID: "2", ChangeSet: &netboxdiodeplugin.ChangeSet{ID: "cs-2", Changes: []netboxdiodeplugin.Change{change}}, + Errors: &netboxdiodeplugin.BulkPlanApplyErrors{Apply: []byte(`["apply boom"]`)}, + }, + }, + }, nil) + + cs1, cs2 := int32(201), int32(202) + mockRepository.EXPECT().BulkCreateDriftDeviations(mock.Anything, mock.MatchedBy(func(driftItems []reconops.DriftDeviationItem) bool { + return len(driftItems) == 2 && + driftItems[0].PriorIngestionLogID == 1 && driftItems[0].NewState == pb.State_APPLIED && + driftItems[1].PriorIngestionLogID == 2 && driftItems[1].NewState == pb.State_FAILED + })).Return([]reconops.DriftDeviationResult{ + {PriorIngestionLogID: 1, NewIngestionLogID: 41, ChangeSetID: &cs1}, + {PriorIngestionLogID: 2, NewIngestionLogID: 42, ChangeSetID: &cs2}, + }, nil) + + // Apply-error annotation targets the new deviation, not the restored prior. + mockRepository.EXPECT().UpdateIngestionLogStateWithError(mock.Anything, int32(42), pb.State_FAILED, mock.Anything).Return(nil) + + results := opsInstance.BulkPlanApply(ctx, items, "") + require.Len(t, results, 2) + + require.NoError(t, results[0].PlanErr) + require.NoError(t, results[0].ApplyErr) + require.Equal(t, int32(41), results[0].IngestionLogID) + + require.NoError(t, results[1].PlanErr) + require.Error(t, results[1].ApplyErr) + require.Equal(t, int32(42), results[1].IngestionLogID) +} + func TestOpsDefaultBranchColdCache(t *testing.T) { // Without Start, the refresher never runs; DefaultBranch returns (nil, nil). logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug, AddSource: false})) diff --git a/diode-server/reconciler/repository.go b/diode-server/reconciler/repository.go index 3b04067b..72e2fe7e 100644 --- a/diode-server/reconciler/repository.go +++ b/diode-server/reconciler/repository.go @@ -20,16 +20,23 @@ type Repository interface { RetrieveDeviationByID(ctx context.Context, externalID string) (*reconcilerpb.Deviation, error) FindPriorIngestionLogByEntityHash(ctx context.Context, entityHash string, currentBranch *string) (*int32, *reconcilerpb.IngestionLog, error) - IncrementDuplicateCount(ctx context.Context, id int32) error TruncateChangeSets(ctx context.Context, ingestionLogID int32, limit int32) error // Bulk operations FindPriorIngestionLogsByEntityHashes(ctx context.Context, entityHashes []string, currentBranch *string) (map[string]*ops.PriorIngestionLog, error) BulkCreateIngestionLogs(ctx context.Context, logs []*reconcilerpb.IngestionLog, sourceMetadata [][]byte, entityHashes []string) (map[string]int32, error) - BulkIncrementDuplicateCounts(ctx context.Context, ids []int32) error + // BulkMarkDuplicates increments duplicate bookkeeping for the given prior + // ingestion logs and requeues drift-eligible ones (APPLIED/FAILED/NO_CHANGES + // -> QUEUED) so they get re-planned. Returns requeued flag by ingestion log ID. + BulkMarkDuplicates(ctx context.Context, ids []int32) (map[int32]bool, error) // Bulk changeset persistence BulkPersistChangeSets(ctx context.Context, items []ops.BulkPersistItem, maxChangeSetsPerLog int32) ([]ops.BulkPersistResult, error) + // BulkCreateDriftDeviations creates, in one transaction, a new ingestion + // log per item (clone of the prior with a fresh external ID) carrying the + // drift change set, and restores each prior log to APPLIED so its history + // stays intact. + BulkCreateDriftDeviations(ctx context.Context, items []ops.DriftDeviationItem) ([]ops.DriftDeviationResult, error) // Inbox processing ClaimQueuedIngestionLogs(ctx context.Context, batchSize int32) ([]ops.QueuedIngestionLog, error) diff --git a/diode-server/telemetry/constants.go b/diode-server/telemetry/constants.go index 188e5d16..8de80480 100644 --- a/diode-server/telemetry/constants.go +++ b/diode-server/telemetry/constants.go @@ -19,4 +19,10 @@ const ( AttributeState = "state" // AttributeDuplicate is a boolean attribute that indicates if an ingestion log was a duplicate AttributeDuplicate = "duplicate" + // AttributeRequestID is the integrator ingest request identifier. + AttributeRequestID = "request_id" + // AttributeEntityCount is the number of entities in a batch. + AttributeEntityCount = "entity_count" + // AttributeStreamLag is queue wait in seconds (consume time minus ingestion_ts). + AttributeStreamLag = "stream_lag" ) diff --git a/diode-server/telemetry/tracing.go b/diode-server/telemetry/tracing.go new file mode 100644 index 00000000..182e274a --- /dev/null +++ b/diode-server/telemetry/tracing.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "context" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" +) + +const tracerName = "github.com/netboxlabs/diode/diode-server" + +// Span names align with diode-pro/docs/observability/tracing.md. +const ( + SpanIngestionHandleStreamMessage = "ingestion.handle_stream_message" + SpanIngestionCreateIngestionLogs = "ingestion.create_ingestion_logs" + SpanRateLimiterWait = "rate_limiter.wait" +) + +var tracer = otel.Tracer(tracerName) + +// StartSpan begins a custom span with optional attributes. +func StartSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) { + return tracer.Start(ctx, name, trace.WithAttributes(attrs...)) +} + +// End finishes a span, recording error status when err is non-nil. +func End(span trace.Span, err error) { + if err != nil { + span.SetStatus(codes.Error, err.Error()) + span.RecordError(err) + } + span.End() +}