From 888708b3f20d0b520021cfcc4ae095ea5ddd2b81 Mon Sep 17 00:00:00 2001 From: Kirill Turanskiy Date: Mon, 13 Jul 2026 05:40:28 +0300 Subject: [PATCH] feat(observability): wire production runtime call sites --- .../sol-synthesis-20260713-r1/summary.json | 28 + .../commands.json | 43 + .../go-test-summary.json | 1622 +++++++++++++++++ .../summary.json | 38 + ...-ops-observability-callsites-sol-review.md | 74 + .../evidence/OBS-CALLSITES-R1.tdd.json | 87 + cmd/engram/main.go | 41 +- cmd/engram/observability_contract_test.go | 118 ++ go.mod | 3 +- go.sum | 6 +- .../grpcserver/observability_contract_test.go | 106 ++ internal/grpcserver/server.go | 12 + internal/grpcserver/version_negotiate.go | 8 +- internal/handlers/codeintel/module.go | 3 + .../codeintel/observability_contract_test.go | 77 + internal/handlers/engramcore/grpcpool.go | 6 + .../engramcore/observability_contract_test.go | 67 + internal/handlers/serverevents/bridge.go | 5 + .../observability_contract_test.go | 67 + internal/module/obs/meter.go | 3 +- internal/module/obs/metrics.go | 20 +- internal/module/obs/runtime.go | 15 +- internal/module/obs/runtime_test.go | 36 + .../database_observability_contract_test.go | 64 + internal/worker/service.go | 20 +- scripts/production-smoke/verify-otlp.ps1 | 1 + 26 files changed, 2558 insertions(+), 12 deletions(-) create mode 100644 .agent/reports/evidence/production-ready/observability/sol-synthesis-20260713-r1/summary.json create mode 100644 .agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/commands.json create mode 100644 .agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/go-test-summary.json create mode 100644 .agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/summary.json create mode 100644 .agent/reviews/2026-07-13-ops-observability-callsites-sol-review.md create mode 100644 .agent/specs/ops-observability-call-sites/evidence/OBS-CALLSITES-R1.tdd.json create mode 100644 cmd/engram/observability_contract_test.go create mode 100644 internal/grpcserver/observability_contract_test.go create mode 100644 internal/handlers/codeintel/observability_contract_test.go create mode 100644 internal/handlers/engramcore/observability_contract_test.go create mode 100644 internal/handlers/serverevents/observability_contract_test.go create mode 100644 internal/worker/database_observability_contract_test.go diff --git a/.agent/reports/evidence/production-ready/observability/sol-synthesis-20260713-r1/summary.json b/.agent/reports/evidence/production-ready/observability/sol-synthesis-20260713-r1/summary.json new file mode 100644 index 000000000..b713d4ea9 --- /dev/null +++ b/.agent/reports/evidence/production-ready/observability/sol-synthesis-20260713-r1/summary.json @@ -0,0 +1,28 @@ +{ + "schema_version": 1, + "gate": "observability-otlp", + "started_at_utc": "2026-07-13T02:21:09.7732887+00:00", + "completed_at_utc": "2026-07-13T02:21:12.4795109+00:00", + "command": "go test ./internal/module/obs -count=1 -json", + "exit_code": 0, + "required_tests": [ + "TestInitNoEndpointIsNoop", + "TestInitForService_ExportsDaemonResourceIdentity", + "TestOTLPExportsStableMetricsAndKeepsHeaderOutOfPayload", + "TestOTLPTLSWithExplicitTrustRoot", + "TestCollectorAuthFailureIsBoundedAndSecretFree", + "TestCollectorBackpressureHonorsDeadline", + "TestTransientCollectorFailureRetriesWithinCallerDeadline", + "TestExporterOutageAndTLSMismatchAreBounded", + "TestShutdownFlushesPendingMetric", + "TestEndpointCredentialsAreRejectedWithoutEcho", + "TestConcurrentInitRecordShutdown", + "TestRuntimeOwnershipIsIdempotentAndReinitializable", + "TestRepeatedLifecycleWhileRecording" + ], + "missing_tests": [], + "failed_tests": [], + "process_residue": [], + "container_residue": [], + "verdict": "PASS" +} diff --git a/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/commands.json b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/commands.json new file mode 100644 index 000000000..4ddc7fa53 --- /dev/null +++ b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/commands.json @@ -0,0 +1,43 @@ +[ + { + "name": "critical-go-test", + "executable": "C:\\Program Files\\Go\\bin\\go.exe", + "arguments": [ + "test", + "-tags=critical", + "-json", + "./tests/critical/...", + "-count=1" + ], + "command": "\"C:\\Program Files\\Go\\bin\\go.exe\" test -tags=critical -json ./tests/critical/... -count=1", + "started_at": "2026-07-13T02:33:41.5597214+00:00", + "finished_at": "2026-07-13T02:37:59.9600691+00:00", + "duration_seconds": 258.4, + "exit_code": 0, + "timed_out": false, + "stdout": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test.stdout.jsonl", + "stderr": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test.stderr.log" + }, + { + "name": "critical-json-parser", + "executable": "C:\\Program Files\\PowerShell\\7\\pwsh.exe", + "arguments": [ + "-NoProfile", + "-File", + "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\scripts\\production-gates\\assert-go-test-json.ps1", + "-InputPath", + ".agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test.stdout.jsonl", + "-SummaryPath", + ".agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test-summary.json", + "-FailOnUnexpectedSkip" + ], + "command": "\"C:\\Program Files\\PowerShell\\7\\pwsh.exe\" -NoProfile -File \"D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\scripts\\production-gates\\assert-go-test-json.ps1\" -InputPath \".agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test.stdout.jsonl\" -SummaryPath \".agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test-summary.json\" -FailOnUnexpectedSkip", + "started_at": "2026-07-13T02:37:59.9725318+00:00", + "finished_at": "2026-07-13T02:38:00.6718968+00:00", + "duration_seconds": 0.699, + "exit_code": 0, + "timed_out": false, + "stdout": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\json-parser.stdout.log", + "stderr": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\json-parser.stderr.log" + } +] diff --git a/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/go-test-summary.json b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/go-test-summary.json new file mode 100644 index 000000000..efa715482 --- /dev/null +++ b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/go-test-summary.json @@ -0,0 +1,1622 @@ +{ + "schema_version": 1, + "verdict": "PASS", + "input_path": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test.stdout.jsonl", + "fail_on_unexpected_skip": true, + "allowed_skip_identities": [], + "counts": { + "packages": 3, + "tests": 197, + "passed": 197, + "failed": 0, + "skipped": 0, + "no_tests": 0, + "zero_tests": 0, + "incomplete": 0, + "unexpected_skips": 0, + "malformed_lines": 0 + }, + "packages": [ + { + "package": "github.com/thebtf/engram/tests/critical", + "outcome": "pass", + "elapsed_seconds": 0.383, + "last_output": "ok \tgithub.com/thebtf/engram/tests/critical\t0.383s", + "tests_observed": 8 + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "outcome": "pass", + "elapsed_seconds": 128.568, + "last_output": "ok \tgithub.com/thebtf/engram/tests/critical/recovery\t128.568s", + "tests_observed": 5 + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "outcome": "pass", + "elapsed_seconds": 257.193, + "last_output": "ok \tgithub.com/thebtf/engram/tests/critical/runtime\t257.193s", + "tests_observed": 184 + } + ], + "tests": [ + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier", + "outcome": "pass", + "elapsed_seconds": 0.01, + "last_output": "--- PASS: TestCritical_AuthTwoTier (0.01s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/anti-stub:_stubbed_validator_flips_success_assertions", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/anti-stub:_stubbed_validator_flips_success_assertions (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/gRPC_accepts_dashboard-issued_keycard", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/gRPC_accepts_dashboard-issued_keycard (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/gRPC_accepts_operator_key", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/gRPC_accepts_operator_key (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/gRPC_rejects_garbage_bearer", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/gRPC_rejects_garbage_bearer (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/gRPC_rejects_missing_bearer", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/gRPC_rejects_missing_bearer (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestCritical_AuthTwoTier/HTTP_bearer_arm:_master_+_keycard_accepted,_garbage_rejected", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestCritical_AuthTwoTier/HTTP_bearer_arm:_master_+_keycard_accepted,_garbage_rejected (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical", + "test": "TestSessionStartAndDocumentQueries_DoNotExposeSiblingProjectsOrPaths", + "outcome": "pass", + "elapsed_seconds": 0.27, + "last_output": "--- PASS: TestSessionStartAndDocumentQueries_DoNotExposeSiblingProjectsOrPaths (0.27s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "test": "TestOperatorCanRestorePostgresBackup_RecoversDurableEngramDataAndRejectsUnsafeRestores", + "outcome": "pass", + "elapsed_seconds": 103.24, + "last_output": "--- PASS: TestOperatorCanRestorePostgresBackup_RecoversDurableEngramDataAndRejectsUnsafeRestores (103.24s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "test": "TestOperatorRecoveryMissingDockerPreservesActionableDependencyError", + "outcome": "pass", + "elapsed_seconds": 0.56, + "last_output": "--- PASS: TestOperatorRecoveryMissingDockerPreservesActionableDependencyError (0.56s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "test": "TestOperatorRecoveryScavengesKilledRunWithoutTouchingLiveOwner", + "outcome": "pass", + "elapsed_seconds": 14.66, + "last_output": "--- PASS: TestOperatorRecoveryScavengesKilledRunWithoutTouchingLiveOwner (14.66s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "test": "TestPreV5UpgradeBehavior", + "outcome": "pass", + "elapsed_seconds": 9.8, + "last_output": "--- PASS: TestPreV5UpgradeBehavior (9.80s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/recovery", + "test": "TestPreV5UpgradeFixtureProvenance", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestPreV5UpgradeFixtureProvenance (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard", + "outcome": "pass", + "elapsed_seconds": 40.28, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard (40.28s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs", + "outcome": "pass", + "elapsed_seconds": 5.84, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs (5.84s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v0.0.0", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v0.0.0 (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3", + "outcome": "pass", + "elapsed_seconds": 0.42, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3 (0.42s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3-alpha-1.2", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3-alpha-1.2 (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v6.43.0-rc.1", + "outcome": "pass", + "elapsed_seconds": 0.42, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/accept_v6.43.0-rc.1 (0.42s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_7630312e322e33", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_7630312e322e33 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_763124287072696e7466247b4946537d494e4a454354454429", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_763124287072696e7466247b4946537d494e4a454354454429 (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e30322e33", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e30322e33 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e32", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e32 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3033", + "outcome": "pass", + "elapsed_seconds": 0.44, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3033 (0.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3320", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3320 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332b6275696c64", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332b6275696c64 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332d3031", + "outcome": "pass", + "elapsed_seconds": 0.43, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332d3031 (0.43s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e333b6563686f2d494e4a4543544544", + "outcome": "pass", + "elapsed_seconds": 0.44, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e333b6563686f2d494e4a4543544544 (0.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/movement_before_and_after_old_guard", + "outcome": "pass", + "elapsed_seconds": 1.51, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/movement_before_and_after_old_guard (1.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix", + "outcome": "pass", + "elapsed_seconds": 2.1, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix (2.10s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/all_destinations_absent", + "outcome": "pass", + "elapsed_seconds": 0.53, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/all_destinations_absent (0.53s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/existing_registry_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/existing_registry_mismatch (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/idempotent_exact_protected_tag", + "outcome": "pass", + "elapsed_seconds": 0.53, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/idempotent_exact_protected_tag (0.53s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/same_release_different_image", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/registry_compare-before-write_matrix/same_release_different_image (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/repository_controlled_single_writer", + "outcome": "pass", + "elapsed_seconds": 0.04, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/repository_controlled_single_writer (0.04s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix", + "outcome": "pass", + "elapsed_seconds": 10.55, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix (10.55s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_digest_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_digest_mismatch (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_expired", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_expired (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_from_another_workflow_run", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_from_another_workflow_run (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_id_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_id_mismatch (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_duplicate_expected_name", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_duplicate_expected_name (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_extra_artifact", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_extra_artifact (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_missing_artifact", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_missing_artifact (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_one_exact_same-run_immutable_artifact", + "outcome": "pass", + "elapsed_seconds": 0.41, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/metadata_one_exact_same-run_immutable_artifact (0.41s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_archive_checksum_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.71, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_archive_checksum_mismatch (0.71s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_exact_regular-file_envelope", + "outcome": "pass", + "elapsed_seconds": 0.78, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_exact_regular-file_envelope (0.78s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_extra_file", + "outcome": "pass", + "elapsed_seconds": 0.66, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_extra_file (0.66s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_immutable_commit_identity", + "outcome": "pass", + "elapsed_seconds": 0.83, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_immutable_commit_identity (0.83s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_link_inside_outer_image_archive", + "outcome": "pass", + "elapsed_seconds": 0.78, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_link_inside_outer_image_archive (0.78s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_manifest_commit_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.77, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_manifest_commit_mismatch (0.77s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_in_bundle", + "outcome": "pass", + "elapsed_seconds": 0.71, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_in_bundle (0.71s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_inside_image_archive", + "outcome": "pass", + "elapsed_seconds": 0.8, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_inside_image_archive (0.80s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_symlink_entry", + "outcome": "pass", + "elapsed_seconds": 0.76, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/same-run_immutable_artifact_bridge_matrix/payload_symlink_entry (0.76s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix", + "outcome": "pass", + "elapsed_seconds": 5.06, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix (5.06s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/bypass", + "outcome": "pass", + "elapsed_seconds": 0.53, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/bypass (0.53s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/disabled", + "outcome": "pass", + "elapsed_seconds": 0.56, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/disabled (0.56s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/duplicate", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/duplicate (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/exact_active_immutable_namespace", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/exact_active_immutable_namespace (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/exclusion", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/exclusion (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing_deletion", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing_deletion (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing_non_fast_forward", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/missing_non_fast_forward (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/wrong_include", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/wrong_include (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/wrong_target", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/tag_ruleset_positive_and_negative_matrix/wrong_target (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix", + "outcome": "pass", + "elapsed_seconds": 14.68, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix (14.68s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/additional_non-authority_status_check_is_allowed", + "outcome": "pass", + "elapsed_seconds": 0.84, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/additional_non-authority_status_check_is_allowed (0.84s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_duplicate_context", + "outcome": "pass", + "elapsed_seconds": 0.66, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_duplicate_context (0.66s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_missing_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_missing_integration_id (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_string_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_string_integration_id (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_wrong_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.56, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_wrong_integration_id (0.56s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/event_action_is_not_completed", + "outcome": "pass", + "elapsed_seconds": 0.61, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/event_action_is_not_completed (0.61s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/event_api_sha_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/event_api_sha_mismatch (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/exact_protected_release_provenance", + "outcome": "pass", + "elapsed_seconds": 0.6, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/exact_protected_release_provenance (0.60s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/fork_head_repository", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/fork_head_repository (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/hostile_shell-like_tag", + "outcome": "pass", + "elapsed_seconds": 0.43, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/hostile_shell-like_tag (0.43s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/inactive_trusted_workflow", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/inactive_trusted_workflow (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/manual_dispatch_spoof", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/manual_dispatch_spoof (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/missing_immutable_tag_ruleset", + "outcome": "pass", + "elapsed_seconds": 0.56, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/missing_immutable_tag_ruleset (0.56s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/missing_protected_main_ruleset", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/missing_protected_main_ruleset (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/noncanonical_tag", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/noncanonical_tag (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_always", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_always (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_duplicated", + "outcome": "pass", + "elapsed_seconds": 0.62, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_duplicated (0.62s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_missing", + "outcome": "pass", + "elapsed_seconds": 0.55, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_missing (0.55s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/run_failed", + "outcome": "pass", + "elapsed_seconds": 0.58, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/run_failed (0.58s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/run_id_event_api_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.58, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/run_id_event_api_mismatch (0.58s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/tag_commit_outside_protected_main", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/tag_commit_outside_protected_main (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/tag_peel_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/tag_peel_mismatch (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_output_rejects_symlink_or_reparse_escape", + "outcome": "pass", + "elapsed_seconds": 1.12, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_output_rejects_symlink_or_reparse_escape (1.12s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_workflow_id_spoof", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_workflow_id_spoof (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_id_event_api_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_id_event_api_mismatch (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_path_spoof", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_path_spoof (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/cancelled_tag_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/cancelled_tag_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/failed_tag_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/failed_tag_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/hostile_pull_request_lookalike_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/hostile_pull_request_lookalike_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/hostile_tag-shaped_push_still_fails_closed_in_preflight", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/hostile_tag-shaped_push_still_fails_closed_in_preflight (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_main_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_main_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_pull_request_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_pull_request_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_tag-shaped_push_enters", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestDockerReleaseRefFreshnessGuard/workflow_run_publisher_selector_matrix/successful_tag-shaped_push_enters (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestOperatorConsoleRuntimeTargetContract", + "outcome": "pass", + "elapsed_seconds": 84.03, + "last_output": "--- PASS: TestOperatorConsoleRuntimeTargetContract (84.03s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestOperatorConsoleRuntimeTargetContract/backend_error-200_never_becomes_healthy", + "outcome": "pass", + "elapsed_seconds": 5.05, + "last_output": "--- PASS: TestOperatorConsoleRuntimeTargetContract/backend_error-200_never_becomes_healthy (5.05s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestOperatorConsoleRuntimeTargetContract/backend_malformed_never_becomes_healthy", + "outcome": "pass", + "elapsed_seconds": 4.8, + "last_output": "--- PASS: TestOperatorConsoleRuntimeTargetContract/backend_malformed_never_becomes_healthy (4.80s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestOperatorConsoleRuntimeTargetContract/backend_root-only_never_becomes_healthy", + "outcome": "pass", + "elapsed_seconds": 4.78, + "last_output": "--- PASS: TestOperatorConsoleRuntimeTargetContract/backend_root-only_never_becomes_healthy (4.78s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestOperatorConsoleRuntimeTargetContract/backend_timeout_never_becomes_healthy", + "outcome": "pass", + "elapsed_seconds": 4.75, + "last_output": "--- PASS: TestOperatorConsoleRuntimeTargetContract/backend_timeout_never_becomes_healthy (4.75s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestPostgresImageContract", + "outcome": "pass", + "elapsed_seconds": 50.15, + "last_output": "--- PASS: TestPostgresImageContract (50.15s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestPostgresImageContract/tmpfs-only_data_is_not_durable", + "outcome": "pass", + "elapsed_seconds": 15.53, + "last_output": "--- PASS: TestPostgresImageContract/tmpfs-only_data_is_not_durable (15.53s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestPostgresImageContract/wrong_locale_never_becomes_ready", + "outcome": "pass", + "elapsed_seconds": 1.01, + "last_output": "--- PASS: TestPostgresImageContract/wrong_locale_never_becomes_ready (1.01s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract", + "outcome": "pass", + "elapsed_seconds": 82.14, + "last_output": "--- PASS: TestServerImageContract (82.14s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/absent-volume_fails_closed", + "outcome": "pass", + "elapsed_seconds": 3.7, + "last_output": "--- PASS: TestServerImageContract/absent-volume_fails_closed (3.70s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/empty-home_fails_closed", + "outcome": "pass", + "elapsed_seconds": 4.09, + "last_output": "--- PASS: TestServerImageContract/empty-home_fails_closed (4.09s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard", + "outcome": "pass", + "elapsed_seconds": 39.18, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard (39.18s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs", + "outcome": "pass", + "elapsed_seconds": 6.03, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs (6.03s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v0.0.0", + "outcome": "pass", + "elapsed_seconds": 0.44, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v0.0.0 (0.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3", + "outcome": "pass", + "elapsed_seconds": 0.43, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3 (0.43s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3-alpha-1.2", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v1.2.3-alpha-1.2 (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v6.43.0-rc.1", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/accept_v6.43.0-rc.1 (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_7630312e322e33", + "outcome": "pass", + "elapsed_seconds": 0.54, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_7630312e322e33 (0.54s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_763124287072696e7466247b4946537d494e4a454354454429", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_763124287072696e7466247b4946537d494e4a454354454429 (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e30322e33", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e30322e33 (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e32", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e32 (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3033", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3033 (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3320", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e3320 (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332b6275696c64", + "outcome": "pass", + "elapsed_seconds": 0.45, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332b6275696c64 (0.45s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332d3031", + "outcome": "pass", + "elapsed_seconds": 0.44, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e332d3031 (0.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e333b6563686f2d494e4a4543544544", + "outcome": "pass", + "elapsed_seconds": 0.44, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/canonical_release_version_and_hostile_Git_refs/reject_76312e322e333b6563686f2d494e4a4543544544 (0.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/movement_before_and_after_old_guard", + "outcome": "pass", + "elapsed_seconds": 1.36, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/movement_before_and_after_old_guard (1.36s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix", + "outcome": "pass", + "elapsed_seconds": 2.02, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix (2.02s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/all_destinations_absent", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/all_destinations_absent (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/existing_registry_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/existing_registry_mismatch (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/idempotent_exact_protected_tag", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/idempotent_exact_protected_tag (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/same_release_different_image", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/registry_compare-before-write_matrix/same_release_different_image (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/repository_controlled_single_writer", + "outcome": "pass", + "elapsed_seconds": 0.05, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/repository_controlled_single_writer (0.05s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix", + "outcome": "pass", + "elapsed_seconds": 10.44, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix (10.44s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_digest_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_digest_mismatch (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_expired", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_expired (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_from_another_workflow_run", + "outcome": "pass", + "elapsed_seconds": 0.54, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_from_another_workflow_run (0.54s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_id_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_artifact_id_mismatch (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_duplicate_expected_name", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_duplicate_expected_name (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_extra_artifact", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_extra_artifact (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_missing_artifact", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_missing_artifact (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_one_exact_same-run_immutable_artifact", + "outcome": "pass", + "elapsed_seconds": 0.41, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/metadata_one_exact_same-run_immutable_artifact (0.41s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_archive_checksum_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.7, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_archive_checksum_mismatch (0.70s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_exact_regular-file_envelope", + "outcome": "pass", + "elapsed_seconds": 0.88, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_exact_regular-file_envelope (0.88s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_extra_file", + "outcome": "pass", + "elapsed_seconds": 0.69, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_extra_file (0.69s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_immutable_commit_identity", + "outcome": "pass", + "elapsed_seconds": 0.76, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_immutable_commit_identity (0.76s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_link_inside_outer_image_archive", + "outcome": "pass", + "elapsed_seconds": 0.77, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_link_inside_outer_image_archive (0.77s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_manifest_commit_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.73, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_manifest_commit_mismatch (0.73s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_in_bundle", + "outcome": "pass", + "elapsed_seconds": 0.71, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_in_bundle (0.71s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_inside_image_archive", + "outcome": "pass", + "elapsed_seconds": 0.71, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_path_traversal_inside_image_archive (0.71s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_symlink_entry", + "outcome": "pass", + "elapsed_seconds": 0.64, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/same-run_immutable_artifact_bridge_matrix/payload_symlink_entry (0.64s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix", + "outcome": "pass", + "elapsed_seconds": 4.66, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix (4.66s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/bypass", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/bypass (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/disabled", + "outcome": "pass", + "elapsed_seconds": 0.46, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/disabled (0.46s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/duplicate", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/duplicate (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/exact_active_immutable_namespace", + "outcome": "pass", + "elapsed_seconds": 0.41, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/exact_active_immutable_namespace (0.41s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/exclusion", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/exclusion (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing_deletion", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing_deletion (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing_non_fast_forward", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/missing_non_fast_forward (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/wrong_include", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/wrong_include (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/wrong_target", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/tag_ruleset_positive_and_negative_matrix/wrong_target (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix", + "outcome": "pass", + "elapsed_seconds": 14.14, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix (14.14s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/additional_non-authority_status_check_is_allowed", + "outcome": "pass", + "elapsed_seconds": 0.58, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/additional_non-authority_status_check_is_allowed (0.58s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_duplicate_context", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_duplicate_context (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_missing_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.52, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_missing_integration_id (0.52s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_string_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_string_integration_id (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_wrong_integration_id", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/authority_guard_wrong_integration_id (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/event_action_is_not_completed", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/event_action_is_not_completed (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/event_api_sha_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.68, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/event_api_sha_mismatch (0.68s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/exact_protected_release_provenance", + "outcome": "pass", + "elapsed_seconds": 0.58, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/exact_protected_release_provenance (0.58s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/fork_head_repository", + "outcome": "pass", + "elapsed_seconds": 0.55, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/fork_head_repository (0.55s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/hostile_shell-like_tag", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/hostile_shell-like_tag (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/inactive_trusted_workflow", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/inactive_trusted_workflow (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/manual_dispatch_spoof", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/manual_dispatch_spoof (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/missing_immutable_tag_ruleset", + "outcome": "pass", + "elapsed_seconds": 0.56, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/missing_immutable_tag_ruleset (0.56s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/missing_protected_main_ruleset", + "outcome": "pass", + "elapsed_seconds": 0.51, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/missing_protected_main_ruleset (0.51s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/noncanonical_tag", + "outcome": "pass", + "elapsed_seconds": 0.49, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/noncanonical_tag (0.49s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_always", + "outcome": "pass", + "elapsed_seconds": 0.55, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_always (0.55s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_duplicated", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_duplicated (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_missing", + "outcome": "pass", + "elapsed_seconds": 0.5, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/recovery_bypass_missing (0.50s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/run_failed", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/run_failed (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/run_id_event_api_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/run_id_event_api_mismatch (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/tag_commit_outside_protected_main", + "outcome": "pass", + "elapsed_seconds": 0.54, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/tag_commit_outside_protected_main (0.54s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/tag_peel_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.6, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/tag_peel_mismatch (0.60s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_output_rejects_symlink_or_reparse_escape", + "outcome": "pass", + "elapsed_seconds": 1.14, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_output_rejects_symlink_or_reparse_escape (1.14s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_workflow_id_spoof", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/trusted_workflow_id_spoof (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_id_event_api_mismatch", + "outcome": "pass", + "elapsed_seconds": 0.47, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_id_event_api_mismatch (0.47s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_path_spoof", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_provenance_and_protected-main_authority_matrix/workflow_path_spoof (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/cancelled_tag_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/cancelled_tag_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/failed_tag_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/failed_tag_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/hostile_pull_request_lookalike_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/hostile_pull_request_lookalike_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/hostile_tag-shaped_push_still_fails_closed_in_preflight", + "outcome": "pass", + "elapsed_seconds": 0.48, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/hostile_tag-shaped_push_still_fails_closed_in_preflight (0.48s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_main_push_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_main_push_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_pull_request_skips", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_pull_request_skips (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_tag-shaped_push_enters", + "outcome": "pass", + "elapsed_seconds": 0.0, + "last_output": "--- PASS: TestServerImageContract/release_ref_freshness_guard/workflow_run_publisher_selector_matrix/successful_tag-shaped_push_enters (0.00s)", + "skip_allowed": false + }, + { + "package": "github.com/thebtf/engram/tests/critical/runtime", + "test": "TestServerImageContract/root-owned-volume_fails_closed", + "outcome": "pass", + "elapsed_seconds": 5.03, + "last_output": "--- PASS: TestServerImageContract/root-owned-volume_fails_closed (5.03s)", + "skip_allowed": false + } + ], + "unexpected_skips": [], + "errors": [] +} diff --git a/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/summary.json b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/summary.json new file mode 100644 index 000000000..6d33aaa03 --- /dev/null +++ b/.agent/reports/evidence/production-ready/release-gates-foundation/critical-suite-runner/sol-synthesis-20260713-observability-r2/summary.json @@ -0,0 +1,38 @@ +{ + "schema_version": 1, + "gate": "critical-suite", + "run_id": "sol-synthesis-20260713-observability-r2", + "started_at": "2026-07-13T02:33:41.4352981+00:00", + "finished_at": "2026-07-13T02:38:00.6881133+00:00", + "duration_seconds": 259.253, + "verdict": "PASS", + "config": { + "path": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\critical-suite.config.yaml", + "sha256": "3F1DF7569119B95A05FD9D61FC0479A79150747B38EFB5A10C45AF511E18268A", + "command": "go test -tags=critical -json ./tests/critical/... -count=1" + }, + "run_pattern": "", + "allowed_skip_identities": null, + "matched_test_files": 6, + "matched_go_files": 6, + "go_test_exit": 0, + "json_parser_exit": 0, + "json_summary": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\go-test-summary.json", + "counts": { + "packages": 3, + "tests": 197, + "passed": 197, + "failed": 0, + "skipped": 0, + "no_tests": 0, + "zero_tests": 0, + "incomplete": 0, + "unexpected_skips": 0, + "malformed_lines": 0 + }, + "child_commands": 2, + "nonzero_child_commands": 0, + "commands": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2\\commands.json", + "errors": [], + "artifact_directory": "D:\\Dev\\engram\\.agent\\worktrees\\prc-sol-synthesis-r1\\.agent\\reports\\evidence\\production-ready\\release-gates-foundation\\critical-suite-runner\\sol-synthesis-20260713-observability-r2" +} diff --git a/.agent/reviews/2026-07-13-ops-observability-callsites-sol-review.md b/.agent/reviews/2026-07-13-ops-observability-callsites-sol-review.md new file mode 100644 index 000000000..c3811d61c --- /dev/null +++ b/.agent/reviews/2026-07-13-ops-observability-callsites-sol-review.md @@ -0,0 +1,74 @@ +# OPS observability call-sites — root post-review + +Date: 2026-07-13 +Candidate: `work/prc-sol-synthesis-r1` +Scope: real gRPC/auth/version/database/index call sites and daemon exporter lifecycle + +## Outcome + +Root changed-code review: **PASS**. +Independent blind judge: **PENDING (review infrastructure unavailable)**. + +The patch closes the previously documented PR-7 call-site blocker without +restoring any v5-demolished path. It uses the supported `otelgrpc` v0.68.0 +StatsHandler integration that exactly matches the repository's OTel v1.43.0 and +gRPC v1.80.0 dependency line. Runtime event attributes are fixed program values; +no token, DSN, user/project/root/run ID, or error text enters metric attributes. + +## Review findings resolved + +1. **Shim telemetry impersonated the daemon.** The first implementation would + initialize `service.name=engram-daemon` in every short-lived client/shim + process. A new RED proved the issue; initialization is now restricted to the + actual `--muxcore-daemon` process. Count-5 and race count-3 are green. +2. **Resource assertion was formatting-sensitive.** Under `-race`, protojson + inserted spaces and the string assertion failed despite the correct payload. + Tests now inspect OTLP protobuf resource attributes structurally. +3. **Graceful-restart hard deadline could be exceeded.** A fresh 5-second + telemetry shutdown context could extend the documented 60-second restart + budget. The graceful path now derives the flush timeout from the existing + restart context; ordinary shutdown retains its own bounded 5-second context. + +## Behavioral-edge review + +- Server transport: a real bufconn RPC emits `rpc.server.call.duration`. +- Auth: missing credentials produce `Unauthenticated` and the bounded + `auth/missing_credentials` event; credential material is absent. +- Version: an authenticated cross-major request returns `compatible=false` and + emits `client_version/incompatible`. +- Both daemon gRPC connection classes execute their production dialers and emit + `rpc.client.call.duration`. +- Database failure is observed at the `gorm.NewStore` production seam without + changing error propagation or exposing the DSN/driver error. +- Background indexing retains its existing terminal state and logs while adding + only `index/run_error` or `index/panic` metrics. +- The server remains `engram-server`; only the persistent local daemon is + `engram-daemon`. Client/shim mode owns no exporter. +- Telemetry shuts down only after the bridge/modules/service have stopped, so + StatsHandlers do not record through a closed provider on the normal paths. + +## Evidence + +- TDD and production mutation proof: + `.agent/specs/ops-observability-call-sites/evidence/OBS-CALLSITES-R1.tdd.json` +- Focused tests: PASS count=5. +- Focused race: PASS count=3 across all changed packages. +- Full changed packages: PASS. +- `go test -p=1 ./... -count=1`: PASS. +- `go vet ./...`: PASS. +- `go build ./...`: PASS. +- OTLP smoke: PASS, no missing required tests. +- Critical suite first run: truthful FAIL 196/197 because `DATABASE_DSN` was + unset and the isolation test failed closed before execution. +- Dedicated `engram_test` database was verified idle; focused isolation test + passed; critical rerun `sol-synthesis-20260713-observability-r2`: PASS 197/197, + 0 failed, 0 skipped, parser exit 0, test exit 0. + +## Independent review status + +AIMux health reported Loom unavailable because its SQLite session store was not +initialized. The review task returned non-retryable `CapabilityMismatch`. +Its solo `peer_review` route produced keyword-only objections unrelated to the +code and is explicitly rejected as blind-judge evidence. This safe-point may be +committed and pushed for PR-based independent review, but it is not the final +blind-judge acceptance record. diff --git a/.agent/specs/ops-observability-call-sites/evidence/OBS-CALLSITES-R1.tdd.json b/.agent/specs/ops-observability-call-sites/evidence/OBS-CALLSITES-R1.tdd.json new file mode 100644 index 000000000..c7777eb8c --- /dev/null +++ b/.agent/specs/ops-observability-call-sites/evidence/OBS-CALLSITES-R1.tdd.json @@ -0,0 +1,87 @@ +{ + "task_id": "OBS-CALLSITES-R1", + "recorded_at": "2026-07-13T02:20:02.4095657Z", + "method": "RED-GREEN-REFACTOR plus production-mutation Prove-It", + "invalidated_attempts": [ + { + "reason": "The first server transport assertion used the removed legacy metric name rpc.server.duration. All production edits from that attempt were reverted before the valid cycle began.", + "disposition": "excluded from evidence" + } + ], + "cycles": [ + { + "surface": "gRPC server transport, auth rejection, and version negotiation", + "test": "TestGRPCObservability_EmitsTransportAuthAndVersionMetrics", + "red_at": "2026-07-13T01:58:26.0261994Z", + "red": "production grpc.Server emitted no rpc.server.call.duration metric", + "green": "ready-made otelgrpc server StatsHandler plus bounded auth/client_version outcomes" + }, + { + "surface": "daemon engramcore gRPC client pool", + "test": "TestDialGRPC_EmitsClientTransportMetric", + "red_at": "2026-07-13T02:01:04.1061959Z", + "red": "production dialGRPC emitted no rpc.client.call.duration metric", + "green": "ready-made otelgrpc client StatsHandler" + }, + { + "surface": "daemon persistent server-events gRPC client", + "test": "TestBridgeDialGRPC_EmitsClientTransportMetric", + "red_at": "2026-07-13T02:04:45.8301567Z", + "red": "production Bridge.dialGRPC emitted no rpc.client.call.duration metric", + "green": "ready-made otelgrpc client StatsHandler" + }, + { + "surface": "PostgreSQL startup", + "test": "TestOpenObservedStore_RecordsInitializationFailure", + "red_at": "2026-07-13T02:06:05.3805569Z", + "red": "production database initialization had no bounded diagnostic event", + "green": "database initialized/initialization_error outcomes without DSN or driver error labels" + }, + { + "surface": "background code-index run", + "test": "TestCodebaseIndex_RecordsBackgroundRunFailure", + "red_at": "2026-07-13T02:07:49.6142498Z", + "red": "production background failure updated only in-memory state and logs", + "green": "index run_error and panic outcomes without project, root, run ID, or error text labels" + }, + { + "surface": "multi-process OTel resource identity", + "test": "TestInitForService_ExportsDaemonResourceIdentity", + "red_at": "2026-07-13T02:09:01.0272375Z", + "red": "InitForService API did not exist, so daemon and server could not be separated by service.name", + "green": "engram-server compatibility wrapper and explicit engram-daemon resource identity" + }, + { + "surface": "local daemon exporter lifecycle", + "test": "TestInitDaemonObservability_ExportsDaemonResourceIdentity", + "red_at": "2026-07-13T02:10:40.6639840Z", + "red": "local daemon had no exporter lifecycle call site", + "green": "daemon-mode init before module lifecycle and bounded shutdown on normal, error, and graceful-restart paths" + }, + { + "surface": "short-lived client/shim isolation", + "test": "TestInitDaemonObservability_ShimModeDoesNotStartExporter", + "red": "client/shim mode incorrectly started and identified an OTLP exporter as engram-daemon", + "green": "only --muxcore-daemon mode owns the daemon exporter" + } + ], + "prove_it": { + "production_mutations": [ + "removed each otelgrpc server/client StatsHandler", + "changed database and index outcomes", + "forced daemon resource identity back to engram-server", + "changed exercised missing-auth and incompatible-version outcomes" + ], + "result": "Every corresponding contract test failed for its intended reason; all mutations were reverted and the focused suite returned green. An unexercised missing-metadata branch mutation did not fail and is not counted as proof." + }, + "verification": { + "focused_count": "PASS, count=5", + "race": "PASS, count=3 across all seven changed packages", + "coverage_note": "Focused package coverage is informational and ranges from 0.2% to 64.7% because several target packages are large; every changed integration seam is executed and has mutation proof.", + "external_sources": [ + "Context7 official OpenTelemetry Go and contrib documentation", + "Parallel official OpenTelemetry RPC metrics specification and pkg.go.dev module metadata", + "Tavily attempted but unavailable because OAuth authorization is required" + ] + } +} diff --git a/cmd/engram/main.go b/cmd/engram/main.go index 16828bea3..8fb1741ea 100644 --- a/cmd/engram/main.go +++ b/cmd/engram/main.go @@ -32,6 +32,7 @@ import ( "github.com/thebtf/engram/internal/module" "github.com/thebtf/engram/internal/module/dispatcher" "github.com/thebtf/engram/internal/module/lifecycle" + "github.com/thebtf/engram/internal/module/obs" "github.com/thebtf/engram/internal/module/registry" "github.com/thebtf/engram/internal/version" muxcontrol "github.com/thebtf/mcp-mux/muxcore/control" @@ -261,11 +262,19 @@ func main() { } logger := newRootLogger() + telemetryCtx, telemetryCancel := context.WithTimeout(context.Background(), 5*time.Second) + telemetry, err := initDaemonObservability(telemetryCtx) + telemetryCancel() + if err != nil { + logger.Error("observability initialization failed", "error", err) + os.Exit(1) + } // --- Framework wiring ------------------------------------------------ reg := registry.New() if err := registerModules(reg); err != nil { logger.Error("module registration failed", "error", err) + shutdownDaemonObservability(logger, telemetry) os.Exit(1) } reg.Freeze() @@ -287,6 +296,7 @@ func main() { if err := pipeline.Start(initCtx, depsProviderFor(logger, daemonCtx)); err != nil { initCancel() logger.Error("lifecycle Start failed", "error", err) + shutdownDaemonObservability(logger, telemetry) os.Exit(1) } initCancel() @@ -308,7 +318,7 @@ func main() { func(cmd string) string { switch cmd { case "graceful-restart": - go handleGracefulRestart(logger, pipeline, disp, filepath.Join(dd, "modules")) + go handleGracefulRestart(logger, pipeline, disp, filepath.Join(dd, "modules"), telemetry) return "ACK" default: return "ERR unknown command" @@ -350,6 +360,7 @@ func main() { if err != nil { logger.Error("engine.New failed", "error", err) _ = pipeline.ShutdownAll(daemonCtx) + shutdownDaemonObservability(logger, telemetry) os.Exit(1) } @@ -376,6 +387,7 @@ func main() { logger.Error("engine.Run terminated", "error", err) sevBridge.Stop() _ = pipeline.ShutdownAll(daemonCtx) + shutdownDaemonObservability(logger, telemetry) os.Exit(1) } @@ -384,6 +396,7 @@ func main() { if err := pipeline.ShutdownAll(daemonCtx); err != nil { logger.Error("lifecycle Shutdown error", "error", err) } + shutdownDaemonObservability(logger, telemetry) } // handleGracefulRestart executes the full graceful-restart sequence: @@ -409,6 +422,7 @@ func handleGracefulRestart( pipeline *lifecycle.Pipeline, disp *dispatcher.Dispatcher, storageDir string, + telemetry *obs.Runtime, ) { const hardDeadline = 60 * time.Second @@ -431,6 +445,7 @@ func handleGracefulRestart( if err := pipeline.ShutdownAll(ctx); err != nil { logger.Warn("ShutdownAll error (continuing)", "error", err) } + shutdownDaemonObservabilityWithin(ctx, logger, telemetry) // Phase 4 — Find new binary. currentExe, err := os.Executable() @@ -475,6 +490,30 @@ func handleGracefulRestart( } } +func initDaemonObservability(ctx context.Context) (*obs.Runtime, error) { + if !isMuxcoreDaemonMode() { + return &obs.Runtime{}, nil + } + runtime, err := obs.InitForService(ctx, "engram-daemon", daemonVersion) + if err != nil { + return nil, err + } + obs.RecordRuntimeEvent(context.Background(), "startup", "started") + return runtime, nil +} + +func shutdownDaemonObservability(logger *slog.Logger, telemetry *obs.Runtime) { + shutdownDaemonObservabilityWithin(context.Background(), logger, telemetry) +} + +func shutdownDaemonObservabilityWithin(parent context.Context, logger *slog.Logger, telemetry *obs.Runtime) { + ctx, cancel := context.WithTimeout(parent, 5*time.Second) + defer cancel() + if err := telemetry.Shutdown(ctx); err != nil { + logger.Error("observability shutdown failed; check collector availability") + } +} + // newRootLogger returns a JSON-format slog logger by default, or a text // logger when ENGRAM_LOG_FORMAT=text is set. Structured by design decision // D12 and NFR-4 (structured logging). diff --git a/cmd/engram/observability_contract_test.go b/cmd/engram/observability_contract_test.go new file mode 100644 index 000000000..7600bd35a --- /dev/null +++ b/cmd/engram/observability_contract_test.go @@ -0,0 +1,118 @@ +package main + +import ( + "context" + "net" + "os" + "sync" + "testing" + "time" + + collector "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + "google.golang.org/grpc" +) + +type daemonMetricReceiver struct { + collector.UnimplementedMetricsServiceServer + + mu sync.Mutex + requests []*collector.ExportMetricsServiceRequest +} + +func (r *daemonMetricReceiver) Export(_ context.Context, request *collector.ExportMetricsServiceRequest) (*collector.ExportMetricsServiceResponse, error) { + r.mu.Lock() + r.requests = append(r.requests, request) + r.mu.Unlock() + return &collector.ExportMetricsServiceResponse{}, nil +} + +func (r *daemonMetricReceiver) snapshot() []*collector.ExportMetricsServiceRequest { + r.mu.Lock() + defer r.mu.Unlock() + return append([]*collector.ExportMetricsServiceRequest(nil), r.requests...) +} + +func TestInitDaemonObservability_ExportsDaemonResourceIdentity(t *testing.T) { + previousArgs := os.Args + os.Args = []string{"engram", muxcoreDaemonFlag} + t.Cleanup(func() { os.Args = previousArgs }) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + receiver := &daemonMetricReceiver{} + server := grpc.NewServer() + collector.RegisterMetricsServiceServer(server, receiver) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://"+listener.Addr().String()) + runtime, err := initDaemonObservability(context.Background()) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := runtime.Shutdown(ctx); err != nil { + t.Fatal(err) + } + + requests := receiver.snapshot() + if len(requests) == 0 { + t.Fatal("collector received no daemon startup metric") + } + if !daemonRequestHasResourceAttribute(requests[0], "service.name", "engram-daemon") { + t.Fatalf("daemon telemetry has the wrong resource identity: %v", requests[0]) + } +} + +func TestInitDaemonObservability_ShimModeDoesNotStartExporter(t *testing.T) { + previousArgs := os.Args + os.Args = []string{"engram"} + t.Cleanup(func() { os.Args = previousArgs }) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + receiver := &daemonMetricReceiver{} + server := grpc.NewServer() + collector.RegisterMetricsServiceServer(server, receiver) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + _ = listener.Close() + }) + + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", "http://"+listener.Addr().String()) + runtime, err := initDaemonObservability(context.Background()) + if err != nil { + t.Fatal(err) + } + if runtime.Enabled() { + t.Fatal("short-lived client/shim process must not start a daemon OTLP exporter") + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatal(err) + } + if len(receiver.snapshot()) != 0 { + t.Fatal("client/shim process exported daemon telemetry") + } +} + +func daemonRequestHasResourceAttribute(request *collector.ExportMetricsServiceRequest, key, want string) bool { + for _, resourceMetrics := range request.GetResourceMetrics() { + for _, attribute := range resourceMetrics.GetResource().GetAttributes() { + if attribute.GetKey() == key && attribute.GetValue().GetStringValue() == want { + return true + } + } + } + return false +} diff --git a/go.mod b/go.mod index efaffe254..7b20672c1 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/swaggo/swag v1.16.6 github.com/thebtf/aimux/loom v0.1.0 github.com/thebtf/mcp-mux/muxcore v0.26.1 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 go.opentelemetry.io/otel v1.43.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 go.opentelemetry.io/otel/metric v1.43.0 @@ -27,7 +28,7 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/crypto v0.52.0 golang.org/x/sync v0.20.0 - google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index a47248d29..31ddf5a4b 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaing github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.43.0 h1:8UQVDcZxOJLtX6gxtDt3vY2WTgvZqMQRzjsqiIHQdkc= @@ -172,8 +174,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9 h1:VPWxll4HlMw1Vs/qXtN7BvhZqsS9cdAittCNvVENElA= google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:7QBABkRtR8z+TEnmXTqIqwJLlzrZKVfAUm7tY3yGv0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9 h1:m8qni9SQFH0tJc1X0vmnpw/0t+AImlSvp30sEupozUg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260401024825-9d38bb4040a9/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d h1:wT2n40TBqFY6wiwazVK9/iTWbsQrgk5ZfCSVFLO9LQA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= diff --git a/internal/grpcserver/observability_contract_test.go b/internal/grpcserver/observability_contract_test.go new file mode 100644 index 000000000..313622bf0 --- /dev/null +++ b/internal/grpcserver/observability_contract_test.go @@ -0,0 +1,106 @@ +package grpcserver + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + "github.com/thebtf/engram/internal/auth" + "github.com/thebtf/engram/internal/module/obs" + pb "github.com/thebtf/engram/proto/engram/v1" +) + +func TestGRPCObservability_EmitsTransportAuthAndVersionMetrics(t *testing.T) { + previousProvider := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + obs.ResetInstrumentsForTesting() + t.Cleanup(func() { + obs.ResetInstrumentsForTesting() + otel.SetMeterProvider(previousProvider) + _ = provider.Shutdown(context.Background()) + }) + + listener := bufconn.Listen(1 << 20) + grpcServer, _ := New( + staticMCPHandler{serverName: "engram", serverVersion: "v5.0.0"}, + auth.NewValidator("master-secret", &stubReader{}), + ) + t.Cleanup(grpcServer.Stop) + go func() { _ = grpcServer.Serve(listener) }() + + connection, err := grpc.NewClient( + "passthrough:///engram-observability", + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { return listener.Dial() }), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + t.Cleanup(func() { _ = connection.Close() }) + client := pb.NewEngramServiceClient(connection) + + _, err = client.NegotiateVersion(context.Background(), &pb.NegotiateVersionRequest{ClientVersion: "v4.9.9"}) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + + authed := metadata.AppendToOutgoingContext(context.Background(), "authorization", "Bearer master-secret") + response, err := client.NegotiateVersion(authed, &pb.NegotiateVersionRequest{ClientVersion: "v4.9.9"}) + require.NoError(t, err) + require.False(t, response.GetCompatible()) + + var collected metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &collected)) + require.True(t, hasMetric(collected, "rpc.server.call.duration"), "real gRPC calls must emit the ready-made otelgrpc server metric") + require.True(t, hasRuntimeEvent(collected, "auth", "missing_credentials"), "auth rejection must be diagnosable without recording credential data") + require.True(t, hasRuntimeEvent(collected, "client_version", "incompatible"), "incompatible clients must be diagnosable with a bounded outcome") +} + +func hasMetric(collected metricdata.ResourceMetrics, name string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name == name { + return true + } + } + } + return false +} + +func hasRuntimeEvent(collected metricdata.ResourceMetrics, component, outcome string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name != "engram_runtime_events_total" { + continue + } + sum, ok := metric.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + for _, point := range sum.DataPoints { + if attributeValue(point.Attributes, "component") == component && attributeValue(point.Attributes, "outcome") == outcome { + return true + } + } + } + } + return false +} + +func attributeValue(set attribute.Set, key string) string { + value, ok := set.Value(attribute.Key(key)) + if !ok { + return "" + } + return value.AsString() +} diff --git a/internal/grpcserver/server.go b/internal/grpcserver/server.go index 74021c18f..f1642ee42 100644 --- a/internal/grpcserver/server.go +++ b/internal/grpcserver/server.go @@ -6,6 +6,7 @@ import ( "strings" "sync" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/genproto/googleapis/rpc/errdetails" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -16,6 +17,7 @@ import ( "github.com/thebtf/engram/internal/auth" engramgorm "github.com/thebtf/engram/internal/db/gorm" "github.com/thebtf/engram/internal/mcp" + "github.com/thebtf/engram/internal/module/obs" "github.com/thebtf/engram/internal/worker/projectevents" pb "github.com/thebtf/engram/proto/engram/v1" ) @@ -73,6 +75,9 @@ func New(handler MCPHandler, validator *auth.Validator) (*grpc.Server, *Server) opts := []grpc.ServerOption{ grpc.MaxRecvMsgSize(16 << 20), // 16 MB grpc.MaxSendMsgSize(16 << 20), + grpc.StatsHandler(otelgrpc.NewServerHandler( + otelgrpc.WithMeterProvider(obs.MeterProvider()), + )), // Always register the interceptors. They are runtime-no-op when the // live validator is nil (auth disabled), and runtime-enforce when // SetValidator promotes the server out of bootstrap. Conditional @@ -255,15 +260,18 @@ func (s *Server) validateBearer(ctx context.Context) (auth.Identity, error) { if v == nil { // Auth disabled deployments skip the interceptor entirely; if we // reach here without a validator, fail closed. + obs.RecordRuntimeEvent(ctx, "auth", "validator_missing") return auth.Identity{}, status.Error(codes.Internal, "auth: validator not configured") } md, ok := metadata.FromIncomingContext(ctx) if !ok { + obs.RecordRuntimeEvent(ctx, "auth", "missing_credentials") return auth.Identity{}, status.Error(codes.Unauthenticated, "missing metadata") } raw := extractBearer(md) if raw == "" { + obs.RecordRuntimeEvent(ctx, "auth", "missing_credentials") return auth.Identity{}, status.Error(codes.Unauthenticated, "missing authorization header") } @@ -272,8 +280,10 @@ func (s *Server) validateBearer(ctx context.Context) (auth.Identity, error) { case err == nil: return id, nil case errors.Is(err, auth.ErrEmptyToken): + obs.RecordRuntimeEvent(ctx, "auth", "missing_credentials") return auth.Identity{}, status.Error(codes.Unauthenticated, "missing authorization header") case errors.Is(err, auth.ErrInvalidCredentials): + obs.RecordRuntimeEvent(ctx, "auth", "invalid_credentials") return auth.Identity{}, status.Error(codes.Unauthenticated, "invalid token") case errors.Is(err, auth.ErrRevoked): // Currently unreachable: gormdb.TokenStore.FindByPrefix already @@ -282,11 +292,13 @@ func (s *Server) validateBearer(ctx context.Context) (auth.Identity, error) { // explicit mapping for the day FindByPrefix changes contract OR // a different TokenStoreReader implementation surfaces revoked // rows for audit logging. + obs.RecordRuntimeEvent(ctx, "auth", "revoked") return auth.Identity{}, status.Error(codes.Unauthenticated, "token revoked") default: // DB error or unexpected bcrypt failure. Surface as Internal so // monitoring distinguishes auth-rejected (Unauthenticated) from // auth-broken (Internal). + obs.RecordRuntimeEvent(ctx, "auth", "store_error") return auth.Identity{}, status.Error(codes.Internal, "auth: store unavailable") } } diff --git a/internal/grpcserver/version_negotiate.go b/internal/grpcserver/version_negotiate.go index 8bb30b096..0928fbb20 100644 --- a/internal/grpcserver/version_negotiate.go +++ b/internal/grpcserver/version_negotiate.go @@ -6,6 +6,7 @@ import ( "strconv" "strings" + "github.com/thebtf/engram/internal/module/obs" pb "github.com/thebtf/engram/proto/engram/v1" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -13,21 +14,25 @@ import ( // NegotiateVersion validates MAJOR-version compatibility between a client and the server. // Versions may optionally start with a leading "v" and must include at least a major segment. -func (s *Server) NegotiateVersion(_ context.Context, req *pb.NegotiateVersionRequest) (*pb.NegotiateVersionResponse, error) { +func (s *Server) NegotiateVersion(ctx context.Context, req *pb.NegotiateVersionRequest) (*pb.NegotiateVersionResponse, error) { if req.GetClientVersion() == "" { + obs.RecordRuntimeEvent(ctx, "client_version", "missing") return nil, status.Error(codes.InvalidArgument, "client_version must not be empty") } if s.handler == nil { + obs.RecordRuntimeEvent(ctx, "client_version", "server_info_unavailable") return nil, status.Error(codes.Unavailable, "server info unavailable") } _, serverVersion := s.handler.ServerInfo() clientMajor, err := parseMajorVersion(req.GetClientVersion()) if err != nil { + obs.RecordRuntimeEvent(ctx, "client_version", "invalid") return nil, status.Errorf(codes.InvalidArgument, "invalid client_version %q: %v", req.GetClientVersion(), err) } serverMajor, err := parseMajorVersion(serverVersion) if err != nil { + obs.RecordRuntimeEvent(ctx, "client_version", "server_version_invalid") return nil, status.Errorf(codes.Internal, "invalid server version %q: %v", serverVersion, err) } @@ -39,6 +44,7 @@ func (s *Server) NegotiateVersion(_ context.Context, req *pb.NegotiateVersionReq if compatible { return response, nil } + obs.RecordRuntimeEvent(ctx, "client_version", "incompatible") response.IncompatReason = fmt.Sprintf( "client major version %d is incompatible with server major version %d; upgrade or downgrade the client to match server version %s", diff --git a/internal/handlers/codeintel/module.go b/internal/handlers/codeintel/module.go index 8127b17ff..a9e5c04b1 100644 --- a/internal/handlers/codeintel/module.go +++ b/internal/handlers/codeintel/module.go @@ -49,6 +49,7 @@ import ( "github.com/thebtf/engram/internal/handlers/engramcore" "github.com/thebtf/engram/internal/module" + "github.com/thebtf/engram/internal/module/obs" muxcore "github.com/thebtf/mcp-mux/muxcore" ) @@ -316,6 +317,7 @@ func (m *Module) handleIndex(_ context.Context, p muxcore.ProjectContext, args j go func() { defer func() { if r := recover(); r != nil { + obs.RecordRuntimeEvent(daemonCtx, "index", "panic") // Panic recovery: log stack and mark state as error. if logger != nil { logger.Error("codeintel: index goroutine panicked", @@ -346,6 +348,7 @@ func (m *Module) handleIndex(_ context.Context, p muxcore.ProjectContext, args j result, err := core.IndexCodebase(daemonCtx, p, root) if err != nil { + obs.RecordRuntimeEvent(daemonCtx, "index", "run_error") if logger != nil { logger.Error("codeintel: index run failed", "project_id", projectID, diff --git a/internal/handlers/codeintel/observability_contract_test.go b/internal/handlers/codeintel/observability_contract_test.go new file mode 100644 index 000000000..6f15bcba6 --- /dev/null +++ b/internal/handlers/codeintel/observability_contract_test.go @@ -0,0 +1,77 @@ +package codeintel_test + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + + "github.com/thebtf/engram/internal/module/obs" + "github.com/thebtf/engram/internal/moduletest" + muxcore "github.com/thebtf/mcp-mux/muxcore" +) + +func TestCodebaseIndex_RecordsBackgroundRunFailure(t *testing.T) { + t.Setenv("ENGRAM_CODE_INTEL_ENABLED", "true") + previousProvider := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + obs.ResetInstrumentsForTesting() + t.Cleanup(func() { + obs.ResetInstrumentsForTesting() + otel.SetMeterProvider(previousProvider) + _ = provider.Shutdown(context.Background()) + }) + + core := &fakeCore{indexErr: errors.New("synthetic index failure")} + mod := newTestModule(core) + h := moduletest.New(t) + require.NoError(t, h.Register(mod)) + h.Freeze() + + project := muxcore.ProjectContext{ID: "proj-observability", Cwd: t.TempDir()} + args, err := json.Marshal(map[string]any{"root": project.Cwd}) + require.NoError(t, err) + _, err = h.CallToolWithProject(context.Background(), project, "codebase_index", args) + require.NoError(t, err) + drainIndex(t, h, project) + + var collected metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &collected)) + require.True(t, codeintelHasRuntimeEvent(collected, "index", "run_error"), "background index failures must emit a bounded diagnostic metric") +} + +func codeintelHasRuntimeEvent(collected metricdata.ResourceMetrics, component, outcome string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name != "engram_runtime_events_total" { + continue + } + sum, ok := metric.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + for _, point := range sum.DataPoints { + if codeintelAttributeValue(point.Attributes, "component") == component && codeintelAttributeValue(point.Attributes, "outcome") == outcome { + return true + } + } + } + } + return false +} + +func codeintelAttributeValue(set attribute.Set, key string) string { + value, ok := set.Value(attribute.Key(key)) + if !ok { + return "" + } + return value.AsString() +} diff --git a/internal/handlers/engramcore/grpcpool.go b/internal/handlers/engramcore/grpcpool.go index 13b3226d7..7f2265099 100644 --- a/internal/handlers/engramcore/grpcpool.go +++ b/internal/handlers/engramcore/grpcpool.go @@ -11,11 +11,14 @@ import ( "sync" "time" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/keepalive" "google.golang.org/grpc/metadata" + + "github.com/thebtf/engram/internal/module/obs" ) // connKey identifies a pooled gRPC connection. The tokenHash axis (FR-7 / @@ -146,6 +149,9 @@ func parseGRPCAddr(serverURL string) (string, error) { func dialGRPC(addr, serverURL, token string) (*grpc.ClientConn, error) { opts := []grpc.DialOption{ grpc.WithNoProxy(), + grpc.WithStatsHandler(otelgrpc.NewClientHandler( + otelgrpc.WithMeterProvider(obs.MeterProvider()), + )), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, Timeout: 10 * time.Second, diff --git a/internal/handlers/engramcore/observability_contract_test.go b/internal/handlers/engramcore/observability_contract_test.go new file mode 100644 index 000000000..a491b83ef --- /dev/null +++ b/internal/handlers/engramcore/observability_contract_test.go @@ -0,0 +1,67 @@ +package engramcore + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc" + + "github.com/thebtf/engram/internal/module/obs" + pb "github.com/thebtf/engram/proto/engram/v1" +) + +type observablePingServer struct { + pb.UnimplementedEngramServiceServer +} + +func (observablePingServer) Ping(context.Context, *pb.PingRequest) (*pb.PingResponse, error) { + return &pb.PingResponse{Status: "ok"}, nil +} + +func TestDialGRPC_EmitsClientTransportMetric(t *testing.T) { + previousProvider := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + obs.ResetInstrumentsForTesting() + t.Cleanup(func() { + obs.ResetInstrumentsForTesting() + otel.SetMeterProvider(previousProvider) + _ = provider.Shutdown(context.Background()) + }) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer() + pb.RegisterEngramServiceServer(server, observablePingServer{}) + t.Cleanup(server.Stop) + go func() { _ = server.Serve(listener) }() + + connection, err := dialGRPC(listener.Addr().String(), "http://"+listener.Addr().String(), "") + require.NoError(t, err) + t.Cleanup(func() { _ = connection.Close() }) + + response, err := pb.NewEngramServiceClient(connection).Ping(context.Background(), &pb.PingRequest{}) + require.NoError(t, err) + require.Equal(t, "ok", response.GetStatus()) + + var collected metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &collected)) + require.True(t, containsMetric(collected, "rpc.client.call.duration"), "production daemon gRPC calls must emit the ready-made otelgrpc client metric") +} + +func containsMetric(collected metricdata.ResourceMetrics, name string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name == name { + return true + } + } + } + return false +} diff --git a/internal/handlers/serverevents/bridge.go b/internal/handlers/serverevents/bridge.go index 7d37e83c3..c555f236a 100644 --- a/internal/handlers/serverevents/bridge.go +++ b/internal/handlers/serverevents/bridge.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" @@ -17,6 +18,7 @@ import ( "github.com/thebtf/engram/internal/config" "github.com/thebtf/engram/internal/module" + "github.com/thebtf/engram/internal/module/obs" "github.com/thebtf/engram/internal/module/registry" pb "github.com/thebtf/engram/proto/engram/v1" ) @@ -435,6 +437,9 @@ func (b *Bridge) dialGRPC() (*grpc.ClientConn, error) { opts := []grpc.DialOption{ grpc.WithNoProxy(), + grpc.WithStatsHandler(otelgrpc.NewClientHandler( + otelgrpc.WithMeterProvider(obs.MeterProvider()), + )), grpc.WithKeepaliveParams(keepalive.ClientParameters{ Time: 30 * time.Second, Timeout: 10 * time.Second, diff --git a/internal/handlers/serverevents/observability_contract_test.go b/internal/handlers/serverevents/observability_contract_test.go new file mode 100644 index 000000000..448a6a64a --- /dev/null +++ b/internal/handlers/serverevents/observability_contract_test.go @@ -0,0 +1,67 @@ +package serverevents + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + "google.golang.org/grpc" + + "github.com/thebtf/engram/internal/module/obs" + pb "github.com/thebtf/engram/proto/engram/v1" +) + +type observableEventsServer struct { + pb.UnimplementedEngramServiceServer +} + +func (observableEventsServer) SyncProjectState(context.Context, *pb.SyncProjectStateRequest) (*pb.SyncProjectStateResponse, error) { + return &pb.SyncProjectStateResponse{}, nil +} + +func TestBridgeDialGRPC_EmitsClientTransportMetric(t *testing.T) { + previousProvider := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + obs.ResetInstrumentsForTesting() + t.Cleanup(func() { + obs.ResetInstrumentsForTesting() + otel.SetMeterProvider(previousProvider) + _ = provider.Shutdown(context.Background()) + }) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + server := grpc.NewServer() + pb.RegisterEngramServiceServer(server, observableEventsServer{}) + t.Cleanup(server.Stop) + go func() { _ = server.Serve(listener) }() + + bridge := &Bridge{serverURL: "http://" + listener.Addr().String()} + connection, err := bridge.dialGRPC() + require.NoError(t, err) + t.Cleanup(func() { _ = connection.Close() }) + + _, err = pb.NewEngramServiceClient(connection).SyncProjectState(context.Background(), &pb.SyncProjectStateRequest{}) + require.NoError(t, err) + + var collected metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &collected)) + require.True(t, serverEventsContainsMetric(collected, "rpc.client.call.duration"), "persistent event bridge calls must emit the ready-made otelgrpc client metric") +} + +func serverEventsContainsMetric(collected metricdata.ResourceMetrics, name string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name == name { + return true + } + } + } + return false +} diff --git a/internal/module/obs/meter.go b/internal/module/obs/meter.go index edd1892b6..fcf6f03a1 100644 --- a/internal/module/obs/meter.go +++ b/internal/module/obs/meter.go @@ -1,7 +1,6 @@ package obs import ( - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" ) @@ -19,5 +18,5 @@ import ( // Passing an empty moduleName is safe: it returns a meter with an unusual scope // name ("github.com/thebtf/engram/") but does not panic. func MeterFor(moduleName string) metric.Meter { - return otel.GetMeterProvider().Meter("github.com/thebtf/engram/" + moduleName) + return MeterProvider().Meter("github.com/thebtf/engram/" + moduleName) } diff --git a/internal/module/obs/metrics.go b/internal/module/obs/metrics.go index 7be36959e..e2b641e0d 100644 --- a/internal/module/obs/metrics.go +++ b/internal/module/obs/metrics.go @@ -37,10 +37,24 @@ const scopeName = "github.com/thebtf/engram/internal/module" // wrapper so that later T064+ work can swap in caching or instrumentation- // version labels at one seam instead of every call site. func meter() metric.Meter { + return meterProviderLocked().Meter(scopeName) +} + +// MeterProvider returns the provider owned by the active Engram observability +// runtime, or the process-global provider when no Engram exporter is active. +// Integrations such as otelgrpc use this accessor so they share the exact +// provider lifecycle without replacing global OpenTelemetry state. +func MeterProvider() metric.MeterProvider { + instrumentsMu.RLock() + defer instrumentsMu.RUnlock() + return meterProviderLocked() +} + +func meterProviderLocked() metric.MeterProvider { if instrumentProvider != nil { - return instrumentProvider.Meter(scopeName) + return instrumentProvider } - return otel.GetMeterProvider().Meter(scopeName) + return otel.GetMeterProvider() } // --------------------------------------------------------------------------- @@ -266,7 +280,7 @@ func RecordRuntimeEvent(ctx context.Context, component, outcome string) { global.runtimeEventsOnce.Do(func() { c, err := meter().Int64Counter( "engram_runtime_events_total", - metric.WithDescription("Engram server lifecycle events labelled by component and outcome"), + metric.WithDescription("Engram runtime lifecycle events labelled by component and outcome"), ) if err != nil { slog.Warn("obs: failed to create engram_runtime_events_total counter", "error", err) diff --git a/internal/module/obs/runtime.go b/internal/module/obs/runtime.go index d2c514583..2e5ee0e85 100644 --- a/internal/module/obs/runtime.go +++ b/internal/module/obs/runtime.go @@ -5,6 +5,7 @@ import ( "errors" "net/url" "os" + "strings" "sync" "time" @@ -40,6 +41,18 @@ var ( // the first configured owner supplies its service version and exporter config. // The process-global OpenTelemetry provider is left untouched. func Init(ctx context.Context, serviceVersion string) (*Runtime, error) { + return InitForService(ctx, serviceName, serviceVersion) +} + +// InitForService installs the same bounded OTLP metrics runtime as Init while +// assigning the process its real resource identity. The daemon and server are +// separate executables, so collectors must not merge their transport metrics +// under one service.name. +func InitForService(ctx context.Context, resourceServiceName, serviceVersion string) (*Runtime, error) { + resourceServiceName = strings.TrimSpace(resourceServiceName) + if resourceServiceName == "" { + return nil, errors.New("observability service name is required") + } endpoint := os.Getenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT") if endpoint == "" { endpoint = os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") @@ -70,7 +83,7 @@ func Init(ctx context.Context, serviceVersion string) (*Runtime, error) { res := resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceName(serviceName), + semconv.ServiceName(resourceServiceName), semconv.ServiceVersion(serviceVersion), ) reader := sdkmetric.NewPeriodicReader(exporter) diff --git a/internal/module/obs/runtime_test.go b/internal/module/obs/runtime_test.go index 018ff46c6..bac02eae9 100644 --- a/internal/module/obs/runtime_test.go +++ b/internal/module/obs/runtime_test.go @@ -224,6 +224,42 @@ func TestOTLPExportsStableMetricsAndKeepsHeaderOutOfPayload(t *testing.T) { } } +func TestInitForService_ExportsDaemonResourceIdentity(t *testing.T) { + clearOTLPEnv(t) + receiver := &metricReceiver{} + t.Setenv("OTEL_EXPORTER_OTLP_METRICS_ENDPOINT", startMetricReceiver(t, receiver)) + + runtime, err := InitForService(context.Background(), "engram-daemon", "vtest") + if err != nil { + t.Fatal(err) + } + RecordRuntimeEvent(context.Background(), "startup", "started") + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if err := runtime.Shutdown(ctx); err != nil { + t.Fatal(err) + } + + requests := receiver.snapshot() + if len(requests) == 0 { + t.Fatal("collector received no daemon metric export") + } + if !requestHasResourceAttribute(requests[0], "service.name", "engram-daemon") { + t.Fatalf("OTLP payload does not identify the daemon service: %v", requests[0]) + } +} + +func requestHasResourceAttribute(request *collector.ExportMetricsServiceRequest, key, want string) bool { + for _, resourceMetrics := range request.GetResourceMetrics() { + for _, attribute := range resourceMetrics.GetResource().GetAttributes() { + if attribute.GetKey() == key && attribute.GetValue().GetStringValue() == want { + return true + } + } + } + return false +} + func TestOTLPTLSWithExplicitTrustRoot(t *testing.T) { clearOTLPEnv(t) receiver := &metricReceiver{} diff --git a/internal/worker/database_observability_contract_test.go b/internal/worker/database_observability_contract_test.go new file mode 100644 index 000000000..c71d4558f --- /dev/null +++ b/internal/worker/database_observability_contract_test.go @@ -0,0 +1,64 @@ +package worker + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + + dbgorm "github.com/thebtf/engram/internal/db/gorm" + "github.com/thebtf/engram/internal/module/obs" +) + +func TestOpenObservedStore_RecordsInitializationFailure(t *testing.T) { + previousProvider := otel.GetMeterProvider() + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + otel.SetMeterProvider(provider) + obs.ResetInstrumentsForTesting() + t.Cleanup(func() { + obs.ResetInstrumentsForTesting() + otel.SetMeterProvider(previousProvider) + _ = provider.Shutdown(context.Background()) + }) + + store, err := openObservedStore(context.Background(), dbgorm.Config{DSN: "://invalid-observability-contract"}) + require.Error(t, err) + require.Nil(t, store) + + var collected metricdata.ResourceMetrics + require.NoError(t, reader.Collect(context.Background(), &collected)) + require.True(t, workerHasRuntimeEvent(collected, "database", "initialization_error"), "database startup failures must emit a bounded diagnostic metric") +} + +func workerHasRuntimeEvent(collected metricdata.ResourceMetrics, component, outcome string) bool { + for _, scope := range collected.ScopeMetrics { + for _, metric := range scope.Metrics { + if metric.Name != "engram_runtime_events_total" { + continue + } + sum, ok := metric.Data.(metricdata.Sum[int64]) + if !ok { + continue + } + for _, point := range sum.DataPoints { + if workerAttributeValue(point.Attributes, "component") == component && workerAttributeValue(point.Attributes, "outcome") == outcome { + return true + } + } + } + } + return false +} + +func workerAttributeValue(set attribute.Set, key string) string { + value, ok := set.Value(attribute.Key(key)) + if !ok { + return "" + } + return value.AsString() +} diff --git a/internal/worker/service.go b/internal/worker/service.go index 2dadbd703..0b0e61a5b 100644 --- a/internal/worker/service.go +++ b/internal/worker/service.go @@ -48,6 +48,7 @@ import ( "github.com/thebtf/engram/internal/injection" "github.com/thebtf/engram/internal/logbuf" "github.com/thebtf/engram/internal/mcp" + "github.com/thebtf/engram/internal/module/obs" "github.com/thebtf/engram/internal/principalmemory" "github.com/thebtf/engram/internal/redaction" "github.com/thebtf/engram/internal/reranking" @@ -781,7 +782,7 @@ func (s *Service) initializeAsync() { } // Open the PostgreSQL connection pool and run pending schema migrations. - store, err := gorm.NewStore(gorm.Config{ + store, err := openObservedStore(s.ctx, gorm.Config{ DSN: s.config.DatabaseDSN, MaxConns: s.config.DatabaseMaxConns, }) @@ -1291,6 +1292,23 @@ func (s *Service) initializeAsync() { s.startWatchers() } +// openObservedStore is the production database initialization seam. It keeps +// diagnostics at the call site that knows whether startup succeeded, while +// deliberately exposing only bounded outcomes (never DSNs or driver errors) +// to metrics. +func openObservedStore(ctx context.Context, cfg gorm.Config) (*gorm.Store, error) { + if ctx == nil { + ctx = context.Background() + } + store, err := gorm.NewStore(cfg) + if err != nil { + obs.RecordRuntimeEvent(ctx, "database", "initialization_error") + return nil, err + } + obs.RecordRuntimeEvent(ctx, "database", "initialized") + return store, nil +} + // startWatchers registers filesystem notification handlers for config hot-reload. // Database-file watching is not applicable for PostgreSQL (server-managed file). func (s *Service) startWatchers() { diff --git a/scripts/production-smoke/verify-otlp.ps1 b/scripts/production-smoke/verify-otlp.ps1 index 1b3bb459e..759e4b92b 100644 --- a/scripts/production-smoke/verify-otlp.ps1 +++ b/scripts/production-smoke/verify-otlp.ps1 @@ -28,6 +28,7 @@ try { $failedTests = @($events | Where-Object { $_.Action -eq "fail" -and $_.Test } | Select-Object -ExpandProperty Test -Unique) $required = @( "TestInitNoEndpointIsNoop", + "TestInitForService_ExportsDaemonResourceIdentity", "TestOTLPExportsStableMetricsAndKeepsHeaderOutOfPayload", "TestOTLPTLSWithExplicitTrustRoot", "TestCollectorAuthFailureIsBoundedAndSecretFree",