From 33ddb7f5ed35bfefbda232aa670b608342c5cfc7 Mon Sep 17 00:00:00 2001 From: EHotwagner Date: Sat, 10 Oct 2026 00:53:04 +0200 Subject: [PATCH] test: qualify retained native target relation through CI projection --- .../roadmaps/utel-operational-completeness.md | 11 + .../TelemetryCiApplicationTests.fs | 216 ++++++++++++++++++ .../FS.GG.Coord.GitHub.Tests/CiReadsTests.fs | 149 ++++++++++++ 3 files changed, 376 insertions(+) diff --git a/docs/roadmaps/utel-operational-completeness.md b/docs/roadmaps/utel-operational-completeness.md index 09a8cb23d..3c5c2af29 100644 --- a/docs/roadmaps/utel-operational-completeness.md +++ b/docs/roadmaps/utel-operational-completeness.md @@ -97,6 +97,17 @@ source/profile/documentation paths. No retained store, package version, installe workspace defaults change. The later event-derived witness design requires separate source/provenance selection and native proof; head/branch/title heuristics are excluded. +PR [#4344](https://github.com/FS-GG/.github/pull/4344) delivered the bounded CI source at +`99715df3d2214de2c86c9ae7c49e6b0ae9ca7f3b` with 48 successful checks and three skips. +Native target run [37997746345, attempt 1](https://github.com/FS-GG/.github/actions/runs/37997746345) +contains one complete PR 4344/head/base/repository relation, unlike the retained empty-array case. +The wired reader tests exercise that sanitized native relation through both collection entrypoints; +the application test uses its literal job/step observations and exact-head profile through a disposable +store, preserving mixed classification and replay without duplicate facts. These controlled populations +do not establish whole-head inventory or installed collection. The empty-array run remains unknown; +prospective automatic-delivery qualification, producer publication and receiver adoption remain pending. +No event-derived witness is selected for the complete native-relation case, and UTEL stays open. + ## Milestones - [x] **UTEL-06.1 — Real observer acceptance and prospective orchestration identities.** Reproduce PR #3347's diff --git a/tests/FS.GG.Coord.Cli.Tests/TelemetryCiApplicationTests.fs b/tests/FS.GG.Coord.Cli.Tests/TelemetryCiApplicationTests.fs index 93021095d..bc23ae4ff 100644 --- a/tests/FS.GG.Coord.Cli.Tests/TelemetryCiApplicationTests.fs +++ b/tests/FS.GG.Coord.Cli.Tests/TelemetryCiApplicationTests.fs @@ -489,3 +489,219 @@ module TelemetryCiApplicationTests = FS.GG.Coord.GitHub.CiReads.targetAssociation "FS-GG/.github" 4342 (row.GetProperty("head_sha").GetString()) (Some "31795a80e706fc4203c27e421f9e3648a1dd1329") row) + + // Sanitized native GET evidence: run37997746345/attempt1 and jobs/check114047987738, + // captured 2026-10-09. This controlled subpopulation does not claim whole-head inventory. + let private retainedPositiveTarget = """ + { + "id": 37997746345, + "run_attempt": 1, + "path": ".github/workflows/routine-eligibility.yml", + "event": "pull_request_target", + "head_sha": "b6be731e8d726fb13d46a034bd0ea60fc01baf2a", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:10:29Z", + "run_started_at": "2026-10-09T22:10:29Z", + "updated_at": "2026-10-09T22:10:36Z", + "pull_requests": [ + { + "number": 4344, + "head": { + "sha": "b6be731e8d726fb13d46a034bd0ea60fc01baf2a", + "repo": { + "id": 1269292704 + } + }, + "base": { + "sha": "a3415321ec33946d66ad1f39469935d01206bb98", + "repo": { + "id": 1269292704 + } + } + } + ], + "repository": { + "id": 1269292704, + "full_name": "FS-GG/.github" + }, + "head_repository": { + "id": 1269292704, + "full_name": "FS-GG/.github" + } + } + """ + let private retainedPositiveJobs = """ + { + "total_count": 1, + "jobs": [ + { + "id": 114047987738, + "name": "routine-eligibility", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:10:30Z", + "started_at": "2026-10-09T22:10:31Z", + "completed_at": "2026-10-09T22:10:36Z", + "check_run_url": "https://api.github.com/repos/FS-GG/.github/check-runs/114047987738", + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2026-10-09T22:10:32Z", + "completed_at": "2026-10-09T22:10:32Z" + }, + { + "name": "Evaluate with the PR base's policy and validator", + "status": "completed", + "conclusion": "success", + "number": 2, + "started_at": "2026-10-09T22:10:32Z", + "completed_at": "2026-10-09T22:10:35Z" + }, + { + "name": "Complete job", + "status": "completed", + "conclusion": "success", + "number": 3, + "started_at": "2026-10-09T22:10:35Z", + "completed_at": "2026-10-09T22:10:35Z" + } + ] + } + ] + } + """ + let private retainedPositiveChecks = """ + { + "total_count": 1, + "check_runs": [ + { + "id": 114047987738, + "name": "routine-eligibility", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:10:31Z", + "completed_at": "2026-10-09T22:10:36Z", + "app": { + "slug": "github-actions" + } + } + ] + } + """ + let private retainedHead = "b6be731e8d726fb13d46a034bd0ea60fc01baf2a" + let private retainedBase = "a3415321ec33946d66ad1f39469935d01206bb98" + let private retainedPr = + $"""{{"head":{{"sha":"{retainedHead}"}},"base":{{"ref":"main","sha":"{retainedBase}"}}}}""" + let private retainedRuns = "{\"total_count\":1,\"workflow_runs\":[" + retainedPositiveTarget + "]}" + let private retainedPositiveProfile = """{ + "schema": "fsgg.telemetry.ci-attribution/1", + "rules": [ + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Install the pinned SDD provider validator", + "classification": "necessary-setup", + "rationale": "Installs the pinned provider needed by the subsequent source validation." + }, + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Restore (locked)", + "classification": "necessary-setup", + "rationale": "Restores the locked dependencies used by the validation suites." + }, + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Build the engine", + "classification": "necessary-setup", + "rationale": "Builds the candidate engine consumed by subsequent production-route qualification." + }, + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Test — core telemetry contracts and source binding", + "classification": "useful-validation", + "rationale": "Runs the independent Core contract suite against the selected source revision." + }, + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Test — standalone telemetry HTTPS contracts and host boundaries", + "classification": "useful-validation", + "rationale": "Runs the standalone telemetry transport and host boundary contract suite." + }, + { + "workflow": ".github/workflows/coord-engine.yml", + "job": "engine", + "step": "Test the shared projection and real loopback browser", + "classification": "mixed", + "rationale": "Bundles npm/browser installation with dashboard and browser validation; native evidence cannot split the interval." + }, + { + "workflow": ".github/workflows/routine-eligibility.yml", + "job": "routine-eligibility", + "step": "Evaluate with the PR base's policy and validator", + "classification": "mixed", + "rationale": "Bundles trusted base/head source fetching and routine policy assessment in one native step." + } + ] +} +""" + + [] + let ``UTEL-CI-03 retained native positive target classifies mixed and replays in disposable store`` () = + let unwrap = function Ok value -> value | Error error -> failwithf "%A" error + let queue = System.Collections.Generic.Queue([ retainedPr; retainedRuns; retainedPositiveJobs; + retainedPositiveChecks; retainedRuns; retainedPositiveChecks ]) + let requests = ResizeArray() + let transport = + { new FS.GG.Coord.GitHub.Transport.ISinglePageGitHubTransport with + member _.SendSingle request = + requests.Add request + Ok { Status = 200; Body = queue.Dequeue(); Headers = Map.empty; ETag = None; NextLink = None } } + let population = FS.GG.Coord.GitHub.CiReads.discoverPopulation transport "https://api.github.com" "FS-GG" ".github" 4344 retainedHead "main" retainedBase false |> unwrap + Assert.Empty queue + Assert.Empty population.Gaps + let profileBytes = System.Text.Encoding.UTF8.GetBytes retainedPositiveProfile + let contents = $"""{{"type":"file","path":".fsgg/telemetry-ci-attribution.json","encoding":"base64","content":"{Convert.ToBase64String profileBytes}"}}""" + queue.Enqueue contents + let selectedProfile = FS.GG.Coord.GitHub.CiReads.readAttributionProfile transport "FS-GG/.github" retainedHead |> unwrap + Assert.Equal(Some profileBytes, selectedProfile) + Assert.Equal("repos/FS-GG/.github/contents/.fsgg/telemetry-ci-attribution.json", requests[6].Path) + Assert.Equal<(string * string) list>([ "ref", retainedHead ], requests[6].Query) + let assignment: FS.GG.Coord.TelemetryCi.Assignment = + { FeatureId = "UTEL"; ItemId = "UTEL-CI-GAPS"; AttemptId = "retained-native-positive"; ParentAttemptId = None; ProducerStream = "routine-delivery" } + let project bytes = TelemetryCiApplication.projectPopulationForTesting bytes assignment population |> unwrap + let missing, available = project None, project selectedProfile + let events batches = + batches |> List.collect (fun (bytes: byte array) -> + use document = JsonDocument.Parse bytes + document.RootElement.GetProperty("events").EnumerateArray() |> Seq.map (fun node -> node.Clone()) |> Seq.toList) + let steps = events available |> List.filter (fun node -> node.GetProperty("kind").GetString() = "ci-step") + Assert.Equal(3, steps.Length) + let mixed = Assert.Single(steps |> List.filter (fun node -> node.GetProperty("classification").GetString() = "mixed")) + Assert.Equal(37997746345L, mixed.GetProperty("runId").GetInt64()) + Assert.Equal(1, mixed.GetProperty("attempt").GetInt32()) + Assert.Equal(114047987738L, mixed.GetProperty("jobId").GetInt64()) + Assert.Equal("Evaluate with the PR base's policy and validator", mixed.GetProperty("name").GetString()) + Assert.All(steps |> List.filter (fun node -> node.GetProperty("number").GetInt32() <> 2), fun node -> Assert.Equal("unclassified", node.GetProperty("classification").GetString())) + Assert.DoesNotContain(events missing, fun node -> node.GetProperty("kind").GetString() = "ci-step") + Assert.Contains(events available, fun node -> node.GetProperty("kind").GetString() = "diagnostic" && node.GetProperty("code").GetString().Contains("3b6ceefa6e1ca1273fdd37bdf1817cd4a79aa771f8e89c5757544f489f0c513c")) + let root = Path.Combine(Path.GetTempPath(), "fsgg-native-target-" + Guid.NewGuid().ToString("N")) + try + let approved = FS.GG.Coord.TelemetryStore.ApprovedLocalDurable + TelemetryStoreApplication.initialize root approved |> unwrap |> ignore + for batch in missing @ available @ available do + TelemetryStoreApplication.ingest root approved batch |> unwrap |> fun result -> Assert.DoesNotContain("conflict", result) + let summary = TelemetryStoreApplication.ciSummary root approved assignment.ItemId |> unwrap + Assert.Contains("\"mixedSeconds\":3", summary) + Assert.Contains("\"runnerSeconds\":5", summary) + Assert.Contains("\"steps\":3", summary) + Assert.Contains("\"usefulValidationSeconds\":null", summary) + finally + if Directory.Exists root then Directory.Delete(root, true) diff --git a/tests/FS.GG.Coord.GitHub.Tests/CiReadsTests.fs b/tests/FS.GG.Coord.GitHub.Tests/CiReadsTests.fs index 125641a32..786e59665 100644 --- a/tests/FS.GG.Coord.GitHub.Tests/CiReadsTests.fs +++ b/tests/FS.GG.Coord.GitHub.Tests/CiReadsTests.fs @@ -589,3 +589,152 @@ let ``UTEL-CI-02 legacy target rerun does not clone latest association into unwi Assert.Single snapshot.Runs |> ignore Assert.Equal(2, snapshot.Runs.Head.Attempt) Assert.Equal(2, fake.Requests.Length) + +// Sanitized native GET evidence: run37997746345/attempt1 and jobs/check114047987738, +// captured 2026-10-09. This controlled subpopulation does not claim whole-head inventory. +let private retainedPositiveTarget = """ +{ + "id": 37997746345, + "run_attempt": 1, + "path": ".github/workflows/routine-eligibility.yml", + "event": "pull_request_target", + "head_sha": "b6be731e8d726fb13d46a034bd0ea60fc01baf2a", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:10:29Z", + "run_started_at": "2026-10-09T22:10:29Z", + "updated_at": "2026-10-09T22:10:36Z", + "pull_requests": [ + { + "number": 4344, + "head": { + "sha": "b6be731e8d726fb13d46a034bd0ea60fc01baf2a", + "repo": { + "id": 1269292704 + } + }, + "base": { + "sha": "a3415321ec33946d66ad1f39469935d01206bb98", + "repo": { + "id": 1269292704 + } + } + } + ], + "repository": { + "id": 1269292704, + "full_name": "FS-GG/.github" + }, + "head_repository": { + "id": 1269292704, + "full_name": "FS-GG/.github" + } +} +""" +let private retainedPositiveJobs = """ +{ + "total_count": 1, + "jobs": [ + { + "id": 114047987738, + "name": "routine-eligibility", + "status": "completed", + "conclusion": "success", + "created_at": "2026-10-09T22:10:30Z", + "started_at": "2026-10-09T22:10:31Z", + "completed_at": "2026-10-09T22:10:36Z", + "check_run_url": "https://api.github.com/repos/FS-GG/.github/check-runs/114047987738", + "steps": [ + { + "name": "Set up job", + "status": "completed", + "conclusion": "success", + "number": 1, + "started_at": "2026-10-09T22:10:32Z", + "completed_at": "2026-10-09T22:10:32Z" + }, + { + "name": "Evaluate with the PR base's policy and validator", + "status": "completed", + "conclusion": "success", + "number": 2, + "started_at": "2026-10-09T22:10:32Z", + "completed_at": "2026-10-09T22:10:35Z" + }, + { + "name": "Complete job", + "status": "completed", + "conclusion": "success", + "number": 3, + "started_at": "2026-10-09T22:10:35Z", + "completed_at": "2026-10-09T22:10:35Z" + } + ] + } + ] +} +""" +let private retainedPositiveChecks = """ +{ + "total_count": 1, + "check_runs": [ + { + "id": 114047987738, + "name": "routine-eligibility", + "status": "completed", + "conclusion": "success", + "started_at": "2026-10-09T22:10:31Z", + "completed_at": "2026-10-09T22:10:36Z", + "app": { + "slug": "github-actions" + } + } + ] +} +""" +let private retainedHead = "b6be731e8d726fb13d46a034bd0ea60fc01baf2a" +let private retainedBase = "a3415321ec33946d66ad1f39469935d01206bb98" +let private retainedPr = + $"""{{"head":{{"sha":"{retainedHead}"}},"base":{{"ref":"main","sha":"{retainedBase}"}}}}""" +let private retainedRuns = "{\"total_count\":1,\"workflow_runs\":[" + retainedPositiveTarget + "]}" + +[] +let ``UTEL-CI-02 retained native positive target passes exact relation and population fences`` () = + use document = System.Text.Json.JsonDocument.Parse retainedPositiveTarget + Assert.Equal(CiReads.Exact, CiReads.targetAssociation "FS-GG/.github" 4344 retainedHead (Some retainedBase) document.RootElement) + Assert.Equal(CiReads.Conflicting, CiReads.targetAssociation "FS-GG/.github" 4344 retainedHead (Some baseSha) document.RootElement) + let fake = Fake [ response retainedPr None; response retainedRuns None; response retainedPositiveJobs None + response retainedPositiveChecks None; response retainedRuns None; response retainedPositiveChecks None ] + match CiReads.discoverPopulation (fake :> ISinglePageGitHubTransport) "https://api.github.com" "FS-GG" ".github" 4344 retainedHead "main" retainedBase false with + | Error error -> failwithf "%A" error + | Ok result -> + Assert.Empty result.Gaps + Assert.Empty result.Pending + Assert.True result.AdmissionWitness + Assert.Equal("exact", result.Snapshot.Binding) + Assert.Equal("complete", result.Snapshot.LineageCoverage) + Assert.Equal("complete", result.CheckCoverage) + let nativeRun = Assert.Single result.Snapshot.Runs + Assert.Equal(37997746345L, nativeRun.Id) + Assert.Equal(1, nativeRun.Attempt) + Assert.Equal(CiReads.Exact, nativeRun.TargetAssociation) + let nativeJob = Assert.Single result.Snapshot.Jobs + Assert.Equal(114047987738L, nativeJob.Id) + Assert.Equal(Some 114047987738L, nativeJob.CheckRunId) + Assert.Equal("routine-eligibility", nativeJob.Name) + Assert.Equal("Evaluate with the PR base's policy and validator", nativeJob.Steps[1].Name) + Assert.Contains(fake.Requests, fun request -> request.Path.EndsWith("/runs/37997746345/attempts/1/jobs")) + Assert.Equal(6, fake.Requests.Length) + +[] +let ``UTEL-CI-02 retained native positive target passes legacy attempt-specific collection`` () = + let fake = Fake [ response retainedPr None; response retainedRuns None; response retainedPositiveJobs None + response "{\"run_attempt\":1}" None ] + match CiReads.collect (fake :> ISinglePageGitHubTransport) "https://api.github.com" "FS-GG" ".github" 4344 retainedHead "routine-eligibility.yml" with + | Error error -> failwithf "%A" error + | Ok snapshot -> + Assert.Equal(CiReads.Exact, snapshot.Runs.Head.TargetAssociation) + Assert.Equal(37997746345L, snapshot.Jobs.Head.RunId) + Assert.Equal(1, snapshot.Jobs.Head.Attempt) + Assert.Equal(3, snapshot.Jobs.Head.Steps.Length) + Assert.Contains(fake.Requests, fun request -> request.Path.EndsWith("/runs/37997746345/attempts/1/jobs"))