Skip to content

Commit 5ab522a

Browse files
committed
feat: add verified compatibility preview
1 parent bdb6411 commit 5ab522a

20 files changed

Lines changed: 2336 additions & 2 deletions

.github/workflows/external-consumer-canary.yml

Lines changed: 89 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -131,9 +131,14 @@ jobs:
131131
if-no-files-found: warn
132132

133133
falco:
134-
name: Falco (scap-open modern_bpf loader)
134+
name: Falco verified compatibility preview
135135
runs-on: ubuntu-latest
136136
timeout-minutes: 60
137+
permissions:
138+
contents: read
139+
id-token: write
140+
attestations: write
141+
artifact-metadata: write
137142
steps:
138143
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
139144
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
@@ -165,12 +170,18 @@ jobs:
165170
-DCREATE_TEST_TARGETS=OFF -DBUILD_BPF=OFF -DBUILD_DRIVER=OFF ..
166171
make scap-open -j"$(nproc)"
167172
file libscap/examples/01-open/scap-open
173+
cd ../..
174+
install -D -m 0755 \
175+
libs/build/libscap/examples/01-open/scap-open \
176+
reports/falco-conformance/subjects/scap-open
168177
- name: Validate scap-open across kernels (its real loader path)
178+
id: validate
179+
continue-on-error: true
169180
uses: ./
170181
with:
171182
command: $BPFCOMPAT_BIN --modern_bpf --num_events 10
172183
command-binary: libs/build/libscap/examples/01-open/scap-open
173-
matrix: matrices/consumer-external.yaml
184+
matrix: conformance/falco-modern-bpf-v0.1/matrix.yaml
174185
out: reports/falco.json
175186
markdown: reports/falco.md
176187
timeout: 20m
@@ -182,13 +193,89 @@ jobs:
182193
actual="$(jq -r '.command.binary.sha256 // empty' reports/falco.json)"
183194
test -n "$actual"
184195
test "$actual" = "$expected"
196+
test "$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')" = "$actual"
197+
- name: Evaluate the Falco conformance profile
198+
id: evaluate
199+
env:
200+
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
201+
run: |
202+
set +e
203+
./bin/bpfcompat conformance evaluate \
204+
--profile conformance/falco-modern-bpf-v0.1/profile.yaml \
205+
--report reports/falco.json \
206+
--out-dir reports/falco-conformance \
207+
--verifier-id https://github.com/Kernel-Guard/bpfcompat/tree/main/conformance/falco-modern-bpf-v0.1 \
208+
--report-url "$RUN_URL"
209+
rc=$?
210+
set -e
211+
212+
test -s reports/falco-conformance/decision.json
213+
test -s reports/falco-conformance/test-result.intoto.json
214+
status="$(jq -er '.status' reports/falco-conformance/decision.json)"
215+
case "$status:$rc" in
216+
conformant:0|nonconformant:2|inconclusive:1) ;;
217+
*) echo "evaluator status/exit mismatch: $status/$rc" >&2; exit 1 ;;
218+
esac
219+
220+
statement_subject="$(jq -er 'select((.subject | length) == 1 and .subject[0].name == "scap-open") | .subject[0].digest.sha256' reports/falco-conformance/test-result.intoto.json)"
221+
report_subject="$(jq -er '.subject.digest.sha256' reports/falco-conformance/decision.json)"
222+
binary_subject="$(sha256sum reports/falco-conformance/subjects/scap-open | awk '{print $1}')"
223+
test "$statement_subject" = "$report_subject"
224+
test "$statement_subject" = "$binary_subject"
225+
226+
jq -e '.predicate' reports/falco-conformance/test-result.intoto.json \
227+
> reports/falco-conformance/test-result.predicate.json
228+
if [[ "$status" == "conformant" ]]; then
229+
test -s reports/falco-conformance/verification-result.intoto.json
230+
jq -e '.predicate' reports/falco-conformance/verification-result.intoto.json \
231+
> reports/falco-conformance/verification-result.predicate.json
232+
else
233+
test ! -e reports/falco-conformance/verification-result.intoto.json
234+
fi
235+
echo "status=$status" >> "$GITHUB_OUTPUT"
236+
- name: Attest the in-toto Test Result
237+
id: test_attestation
238+
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
239+
with:
240+
subject-path: reports/falco-conformance/subjects/scap-open
241+
predicate-type: https://in-toto.io/attestation/test-result/v0.1
242+
predicate-path: reports/falco-conformance/test-result.predicate.json
243+
- name: Attest the passing verification result
244+
id: verification_attestation
245+
if: steps.evaluate.outputs.status == 'conformant'
246+
uses: actions/attest@508db95dd578ae2727ebd6217d5ba78e4fbda05d # v4.2.1
247+
with:
248+
subject-path: reports/falco-conformance/subjects/scap-open
249+
predicate-type: https://in-toto.io/attestation/svr/v0.2
250+
predicate-path: reports/falco-conformance/verification-result.predicate.json
251+
- name: Preserve signed attestation bundles
252+
if: steps.test_attestation.outcome == 'success'
253+
env:
254+
TEST_BUNDLE: ${{ steps.test_attestation.outputs.bundle-path }}
255+
VERIFICATION_BUNDLE: ${{ steps.verification_attestation.outputs.bundle-path }}
256+
run: |
257+
install -m 0644 "$TEST_BUNDLE" \
258+
reports/falco-conformance/test-result.sigstore.json
259+
if [[ -n "$VERIFICATION_BUNDLE" ]]; then
260+
install -m 0644 "$VERIFICATION_BUNDLE" \
261+
reports/falco-conformance/verification-result.sigstore.json
262+
fi
185263
- name: Upload report
186264
if: always()
187265
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
188266
with:
189267
name: canary-falco-${{ github.run_id }}
190268
path: reports/
191269
if-no-files-found: warn
270+
- name: Enforce the conformance decision
271+
if: always()
272+
env:
273+
STATUS: ${{ steps.evaluate.outputs.status }}
274+
run: |
275+
test "$STATUS" = "conformant" || {
276+
echo "Falco conformance decision: ${STATUS:-evaluation unavailable}" >&2
277+
exit 1
278+
}
192279
193280
report:
194281
name: File or close the canary tracking issue

README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -711,6 +711,7 @@ Reference matrices (real, reproducible artifacts):
711711

712712
- [`docs/integrations/`](docs/integrations/) — copy-paste templates: Go loader, Rust/Aya, OCI gadgets, and an in-process pre-load gate
713713
- [`docs/case-study-falco-modern-bpf.md`](docs/case-study-falco-modern-bpf.md) — Falco `modern_bpf` across 5 kernels
714+
- [`docs/verified-compatibility-preview.md`](docs/verified-compatibility-preview.md) — versioned Falco conformance profile, decisions, and signed evidence
714715
- [`docs/case-study-enterprise-kernels.md`](docs/case-study-enterprise-kernels.md) — RHEL/Oracle/Amazon/SUSE backported tier
715716
- [`docs/case-study-inspektor-gadget.md`](docs/case-study-inspektor-gadget.md) — published gadgets from OCI, zero config
716717
- [`docs/evidence-rhcos.md`](docs/evidence-rhcos.md) — RHEL CoreOS / OpenShift 4.14·4.16·4.18 × 6 artifacts (x86_64) + a real aarch64 boot

cmd/bpfcompat/conformance.go

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
package main
2+
3+
import (
4+
"errors"
5+
"flag"
6+
"fmt"
7+
"os"
8+
"strings"
9+
10+
"github.com/kernel-guard/bpfcompat/internal/conformance"
11+
"github.com/kernel-guard/bpfcompat/internal/runner"
12+
)
13+
14+
func runConformance(args []string) int {
15+
if len(args) == 0 {
16+
printConformanceUsage()
17+
return runner.ExitToolError
18+
}
19+
switch args[0] {
20+
case "evaluate":
21+
return runConformanceEvaluate(args[1:])
22+
case "-h", "--help", "help":
23+
printConformanceUsage()
24+
return runner.ExitSuccess
25+
default:
26+
fmt.Fprintf(os.Stderr, "unknown conformance subcommand: %s\n\n", args[0])
27+
printConformanceUsage()
28+
return runner.ExitToolError
29+
}
30+
}
31+
32+
func runConformanceEvaluate(args []string) int {
33+
fs := flag.NewFlagSet("conformance evaluate", flag.ContinueOnError)
34+
fs.SetOutput(os.Stderr)
35+
profilePath := fs.String("profile", "", "Path to a versioned conformance profile YAML")
36+
reportPath := fs.String("report", "", "Path to the bpfcompat JSON report to evaluate")
37+
outDir := fs.String("out-dir", "", "Directory for decision and in-toto statement outputs")
38+
verifierID := fs.String("verifier-id", "", "Absolute HTTPS URI identifying the verifier")
39+
reportURL := fs.String("report-url", "", "Optional absolute HTTPS URL for the originating test run")
40+
fs.Usage = func() {
41+
fmt.Fprintf(fs.Output(), "Usage:\n bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri> [--report-url <url>]\n\n")
42+
fs.PrintDefaults()
43+
}
44+
if err := fs.Parse(args); err != nil {
45+
if errors.Is(err, flag.ErrHelp) {
46+
return runner.ExitSuccess
47+
}
48+
return runner.ExitToolError
49+
}
50+
if fs.NArg() != 0 {
51+
fmt.Fprintf(os.Stderr, "unexpected positional arguments: %v\n", fs.Args())
52+
return runner.ExitToolError
53+
}
54+
for name, value := range map[string]string{
55+
"--profile": *profilePath,
56+
"--report": *reportPath,
57+
"--out-dir": *outDir,
58+
"--verifier-id": *verifierID,
59+
} {
60+
if strings.TrimSpace(value) == "" {
61+
fmt.Fprintf(os.Stderr, "%s is required\n", name)
62+
return runner.ExitToolError
63+
}
64+
}
65+
66+
profile, err := conformance.LoadProfile(*profilePath)
67+
if err != nil {
68+
fmt.Fprintf(os.Stderr, "load conformance profile: %v\n", err)
69+
return runner.ExitToolError
70+
}
71+
report, err := conformance.LoadReport(*reportPath)
72+
if err != nil {
73+
fmt.Fprintf(os.Stderr, "load conformance report: %v\n", err)
74+
return runner.ExitToolError
75+
}
76+
evaluation, err := conformance.Evaluate(profile, report, conformance.EvaluateOptions{
77+
VerifierID: strings.TrimSpace(*verifierID),
78+
ReportURL: strings.TrimSpace(*reportURL),
79+
})
80+
if err != nil {
81+
fmt.Fprintf(os.Stderr, "evaluate conformance report: %v\n", err)
82+
return runner.ExitToolError
83+
}
84+
paths, err := conformance.WriteEvaluation(*outDir, evaluation)
85+
if err != nil {
86+
fmt.Fprintf(os.Stderr, "write conformance evidence: %v\n", err)
87+
return runner.ExitToolError
88+
}
89+
90+
fmt.Printf("Conformance: %s\n", evaluation.Decision.Status)
91+
fmt.Printf("Profile: %s sha256:%s\n", evaluation.Decision.Profile.ID, evaluation.Decision.Profile.SHA256)
92+
fmt.Printf("Subject: %s sha256:%s\n", evaluation.Decision.Subject.Name, evaluation.Decision.Subject.Digest["sha256"])
93+
fmt.Printf("Decision: %s\n", paths.Decision)
94+
fmt.Printf("Test Result: %s\n", paths.TestResult)
95+
if paths.VerificationResult != "" {
96+
fmt.Printf("Verification Result: %s\n", paths.VerificationResult)
97+
} else {
98+
fmt.Println("Verification Result: not issued")
99+
}
100+
101+
switch evaluation.Decision.Status {
102+
case conformance.StatusConformant:
103+
return runner.ExitSuccess
104+
case conformance.StatusNonconformant:
105+
return runner.ExitCompatibilityFailure
106+
default:
107+
return runner.ExitToolError
108+
}
109+
}
110+
111+
func printConformanceUsage() {
112+
fmt.Println("Usage:")
113+
fmt.Println(" bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri> [--report-url <url>]")
114+
}

cmd/bpfcompat/main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ func run(args []string) int {
6161
return runKernelFreshness(args[1:])
6262
case "kernel-sweep":
6363
return runKernelSweep(args[1:])
64+
case "conformance":
65+
return runConformance(args[1:])
6466
case "version", "--version":
6567
return runVersion(args[1:])
6668
case "env":
@@ -1456,6 +1458,7 @@ func printRootUsage() {
14561458
fmt.Println(" bpfcompat report-summary --report <file>")
14571459
fmt.Println(" bpfcompat kernel-freshness [--baselines <file>] [--fail-on-stale] [--update-from-report <file>]")
14581460
fmt.Println(" bpfcompat kernel-sweep --profile <id> [--count N] [--series <prefix>]")
1461+
fmt.Println(" bpfcompat conformance evaluate --profile <file> --report <file> --out-dir <dir> --verifier-id <uri>")
14591462
fmt.Println(" bpfcompat version [--json]")
14601463
fmt.Println(" bpfcompat env [--markdown|--json]")
14611464
}

cmd/bpfcompat/main_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,3 +109,15 @@ func TestRunTestCommandDispatch(t *testing.T) {
109109
t.Fatalf("run test-command -h = %d, want 0", got)
110110
}
111111
}
112+
113+
func TestRunConformanceDispatchAndValidation(t *testing.T) {
114+
if got := run([]string{"conformance", "evaluate", "-h"}); got != runner.ExitSuccess {
115+
t.Fatalf("run conformance evaluate -h = %d, want 0", got)
116+
}
117+
if got := run([]string{"conformance", "evaluate"}); got != runner.ExitToolError {
118+
t.Fatalf("run conformance evaluate = %d, want tool error", got)
119+
}
120+
if got := run([]string{"conformance", "unknown"}); got != runner.ExitToolError {
121+
t.Fatalf("run conformance unknown = %d, want tool error", got)
122+
}
123+
}

conformance/README.md

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# bpfcompat Verified Compatibility Preview
2+
3+
This directory contains versioned, machine-evaluable compatibility profiles.
4+
It is a preview conformance program, not an independent certification program.
5+
6+
A conformance decision is scoped to:
7+
8+
- one immutable subject digest;
9+
- one versioned profile and matrix;
10+
- one bpfcompat report and its exact test invocation;
11+
- the required kernels actually exercised by that report; and
12+
- the time and verifier identity recorded in the decision.
13+
14+
The evaluator has three outcomes:
15+
16+
- `conformant`: every required assertion passed;
17+
- `nonconformant`: the tested product behavior failed at least one required
18+
assertion; and
19+
- `inconclusive`: the test contract, evidence, or infrastructure was incomplete.
20+
21+
Only `conformant` decisions produce an in-toto Simple Verification Result.
22+
Every valid evaluation produces a decision and an in-toto Test Result, including
23+
failed and inconclusive evaluations.
24+
25+
The first profile is [Falco modern_bpf v0.1](falco-modern-bpf-v0.1/spec.md).
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# Canonical required matrix for the Falco modern_bpf compatibility preview.
2+
# This matches the upstream falcosecurity/libs proof-of-concept matrix. A
3+
# credential covers only these required profiles, even if a report contains
4+
# additional informational targets.
5+
name: falco-modern-bpf-v0.1
6+
profiles:
7+
- id: ubuntu-22.04-5.15
8+
required: true
9+
- id: debian-12-6.1
10+
required: true
11+
- id: ubuntu-24.04-6.8
12+
required: true
13+
- id: almalinux-8-4.18
14+
required: true
15+
- id: almalinux-9-5.14
16+
required: true

0 commit comments

Comments
 (0)