diff --git a/cmd/clawscan/main.go b/cmd/clawscan/main.go index 1b680c9..ceaf074 100644 --- a/cmd/clawscan/main.go +++ b/cmd/clawscan/main.go @@ -800,7 +800,7 @@ OpenClaw install policy: Benchmark command flags: --split Benchmark split. Defaults to benchmark for SkillTrustBench and eval_holdout for clawhub-security-signals. - --ids Run selected benchmark IDs from a text file or JSONL id source. SkillTrustBench only. + --ids Run selected benchmark IDs from a streamed text or JSONL source (max 5520 IDs). SkillTrustBench only. --limit Maximum benchmark rows to run. 0 means all rows. --offset Benchmark row offset. Defaults to 0. --predictions-output Write benchmark predictions JSONL. Defaults next to --output for clawhub-security-signals. diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 77682c0..2dfd08a 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -12,7 +12,9 @@ clawscan benchmark SkillTrustBench \ ``` Use `--ids ` with SkillTrustBench to run a fixed subset from a -plain text ID list or JSONL rows with an `id` field. +plain text ID list or JSONL rows with an `id` field. The loader streams the +source (file or HTTP) and accepts at most 5,520 unique IDs, the size of the +pinned SkillTrustBench full set. ## Available benchmarks diff --git a/internal/runner/benchmark.go b/internal/runner/benchmark.go index 1c9f115..e0b0961 100644 --- a/internal/runner/benchmark.go +++ b/internal/runner/benchmark.go @@ -40,6 +40,18 @@ const ( huggingFaceRowsEndpoint = "https://datasets-server.huggingface.co/rows" huggingFaceRowsPageSize = 100 huggingFaceRowsMaxAttempts = 6 + // maxSkillTrustBenchIDSelection is the pinned SkillTrustBench full set + // (5,520 cases). --ids is SkillTrustBench-only, so a valid selection + // cannot contain more unique IDs than that set. + maxSkillTrustBenchIDSelection = 5520 + // maxBenchmarkIDBytes caps one extracted id. Documented SkillTrustBench + // ids are case_NNNNN. The scanner still allows 1 MiB records, so this + // stops a hostile source from retaining megabyte-sized unique ids. + maxBenchmarkIDBytes = 256 + // maxBenchmarkIDSelectionBytes caps retained id text (not the JSONL + // stream). 256 KiB holds the 5,520-id set with headroom; it is not a + // file-size limit (the full JSONL is about 1.3 MiB). + maxBenchmarkIDSelectionBytes = 256 * 1024 ) var huggingFaceRowsRetryDelay = 2 * time.Second @@ -312,11 +324,12 @@ func LoadBenchmarkIDSelection(source string) (BenchmarkIDSelection, error) { if source == "" { return BenchmarkIDSelection{}, errors.New("--ids source is required") } - data, err := readBenchmarkIDSource(source) + reader, err := openBenchmarkIDSource(source) if err != nil { return BenchmarkIDSelection{}, err } - ids, err := parseBenchmarkIDs(source, data) + defer reader.Close() + ids, err := parseBenchmarkIDs(source, reader) if err != nil { return BenchmarkIDSelection{}, err } @@ -329,31 +342,32 @@ func LoadBenchmarkIDSelection(source string) (BenchmarkIDSelection, error) { }, nil } -func readBenchmarkIDSource(source string) ([]byte, error) { +func openBenchmarkIDSource(source string) (io.ReadCloser, error) { if parsed, err := url.Parse(source); err == nil && (parsed.Scheme == "http" || parsed.Scheme == "https") { client := &http.Client{Timeout: 60 * time.Second} resp, err := client.Get(source) if err != nil { return nil, fmt.Errorf("read --ids source %s: %w", source, err) } - defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode > 299 { + resp.Body.Close() return nil, fmt.Errorf("read --ids source %s: HTTP %d", source, resp.StatusCode) } - return io.ReadAll(resp.Body) + return resp.Body, nil } - data, err := os.ReadFile(source) + file, err := os.Open(source) if err != nil { return nil, fmt.Errorf("read --ids source %s: %w", source, err) } - return data, nil + return file, nil } -func parseBenchmarkIDs(source string, data []byte) ([]string, error) { - scanner := bufio.NewScanner(strings.NewReader(string(data))) +func parseBenchmarkIDs(source string, reader io.Reader) ([]string, error) { + scanner := bufio.NewScanner(reader) scanner.Buffer(make([]byte, 1024), 1024*1024) var ids []string seen := map[string]bool{} + retained := 0 lineNumber := 0 for scanner.Scan() { lineNumber++ @@ -368,8 +382,18 @@ func parseBenchmarkIDs(source string, data []byte) ([]string, error) { if seen[id] { return nil, fmt.Errorf("--ids source %s line %d duplicates benchmark id %s", source, lineNumber, id) } + if len(id) > maxBenchmarkIDBytes { + return nil, fmt.Errorf("--ids source %s line %d exceeds the %d-byte benchmark id limit", source, lineNumber, maxBenchmarkIDBytes) + } + if retained+len(id) > maxBenchmarkIDSelectionBytes { + return nil, fmt.Errorf("--ids source %s exceeds the %d-byte retained-id budget", source, maxBenchmarkIDSelectionBytes) + } seen[id] = true ids = append(ids, id) + retained += len(id) + if len(ids) > maxSkillTrustBenchIDSelection { + return nil, fmt.Errorf("--ids source %s exceeds the %d-id SkillTrustBench selection limit", source, maxSkillTrustBenchIDSelection) + } } if err := scanner.Err(); err != nil { return nil, fmt.Errorf("read --ids source %s: %w", source, err) diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index 8ee851d..758fe30 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -801,6 +801,91 @@ func TestLoadBenchmarkIDSelectionRejectsBadSources(t *testing.T) { } } +func TestLoadBenchmarkIDSelectionAcceptsJSONLLargerThan256KiB(t *testing.T) { + payload := oversizedBenchmarkIDJSONL(t, 400) + if len(payload) <= 256*1024 { + t.Fatalf("fixture is %d bytes, want more than 256 KiB", len(payload)) + } + + path := filepath.Join(t.TempDir(), "ids.jsonl") + if err := os.WriteFile(path, payload, 0o644); err != nil { + t.Fatal(err) + } + fileSelection, err := LoadBenchmarkIDSelection(path) + if err != nil { + t.Fatal(err) + } + if len(fileSelection.IDs) != 400 { + t.Fatalf("file ids = %d, want 400", len(fileSelection.IDs)) + } + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(payload) + })) + defer server.Close() + + httpSelection, err := LoadBenchmarkIDSelection(server.URL + "/ids.jsonl") + if err != nil { + t.Fatal(err) + } + if len(httpSelection.IDs) != 400 { + t.Fatalf("http ids = %d, want 400", len(httpSelection.IDs)) + } +} + +func TestLoadBenchmarkIDSelectionRejectsOversizedRetainedIDs(t *testing.T) { + huge := strings.Repeat("a", maxBenchmarkIDBytes+1) + path := filepath.Join(t.TempDir(), "huge-id.txt") + if err := os.WriteFile(path, []byte(huge+"\n"), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadBenchmarkIDSelection(path) + if err == nil || !strings.Contains(err.Error(), "256-byte benchmark id limit") { + t.Fatalf("err = %v, want 256-byte benchmark id limit", err) + } + + var body strings.Builder + // 2000 IDs * 200 bytes is under the 5,520 count cap but over the + // retained-id budget (256 KiB). + chunk := strings.Repeat("b", 200) + for i := 0; i < 2000; i++ { + fmt.Fprintf(&body, "%s-%04d\n", chunk, i) + } + aggPath := filepath.Join(t.TempDir(), "agg-ids.txt") + if err := os.WriteFile(aggPath, []byte(body.String()), 0o644); err != nil { + t.Fatal(err) + } + _, err = LoadBenchmarkIDSelection(aggPath) + if err == nil || !strings.Contains(err.Error(), "262144-byte retained-id budget") { + t.Fatalf("err = %v, want 262144-byte retained-id budget", err) + } +} + +func TestLoadBenchmarkIDSelectionRejectsMoreIDsThanPinnedSet(t *testing.T) { + var body strings.Builder + for i := 0; i < maxSkillTrustBenchIDSelection+1; i++ { + fmt.Fprintf(&body, "case_%05d\n", i) + } + path := filepath.Join(t.TempDir(), "ids.txt") + if err := os.WriteFile(path, []byte(body.String()), 0o644); err != nil { + t.Fatal(err) + } + _, err := LoadBenchmarkIDSelection(path) + if err == nil || !strings.Contains(err.Error(), "5520-id") { + t.Fatalf("err = %v, want 5520-id selection limit", err) + } +} + +func oversizedBenchmarkIDJSONL(t *testing.T, count int) []byte { + t.Helper() + var body strings.Builder + pad := strings.Repeat("x", 700) + for i := 0; i < count; i++ { + fmt.Fprintf(&body, `{"id":"case_%05d","judgment":"normal","pad":"%s"}`+"\n", i, pad) + } + return []byte(body.String()) +} + func TestRunSkillTrustBenchBenchmarkRejectsMissingSelectedID(t *testing.T) { dir := t.TempDir() idsPath := filepath.Join(dir, "ids.txt") diff --git a/skills/clawscan-cli/SKILL.md b/skills/clawscan-cli/SKILL.md index c8a742e..2e0e24a 100644 --- a/skills/clawscan-cli/SKILL.md +++ b/skills/clawscan-cli/SKILL.md @@ -213,7 +213,8 @@ SkillTrustBench uses split `benchmark`. The first live run downloads and caches into temporary scan targets. Use `--ids ` with SkillTrustBench to run a fixed subset from a -plain text file with one ID per line or JSONL rows with an `id` field. `--ids` +plain text file with one ID per line or JSONL rows with an `id` field. The +source is streamed and may contain at most 5,520 unique IDs. `--ids` preserves source order, records `idsSource`, `idsCount`, and `idsSha256` in the artifact, and is mutually exclusive with `--limit` and `--offset`.