metadata_croissant_draft.json

{
  "@context": {
    "@language": "en",
    "@vocab": "https://schema.org/",
    "cr": "http://mlcommons.org/croissant/",
    "rai": "http://mlcommons.org/croissant/RAI/",
    "sc": "https://schema.org/"
  },
  "@type": "cr:Dataset",
  "name": "AgentCloseoutBench-Sycophancy-HeldOut",
  "description": "Taxonomy-grounded sycophancy held-out corpus for evaluating the no-sycophancy dark-pattern detection hook in AgentCloseoutBench. Covers five sycophancy modes (opener-praise, rebuttal flips, multi-turn capitulation, social/emotional validation, false-statement validation) drawn from four 2026 research taxonomies (SycEval, SyConBench, ELEPHANT, BrokenMath). All text is original, synthesized from taxonomy structure only. Judge-validated with dual independent LLM judges (inter-Claude proxy). Headline finding: the hook's 0.667 TRAIN F1 does not survive; v2 scores F1=0.298 (P=1.000, R=0.175) on heldout_test and F1=0.231 (P=1.000, R=0.130) on freshtest.",
  "license": "https://www.apache.org/licenses/LICENSE-2.0",
  "url": "https://huggingface.co/datasets/waitdeadai/agent-closeout-bench",
  "version": "3.0.0",
  "isAccessibleForFree": true,
  "keywords": [
    "sycophancy",
    "llm-safety",
    "dark-patterns",
    "held-out",
    "ai-safety",
    "agentic-ai",
    "benchmark",
    "closeout-detection"
  ],
  "creator": {
    "@type": "Organization",
    "name": "waitdeadai",
    "url": "https://github.com/waitdeadai"
  },
  "distribution": [
    {
      "@type": "cr:FileObject",
      "@id": "heldout-positives-jsonl",
      "name": "heldout_positives",
      "description": "Held-out test set: 58 records (40 sycophancy positives + 18 controls) across five taxonomy sources. Split=heldout_test. Used in v3 and v4 evaluations.",
      "contentUrl": "data/sycophancy/heldout_positives.jsonl",
      "encodingFormat": "application/jsonlines",
      "sha256": "to-be-computed-on-upload"
    },
    {
      "@type": "cr:FileObject",
      "@id": "freshtest-jsonl",
      "name": "freshtest",
      "description": "Fresh generalization test: 35 records (23 positives + 12 controls) hand-authored with phrasing distinct from the v3 corpus and v4 regex tiers. Split=freshtest. Used in v4 evaluation.",
      "contentUrl": "data/sycophancy/freshtest.jsonl",
      "encodingFormat": "application/jsonlines",
      "sha256": "to-be-computed-on-upload"
    }
  ],
  "recordSet": [
    {
      "@type": "cr:RecordSet",
      "@id": "heldout_test_records",
      "name": "heldout_test_records",
      "description": "Records from heldout_positives.jsonl (split=heldout_test). Includes source_license, judge_primary_reason, and human_label fields not present in freshtest.",
      "source": {
        "cr:fileObject": {"@id": "heldout-positives-jsonl"}
      },
      "field": [
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/id",
          "name": "id",
          "description": "Unique record identifier, e.g. syc-heldout-001.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.id"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/closeout_text",
          "name": "closeout_text",
          "description": "The agent closeout text to classify.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.closeout_text"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/label_construction",
          "name": "label_construction",
          "description": "Construction label: 1 = sycophantic, 0 = control.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.label_construction"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/source_taxonomy",
          "name": "source_taxonomy",
          "description": "Sycophancy taxonomy source: SycEval / SyConBench / ELEPHANT / BrokenMath / DarkBench-style / control.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.source_taxonomy"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/subtype",
          "name": "subtype",
          "description": "Fine-grained subtype within the taxonomy.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.subtype"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/surface",
          "name": "surface",
          "description": "Text surface type: chat / multi_turn / closeout.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.surface"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/provenance",
          "name": "provenance",
          "description": "Construction provenance string (e.g. synthesized-from-taxonomy:SycEval:regressive_capitulation).",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.provenance"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/source_license",
          "name": "source_license",
          "description": "License of the original authored text (CC-BY-4.0 with taxonomy-only use notation).",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.source_license"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/split",
          "name": "split",
          "description": "Dataset split name. Always heldout_test for this file.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.split"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/judge_primary",
          "name": "judge_primary",
          "description": "Primary LLM judge label (claude-sonnet-4-6, rubric A): 1 = sycophantic, 0 = not sycophantic.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.judge_primary"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/judge_secondary",
          "name": "judge_secondary",
          "description": "Secondary LLM judge label (claude-haiku-4-5, rubric B): 1 = sycophantic, 0 = not sycophantic.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.judge_secondary"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/judge_primary_reason",
          "name": "judge_primary_reason",
          "description": "Primary judge reasoning string; null when not captured.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.judge_primary_reason"}}
        },
        {
          "@type": "cr:Field",
          "@id": "heldout_test_records/human_label",
          "name": "human_label",
          "description": "Human annotator label; null = not yet filled. See annotations/sycophancy_heldout_blind_sheet.csv for the blind validation sheet.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "heldout-positives-jsonl"}, "cr:extract": {"cr:jsonPath": "$.human_label"}}
        }
      ]
    },
    {
      "@type": "cr:RecordSet",
      "@id": "freshtest_records",
      "name": "freshtest_records",
      "description": "Records from freshtest.jsonl (split=freshtest). Hand-authored with novel phrasing distinct from heldout_test and v4 regex tiers. Does not include source_license, judge_primary_reason, or human_label fields.",
      "source": {
        "cr:fileObject": {"@id": "freshtest-jsonl"}
      },
      "field": [
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/id",
          "name": "id",
          "description": "Unique record identifier, e.g. syc-fresh-001.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.id"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/closeout_text",
          "name": "closeout_text",
          "description": "The agent closeout text to classify.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.closeout_text"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/label_construction",
          "name": "label_construction",
          "description": "Construction label: 1 = sycophantic, 0 = control.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.label_construction"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/source_taxonomy",
          "name": "source_taxonomy",
          "description": "Sycophancy taxonomy source: SyConBench / SycEval / ELEPHANT / BrokenMath / DarkBench / control.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.source_taxonomy"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/subtype",
          "name": "subtype",
          "description": "Fine-grained subtype within the taxonomy.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.subtype"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/surface",
          "name": "surface",
          "description": "Text surface type: chat.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.surface"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/provenance",
          "name": "provenance",
          "description": "Construction provenance. Always: hand-authored-fresh-test (distinct phrasing from tuning set).",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.provenance"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/split",
          "name": "split",
          "description": "Dataset split name. Always freshtest for this file.",
          "dataType": "sc:Text",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.split"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/judge_primary",
          "name": "judge_primary",
          "description": "Primary LLM judge label (claude-sonnet-4-6, rubric A): 1 = sycophantic, 0 = not sycophantic.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.judge_primary"}}
        },
        {
          "@type": "cr:Field",
          "@id": "freshtest_records/judge_secondary",
          "name": "judge_secondary",
          "description": "Secondary LLM judge label (claude-haiku-4-5, rubric B): 1 = sycophantic, 0 = not sycophantic.",
          "dataType": "sc:Integer",
          "source": {"cr:fileObject": {"@id": "freshtest-jsonl"}, "cr:extract": {"cr:jsonPath": "$.judge_secondary"}}
        }
      ]
    }
  ],
  "rai:dataCollection": "Synthesized original text constructed from 2026 sycophancy taxonomy structure (SycEval, SyConBench, ELEPHANT, BrokenMath). No verbatim text from source papers. freshtest split hand-authored with phrasing distinct from heldout_test and v4 regex tiers.",
  "rai:dataCollectionType": "expert synthesis + hand-authoring",
  "rai:dataCollectionRawData": "No raw private transcripts are included. Recovered local transcripts are stored as excluded development evidence and are not in this public release.",
  "rai:dataPreprocessingProtocol": "Records generated from taxonomy structure with per-source quotas, schema validation, split assignment, duplicate checks, and sensitive-token scans. Judge labels frozen into JSONL at construction time.",
  "rai:dataManipulationProtocol": "Positive and negative records balanced by taxonomy source and label. Controls include honest disagreement, validation-then-disagreement redemption, and neutral closeout patterns.",
  "rai:dataAnnotationProtocol": "Dual independent LLM judges (claude-sonnet-4-6 rubric A + claude-haiku-4-5 rubric B) used as inter-Claude proxy validation. Human annotation deferred; blind sheet available in annotations/sycophancy_heldout_blind_sheet.csv. Both judges are Claude models — kappa values are NOT cross-provider or human validation.",
  "rai:annotationsPerItem": "2 LLM judge labels per item. 0 human annotations in current release; human validation in progress via blind sheet.",
  "rai:dataUseCases": "Evaluating sycophancy detection hooks and dark-pattern classifiers for agentic AI closeout text. Reproducing no-sycophancy v2/v4 diagnostic. Motivating semantic (vs. lexical) sycophancy detection research.",
  "rai:dataLimitations": "English-only. Claude Code lifecycle-specific synthetic text. Inter-Claude judge proxy (not human/cross-provider). BrokenMath false-statement validation not addressable by regex. No statistical significance on v4 fresh-set gain. Contamination-prone public splits. Human labels not yet available.",
  "rai:personalSensitiveInformation": "The public corpus contains no real-session personal data. All text is synthesized original content."
}