Skip to content

Commit d7209c1

Browse files
committed
fix(examples): redact camelcase audit secrets
1 parent 590dd8e commit d7209c1

2 files changed

Lines changed: 69 additions & 35 deletions

File tree

examples/optimization/eval_optimize_loop/eval_loop/pipeline.py

Lines changed: 33 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,36 @@
5252
from .writeback import snapshot_prompt_files
5353

5454
_REPOSITORY_ROOT = Path(__file__).resolve().parents[4]
55+
_SENSITIVE_CONFIG_KEY_SUFFIXES = (
56+
"apikey",
57+
"authorization",
58+
"authtoken",
59+
"bearertoken",
60+
"clientsecret",
61+
"credential",
62+
"credentials",
63+
"idtoken",
64+
"password",
65+
"passwd",
66+
"privatekey",
67+
"refreshtoken",
68+
"secret",
69+
"secretaccesskey",
70+
"secretkey",
71+
"signingkey",
72+
"sshkey",
73+
"token",
74+
"accesstoken",
75+
)
76+
_SENSITIVE_CONFIG_KEY_MARKERS = (
77+
"apikey",
78+
"password",
79+
"passwd",
80+
"credential",
81+
"secret",
82+
"authorization",
83+
"privatekey",
84+
)
5585

5686

5787
@dataclass(frozen=True)
@@ -1580,36 +1610,9 @@ def _pretty_json_sha256(value: Any) -> str:
15801610

15811611

15821612
def _is_secret_key(lowered: str) -> bool:
1583-
normalized = lowered.replace("-", "_").replace(" ", "_")
1584-
if normalized in {
1585-
"api_key",
1586-
"apikey",
1587-
"token",
1588-
"access_token",
1589-
"refresh_token",
1590-
"auth_token",
1591-
"password",
1592-
"passwd",
1593-
"credentials",
1594-
"credential",
1595-
"secret",
1596-
"authorization",
1597-
"private_key",
1598-
"signing_key",
1599-
"ssh_key",
1600-
}:
1601-
return True
1602-
if normalized.endswith("_token"):
1603-
return True
1604-
return any(marker in normalized for marker in (
1605-
"api_key",
1606-
"password",
1607-
"passwd",
1608-
"credential",
1609-
"secret",
1610-
"authorization",
1611-
"private_key",
1612-
))
1613+
normalized = "".join(character for character in lowered if character.isalnum())
1614+
return normalized.endswith(_SENSITIVE_CONFIG_KEY_SUFFIXES) or any(marker in normalized
1615+
for marker in _SENSITIVE_CONFIG_KEY_MARKERS)
16131616

16141617

16151618
def _sanitize_public_value(value: Any) -> Any:

examples/optimization/eval_optimize_loop/tests/test_pipeline_orchestration.py

Lines changed: 36 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1681,11 +1681,19 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat
16811681
"api_key": "plain-api-key",
16821682
"nested": {
16831683
"access_token": "plain-token",
1684+
"accessToken": "plain-camel-access-token",
1685+
"authToken": "plain-camel-auth-token",
1686+
"bearerToken": "plain-camel-bearer-token",
1687+
"idToken": "plain-camel-id-token",
1688+
"githubToken": "plain-camel-github-token",
1689+
"passwordValue": "plain-camel-password-value",
16841690
"credentials": {
16851691
"password": "plain-password"
16861692
},
16871693
"github_token": "plain-github-token",
16881694
"private_key": "plain-private-key",
1695+
"max_tokens": 2048,
1696+
"token_budget": 4096,
16891697
"cache_path": str(tmp_path / "private-cache"),
16901698
},
16911699
}
@@ -1700,6 +1708,14 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat
17001708
assert "plain-password" not in payload
17011709
assert "plain-github-token" not in payload
17021710
assert "plain-private-key" not in payload
1711+
assert "plain-camel-access-token" not in payload
1712+
assert "plain-camel-auth-token" not in payload
1713+
assert "plain-camel-bearer-token" not in payload
1714+
assert "plain-camel-id-token" not in payload
1715+
assert "plain-camel-github-token" not in payload
1716+
assert "plain-camel-password-value" not in payload
1717+
assert report.audit["config_snapshot"]["nested"]["max_tokens"] == 2048
1718+
assert report.audit["config_snapshot"]["nested"]["token_budget"] == 4096
17031719
assert str(tmp_path.resolve()) not in payload
17041720
assert report.run["reproducibility_shell"] == "powershell"
17051721
assert "$EXTERNAL" in report.run["reproducibility_command"]
@@ -1713,11 +1729,26 @@ async def test_public_report_redacts_secrets_and_absolute_personal_paths(tmp_pat
17131729
request.run_id,
17141730
"config.snapshot.json",
17151731
).read_text(encoding="utf-8")
1716-
assert "plain-api-key" not in persisted + config_snapshot
1717-
assert "plain-token" not in persisted + config_snapshot
1718-
assert "plain-password" not in persisted + config_snapshot
1719-
assert "plain-github-token" not in persisted + config_snapshot
1720-
assert "plain-private-key" not in persisted + config_snapshot
1732+
final_run_dir = Path(request.output_dir) / "runs" / request.run_id
1733+
persisted_bytes = persisted.encode("utf-8") + persisted_markdown.encode("utf-8")
1734+
persisted_bytes += b"".join(path.read_bytes() for path in final_run_dir.rglob("*") if path.is_file())
1735+
for secret in (
1736+
"plain-api-key",
1737+
"plain-token",
1738+
"plain-password",
1739+
"plain-github-token",
1740+
"plain-private-key",
1741+
"plain-camel-access-token",
1742+
"plain-camel-auth-token",
1743+
"plain-camel-bearer-token",
1744+
"plain-camel-id-token",
1745+
"plain-camel-github-token",
1746+
"plain-camel-password-value",
1747+
):
1748+
assert secret.encode("utf-8") not in persisted_bytes
1749+
redacted_snapshot = json.loads(config_snapshot)
1750+
assert redacted_snapshot["nested"]["max_tokens"] == 2048
1751+
assert redacted_snapshot["nested"]["token_budget"] == 4096
17211752
assert "```powershell" in persisted_markdown
17221753
assert str(tmp_path.resolve()) not in persisted_markdown
17231754
assert hashlib.sha256(

0 commit comments

Comments
 (0)