From 3803af26d6f2c05174fc0e77a5b1af7d6e7e9c9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 20:27:55 +0000 Subject: [PATCH 01/25] Add opt-in local PII redaction before prompts leave the proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scrub high-impact PII from chat-completion prompts locally, before the request is HPKE-sealed to the TEE — defense-in-depth on top of Veil's existing end-to-end privacy. Off by default; enable with --pii-scrub / OG_VEIL_PII_SCRUB. Two layered tiers: - Regex (always on when enabled, zero new core deps): email, US SSN, and bank numbers (credit cards Luhn-checked, IBANs mod-97-checked, plus labelled routing/account numbers). DOB caught by context cues. - NER (optional [pii] extra: Presidio + spaCy en_core_web_sm): free-form addresses/locations, with an optional --pii-all-dates mode. Redaction is irreversible (no de-anonymization), so the TEE's signed output_hash covers exactly what it ran. Wired into Gateway.chat() ahead of the retry loop; covers string and multimodal text message content. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- README.md | 36 ++++ pyproject.toml | 11 + tests/test_pii.py | 112 ++++++++++ uv.lock | 520 +++++++++++++++++++++++++++++++++++++++++++++- veil/cli.py | 23 ++ veil/config.py | 15 ++ veil/gateway.py | 11 + veil/pii.py | 275 ++++++++++++++++++++++++ 8 files changed, 1002 insertions(+), 1 deletion(-) create mode 100644 tests/test_pii.py create mode 100644 veil/pii.py diff --git a/README.md b/README.md index b392605..9d84c52 100644 --- a/README.md +++ b/README.md @@ -173,6 +173,42 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). | `OG_VEIL_TEE_ID` | `--tee-id` | — | Pin a specific registry TEE. | | `OG_VEIL_EXPECTED_PCR_HASH` | `--expected-pcr` | — | Refuse any TEE whose `pcrHash` differs. | | `OG_VEIL_APP_URL` | `--app-url` | `https://chat.opengradient.ai` | Chat app origin for login. | +| `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Redact high-impact PII from prompts locally before they leave the machine. | +| `OG_VEIL_PII_REDACT_ALL_DATES` | `--pii-all-dates` | off | With scrubbing on, redact *every* date as a DOB (aggressive) instead of only birth-date-cued ones. | + +### Local PII redaction (opt-in) + +Veil already keeps prompts private end-to-end — OHTTP splits *who you are* from +*what you ask*, and the enclave is attested and reproducible. Local PII scrubbing +is **defense-in-depth** on top of that: when enabled, high-impact PII is +irreversibly replaced with `[REDACTED_*]` tags *before* the prompt is encrypted to +the TEE, so the raw values never leave your machine. Handy for compliance, +data-residency, and keeping PII out of any model-side logging. + +```sh +og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 +``` + +Two layered tiers: + +- **Regex (always on when enabled, zero extra deps)** — email, US SSN, and bank + numbers (credit cards Luhn-checked, IBANs mod-97-checked, plus routing/account + numbers when labelled). Dates of birth are caught by context (`DOB:`, `born on …`). +- **Addresses (optional)** — free-form street addresses are prose that regex can't + see, so they need a lightweight local NER model. Install the extra once: + + ```sh + pip install 'opengradient-veil[pii]' + python -m spacy download en_core_web_sm + ``` + + With the extra present, `--pii-scrub` also redacts addresses/locations. Without + it, scrubbing still runs (regex tier) and logs that address coverage is off. + +Redaction is **irreversible** — there's no de-anonymization step, so the TEE's +signed `output_hash` covers exactly what it ran. This is risk-reduction, not a +guarantee: NER misses a fraction of addresses each run, and bare (unlabelled) +account numbers can slip through. ## Notes & limitations diff --git a/pyproject.toml b/pyproject.toml index 474d157..46bc1d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,17 @@ dependencies = [ "requests>=2.32.0", ] +[project.optional-dependencies] +# Adds free-form address/location (and optional all-date) PII redaction on top of +# the always-available, zero-dependency regex tier. Microsoft Presidio drives a +# spaCy NER model; after installing this extra, download the model once with: +# python -m spacy download en_core_web_sm +pii = [ + "presidio-analyzer>=2.2.0", + "presidio-anonymizer>=2.2.0", + "spacy>=3.7.0", +] + [project.urls] Homepage = "https://opengradient.ai" Repository = "https://github.com/OpenGradient/veil" diff --git a/tests/test_pii.py b/tests/test_pii.py new file mode 100644 index 0000000..4133730 --- /dev/null +++ b/tests/test_pii.py @@ -0,0 +1,112 @@ +"""PII redaction tests — the always-on regex tier and request rewriting. + +The optional NER tier (Presidio + spaCy) is exercised only when the [pii] extra +is installed, so addresses are not asserted here; these cover the zero-dependency +behavior that ships in the base install. +""" + +from __future__ import annotations + +from veil.pii import ( + ADDRESS_TAG, + BANK_TAG, + DOB_TAG, + EMAIL_TAG, + SSN_TAG, + Redactor, + build_redactor, +) + +# A Redactor with no NER engine loaded — pure regex tier. (If Presidio happens to +# be installed in the dev env it would only *add* address coverage, never change +# these structured-PII assertions.) +R = Redactor() + + +def test_email_redacted(): + assert R.scrub_text("ping me at jane.doe+x@example.co.uk please") == ( + f"ping me at {EMAIL_TAG} please" + ) + + +def test_dashed_ssn_redacted(): + assert R.scrub_text("my ssn is 123-45-6789") == f"my ssn is {SSN_TAG}" + + +def test_bare_ssn_only_with_label(): + # Bare 9-digit runs are left alone (too ambiguous)… + assert "987654321" in R.scrub_text("order 987654321 shipped") + # …but a labelled one is redacted, keeping the label. + out = R.scrub_text("SSN: 987654321") + assert "987654321" not in out and SSN_TAG in out and out.lower().startswith("ssn") + + +def test_valid_credit_card_redacted_invalid_kept(): + # 4111 1111 1111 1111 is the canonical Luhn-valid test number. + assert R.scrub_text("card 4111 1111 1111 1111") == f"card {BANK_TAG}" + # One digit off → fails Luhn → not redacted. + assert "4111 1111 1111 1112" in R.scrub_text("card 4111 1111 1111 1112") + + +def test_valid_iban_redacted(): + out = R.scrub_text("send to GB82 WEST 1234 5698 7654 32 today") + assert BANK_TAG in out and "WEST" not in out + + +def test_invalid_iban_kept(): + assert "GB00WEST12345698765432" in R.scrub_text("ref GB00WEST12345698765432") + + +def test_labelled_account_number_redacted(): + out = R.scrub_text("account number: 0012345678") + assert "0012345678" not in out and BANK_TAG in out + + +def test_dob_context_redacts_only_cued_dates(): + out = R.scrub_text("DOB: 04/12/1990, meeting on 06/01/2026") + assert DOB_TAG in out + assert "04/12/1990" not in out + # A non-birth date is untouched without the all-dates toggle. + assert "06/01/2026" in out + + +def test_scrub_request_string_content(): + body = { + "model": "gpt-4.1", + "messages": [ + {"role": "system", "content": "be helpful"}, + {"role": "user", "content": "email me at a@b.com"}, + ], + } + out = R.scrub_request(body) + assert out["messages"][1]["content"] == f"email me at {EMAIL_TAG}" + # Original body is not mutated. + assert body["messages"][1]["content"] == "email me at a@b.com" + + +def test_scrub_request_multimodal_parts(): + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "reach me at a@b.com"}, + {"type": "image_url", "image_url": {"url": "http://x/y.png"}}, + ], + } + ] + } + out = R.scrub_request(body) + parts = out["messages"][0]["content"] + assert parts[0]["text"] == f"reach me at {EMAIL_TAG}" + assert parts[1] == {"type": "image_url", "image_url": {"url": "http://x/y.png"}} + + +def test_build_redactor_disabled_returns_none(): + assert build_redactor(enabled=False) is None + assert build_redactor(enabled=True) is not None + + +def test_address_tag_is_distinct(): + # Sanity: tags are unique strings so downstream tooling can grep them. + assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, DOB_TAG, ADDRESS_TAG}) == 5 diff --git a/uv.lock b/uv.lock index c576016..4599bdf 100644 --- a/uv.lock +++ b/uv.lock @@ -305,6 +305,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, ] +[[package]] +name = "blis" +version = "0.7.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.0rc1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/8c/60c85350f2e1c9647df580083a0f6acc686ef32d1a91f4ab0c624b3ff867/blis-0.7.11.tar.gz", hash = "sha256:cec6d48f75f7ac328ae1b6fbb372dde8c8a57c89559172277f66e01ff08d4d42", size = 2897107, upload-time = "2023-09-22T06:28:25.103Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/59/c8010f380a16709e6d3ef5534845d1ca1e689079914ec67ab60f57edfc37/blis-0.7.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1b68df4d01d62f9adaef3dad6f96418787265a6878891fc4e0fabafd6d02afba", size = 6123547, upload-time = "2023-09-22T06:27:28.47Z" }, + { url = "https://files.pythonhosted.org/packages/a8/73/0a9d4e7f6e78ef270e3a4532b17e060a02087590cf615ba9943fd1a283e9/blis-0.7.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:162e60d941a8151418d558a94ee5547cb1bbeed9f26b3b6f89ec9243f111a201", size = 1106895, upload-time = "2023-09-22T06:27:30.964Z" }, + { url = "https://files.pythonhosted.org/packages/51/f7/a5d9a0be0729f4172248dbae74d7e02b139b3a32cc29650d3ade7ab91fea/blis-0.7.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:686a7d0111d5ba727cd62f374748952fd6eb74701b18177f525b16209a253c01", size = 1707389, upload-time = "2023-09-22T06:27:32.321Z" }, + { url = "https://files.pythonhosted.org/packages/dc/23/eb01450dc284a7ea8ebc0e5296f1f8fdbe5299169f4c318f836b4284a119/blis-0.7.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0421d6e44cda202b113a34761f9a062b53f8c2ae8e4ec8325a76e709fca93b6e", size = 10172888, upload-time = "2023-09-22T06:27:34.529Z" }, + { url = "https://files.pythonhosted.org/packages/2f/09/da0592c74560cc33396504698122f7a56747c82a5e072ca7d2c3397898e1/blis-0.7.11-cp311-cp311-win_amd64.whl", hash = "sha256:0dc9dcb3843045b6b8b00432409fd5ee96b8344a324e031bfec7303838c41a1a", size = 6602835, upload-time = "2023-09-22T06:27:37.46Z" }, + { url = "https://files.pythonhosted.org/packages/e2/12/90897bc489626cb71e51ce8bb89e492fabe96a57811e53159c0f74ae90ec/blis-0.7.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dadf8713ea51d91444d14ad4104a5493fa7ecc401bbb5f4a203ff6448fadb113", size = 6121528, upload-time = "2023-09-22T06:27:39.451Z" }, + { url = "https://files.pythonhosted.org/packages/e2/5d/67a3f6b6108c39d3fd1cf55a7dca9267152190dad419c9de6d764b3708ca/blis-0.7.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5bcdaf370f03adaf4171d6405a89fa66cb3c09399d75fc02e1230a78cd2759e4", size = 1105039, upload-time = "2023-09-22T06:27:41.143Z" }, + { url = "https://files.pythonhosted.org/packages/03/62/0d214dde0703863ed2d3dabb3f10606f7f55ac4eb07a52c3906601331b63/blis-0.7.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7de19264b1d49a178bf8035406d0ae77831f3bfaa3ce02942964a81a202abb03", size = 1701009, upload-time = "2023-09-22T06:27:42.672Z" }, + { url = "https://files.pythonhosted.org/packages/66/aa/bcbd1c6b1c7dfd717ff5c899a1c8adcc6b3e391fb7a0b00fdc64e4e54235/blis-0.7.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ea55c6a4a60fcbf6a0fdce40df6e254451ce636988323a34b9c94b583fc11e5", size = 10161187, upload-time = "2023-09-22T06:27:44.183Z" }, + { url = "https://files.pythonhosted.org/packages/9a/91/4aea63dccee6491a54c630d9817656a886e086ab97222e2d8101d8cdf894/blis-0.7.11-cp312-cp312-win_amd64.whl", hash = "sha256:5a305dbfc96d202a20d0edd6edf74a406b7e1404f4fa4397d24c68454e60b1b4", size = 6624079, upload-time = "2023-09-22T06:27:46.719Z" }, +] + +[[package]] +name = "catalogue" +version = "2.0.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/b4/244d58127e1cdf04cf2dc7d9566f0d24ef01d5ce21811bab088ecc62b5ea/catalogue-2.0.10.tar.gz", hash = "sha256:4f56daa940913d3f09d589c191c74e5a6d51762b3a9e37dd53b7437afd6cda15", size = 19561, upload-time = "2023-09-25T06:29:24.962Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9e/96/d32b941a501ab566a16358d68b6eb4e4acc373fab3c3c4d7d9e649f7b4bb/catalogue-2.0.10-py3-none-any.whl", hash = "sha256:58c2de0020aa90f4a2da7dfad161bf7b3b054c86a5f09fcedc0b2b740c109a9f", size = 17325, upload-time = "2023-09-25T06:29:23.337Z" }, +] + [[package]] name = "certifi" version = "2026.5.20" @@ -533,6 +564,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/0d/67e5b4109ea4a837e80daa87c2c696711955e40449a97e8926672534def2/click-8.4.1-py3-none-any.whl", hash = "sha256:482be17c6991b8c19c5429a1e995d9b0efdbb63172824c41f99965dc0ade8ec2", size = 116639, upload-time = "2026-05-22T04:08:35.26Z" }, ] +[[package]] +name = "cloudpathlib" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/46/1dac9e937dc598ab08a3f66925e11a3188a423e6bacf0b00b080448149eb/cloudpathlib-0.16.0.tar.gz", hash = "sha256:cdfcd35d46d529587d744154a0bdf962aca953b725c8784cd2ec478354ea63a3", size = 38009, upload-time = "2023-10-10T00:36:20.003Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/6e/45b57a7d4573d85d0b0a39d99673dc1f5eea9d92a1a4603b35e968fbf89a/cloudpathlib-0.16.0-py3-none-any.whl", hash = "sha256:f46267556bf91f03db52b5df7a152548596a15aabca1c8731ef32b0b25a1a6a3", size = 45021, upload-time = "2023-10-10T00:36:17.522Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -542,6 +582,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "confection" +version = "0.1.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "srsly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/51/d3/57c6631159a1b48d273b40865c315cf51f89df7a9d1101094ef12e3a37c2/confection-0.1.5.tar.gz", hash = "sha256:8e72dd3ca6bd4f48913cd220f10b8275978e740411654b6e8ca6d7008c590f0e", size = 38924, upload-time = "2024-05-31T16:17:01.559Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/00/3106b1854b45bd0474ced037dfe6b73b90fe68a68968cef47c23de3d43d2/confection-0.1.5-py3-none-any.whl", hash = "sha256:e29d3c3f8eac06b3f77eb9dfb4bf2fc6bcc9622a98ca00a698e3d019c6430b14", size = 35451, upload-time = "2024-05-31T16:16:59.075Z" }, +] + [[package]] name = "cryptography" version = "46.0.7" @@ -601,6 +654,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/2a/1b016902351a523aa2bd446b50a5bc1175d7a7d1cf90fe2ef904f9b84ebc/cryptography-46.0.7-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:258514877e15963bd43b558917bc9f54cf7cf866c38aa576ebf47a77ddbc43a4", size = 3412829, upload-time = "2026-04-08T01:57:48.874Z" }, ] +[[package]] +name = "cymem" +version = "2.0.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/2f0fbb32535c3731b7c2974c569fb9325e0a38ed5565a08e1139a3b71e82/cymem-2.0.13.tar.gz", hash = "sha256:1c91a92ae8c7104275ac26bd4d29b08ccd3e7faff5893d3858cb6fadf1bc1588", size = 12320, upload-time = "2025-11-14T14:58:36.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/64/1db41f7576a6b69f70367e3c15e968fd775ba7419e12059c9966ceb826f8/cymem-2.0.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:673183466b0ff2e060d97ec5116711d44200b8f7be524323e080d215ee2d44a5", size = 43587, upload-time = "2025-11-14T14:57:22.39Z" }, + { url = "https://files.pythonhosted.org/packages/81/13/57f936fc08551323aab3f92ff6b7f4d4b89d5b4e495c870a67cb8d279757/cymem-2.0.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bee2791b3f6fc034ce41268851462bf662ff87e8947e35fb6dd0115b4644a61f", size = 43139, upload-time = "2025-11-14T14:57:23.363Z" }, + { url = "https://files.pythonhosted.org/packages/32/a6/9345754be51e0479aa387b7b6cffc289d0fd3201aaeb8dade4623abd1e02/cymem-2.0.13-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f3aee3adf16272bca81c5826eed55ba3c938add6d8c9e273f01c6b829ecfde22", size = 245063, upload-time = "2025-11-14T14:57:24.839Z" }, + { url = "https://files.pythonhosted.org/packages/d6/01/6bc654101526fa86e82bf6b05d99b2cd47c30a333cfe8622c26c0592beb2/cymem-2.0.13-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:30c4e75a3a1d809e89106b0b21803eb78e839881aa1f5b9bd27b454bc73afde3", size = 244496, upload-time = "2025-11-14T14:57:26.42Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fb/853b7b021e701a1f41687f3704d5f469aeb2a4f898c3fbb8076806885955/cymem-2.0.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ec99efa03cf8ec11c8906aa4d4cc0c47df393bc9095c9dd64b89b9b43e220b04", size = 243287, upload-time = "2025-11-14T14:57:27.542Z" }, + { url = "https://files.pythonhosted.org/packages/d4/2b/0e4664cafc581de2896d75000651fd2ce7094d33263f466185c28ffc96e4/cymem-2.0.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c90a6ecba994a15b17a3f45d7ec74d34081df2f73bd1b090e2adc0317e4e01b6", size = 248287, upload-time = "2025-11-14T14:57:29.055Z" }, + { url = "https://files.pythonhosted.org/packages/21/0f/f94c6950edbfc2aafb81194fc40b6cacc8e994e9359d3cb4328c5705b9b5/cymem-2.0.13-cp311-cp311-win_amd64.whl", hash = "sha256:ce821e6ba59148ed17c4567113b8683a6a0be9c9ac86f14e969919121efb61a5", size = 40116, upload-time = "2025-11-14T14:57:30.592Z" }, + { url = "https://files.pythonhosted.org/packages/00/df/2455eff6ac0381ff165db6883b311f7016e222e3dd62185517f8e8187ed0/cymem-2.0.13-cp311-cp311-win_arm64.whl", hash = "sha256:0dca715e708e545fd1d97693542378a00394b20a37779c1ae2c8bdbb43acef79", size = 36349, upload-time = "2025-11-14T14:57:31.573Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/478a2911ab5028cb710b4900d64aceba6f4f882fcb13fd8d40a456a1b6dc/cymem-2.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8afbc5162a0fe14b6463e1c4e45248a1b2fe2cbcecc8a5b9e511117080da0eb", size = 43745, upload-time = "2025-11-14T14:57:32.52Z" }, + { url = "https://files.pythonhosted.org/packages/f9/71/f0f8adee945524774b16af326bd314a14a478ed369a728a22834e6785a18/cymem-2.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:c9251d889348fe79a75e9b3e4d1b5fa651fca8a64500820685d73a3acc21b6a8", size = 42927, upload-time = "2025-11-14T14:57:33.827Z" }, + { url = "https://files.pythonhosted.org/packages/62/6d/159780fe162ff715d62b809246e5fc20901cef87ca28b67d255a8d741861/cymem-2.0.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:742fc19764467a49ed22e56a4d2134c262d73a6c635409584ae3bf9afa092c33", size = 258346, upload-time = "2025-11-14T14:57:34.917Z" }, + { url = "https://files.pythonhosted.org/packages/eb/12/678d16f7aa1996f947bf17b8cfb917ea9c9674ef5e2bd3690c04123d5680/cymem-2.0.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f190a92fe46197ee64d32560eb121c2809bb843341733227f51538ce77b3410d", size = 260843, upload-time = "2025-11-14T14:57:36.503Z" }, + { url = "https://files.pythonhosted.org/packages/31/5d/0dd8c167c08cd85e70d274b7235cfe1e31b3cebc99221178eaf4bbb95c6f/cymem-2.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d670329ee8dbbbf241b7c08069fe3f1d3a1a3e2d69c7d05ea008a7010d826298", size = 254607, upload-time = "2025-11-14T14:57:38.036Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c9/d6514a412a1160aa65db539836b3d47f9b59f6675f294ec34ae32f867c82/cymem-2.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a84ba3178d9128b9ffb52ce81ebab456e9fe959125b51109f5b73ebdfc6b60d6", size = 262421, upload-time = "2025-11-14T14:57:39.265Z" }, + { url = "https://files.pythonhosted.org/packages/dd/fe/3ee37d02ca4040f2fb22d34eb415198f955862b5dd47eee01df4c8f5454c/cymem-2.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:2ff1c41fd59b789579fdace78aa587c5fc091991fa59458c382b116fc36e30dc", size = 40176, upload-time = "2025-11-14T14:57:40.706Z" }, + { url = "https://files.pythonhosted.org/packages/94/fb/1b681635bfd5f2274d0caa8f934b58435db6c091b97f5593738065ddb786/cymem-2.0.13-cp312-cp312-win_arm64.whl", hash = "sha256:6bbd701338df7bf408648191dff52472a9b334f71bcd31a21a41d83821050f67", size = 35959, upload-time = "2025-11-14T14:57:41.682Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0f/95a4d1e3bebfdfa7829252369357cf9a764f67569328cd9221f21e2c952e/cymem-2.0.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:891fd9030293a8b652dc7fb9fdc79a910a6c76fc679cd775e6741b819ffea476", size = 43478, upload-time = "2025-11-14T14:57:42.682Z" }, + { url = "https://files.pythonhosted.org/packages/bf/a0/8fc929cc29ae466b7b4efc23ece99cbd3ea34992ccff319089c624d667fd/cymem-2.0.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:89c4889bd16513ce1644ccfe1e7c473ba7ca150f0621e66feac3a571bde09e7e", size = 42695, upload-time = "2025-11-14T14:57:43.741Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b3/deeb01354ebaf384438083ffe0310209ef903db3e7ba5a8f584b06d28387/cymem-2.0.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:45dcaba0f48bef9cc3d8b0b92058640244a95a9f12542210b51318da97c2cf28", size = 250573, upload-time = "2025-11-14T14:57:44.81Z" }, + { url = "https://files.pythonhosted.org/packages/36/36/bc980b9a14409f3356309c45a8d88d58797d02002a9d794dd6c84e809d3a/cymem-2.0.13-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e96848faaafccc0abd631f1c5fb194eac0caee4f5a8777fdbb3e349d3a21741c", size = 254572, upload-time = "2025-11-14T14:57:46.023Z" }, + { url = "https://files.pythonhosted.org/packages/fd/dd/a12522952624685bd0f8968e26d2ed6d059c967413ce6eb52292f538f1b0/cymem-2.0.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e02d3e2c3bfeb21185d5a4a70790d9df40629a87d8d7617dc22b4e864f665fa3", size = 248060, upload-time = "2025-11-14T14:57:47.605Z" }, + { url = "https://files.pythonhosted.org/packages/08/11/5dc933ddfeb2dfea747a0b935cb965b9a7580b324d96fc5f5a1b5ff8df29/cymem-2.0.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fece5229fd5ecdcd7a0738affb8c59890e13073ae5626544e13825f26c019d3c", size = 254601, upload-time = "2025-11-14T14:57:48.861Z" }, + { url = "https://files.pythonhosted.org/packages/70/66/d23b06166864fa94e13a98e5922986ce774832936473578febce64448d75/cymem-2.0.13-cp313-cp313-win_amd64.whl", hash = "sha256:38aefeb269597c1a0c2ddf1567dd8605489b661fa0369c6406c1acd433b4c7ba", size = 40103, upload-time = "2025-11-14T14:57:50.396Z" }, + { url = "https://files.pythonhosted.org/packages/2f/9e/c7b21271ab88a21760f3afdec84d2bc09ffa9e6c8d774ad9d4f1afab0416/cymem-2.0.13-cp313-cp313-win_arm64.whl", hash = "sha256:717270dcfd8c8096b479c42708b151002ff98e434a7b6f1f916387a6c791e2ad", size = 36016, upload-time = "2025-11-14T14:57:51.611Z" }, + { url = "https://files.pythonhosted.org/packages/7f/28/d3b03427edc04ae04910edf1c24b993881c3ba93a9729a42bcbb816a1808/cymem-2.0.13-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:7e1a863a7f144ffb345397813701509cfc74fc9ed360a4d92799805b4b865dd1", size = 46429, upload-time = "2025-11-14T14:57:52.582Z" }, + { url = "https://files.pythonhosted.org/packages/35/a9/7ed53e481f47ebfb922b0b42e980cec83e98ccb2137dc597ea156642440c/cymem-2.0.13-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c16cb80efc017b054f78998c6b4b013cef509c7b3d802707ce1f85a1d68361bf", size = 46205, upload-time = "2025-11-14T14:57:53.64Z" }, + { url = "https://files.pythonhosted.org/packages/61/39/a3d6ad073cf7f0fbbb8bbf09698c3c8fac11be3f791d710239a4e8dd3438/cymem-2.0.13-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0d78a27c88b26c89bd1ece247d1d5939dba05a1dae6305aad8fd8056b17ddb51", size = 296083, upload-time = "2025-11-14T14:57:55.922Z" }, + { url = "https://files.pythonhosted.org/packages/36/0c/20697c8bc19f624a595833e566f37d7bcb9167b0ce69de896eba7cfc9c2d/cymem-2.0.13-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d36710760f817194dacb09d9fc45cb6a5062ed75e85f0ef7ad7aeeb13d80cc3", size = 286159, upload-time = "2025-11-14T14:57:57.106Z" }, + { url = "https://files.pythonhosted.org/packages/82/d4/9326e3422d1c2d2b4a8fb859bdcce80138f6ab721ddafa4cba328a505c71/cymem-2.0.13-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c8f30971cadd5dcf73bcfbbc5849b1f1e1f40db8cd846c4aa7d3b5e035c7b583", size = 288186, upload-time = "2025-11-14T14:57:58.334Z" }, + { url = "https://files.pythonhosted.org/packages/ed/bc/68da7dd749b72884dc22e898562f335002d70306069d496376e5ff3b6153/cymem-2.0.13-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:9d441d0e45798ec1fd330373bf7ffa6b795f229275f64016b6a193e6e2a51522", size = 290353, upload-time = "2025-11-14T14:58:00.562Z" }, + { url = "https://files.pythonhosted.org/packages/50/23/dbf2ad6ecd19b99b3aab6203b1a06608bbd04a09c522d836b854f2f30f73/cymem-2.0.13-cp313-cp313t-win_amd64.whl", hash = "sha256:d1c950eebb9f0f15e3ef3591313482a5a611d16fc12d545e2018cd607f40f472", size = 44764, upload-time = "2025-11-14T14:58:01.793Z" }, + { url = "https://files.pythonhosted.org/packages/54/3f/35701c13e1fc7b0895198c8b20068c569a841e0daf8e0b14d1dc0816b28f/cymem-2.0.13-cp313-cp313t-win_arm64.whl", hash = "sha256:042e8611ef862c34a97b13241f5d0da86d58aca3cecc45c533496678e75c5a1f", size = 38964, upload-time = "2025-11-14T14:58:02.87Z" }, + { url = "https://files.pythonhosted.org/packages/a7/2e/f0e1596010a9a57fa9ebd124a678c07c5b2092283781ae51e79edcf5cb98/cymem-2.0.13-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d2a4bf67db76c7b6afc33de44fb1c318207c3224a30da02c70901936b5aafdf1", size = 43812, upload-time = "2025-11-14T14:58:04.227Z" }, + { url = "https://files.pythonhosted.org/packages/bc/45/8ccc21df08fcbfa6aa3efeb7efc11a1c81c90e7476e255768bb9c29ba02a/cymem-2.0.13-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:92a2ce50afa5625fb5ce7c9302cee61e23a57ccac52cd0410b4858e572f8614b", size = 42951, upload-time = "2025-11-14T14:58:05.424Z" }, + { url = "https://files.pythonhosted.org/packages/01/8c/fe16531631f051d3d1226fa42e2d76fd2c8d5cfa893ec93baee90c7a9d90/cymem-2.0.13-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bc116a70cc3a5dc3d1684db5268eff9399a0be8603980005e5b889564f1ea42f", size = 249878, upload-time = "2025-11-14T14:58:06.95Z" }, + { url = "https://files.pythonhosted.org/packages/47/4b/39d67b80ffb260457c05fcc545de37d82e9e2dbafc93dd6b64f17e09b933/cymem-2.0.13-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:68489bf0035c4c280614067ab6a82815b01dc9fcd486742a5306fe9f68deb7ef", size = 252571, upload-time = "2025-11-14T14:58:08.232Z" }, + { url = "https://files.pythonhosted.org/packages/53/0e/76f6531f74dfdfe7107899cce93ab063bb7ee086ccd3910522b31f623c08/cymem-2.0.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:03cb7bdb55718d5eb6ef0340b1d2430ba1386db30d33e9134d01ba9d6d34d705", size = 248555, upload-time = "2025-11-14T14:58:09.429Z" }, + { url = "https://files.pythonhosted.org/packages/c7/7c/eee56757db81f0aefc2615267677ae145aff74228f529838425057003c0d/cymem-2.0.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:1710390e7fb2510a8091a1991024d8ae838fd06b02cdfdcd35f006192e3c6b0e", size = 254177, upload-time = "2025-11-14T14:58:10.594Z" }, + { url = "https://files.pythonhosted.org/packages/77/e0/a4b58ec9e53c836dce07ef39837a64a599f4a21a134fc7ca57a3a8f9a4b5/cymem-2.0.13-cp314-cp314-win_amd64.whl", hash = "sha256:ac699c8ec72a3a9de8109bd78821ab22f60b14cf2abccd970b5ff310e14158ed", size = 40853, upload-time = "2025-11-14T14:58:12.116Z" }, + { url = "https://files.pythonhosted.org/packages/61/81/9931d1f83e5aeba175440af0b28f0c2e6f71274a5a7b688bc3e907669388/cymem-2.0.13-cp314-cp314-win_arm64.whl", hash = "sha256:90c2d0c04bcda12cd5cebe9be93ce3af6742ad8da96e1b1907e3f8e00291def1", size = 36970, upload-time = "2025-11-14T14:58:13.114Z" }, + { url = "https://files.pythonhosted.org/packages/b7/ef/af447c2184dec6dec973be14614df8ccb4d16d1c74e0784ab4f02538433c/cymem-2.0.13-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ff036bbc1464993552fd1251b0a83fe102af334b301e3896d7aa05a4999ad042", size = 46804, upload-time = "2025-11-14T14:58:14.113Z" }, + { url = "https://files.pythonhosted.org/packages/8c/95/e10f33a8d4fc17f9b933d451038218437f9326c2abb15a3e7f58ce2a06ec/cymem-2.0.13-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:fb8291691ba7ff4e6e000224cc97a744a8d9588418535c9454fd8436911df612", size = 46254, upload-time = "2025-11-14T14:58:15.156Z" }, + { url = "https://files.pythonhosted.org/packages/e7/7a/5efeb2d2ea6ebad2745301ad33a4fa9a8f9a33b66623ee4d9185683007a6/cymem-2.0.13-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d8d06ea59006b1251ad5794bcc00121e148434826090ead0073c7b7fedebe431", size = 296061, upload-time = "2025-11-14T14:58:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/0b/28/2a3f65842cc8443c2c0650cf23d525be06c8761ab212e0a095a88627be1b/cymem-2.0.13-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c0046a619ecc845ccb4528b37b63426a0cbcb4f14d7940add3391f59f13701e6", size = 285784, upload-time = "2025-11-14T14:58:17.412Z" }, + { url = "https://files.pythonhosted.org/packages/98/73/dd5f9729398f0108c2e71d942253d0d484d299d08b02e474d7cfc43ed0b0/cymem-2.0.13-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:18ad5b116a82fa3674bc8838bd3792891b428971e2123ae8c0fd3ca472157c5e", size = 288062, upload-time = "2025-11-14T14:58:20.225Z" }, + { url = "https://files.pythonhosted.org/packages/5a/01/ffe51729a8f961a437920560659073e47f575d4627445216c1177ecd4a41/cymem-2.0.13-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:666ce6146bc61b9318aa70d91ce33f126b6344a25cf0b925621baed0c161e9cc", size = 290465, upload-time = "2025-11-14T14:58:21.815Z" }, + { url = "https://files.pythonhosted.org/packages/fd/ac/c9e7d68607f71ef978c81e334ab2898b426944c71950212b1467186f69f9/cymem-2.0.13-cp314-cp314t-win_amd64.whl", hash = "sha256:84c1168c563d9d1e04546cb65e3e54fde2bf814f7c7faf11fc06436598e386d1", size = 46665, upload-time = "2025-11-14T14:58:23.512Z" }, + { url = "https://files.pythonhosted.org/packages/66/66/150e406a2db5535533aa3c946de58f0371f2e412e23f050c704588023e6e/cymem-2.0.13-cp314-cp314t-win_arm64.whl", hash = "sha256:e9027764dc5f1999fb4b4cabee1d0322c59e330c0a6485b436a68275f614277f", size = 39715, upload-time = "2025-11-14T14:58:24.773Z" }, +] + [[package]] name = "cytoolz" version = "1.1.0" @@ -871,6 +980,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/45/a20b907227b9d1aea2e36f7b12818d055629ca9bc65fc282b45738f28ca3/eth_utils-6.0.0-py3-none-any.whl", hash = "sha256:63cf48ee32c45541cb5748751909a8345c470432fb6f0fed4bd7c53fd6400469", size = 102473, upload-time = "2026-03-25T17:11:49.953Z" }, ] +[[package]] +name = "filelock" +version = "3.29.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/f5/3557bf28e0f1943e4849154c821533706e6dea010f96fb6aa0b6949037d1/filelock-3.29.3.tar.gz", hash = "sha256:7fc1b3f39cf172fd8203812043c57b8a65aef9969f38b6704f628b881f761a84", size = 61956, upload-time = "2026-06-10T17:37:11.832Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/8f/b61d427c4f49a8bdadc93f4e7e74df8a6df6f77ee6e26bf0df53d3925363/filelock-3.29.3-py3-none-any.whl", hash = "sha256:e58333029cc9b925f39aad59b1d8f0a1ad836af4e60d7217f4a4dba87461261d", size = 42324, upload-time = "2026-06-10T17:37:10.37Z" }, +] + [[package]] name = "firebase-rest-api" version = "1.11.0" @@ -1494,6 +1612,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/9c/06dfcc88d02a6364e8d864c421ddd3736305cb0a6c853f75c302c80fe17c/langchain_protocol-0.0.16-py3-none-any.whl", hash = "sha256:3658c142c5d0fb3a023a4be442ce4c15c6d626aab6135eb79a76dc64ad19c3c3", size = 7037, upload-time = "2026-05-28T23:05:10.163Z" }, ] +[[package]] +name = "langcodes" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/75/f9edc5d72945019312f359e69ded9f82392a81d49c5051ed3209b100c0d2/langcodes-3.5.1.tar.gz", hash = "sha256:40bff315e01b01d11c2ae3928dd4f5cbd74dd38f9bd912c12b9a3606c143f731", size = 191084, upload-time = "2025-12-02T16:22:01.627Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/c1/d10b371bcba7abce05e2b33910e39c33cfa496a53f13640b7b8e10bb4d2b/langcodes-3.5.1-py3-none-any.whl", hash = "sha256:b6a9c25c603804e2d169165091d0cdb23934610524a21d226e4f463e8e958a72", size = 183050, upload-time = "2025-12-02T16:21:59.954Z" }, +] + [[package]] name = "langgraph" version = "1.2.4" @@ -1838,6 +1965,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] +[[package]] +name = "murmurhash" +version = "1.0.15" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/23/2e/88c147931ea9725d634840d538622e94122bceaf346233349b7b5c62964b/murmurhash-1.0.15.tar.gz", hash = "sha256:58e2b27b7847f9e2a6edf10b47a8c8dd70a4705f45dccb7bf76aeadacf56ba01", size = 13291, upload-time = "2025-11-14T09:51:15.272Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/ca/77d3e69924a8eb4508bb4f0ad34e46adbeedeb93616a71080e61e53dad71/murmurhash-1.0.15-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f32307fb9347680bb4fe1cbef6362fb39bd994f1b59abd8c09ca174e44199081", size = 27397, upload-time = "2025-11-14T09:50:03.077Z" }, + { url = "https://files.pythonhosted.org/packages/e6/53/a936f577d35b245d47b310f29e5e9f09fcac776c8c992f1ab51a9fb0cee2/murmurhash-1.0.15-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:539d8405885d1d19c005f3a2313b47e8e54b0ee89915eb8dfbb430b194328e6c", size = 27692, upload-time = "2025-11-14T09:50:04.144Z" }, + { url = "https://files.pythonhosted.org/packages/4d/64/5f8cfd1fd9cbeb43fcff96672f5bd9e7e1598d1c970f808ecd915490dc20/murmurhash-1.0.15-cp311-cp311-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:c4cd739a00f5a4602201b74568ddabae46ec304719d9be752fd8f534a9464b5e", size = 128396, upload-time = "2025-11-14T09:50:05.268Z" }, + { url = "https://files.pythonhosted.org/packages/ac/10/d9ce29d559a75db0d8a3f13ea12c7f541ec9de2afca38dc70418b890eedb/murmurhash-1.0.15-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:44d211bcc3ec203c47dac06f48ee871093fcbdffa6652a6cc5ea7180306680a8", size = 128687, upload-time = "2025-11-14T09:50:06.527Z" }, + { url = "https://files.pythonhosted.org/packages/48/cd/dc97ab7e68cdfa1537a56e36dbc846c5a66701cc39ecee2d4399fe61996c/murmurhash-1.0.15-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f9bf47101354fb1dc4b2e313192566f04ba295c28a37e2f71c692759acc1ba3c", size = 128198, upload-time = "2025-11-14T09:50:08.062Z" }, + { url = "https://files.pythonhosted.org/packages/53/73/32f2aaa22c1e4afae337106baf0c938abf36a6cc879cfee83a00461bbbf7/murmurhash-1.0.15-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3c69b4d3bcd6233782a78907fe10b9b7a796bdc5d28060cf097d067bec280a5d", size = 127214, upload-time = "2025-11-14T09:50:09.265Z" }, + { url = "https://files.pythonhosted.org/packages/82/ed/812103a7f353eba2d83655b08205e13a38c93b4db0692f94756e1eb44516/murmurhash-1.0.15-cp311-cp311-win_amd64.whl", hash = "sha256:e43a69496342ce530bdd670264cb7c8f45490b296e4764c837ce577e3c7ebd53", size = 25241, upload-time = "2025-11-14T09:50:10.373Z" }, + { url = "https://files.pythonhosted.org/packages/eb/5f/2c511bdd28f7c24da37a00116ffd0432b65669d098f0d0260c66ac0ffdc2/murmurhash-1.0.15-cp311-cp311-win_arm64.whl", hash = "sha256:f3e99a6ee36ef5372df5f138e3d9c801420776d3641a34a49e5c2555f44edba7", size = 23216, upload-time = "2025-11-14T09:50:11.651Z" }, + { url = "https://files.pythonhosted.org/packages/b6/46/be8522d3456fdccf1b8b049c6d82e7a3c1114c4fc2cfe14b04cba4b3e701/murmurhash-1.0.15-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d37e3ae44746bca80b1a917c2ea625cf216913564ed43f69d2888e5df97db0cb", size = 27884, upload-time = "2025-11-14T09:50:13.133Z" }, + { url = "https://files.pythonhosted.org/packages/ed/cc/630449bf4f6178d7daf948ce46ad00b25d279065fc30abd8d706be3d87e0/murmurhash-1.0.15-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0861cb11039409eaf46878456b7d985ef17b6b484103a6fc367b2ecec846891d", size = 27855, upload-time = "2025-11-14T09:50:14.859Z" }, + { url = "https://files.pythonhosted.org/packages/ff/30/ea8f601a9bf44db99468696efd59eb9cff1157cd55cb586d67116697583f/murmurhash-1.0.15-cp312-cp312-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:5a301decfaccfec70fe55cb01dde2a012c3014a874542eaa7cc73477bb749616", size = 134088, upload-time = "2025-11-14T09:50:15.958Z" }, + { url = "https://files.pythonhosted.org/packages/c9/de/c40ce8c0877d406691e735b8d6e9c815f36a82b499d358313db5dbe219d7/murmurhash-1.0.15-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:32c6fde7bd7e9407003370a07b5f4addacabe1556ad3dc2cac246b7a2bba3400", size = 133978, upload-time = "2025-11-14T09:50:17.572Z" }, + { url = "https://files.pythonhosted.org/packages/47/84/bd49963ecd84ebab2fe66595e2d1ed41d5e8b5153af5dc930f0bd827007c/murmurhash-1.0.15-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5d8b43a7011540dc3c7ce66f2134df9732e2bc3bbb4a35f6458bc755e48bde26", size = 132956, upload-time = "2025-11-14T09:50:18.742Z" }, + { url = "https://files.pythonhosted.org/packages/4f/7c/2530769c545074417c862583f05f4245644599f1e9ff619b3dfe2969aafc/murmurhash-1.0.15-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:43bf4541892ecd95963fcd307bf1c575fc0fee1682f41c93007adee71ca2bb40", size = 134184, upload-time = "2025-11-14T09:50:19.941Z" }, + { url = "https://files.pythonhosted.org/packages/84/a4/b249b042f5afe34d14ada2dc4afc777e883c15863296756179652e081c44/murmurhash-1.0.15-cp312-cp312-win_amd64.whl", hash = "sha256:f4ac15a2089dc42e6eb0966622d42d2521590a12c92480aafecf34c085302cca", size = 25647, upload-time = "2025-11-14T09:50:21.049Z" }, + { url = "https://files.pythonhosted.org/packages/13/bf/028179259aebc18fd4ba5cae2601d1d47517427a537ab44336446431a215/murmurhash-1.0.15-cp312-cp312-win_arm64.whl", hash = "sha256:4a70ca4ae19e600d9be3da64d00710e79dde388a4d162f22078d64844d0ebdda", size = 23338, upload-time = "2025-11-14T09:50:22.359Z" }, + { url = "https://files.pythonhosted.org/packages/29/2f/ba300b5f04dae0409202d6285668b8a9d3ade43a846abee3ef611cb388d5/murmurhash-1.0.15-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fe50dc70e52786759358fd1471e309b94dddfffb9320d9dfea233c7684c894ba", size = 27861, upload-time = "2025-11-14T09:50:23.804Z" }, + { url = "https://files.pythonhosted.org/packages/34/02/29c19d268e6f4ea1ed2a462c901eed1ed35b454e2cbc57da592fad663ac6/murmurhash-1.0.15-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1349a7c23f6092e7998ddc5bd28546cc31a595afc61e9fdb3afc423feec3d7ad", size = 27840, upload-time = "2025-11-14T09:50:25.146Z" }, + { url = "https://files.pythonhosted.org/packages/e2/63/58e2de2b5232cd294c64092688c422196e74f9fa8b3958bdf02d33df24b9/murmurhash-1.0.15-cp313-cp313-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:b3ba6d05de2613535b5a9227d4ad8ef40a540465f64660d4a8800634ae10e04f", size = 133080, upload-time = "2025-11-14T09:50:26.566Z" }, + { url = "https://files.pythonhosted.org/packages/aa/9a/d13e2e9f8ba1ced06840921a50f7cece0a475453284158a3018b72679761/murmurhash-1.0.15-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:fa1b70b3cc2801ab44179c65827bbd12009c68b34e9d9ce7125b6a0bd35af63c", size = 132648, upload-time = "2025-11-14T09:50:27.788Z" }, + { url = "https://files.pythonhosted.org/packages/b2/e1/47994f1813fa205c84977b0ff51ae6709f8539af052c7491a5f863d82bdc/murmurhash-1.0.15-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:213d710fb6f4ef3bc11abbfad0fa94a75ffb675b7dc158c123471e5de869f9af", size = 131502, upload-time = "2025-11-14T09:50:29.339Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ea/90c1fd00b4aeb704fb5e84cd666b33ffd7f245155048071ffbb51d2bb57d/murmurhash-1.0.15-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b65a5c4e7f5d71f7ccac2d2b60bdf7092d7976270878cfec59d5a66a533db823", size = 132736, upload-time = "2025-11-14T09:50:30.545Z" }, + { url = "https://files.pythonhosted.org/packages/00/db/da73462dbfa77f6433b128d2120ba7ba300f8c06dc4f4e022c38d240a5f5/murmurhash-1.0.15-cp313-cp313-win_amd64.whl", hash = "sha256:9aba94c5d841e1904cd110e94ceb7f49cfb60a874bbfb27e0373622998fb7c7c", size = 25682, upload-time = "2025-11-14T09:50:31.624Z" }, + { url = "https://files.pythonhosted.org/packages/bb/83/032729ef14971b938fbef41ee125fc8800020ee229bd35178b6ede8ee934/murmurhash-1.0.15-cp313-cp313-win_arm64.whl", hash = "sha256:263807eca40d08c7b702413e45cca75ecb5883aa337237dc5addb660f1483378", size = 23370, upload-time = "2025-11-14T09:50:33.264Z" }, + { url = "https://files.pythonhosted.org/packages/10/83/7547d9205e9bd2f8e5dfd0b682cc9277594f98909f228eb359489baec1df/murmurhash-1.0.15-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:694fd42a74b7ce257169d14c24aa616aa6cd4ccf8abe50eca0557e08da99d055", size = 29955, upload-time = "2025-11-14T09:50:34.488Z" }, + { url = "https://files.pythonhosted.org/packages/b7/c7/3afd5de7a5b3ae07fe2d3a3271b327ee1489c58ba2b2f2159bd31a25edb9/murmurhash-1.0.15-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a2ea4546ba426390beff3cd10db8f0152fdc9072c4f2583ec7d8aa9f3e4ac070", size = 30108, upload-time = "2025-11-14T09:50:35.53Z" }, + { url = "https://files.pythonhosted.org/packages/02/69/d6637ee67d78ebb2538c00411f28ea5c154886bbe1db16c49435a8a4ab16/murmurhash-1.0.15-cp313-cp313t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:34e5a91139c40b10f98d0b297907f5d5267b4b1b2e5dd2eb74a021824f751b98", size = 164054, upload-time = "2025-11-14T09:50:36.591Z" }, + { url = "https://files.pythonhosted.org/packages/ab/4c/89e590165b4c7da6bf941441212a721a270195332d3aacfdfdf527d466ca/murmurhash-1.0.15-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:dc35606868a5961cf42e79314ca0bddf5a400ce377b14d83192057928d6252ec", size = 168153, upload-time = "2025-11-14T09:50:37.856Z" }, + { url = "https://files.pythonhosted.org/packages/07/7a/95c42df0c21d2e413b9fcd17317a7587351daeb264dc29c6aec1fdbd26f8/murmurhash-1.0.15-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:43cc6ac3b91ca0f7a5ae9c063ba4d6c26972c97fd7c25280ecc666413e4c5535", size = 164345, upload-time = "2025-11-14T09:50:39.346Z" }, + { url = "https://files.pythonhosted.org/packages/d0/22/9d02c880a88b83bb3ce7d6a38fb727373ab78d82e5f3d8d9fc5612219f90/murmurhash-1.0.15-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:847d712136cb462f0e4bd6229ee2d9eb996d8854eb8312dff3d20c8f5181fda5", size = 161990, upload-time = "2025-11-14T09:50:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/9a/e3/750232524e0dc262e8dcede6536dafc766faadd9a52f1d23746b02948ad8/murmurhash-1.0.15-cp313-cp313t-win_amd64.whl", hash = "sha256:2680851af6901dbe66cc4aa7ef8e263de47e6e1b425ae324caa571bdf18f8d58", size = 28812, upload-time = "2025-11-14T09:50:41.971Z" }, + { url = "https://files.pythonhosted.org/packages/ff/89/4ad9d215ef6ade89f27a72dc4e86b98ef1a43534cc3e6a6900a362a0bf0a/murmurhash-1.0.15-cp313-cp313t-win_arm64.whl", hash = "sha256:189a8de4d657b5da9efd66601b0636330b08262b3a55431f2379097c986995d0", size = 25398, upload-time = "2025-11-14T09:50:43.023Z" }, + { url = "https://files.pythonhosted.org/packages/1c/69/726df275edf07688146966e15eaaa23168100b933a2e1a29b37eb56c6db8/murmurhash-1.0.15-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7c4280136b738e85ff76b4bdc4341d0b867ee753e73fd8b6994288080c040d0b", size = 28029, upload-time = "2025-11-14T09:50:44.124Z" }, + { url = "https://files.pythonhosted.org/packages/59/8f/24ecf9061bc2b20933df8aba47c73e904274ea8811c8300cab92f6f82372/murmurhash-1.0.15-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d4d681f474830489e2ec1d912095cfff027fbaf2baa5414c7e9d25b89f0fab68", size = 27912, upload-time = "2025-11-14T09:50:45.266Z" }, + { url = "https://files.pythonhosted.org/packages/ba/26/fff3caba25aa3c0622114e03c69fb66c839b22335b04d7cce91a3a126d44/murmurhash-1.0.15-cp314-cp314-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d7e47c5746785db6a43b65fac47b9e63dd71dfbd89a8c92693425b9715e68c6e", size = 131847, upload-time = "2025-11-14T09:50:46.819Z" }, + { url = "https://files.pythonhosted.org/packages/df/e4/0f2b9fc533467a27afb4e906c33f32d5f637477de87dd94690e0c44335a6/murmurhash-1.0.15-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e8e674f02a99828c8a671ba99cd03299381b2f0744e6f25c29cadfc6151dc724", size = 132267, upload-time = "2025-11-14T09:50:48.298Z" }, + { url = "https://files.pythonhosted.org/packages/da/bf/9d1c107989728ec46e25773d503aa54070b32822a18cfa7f9d5f41bc17a5/murmurhash-1.0.15-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:26fd7c7855ac4850ad8737991d7b0e3e501df93ebaf0cf45aa5954303085fdba", size = 131894, upload-time = "2025-11-14T09:50:49.485Z" }, + { url = "https://files.pythonhosted.org/packages/0d/81/dcf27c71445c0e993b10e33169a098ca60ee702c5c58fcbde205fa6332a6/murmurhash-1.0.15-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:cb8ebafae60d5f892acff533cc599a359954d8c016a829514cb3f6e9ee10f322", size = 132054, upload-time = "2025-11-14T09:50:50.747Z" }, + { url = "https://files.pythonhosted.org/packages/bc/32/e874a14b2d2246bd2d16f80f49fad393a3865d4ee7d66d2cae939a67a29a/murmurhash-1.0.15-cp314-cp314-win_amd64.whl", hash = "sha256:898a629bf111f1aeba4437e533b5b836c0a9d2dd12d6880a9c75f6ca13e30e22", size = 26579, upload-time = "2025-11-14T09:50:52.278Z" }, + { url = "https://files.pythonhosted.org/packages/af/8e/4fca051ed8ae4d23a15aaf0a82b18cb368e8cf84f1e3b474d5749ec46069/murmurhash-1.0.15-cp314-cp314-win_arm64.whl", hash = "sha256:88dc1dd53b7b37c0df1b8b6bce190c12763014492f0269ff7620dc6027f470f4", size = 24341, upload-time = "2025-11-14T09:50:53.295Z" }, + { url = "https://files.pythonhosted.org/packages/38/9c/c72c2a4edd86aac829337ab9f83cf04cdb15e5d503e4c9a3a243f30a261c/murmurhash-1.0.15-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:6cb4e962ec4f928b30c271b2d84e6707eff6d942552765b663743cfa618b294b", size = 30146, upload-time = "2025-11-14T09:50:54.705Z" }, + { url = "https://files.pythonhosted.org/packages/ac/d7/72b47ebc86436cd0aa1fd4c6e8779521ec389397ac11389990278d0f7a47/murmurhash-1.0.15-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5678a3ea4fbf0cbaaca2bed9b445f556f294d5f799c67185d05ffcb221a77faf", size = 30141, upload-time = "2025-11-14T09:50:55.829Z" }, + { url = "https://files.pythonhosted.org/packages/64/bb/6d2f09135079c34dc2d26e961c52742d558b320c61503f273eab6ba743d9/murmurhash-1.0.15-cp314-cp314t-manylinux1_x86_64.manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ef19f38c6b858eef83caf710773db98c8f7eb2193b4c324650c74f3d8ba299e0", size = 163898, upload-time = "2025-11-14T09:50:56.946Z" }, + { url = "https://files.pythonhosted.org/packages/b9/e2/9c1b462e33f9cb2d632056f07c90b502fc20bd7da50a15d0557343bd2fed/murmurhash-1.0.15-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:22aa3ceaedd2e57078b491ed08852d512b84ff4ff9bb2ff3f9bf0eec7f214c9e", size = 168040, upload-time = "2025-11-14T09:50:58.234Z" }, + { url = "https://files.pythonhosted.org/packages/e8/73/8694db1408fcdfa73589f7df6c445437ea146986fa1e393ec60d26d6e30c/murmurhash-1.0.15-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bba0e0262c0d08682b028cb963ac477bd9839029486fa1333fc5c01fb6072749", size = 164239, upload-time = "2025-11-14T09:50:59.95Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f9/8e360bdfc3c44e267e7e046f0e0b9922766da92da26959a6963f597e6bb5/murmurhash-1.0.15-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4fd8189ee293a09f30f4931408f40c28ccd42d9de4f66595f8814879339378bc", size = 161811, upload-time = "2025-11-14T09:51:01.289Z" }, + { url = "https://files.pythonhosted.org/packages/f9/31/97649680595b1096803d877ababb9a67c07f4378f177ec885eea28b9db6d/murmurhash-1.0.15-cp314-cp314t-win_amd64.whl", hash = "sha256:66395b1388f7daa5103db92debe06842ae3be4c0749ef6db68b444518666cdcc", size = 29817, upload-time = "2025-11-14T09:51:02.493Z" }, + { url = "https://files.pythonhosted.org/packages/76/66/4fce8755f25d77324401886c00017c556be7ca3039575b94037aff905385/murmurhash-1.0.15-cp314-cp314t-win_arm64.whl", hash = "sha256:c22e56c6a0b70598a66e456de5272f76088bc623688da84ef403148a6d41851d", size = 26219, upload-time = "2025-11-14T09:51:03.563Z" }, +] + [[package]] name = "mypy" version = "2.1.0" @@ -2111,7 +2294,7 @@ wheels = [ [[package]] name = "opengradient-veil" -version = "0.2.4" +version = "0.2.5" source = { editable = "." } dependencies = [ { name = "click" }, @@ -2120,6 +2303,13 @@ dependencies = [ { name = "requests" }, ] +[package.optional-dependencies] +pii = [ + { name = "presidio-analyzer" }, + { name = "presidio-anonymizer" }, + { name = "spacy" }, +] + [package.dev-dependencies] dev = [ { name = "mypy" }, @@ -2134,8 +2324,12 @@ requires-dist = [ { name = "click", specifier = ">=8.1.0" }, { name = "flask", specifier = ">=3.0.0" }, { name = "opengradient", specifier = ">=1.1.0" }, + { name = "presidio-analyzer", marker = "extra == 'pii'", specifier = ">=2.2.0" }, + { name = "presidio-anonymizer", marker = "extra == 'pii'", specifier = ">=2.2.0" }, { name = "requests", specifier = ">=2.32.0" }, + { name = "spacy", marker = "extra == 'pii'", specifier = ">=3.7.0" }, ] +provides-extras = ["pii"] [package.metadata.requires-dev] dev = [ @@ -2290,6 +2484,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] +[[package]] +name = "phonenumbers" +version = "9.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/11/ba7611cadc8c99b797416d0dada535c02890dc21b759dfef60a3751d2e20/phonenumbers-9.0.32.tar.gz", hash = "sha256:108ad0237202d2f6cf4b342fac411f22808d85187c3a366152a2af7ed3202a8e", size = 2306598, upload-time = "2026-06-05T05:48:38.909Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c6/e9/1ca526105e792b7ed752fd0ff5732ab01185f2deb719ae6f30183fb21143/phonenumbers-9.0.32-py2.py3-none-any.whl", hash = "sha256:fbcd40d3b11920ee77b1d34a54c3b64c6648a90b8e37222eb62e7fafb87db3e7", size = 2595438, upload-time = "2026-06-05T05:48:36.066Z" }, +] + [[package]] name = "pkce" version = "1.0.3" @@ -2308,6 +2511,85 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "preshed" +version = "3.0.13" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cymem" }, + { name = "murmurhash" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/75/fe6b7bbd0dea530a001b0e24c331b21a0be2786e402abf3c57f5dce43d4b/preshed-3.0.13.tar.gz", hash = "sha256:d75f718bbfd97e992f7827e0fa7faf6a91bdd9c922d5baa4b50d62731396cb89", size = 18338, upload-time = "2026-03-23T08:57:31.378Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/d1/7bc39738388b38ff48cecbb326a9b2bb3f422bb32097be92e010f3162395/preshed-3.0.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5268c0e6fa96f50cdf87f516c2d4b32563c12706ee768e75c00e8d0098acd545", size = 136718, upload-time = "2026-03-23T08:56:23.889Z" }, + { url = "https://files.pythonhosted.org/packages/f6/65/de465b6801740140c2b5d2db6c312ca7937dcfd0442f1ae7d50dee529544/preshed-3.0.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df642547a1a94079978a0ea8f4593ab4b8d3bd43f767bef0ef64d9a214f8c4c9", size = 137261, upload-time = "2026-03-23T08:56:25.303Z" }, + { url = "https://files.pythonhosted.org/packages/89/83/478ee078746a4a413c841542caebd2ea74b659475b8bf5f2e3724b6fe655/preshed-3.0.13-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:09397592d333a77f88454e72b7f1f941b2afaf040b392b9e74898dbc4648cdf5", size = 821010, upload-time = "2026-03-23T08:56:26.455Z" }, + { url = "https://files.pythonhosted.org/packages/ee/2e/1ac761e973966893cd3a0ad3256360365276e2d1e779e351448981a1156a/preshed-3.0.13-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f8e6fe0620ed0f96a246d46447055c447e071cd8222731a045c235e8a758c918", size = 823096, upload-time = "2026-03-23T08:56:28.126Z" }, + { url = "https://files.pythonhosted.org/packages/3c/51/7824cfd85dd7fe547888de20228ebd87d9acd3708206d30b82211e382d23/preshed-3.0.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:502f93f49a22788203f02d3067d4ea077a0cca3864de6a792eae12e7ce589e14", size = 1812148, upload-time = "2026-03-23T08:56:29.755Z" }, + { url = "https://files.pythonhosted.org/packages/34/48/32160a24705d56179de6af838c10a0c735c955dae5f9e4bb344750b79bc2/preshed-3.0.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:acd4d89abeca3678c5d8c89b3cd351314465bc67c7fa053d2644f8513e543386", size = 1881154, upload-time = "2026-03-23T08:56:31.49Z" }, + { url = "https://files.pythonhosted.org/packages/ed/22/0344b50f8b1ad9e3aac08099c47e1aba91c81602fd117d2673f6606ecae6/preshed-3.0.13-cp311-cp311-win_amd64.whl", hash = "sha256:de87fbabb0f37c3c92d4dd9b94fc82ab73cdab4247cdfbd57ab3926caa983919", size = 122219, upload-time = "2026-03-23T08:56:32.74Z" }, + { url = "https://files.pythonhosted.org/packages/33/c4/812eeaa568510f396e27edab01100ca71418f032fd7098b107f12e572361/preshed-3.0.13-cp311-cp311-win_arm64.whl", hash = "sha256:5e2753779832e411e93eb727f3d409c0a6b7408e5ce4dd868076d8ece48c7693", size = 109308, upload-time = "2026-03-23T08:56:33.839Z" }, + { url = "https://files.pythonhosted.org/packages/39/fb/ccff23c44c04088c248539005fcda78b9014512a34d170c5360f02ad908b/preshed-3.0.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5d14eea14bd01291388928991d7df7d60b9fd19ae970e55006eb4d29b0c1e8eb", size = 138497, upload-time = "2026-03-23T08:56:35.321Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ce/cad5a8145881a771e6c0d002f2e585fc19b962f120860b54d32af5baa342/preshed-3.0.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f05b08ce92399c0655b5e0eb5a1cc1f9e295703ed3aabdfaf6538dfa8ae23d57", size = 138010, upload-time = "2026-03-23T08:56:36.399Z" }, + { url = "https://files.pythonhosted.org/packages/a7/a2/c5fed4fb3e946699259d11e4036a3cfdd8c89b3e542e3077d46781642425/preshed-3.0.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:62cf7f3113132891d6bba70ff547ad81c6fe50a31930bbbb8499f1d47cd122b7", size = 861498, upload-time = "2026-03-23T08:56:37.67Z" }, + { url = "https://files.pythonhosted.org/packages/51/94/8c9bc48a6ea4903f53a1a0031ce8e35687526949f25821762ef21493c007/preshed-3.0.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8b8de3f58043070a354477995acdd98626ce43e4193c708ebd0f694e467f5155", size = 868988, upload-time = "2026-03-23T08:56:39.324Z" }, + { url = "https://files.pythonhosted.org/packages/b6/df/ecd2f40055ff52527ca117ffbfafb888c1a3079b59fbabe03c5b8f9b7240/preshed-3.0.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:183b339956a9e1d7a4a00038a3b9587a734db9e8bd915939a49791bd1b372156", size = 1847382, upload-time = "2026-03-23T08:56:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/e6/88/bdb244e40284ded3632a9f88c23bc80230bd7b2ae4a8b7f2cc91adead7a8/preshed-3.0.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e77bed56aded7cbe5d28d6bd2178bc5b13eda0e0e464dab205fb578fa915000", size = 1919236, upload-time = "2026-03-23T08:56:42.616Z" }, + { url = "https://files.pythonhosted.org/packages/a0/c9/c91ea56342e6c364fc69b444a1ac5432327857199c44032c9cc9dc4c3a23/preshed-3.0.13-cp312-cp312-win_amd64.whl", hash = "sha256:04d8f13f2986e5d11af5ac51f55ce3106c70c41b483d20ea392e6180bdd0f870", size = 122938, upload-time = "2026-03-23T08:56:44.271Z" }, + { url = "https://files.pythonhosted.org/packages/b2/0b/6a99d99619fd83b14c696e2489caed7070647488d4d3ac0b723d35db2de0/preshed-3.0.13-cp312-cp312-win_arm64.whl", hash = "sha256:19318dc1cd8cac6663c6c830bf7e0002d2de853769fb03e056774e97c21bedfd", size = 109194, upload-time = "2026-03-23T08:56:45.346Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2a/401158195d6dc7f6aef0b354d74d0e95c9da124499448c2b3dbb95b71204/preshed-3.0.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0d0c14187dc0078d8a63bf190ec045a4d13e7748b6caeb557a7d575e411410b", size = 137289, upload-time = "2026-03-23T08:56:46.516Z" }, + { url = "https://files.pythonhosted.org/packages/88/8f/e20e64573988528785447a6893b2e7ab287ecfd85b3888e978b28812fd20/preshed-3.0.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7770987c2e57497cd26124a9be5f652b5b3ccd0def89859ab0da8bca6144a3de", size = 136847, upload-time = "2026-03-23T08:56:47.572Z" }, + { url = "https://files.pythonhosted.org/packages/b9/72/18168f881359c4482d312f8dc196371bdd61c1583a52b34390da4c88bbea/preshed-3.0.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4a7bc48220de579be6bdb0a8715482cf36e2a625a6fd5ad26c9f43485a4a23b5", size = 831478, upload-time = "2026-03-23T08:56:48.769Z" }, + { url = "https://files.pythonhosted.org/packages/fd/3a/3543476091087102775568cea9885dde3453569e9aeee365809108de572f/preshed-3.0.13-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5c8462472f790c16708306aef3a102a762bd19dfe3d2f8ee08bd5e12f51b835", size = 839913, upload-time = "2026-03-23T08:56:49.937Z" }, + { url = "https://files.pythonhosted.org/packages/cf/65/b13f01329decc44ef53cfb6b4601ba85382dcb2a4ec78d9250f03a418066/preshed-3.0.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c046736239cc8d72670749b79b526e4111839a2fc461a58545d212797649129c", size = 1816452, upload-time = "2026-03-23T08:56:51.233Z" }, + { url = "https://files.pythonhosted.org/packages/d1/c7/f1a996c6832234efd4d543041b582418d41ac480ee55c557ec9e65344637/preshed-3.0.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7c333f18e9a81c8a6de0603fd8781e17115324b117c445ca91abdf7bfb1abe49", size = 1888978, upload-time = "2026-03-23T08:56:52.591Z" }, + { url = "https://files.pythonhosted.org/packages/e3/b9/96fb71499049885ce19545903fdd38877bbc2be0da47e37c04d01f3e9f66/preshed-3.0.13-cp313-cp313-win_amd64.whl", hash = "sha256:461327f8dd36520dcf1fd55a671e0c3c2c97a2d95e22fc85faa31173f4785dda", size = 122134, upload-time = "2026-03-23T08:56:54.392Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a7/32a4903019d936a2316fdd330bedddac287ac26326107d24fb76a1fbc60a/preshed-3.0.13-cp313-cp313-win_arm64.whl", hash = "sha256:35d6c5acb3ee3b12b87a551913063f0cec784055c2af16e028c19fe875f079d0", size = 108497, upload-time = "2026-03-23T08:56:55.816Z" }, + { url = "https://files.pythonhosted.org/packages/bb/b5/993886c98f5caaa6f07a648cac97a7c62a3093091cad65e1e43a1bd41cc4/preshed-3.0.13-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d2f1efae396cadab5f3890a2fd43d2ee65373ef9096ccbb805e51e8d8bcc563b", size = 137882, upload-time = "2026-03-23T08:56:56.878Z" }, + { url = "https://files.pythonhosted.org/packages/c6/86/b7fd137cbf140afd6c45e895946068a15f5b55642916de0075e6eb18581c/preshed-3.0.13-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8d6acc1f5031a535a55a6f7148e2f274554a8343a16309c700cebea0fe7aee8c", size = 138233, upload-time = "2026-03-23T08:56:58.318Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ca/21a7e79625614134273dfed32bca5bb4c2ec1313e33fbd12d41657536f1f/preshed-3.0.13-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7da9d931e7660dcdd757e5870269f0c159126d682ed73ed313971d199eb0f334", size = 834835, upload-time = "2026-03-23T08:56:59.48Z" }, + { url = "https://files.pythonhosted.org/packages/8f/3a/2dbd299516461831ae90e0d5b0637137bf28520c4e6dd0b01d6f1886659a/preshed-3.0.13-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d4ae5cfe075bb7a07982e382bca44f41ddf041f4d24cbd358e8cccfc049259b8", size = 834928, upload-time = "2026-03-23T08:57:01.075Z" }, + { url = "https://files.pythonhosted.org/packages/7c/d3/af654eba4f6587c4ee02c5043e62c194b0a1c4431ffef0c67b9518f6b61c/preshed-3.0.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7557963d0125a3a7bcdb2eb6948f3e45da31b5a7f066b55320de3dea22d7557f", size = 1820368, upload-time = "2026-03-23T08:57:02.351Z" }, + { url = "https://files.pythonhosted.org/packages/bf/9b/ebcb2b9e8cb881e40b55b0bf450f8a6b187e2ef3ae0c685cce81d2d85026/preshed-3.0.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c4bc60dc994864095d784b7e4d77dba3e64188d169ac88722b699d175561fddb", size = 1888251, upload-time = "2026-03-23T08:57:04.158Z" }, + { url = "https://files.pythonhosted.org/packages/97/f7/c6c012779edcaa6e2cd092c554e98dc53e77f41205b07208655ba77e2327/preshed-3.0.13-cp314-cp314-win_amd64.whl", hash = "sha256:208dcebbe294bf1881ce33fb015d56ab2a7587aece85a09147727174207892e4", size = 125211, upload-time = "2026-03-23T08:57:05.83Z" }, + { url = "https://files.pythonhosted.org/packages/f8/82/390ef87d732ef64e673ef6bf9e5d898453986e979efa50fb3a400e2c0766/preshed-3.0.13-cp314-cp314-win_arm64.whl", hash = "sha256:cf8e1a7a1823b2a7765121446c630140ac6e8650c07a6efbf375e168d1fef4f7", size = 111942, upload-time = "2026-03-23T08:57:06.996Z" }, + { url = "https://files.pythonhosted.org/packages/80/3a/a9dde3167bcecb27ae82ce4567b5ab1aa3989113ae6814c092ce223cc4ef/preshed-3.0.13-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:9ca43ecbc3783eda4d6ab3416ae2ecd9ef23dca5f53995843f69f7457bcd0677", size = 144997, upload-time = "2026-03-23T08:57:08.064Z" }, + { url = "https://files.pythonhosted.org/packages/74/d4/22d9355b50b6a13b407dcad0a81df83fb1d5602092d1f05834674dde8fda/preshed-3.0.13-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c8596e41a258ff213553a441e0bb3eb388fd8158e84a7bf3aae6d8ede2c166d3", size = 147294, upload-time = "2026-03-23T08:57:09.411Z" }, + { url = "https://files.pythonhosted.org/packages/70/42/a225ee83fdb306d2a503f21a627953b820f4e079c90c8a84338957cb8ff5/preshed-3.0.13-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4f8856ca3d88e9b250630d70abb4f260d8933151ddfb413024784b25b009868e", size = 952110, upload-time = "2026-03-23T08:57:10.592Z" }, + { url = "https://files.pythonhosted.org/packages/40/ba/09a9dfe3d22d7e745483fd5d7f2a82cd4d39c161f7d2daa0faa4bd6402be/preshed-3.0.13-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0e5b2865aecbd2e1e10e5d19bb8bfad765863c1307c6c3e51f2a08bd64122409", size = 932217, upload-time = "2026-03-23T08:57:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5c/e10e2e05133e7fcbd7c40536af1148c82dd24357b8f5726e2c7bc51cfd53/preshed-3.0.13-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:09f96b477c987755b3c945df214ea1c1c80bfb350e9f34e78da89585535b77e8", size = 1896542, upload-time = "2026-03-23T08:57:13.525Z" }, + { url = "https://files.pythonhosted.org/packages/37/aa/51e5b4109a4cdfae28c3613eeeb10764a3794ebef8de93ffbb109465bea3/preshed-3.0.13-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:670db59a52e1823b5f088c764df474e65b686592d4093adbeef14581c95ee2cb", size = 1959473, upload-time = "2026-03-23T08:57:15.706Z" }, + { url = "https://files.pythonhosted.org/packages/0e/6a/1d966f367a14c703dde629d150d996c1b727d442f620300b21c9ec1a24d1/preshed-3.0.13-cp314-cp314t-win_amd64.whl", hash = "sha256:b03e21b0bf95eb56e23973f32cabb930e94f352228652f81c0955dbd6967d904", size = 146229, upload-time = "2026-03-23T08:57:17.457Z" }, + { url = "https://files.pythonhosted.org/packages/22/80/368139067603e590a000122355f9c8576c8ebed4fb0b8849feaa2698489d/preshed-3.0.13-cp314-cp314t-win_arm64.whl", hash = "sha256:b980f3ea9bb74b7f94464bc3d6eb3c9162b6b79b531febd14c6465c24344d2cc", size = 119339, upload-time = "2026-03-23T08:57:18.882Z" }, +] + +[[package]] +name = "presidio-analyzer" +version = "2.2.362" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "phonenumbers" }, + { name = "pydantic" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "spacy" }, + { name = "tldextract" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/8a/d6cf4bddbb11d9c78f5c9342bbbcde4f90fe4e3c7de114a836ec4ed8a6cf/presidio_analyzer-2.2.362-py3-none-any.whl", hash = "sha256:4c36438924b1fcb4df92ea5cf2d8dc57508808e116b10923c983b8732aa07d90", size = 201084, upload-time = "2026-03-15T12:40:43.801Z" }, +] + +[[package]] +name = "presidio-anonymizer" +version = "2.2.362" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/01/1c3404e8da63a979ae951b8822576bc0e52ca3a46a624ac9439786990c6d/presidio_anonymizer-2.2.362-py3-none-any.whl", hash = "sha256:b2d81542cb797360d18506a1aa73ea0dfacc44abca15248c81dd062f582e6b2b", size = 36603, upload-time = "2026-03-15T12:40:38.651Z" }, +] + [[package]] name = "propcache" version = "0.5.2" @@ -2891,6 +3173,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[[package]] +name = "requests-file" +version = "3.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3c/f8/5dc70102e4d337063452c82e1f0d95e39abfe67aa222ed8a5ddeb9df8de8/requests_file-3.0.1.tar.gz", hash = "sha256:f14243d7796c588f3521bd423c5dea2ee4cc730e54a3cac9574d78aca1272576", size = 6967, upload-time = "2025-10-20T18:56:42.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/d5/de8f089119205a09da657ed4784c584ede8381a0ce6821212a6d4ca47054/requests_file-3.0.1-py2.py3-none-any.whl", hash = "sha256:d0f5eb94353986d998f80ac63c7f146a307728be051d4d1cd390dbdb59c10fa2", size = 4514, upload-time = "2025-10-20T18:56:41.184Z" }, +] + [[package]] name = "requests-toolbelt" version = "1.0.0" @@ -3077,6 +3371,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2d/c7/c53e8dbff9c9dc4b7928773421ae294a5d28fcb8dcda1a089579d3a7e510/ruff-0.15.17-py3-none-win_arm64.whl", hash = "sha256:f3be1fbb34bcdfd146240d8fb92a709d4c2c8191348580a3c044ec60fa0b4456", size = 11355275, upload-time = "2026-06-11T17:54:43.635Z" }, ] +[[package]] +name = "setuptools" +version = "82.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -3086,6 +3389,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, ] +[[package]] +name = "smart-open" +version = "6.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/69/bf2e8a00fbf9bf9f27734c4f3f2030fb422c4d8b1594bb9fc763561a4ec2/smart_open-6.4.0.tar.gz", hash = "sha256:be3c92c246fbe80ebce8fbacb180494a481a77fcdcb7c1aadb2ea5b9c2bee8b9", size = 65491, upload-time = "2023-09-07T02:59:51.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/d9/d97f1db64b09278aba64e8c81b5d322d436132df5741c518f3823824fae0/smart_open-6.4.0-py3-none-any.whl", hash = "sha256:8d3ef7e6997e8e42dd55c74166ed21e6ac70664caa32dd940b26d54a8f6b4142", size = 57048, upload-time = "2023-09-07T02:59:54.467Z" }, +] + [[package]] name = "sniffio" version = "1.3.1" @@ -3095,6 +3407,119 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "spacy" +version = "4.0.0.dev3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, + { name = "cymem" }, + { name = "jinja2" }, + { name = "langcodes" }, + { name = "murmurhash" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.0rc1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "setuptools" }, + { name = "smart-open" }, + { name = "spacy-legacy" }, + { name = "spacy-loggers" }, + { name = "srsly" }, + { name = "thinc" }, + { name = "tqdm" }, + { name = "typer" }, + { name = "wasabi" }, + { name = "weasel" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/89/11f8e53b4517400be6f869286c5f45c7923a5887e18aac3a91b7841a82af/spacy-4.0.0.dev3.tar.gz", hash = "sha256:a528120ef136ad525654386173959167912076c92857ddd99628647e2d856a5e", size = 1287081, upload-time = "2024-04-22T09:56:42.929Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/0d/5ebc3da3e4983968f2da9dcadb6b6722124b0a50f5cee21af14aa96962d2/spacy-4.0.0.dev3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ec399ae3166e86da282f5e1dce0105dc953488a01398c2016b81cb084ba4153", size = 6857652, upload-time = "2024-04-22T09:56:01.519Z" }, + { url = "https://files.pythonhosted.org/packages/41/e8/791aa1226870a569921879a419374471d05ff1897708a61707eac65a1da3/spacy-4.0.0.dev3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a3f2ec89f6ca7f6d4157f60843f3e9f0cecb6e7afb3bec54723a01e9e80f0f1", size = 6622584, upload-time = "2024-04-22T09:56:04.325Z" }, + { url = "https://files.pythonhosted.org/packages/19/5c/5a29034b10a113ae96abf4029050772aee6e1619ba85ef70e6fa9d39a2b6/spacy-4.0.0.dev3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da6901d541337bc7799849cc6990d8cb52a83a41f8790cdb3915abdfa9dd603", size = 6326685, upload-time = "2024-04-22T09:56:07.217Z" }, + { url = "https://files.pythonhosted.org/packages/23/73/4a8525bba6368e84877d1af39d1dac21767c5ca6c8c9b7a2f1f945835f94/spacy-4.0.0.dev3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68ca7d48dfb3a70bc5650565bf610f14332e88c0e9dad468dca63098e8776f04", size = 6680963, upload-time = "2024-04-22T09:56:09.519Z" }, + { url = "https://files.pythonhosted.org/packages/b5/25/3bdcc988f30e717c00e90e62795d90250b37e32781e1b812decfb13a374d/spacy-4.0.0.dev3-cp311-cp311-win_amd64.whl", hash = "sha256:24457f0b228f0bb7b6a0a1a9c64b19d6a24356527b364494bfc9f2bb2b07d330", size = 12252349, upload-time = "2024-04-22T09:56:11.676Z" }, + { url = "https://files.pythonhosted.org/packages/e3/18/7b5f3a9841cc14bdcee3a2192aa41a8a6b01ec83ad919581c4b74d70fc41/spacy-4.0.0.dev3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e221723b9194edc74f3b70e73d39531525fd7656b8ff6e9e652cefe45f93d119", size = 6417309, upload-time = "2024-04-22T09:56:14.735Z" }, + { url = "https://files.pythonhosted.org/packages/46/cd/b5a272123347ca4c00a0efb564257b666d19e924a1bbdec4035e09f305e6/spacy-4.0.0.dev3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:554f0818313b57ab72a7b801d19c49e52535019dede482a150656ad9b46553ca", size = 6207331, upload-time = "2024-04-22T09:56:17.397Z" }, + { url = "https://files.pythonhosted.org/packages/cb/fb/1120d9e3db87ead4ffa5e51691e4cbb4a3e9e3331b09a6585f39f2dcd13e/spacy-4.0.0.dev3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64e13f03eeef66168f883e285f6a6cbeba9452f2cae2f9cd14de8579ad3985b0", size = 6149396, upload-time = "2024-04-22T09:56:20.299Z" }, + { url = "https://files.pythonhosted.org/packages/31/ba/2053edb0670480530a688dacae72c1a3e76c51482e1ed1fc751861664b3e/spacy-4.0.0.dev3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bfc7f789a696f9d4fddf405522e871c016a8397ed631f6a5b67d7d564ba4de1", size = 6543611, upload-time = "2024-04-22T09:56:23.315Z" }, + { url = "https://files.pythonhosted.org/packages/70/fd/39947ce507ca2724d2bdd219933144dbc5922a42f6455a57368ffbefdfdb/spacy-4.0.0.dev3-cp312-cp312-win_amd64.whl", hash = "sha256:f9ba698347e4fa46bb1ecdec1b5c942bf41a32868fac38a870907a5b72a3cf8d", size = 11836894, upload-time = "2024-04-22T09:56:25.847Z" }, +] + +[[package]] +name = "spacy-legacy" +version = "4.0.0.dev1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tqdm" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/c7/1f50e31b0db55eaa83fbfeaa219f68fad2fdc2b7262ae318d253e550630c/spacy-legacy-4.0.0.dev1.tar.gz", hash = "sha256:c3d778162cdf84f10cf681101ddc51bba322c4edf0ab49d09a09182a09603dce", size = 24285, upload-time = "2024-01-25T15:03:46.967Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/92/5e779b0334cae2231440e25a5f004315e69b75b37def4d2695a2407b081a/spacy_legacy-4.0.0.dev1-py2.py3-none-any.whl", hash = "sha256:1be48d206439a5288c14d90b2c357e3a06233ddb40e1d13c41286d2cb91ce601", size = 30313, upload-time = "2024-01-25T15:03:45.71Z" }, +] + +[[package]] +name = "spacy-loggers" +version = "1.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/3d/926db774c9c98acf66cb4ed7faf6c377746f3e00b84b700d0868b95d0712/spacy-loggers-1.0.5.tar.gz", hash = "sha256:d60b0bdbf915a60e516cc2e653baeff946f0cfc461b452d11a4d5458c6fe5f24", size = 20811, upload-time = "2023-09-11T12:26:52.323Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/78/d1a1a026ef3af911159398c939b1509d5c36fe524c7b644f34a5146c4e16/spacy_loggers-1.0.5-py3-none-any.whl", hash = "sha256:196284c9c446cc0cdb944005384270d775fdeaf4f494d8e269466cfa497ef645", size = 22343, upload-time = "2023-09-11T12:26:50.586Z" }, +] + +[[package]] +name = "srsly" +version = "2.5.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "catalogue" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/db/f794f219a6c788b881252d2536a8c4a97d2bdaadc690391e1cb53d123d71/srsly-2.5.3.tar.gz", hash = "sha256:08f98dbecbff3a31466c4ae7c833131f59d3655a0ad8ac749e6e2c149e2b0680", size = 490881, upload-time = "2026-03-23T11:56:59.865Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/36/5d7bb412d52e9cca787f9bfe838b596367189b254e50bf90f234a97184bf/srsly-2.5.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:785a09216ac31570fb301ddb9f61ee73d1f18f8b9561f712dce0b8ac8628bc88", size = 656760, upload-time = "2026-03-23T11:55:47.155Z" }, + { url = "https://files.pythonhosted.org/packages/d6/dc/124f008cd2be3e887e972cbdeb17c5aee0f42093eca02c7cfd63bb5daf19/srsly-2.5.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0017c7d2a0cd9a4f1bdc00d946b45edcf90bb0e271e8f084c1ce542bf6708c32", size = 657503, upload-time = "2026-03-23T11:55:48.681Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/2c97244ebab125d55f1bfb7bb94e9572b3e819410dffd6a040eca1112350/srsly-2.5.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:66ebae2c70305987341519ec1a720072a3cb3e4b1d52ac0e9e841f4d02658d3d", size = 1139161, upload-time = "2026-03-23T11:55:50.179Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ea/ecd396188f7591d80b89665f7af9e3ae02e42683daef57033ad7993ad3f9/srsly-2.5.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4ca4a068f6e14d84113a02fcb875c6b50a6285a12938c0e7a157eb3a63c50a86", size = 1142438, upload-time = "2026-03-23T11:55:52.607Z" }, + { url = "https://files.pythonhosted.org/packages/9c/65/143e2e143c53d498ad0956f69d0e09189aa7a6e0ee6017758c285ba1ab2d/srsly-2.5.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e283fa2a8f7350fb9fb70ecdee28d59d39c92f4c7f1cc90a44d6b86db3b3a8b3", size = 1101783, upload-time = "2026-03-23T11:55:53.906Z" }, + { url = "https://files.pythonhosted.org/packages/6b/86/1392a5593de0cd3d08c2d6c071b877c84358a37f63172c4e9cb71706842d/srsly-2.5.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9ffc97e22730ea97b00f7c303ccc60b1305e786afadb2a4a46578dafa4d29da0", size = 1115876, upload-time = "2026-03-23T11:55:55.624Z" }, + { url = "https://files.pythonhosted.org/packages/d4/a5/6193aa4c08e488821538fcbce2282449e228fd2183ed67d118bb5ccd8b54/srsly-2.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:f09b551f6c3e334652831ac68c770ee4284741ce0a3895bf1ccf2a1178d66cdd", size = 651733, upload-time = "2026-03-23T11:55:56.964Z" }, + { url = "https://files.pythonhosted.org/packages/66/a8/a73181743b6d237026615ca75c3fb3e4780736f1390550a7350d0c7f1149/srsly-2.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:21cf09e417d3e4f3fbf7dd337fd6d948c97abd01896b9b4cb80e81cd9778a73a", size = 639124, upload-time = "2026-03-23T11:55:58.532Z" }, + { url = "https://files.pythonhosted.org/packages/02/cc/e9f7fcec4cc92ad8bad6316c4241638b8cf7380382d4489d94ec6c436452/srsly-2.5.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:71e51c046ccbeefb86524c6b1e17574f579c6ac4dc8ea4a09437d3e8f88342d3", size = 658379, upload-time = "2026-03-23T11:55:59.85Z" }, + { url = "https://files.pythonhosted.org/packages/21/e4/fea4512e9785f58509b2cf67d993323848e583161b5fcfdc7dd9d7c1f3df/srsly-2.5.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f73c0db911552e94fe2016e1759d261d2f47926f68826664cada3723c87006a", size = 658513, upload-time = "2026-03-23T11:56:01.239Z" }, + { url = "https://files.pythonhosted.org/packages/20/b1/53591681b6ff2699a4f97b2d5552ba196eaa6a979b0873605f4c04b5f7ee/srsly-2.5.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5c1ac27ae5f4bb9163c7d2c45fc8ec173aac3d92e32086d9472b326c5c6e570e", size = 1172265, upload-time = "2026-03-23T11:56:02.589Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c9/741e29f534919a944a16da4184924b1d3404c4bf60716ab2b91be771d1e3/srsly-2.5.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:99026bcd9cbd3211cc36517400b04ca0fc5d3e412b14daf84ee6e65f67d9a2d8", size = 1180873, upload-time = "2026-03-23T11:56:03.944Z" }, + { url = "https://files.pythonhosted.org/packages/89/57/5554f786eccf78b2750d6ac63be126e1b67badec2cb409dd611cf6f8c52b/srsly-2.5.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:07d682679e639eb46ff7e6da4a92714f4d5ffe351d088ee66f221e9b1f8865bb", size = 1120437, upload-time = "2026-03-23T11:56:05.283Z" }, + { url = "https://files.pythonhosted.org/packages/eb/95/9b4f73b1be3692f86d72ccc131c8e50f26f824d5c8830a59390bcc5b60ef/srsly-2.5.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8e0542d85d6b55cf2934050d6ffcb1cd76c768dcf9572e7467002cf087bb366d", size = 1137376, upload-time = "2026-03-23T11:56:06.613Z" }, + { url = "https://files.pythonhosted.org/packages/5a/de/89ca640ca1953c4612279ce515d0af35658df3c06cdb324329bc91b4a7e1/srsly-2.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:598f1e494c18cacb978299d77125415a586417081959f8ec3f068b32d97f8933", size = 652459, upload-time = "2026-03-23T11:56:07.994Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/7ab6d49e36d9cc72ee15746cabd116eb6f338be8a06c1882968ee9d6c7d7/srsly-2.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:4b1b721cd3ad1a9b2343519aadc786a4d09d5c0666962d49852eb12d6ec3fe26", size = 638411, upload-time = "2026-03-23T11:56:09.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/5c/12901e3794f4158abc6da750725aad6c2afddb1e4227b300fe7c71f66957/srsly-2.5.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e67b6bbacbfadea5e100266d2797f2d4cec9883ea4dc84a5537673850036a8d8", size = 656750, upload-time = "2026-03-23T11:56:10.708Z" }, + { url = "https://files.pythonhosted.org/packages/04/61/181c26370995f96f56f1b64b801e3ca1e0d703fc36506ae28606d62369fb/srsly-2.5.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:348c231b4477d8fe86603131d0f166d2feac9c372704dfc4398be71cc5b6fb07", size = 656746, upload-time = "2026-03-23T11:56:12.28Z" }, + { url = "https://files.pythonhosted.org/packages/77/c6/35876c78889f8ffe11ed3521644e666c3aef20ea31527b70f47456cf35c2/srsly-2.5.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b0938c2978c91ae1ef9c1f2ba35abb86330e198fb23469e356eba311e02233ee", size = 1155762, upload-time = "2026-03-23T11:56:14.075Z" }, + { url = "https://files.pythonhosted.org/packages/3e/da/40b71ca9906c8eb8f8feb6ac11d33dad458c85a56e1de764b96d402168a0/srsly-2.5.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5f6a837954429ecbe6dcdd27390d2fb4c7d01a3f99c9ffcf9ce66b2a6dd1b738", size = 1161092, upload-time = "2026-03-23T11:56:15.778Z" }, + { url = "https://files.pythonhosted.org/packages/dc/14/c0dd30cc8b93ce8137ff4766f743c882440ce49195fffc5d50eaeef311a6/srsly-2.5.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3576c125c486ce2958c2047e8858fe3cfc9ea877adfa05203b0986f9badee355", size = 1109984, upload-time = "2026-03-23T11:56:17.056Z" }, + { url = "https://files.pythonhosted.org/packages/08/f3/34354f183d8faafc631585571224b54d1b4b67e796972c36519c074ca355/srsly-2.5.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5fb59c42922e095d1ea36085c55bc16e2adb06a7bfe57b24d381e0194ae699f2", size = 1128409, upload-time = "2026-03-23T11:56:18.761Z" }, + { url = "https://files.pythonhosted.org/packages/a4/d9/5531f8a19492060b4e76e4ab06aca6f096fb5128fe18cc813d1772daf653/srsly-2.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:111805927f05f5db440aeeacb85ce43da0b19ce7b2a09567a9ef8d30f3cc4d83", size = 650820, upload-time = "2026-03-23T11:56:20.096Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/62fb7a971eca29e12f03fb9ddacb058548c14d33e5b5675ff0f85839cc7b/srsly-2.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:0f106b0a700ab56e4a7c431b0f1444009ab6cb332edc7bbf6811c2a43f4722cb", size = 637278, upload-time = "2026-03-23T11:56:21.439Z" }, + { url = "https://files.pythonhosted.org/packages/e1/5b/e4ef43c2a381711230af98d4c94a5323df48d6a7899ee652e05bf889290e/srsly-2.5.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:39c13d552a9f9674a12cdcdc66b0c2f02f3430d0cd04c5f9cf598824c2bd3d65", size = 661294, upload-time = "2026-03-23T11:56:23.29Z" }, + { url = "https://files.pythonhosted.org/packages/92/2d/ebce7f3717e52cd0a01f4ec570f388f3b7098526794fcf1ad734e0b8f852/srsly-2.5.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:14c930767cc169611a2dc14e23bc7638cfb616d6f79029700ade033607343540", size = 660952, upload-time = "2026-03-23T11:56:24.908Z" }, + { url = "https://files.pythonhosted.org/packages/22/47/a8f3e9b214be2624c8e8a78d38ca7b1d4e26b92d57018412e4bfc4abe89a/srsly-2.5.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2f2d464f0d0237e32fb53f0ec6f05418652c550e772b50e9918e83a1577cba4d", size = 1154554, upload-time = "2026-03-23T11:56:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/d6/71/2a89dc3180a51e633a87a079ca064225f4aaf46c7b2a5fc720e28f261d98/srsly-2.5.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d18933248a5bb0ad56a1bae6003a9a7f37daac2ecb0c5bcbfaaf081b317e1c84", size = 1155746, upload-time = "2026-03-23T11:56:28.102Z" }, + { url = "https://files.pythonhosted.org/packages/b8/36/72e5ce3153927ca404b6f5bf5280e6ff3399c11557df472b153945468e0a/srsly-2.5.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7ea5412ea229e571ac9738cbe14f845cc06c8e4e956afb5f42061ccd087ef31f", size = 1112374, upload-time = "2026-03-23T11:56:29.591Z" }, + { url = "https://files.pythonhosted.org/packages/04/b2/0895de109c28eca0d41a811ab7c076d4e4a505e8466f06bae22f5180a1dd/srsly-2.5.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8d3988970b4cf7d03bdd5b5169302ff84562dd2e1e0f84aeb34df3e5b5dc19bf", size = 1127732, upload-time = "2026-03-23T11:56:31.458Z" }, + { url = "https://files.pythonhosted.org/packages/c7/79/a37fa7759797fbdfe0a2e029ab13e78b1e81e191220d2bb8ff57d869aefb/srsly-2.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:6a02d7dcc16126c8fae1c1c09b2072798a1dc482ab5f9c52b12c7114dac47325", size = 656467, upload-time = "2026-03-23T11:56:33.14Z" }, + { url = "https://files.pythonhosted.org/packages/d7/25/0dae019b3b90ad9037f91de4c390555cdaac9460a93ad62b02b03babdff5/srsly-2.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:1c9129c4abe31903ff7996904a51afdd5428060de6c3d12af49a4da5e8df2821", size = 643040, upload-time = "2026-03-23T11:56:34.448Z" }, + { url = "https://files.pythonhosted.org/packages/3a/44/72dd5285b2e05435d98b0797f101d91d9b345d491ddc1fdb9bd09e27ccb8/srsly-2.5.3-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:29d5d01ba4c2e9c01f936e5e6d5babc4a47b38c9cbd6e1ec23f6d5a49df32605", size = 666200, upload-time = "2026-03-23T11:56:35.753Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ad/002c71b87fc3f648c9bf0ec47de0c3822bf2c95c8896a589dd03e7fd3977/srsly-2.5.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5c8df4039426d99f0148b5743542842ab96b82daded0b342555e15a639927757", size = 667409, upload-time = "2026-03-23T11:56:37.172Z" }, + { url = "https://files.pythonhosted.org/packages/2a/35/2cea3d5e80aeecfc4ece9e7e1783e7792cc3bad7ab85ab585882e1db4e38/srsly-2.5.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:06a43d63bde2e8cccadb953d7fff70b18196ca286b65dd2ad16006d65f3f8166", size = 1265941, upload-time = "2026-03-23T11:56:38.825Z" }, + { url = "https://files.pythonhosted.org/packages/aa/38/8a4d7e86dd0370a2e5af251b646000197bb5b7e0f9aa360c71bbfb253d0d/srsly-2.5.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:808cfafc047f0dec507a34c8fa8e4cda5722737fd33577df73452f52f7aca644", size = 1250693, upload-time = "2026-03-23T11:56:40.449Z" }, + { url = "https://files.pythonhosted.org/packages/99/05/340129de5ea7b237271b12f8a6962cfa7eb0c5a3056794626d348c5ae7c7/srsly-2.5.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:71d4cbe2b2a1335c76ed0acae2dc862163787d8b01a705e1949796907ed94ccd", size = 1242408, upload-time = "2026-03-23T11:56:41.8Z" }, + { url = "https://files.pythonhosted.org/packages/01/cb/d7fee7ab27c6aa2e3f865fb7b50ba18c81a4c763bba12bdf53df246441bc/srsly-2.5.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:565f69083d33cb329cfc74317da937fb3270c0f40fabc1b4488702d8074b4a3e", size = 1242749, upload-time = "2026-03-23T11:56:43.246Z" }, + { url = "https://files.pythonhosted.org/packages/d8/d1/9bad3a0f2fa7b72f4e0cf1d267b00513092d20ef538c47f72823ae4f7656/srsly-2.5.3-cp314-cp314t-win_amd64.whl", hash = "sha256:8ac016ffaeac35bc010992b71bf8afdd39d458f201c8138d84cf78778a936e6c", size = 673783, upload-time = "2026-03-23T11:56:44.875Z" }, + { url = "https://files.pythonhosted.org/packages/2a/ae/57d1d7af907e20c077e113e0e4976f87b82c0a415403d99284a262229dd0/srsly-2.5.3-cp314-cp314t-win_arm64.whl", hash = "sha256:d822083fe26ec6728bd8c273ac121fc4ab3864a0fdf0cf0ff3efb188fcd209ed", size = 650229, upload-time = "2026-03-23T11:56:46.148Z" }, +] + [[package]] name = "tenacity" version = "9.1.4" @@ -3104,6 +3529,54 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "thinc" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blis" }, + { name = "catalogue" }, + { name = "confection" }, + { name = "cymem" }, + { name = "murmurhash" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.5.0rc1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "packaging" }, + { name = "preshed" }, + { name = "pydantic" }, + { name = "setuptools" }, + { name = "srsly" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/20/7e/7706f5ca67e0f8f965985e59e2a359b9f5919621eb4e2cb73524ed4676f9/thinc-9.0.0.tar.gz", hash = "sha256:bbe7fc3dca4f2b89a2c1429e8bf1c9dc20b0faad13f0ac29053bbb3c2531ad57", size = 191333, upload-time = "2024-04-19T11:40:46.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ac/7b/5885c95c391d6c905630e4240f179faea8ae07774ea55d43d8672c254c34/thinc-9.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68edaf4d75e6c4db8ba71c672d4735275ee9fab0784c2732f5c2517943f292e3", size = 874482, upload-time = "2024-04-19T11:40:05.35Z" }, + { url = "https://files.pythonhosted.org/packages/ac/9b/09ae5fc3f31d1f6facf191277254e3a42270643d803d1968db2131a12c67/thinc-9.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b1356e72055155b85644a436dd4967f209cc61faf0a4c0a515f867b107b01a2c", size = 789772, upload-time = "2024-04-19T11:40:08.003Z" }, + { url = "https://files.pythonhosted.org/packages/99/00/fd907206984e3067e0c82c2169921e99a825fb2877b388dc12ec2482a657/thinc-9.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd08923a591e7a0776518f2ff641d57af09cf28f0615f6d9f09932c17b62342", size = 786364, upload-time = "2024-04-19T11:40:10.626Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b6/ec656d330c505d3aad89c451fe9d29bd89d7b95fb8a4c8ae86e938239b6c/thinc-9.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e7f39a5c36aed1266aa883b48052cc520c7fd3296a3249bfd9a4bfc536f8fdb", size = 831750, upload-time = "2024-04-19T11:40:12.558Z" }, + { url = "https://files.pythonhosted.org/packages/28/aa/7b2f22eeff342eb30cd65e3e93e22cd69d56d53ca7f4505f0b4befa0f3c5/thinc-9.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f37359839b4de41166d5c4e7b1535273484420b7708f94122822c52edebc1805", size = 1275633, upload-time = "2024-04-19T11:40:14.553Z" }, + { url = "https://files.pythonhosted.org/packages/f5/e3/f7649487a70b7a795c437c0e7efca6b2ea3f8aeeaed92c714a4fd2906f7b/thinc-9.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:99d6c1044a3c51cda7e913e56d19db455a690b23b0700e5b8e1c8f6d700ede6f", size = 834768, upload-time = "2024-04-19T11:40:16.642Z" }, + { url = "https://files.pythonhosted.org/packages/b7/08/5f0d84ae552939bc0fecf4b343c7722cb682a3f54ce25c7409aa3228205a/thinc-9.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cb96d1267dd1bae0f13015c09ee187ddb67fbef04243bd841c0a4bf286a7c74", size = 766119, upload-time = "2024-04-19T11:40:19.641Z" }, + { url = "https://files.pythonhosted.org/packages/1f/44/f9c9713fa66b17fd22aa11ee82bfba885d684804abafd7057989b539fef6/thinc-9.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c020be88bb2c342cc10083ca2ec0db10ca9c730af540f65ae4061bf7a202950d", size = 740170, upload-time = "2024-04-19T11:40:22.601Z" }, + { url = "https://files.pythonhosted.org/packages/3d/98/df43d0625b1ed7923b6f079dd55657665c61bb229ef5f739490f483b1a45/thinc-9.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e18f9e2dc7282d8885a9bb2269b6a3dc1b0036e4752aaf35ccdb6017baf83cc", size = 781873, upload-time = "2024-04-19T11:40:25.119Z" }, + { url = "https://files.pythonhosted.org/packages/bc/fe/31364ff4d9cc5fc9c308b365da2771e0da1cadb287c1781b2842326e17d7/thinc-9.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:67848dd4cbb2c269d044a41af3a3e3bd28aa068a0b679340c8753d5dac0c79fb", size = 1253617, upload-time = "2024-04-19T11:40:28.084Z" }, +] + +[[package]] +name = "tldextract" +version = "5.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, + { name = "idna" }, + { name = "requests" }, + { name = "requests-file" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/65/7b/644fbbb49564a6cb124a8582013315a41148dba2f72209bba14a84242bf0/tldextract-5.3.1.tar.gz", hash = "sha256:a72756ca170b2510315076383ea2993478f7da6f897eef1f4a5400735d5057fb", size = 126105, upload-time = "2025-12-28T23:58:05.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/42/0e49d6d0aac449ca71952ec5bae764af009754fcb2e76a5cc097543747b3/tldextract-5.3.1-py3-none-any.whl", hash = "sha256:6bfe36d518de569c572062b788e16a659ccaceffc486d243af0484e8ecf432d9", size = 105886, upload-time = "2025-12-28T23:58:04.071Z" }, +] + [[package]] name = "toolz" version = "1.1.0" @@ -3125,6 +3598,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/75/1a0392bcc21c44dcdf87b3cf2d137e7829be2c083a1e38d44efca3d57a16/tqdm-4.68.2-py3-none-any.whl", hash = "sha256:d4240441fb5353290b87d6a85968c9decc131a99b8c7faa28269d829de669ede", size = 78578, upload-time = "2026-06-09T13:26:40.731Z" }, ] +[[package]] +name = "typer" +version = "0.9.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/7d/b1e0399aa5e27071f0042784681d28417f3e526c61f62c8e3635ee5ad334/typer-0.9.4.tar.gz", hash = "sha256:f714c2d90afae3a7929fcd72a3abb08df305e1ff61719381384211c4070af57f", size = 276061, upload-time = "2024-03-23T17:07:55.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/39/82c9d3e10979851847361d922a373bdfef4091020da7f893acfaf07c0225/typer-0.9.4-py3-none-any.whl", hash = "sha256:aa6c4a4e2329d868b80ecbaf16f807f2b54e192209d7ac9dd42691d63f7a54eb", size = 45973, upload-time = "2024-03-23T17:07:53.985Z" }, +] + [[package]] name = "types-requests" version = "2.33.0.20260518" @@ -3267,6 +3753,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/44/e7/3937b9a9d6745b94dbe7b86531e098db8c53b77c8d07df7daa9577a47b8e/uuid_utils-0.16.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:680799a9ade01d69c53cb9d41392ced24919d4f600bfab5060b61fca37510097", size = 178508, upload-time = "2026-05-19T07:43:43.774Z" }, ] +[[package]] +name = "wasabi" +version = "1.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/f9/054e6e2f1071e963b5e746b48d1e3727470b2a490834d18ad92364929db3/wasabi-1.1.3.tar.gz", hash = "sha256:4bb3008f003809db0c3e28b4daf20906ea871a2bb43f9914197d540f4f2e0878", size = 30391, upload-time = "2024-05-31T16:56:18.99Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/7c/34330a89da55610daa5f245ddce5aab81244321101614751e7537f125133/wasabi-1.1.3-py3-none-any.whl", hash = "sha256:f76e16e8f7e79f8c4c8be49b4024ac725713ab10cd7f19350ad18a8e3f71728c", size = 27880, upload-time = "2024-05-31T16:56:16.699Z" }, +] + +[[package]] +name = "weasel" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cloudpathlib" }, + { name = "confection" }, + { name = "packaging" }, + { name = "pydantic" }, + { name = "requests" }, + { name = "smart-open" }, + { name = "srsly" }, + { name = "typer" }, + { name = "wasabi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/15/5be22cbde3fa1704da51951bb0b3dcdf33a7930449e156440e518df11391/weasel-0.3.4.tar.gz", hash = "sha256:eb16f92dc9f1a3ffa89c165e3a9acd28018ebb656e0da4da02c0d7d8ae3f6178", size = 38252, upload-time = "2023-11-06T16:14:16.487Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/e5/b63b8e255d89ba4155972990d42523251d4d1368c4906c646597f63870e2/weasel-0.3.4-py3-none-any.whl", hash = "sha256:ee48a944f051d007201c2ea1661d0c41035028c5d5a8bcb29a0b10f1100206ae", size = 50108, upload-time = "2023-11-06T16:14:15.198Z" }, +] + [[package]] name = "web3" version = "8.0.0b3" diff --git a/veil/cli.py b/veil/cli.py index 46fa9a2..c232c11 100644 --- a/veil/cli.py +++ b/veil/cli.py @@ -81,6 +81,19 @@ def setup(app_url: str, no_browser: bool, yes: bool) -> None: @click.option( "--expected-pcr", default=None, help="Refuse any TEE whose registry pcrHash differs from this." ) +@click.option( + "--pii-scrub", + is_flag=True, + default=False, + help="Redact high-impact PII (email, SSN, bank numbers, DOB; addresses with the [pii] extra) " + "from prompts locally before they leave this machine.", +) +@click.option( + "--pii-all-dates", + is_flag=True, + default=False, + help="With --pii-scrub, redact every date as a DOB (aggressive) instead of only birth-date-cued ones.", +) @click.option( "--app-url", default=DEFAULT_APP_URL, show_default=True, help="Chat app web origin for login." ) @@ -104,6 +117,8 @@ def serve( port: int | None, tee_id: str | None, expected_pcr: str | None, + pii_scrub: bool, + pii_all_dates: bool, app_url: str, no_browser: bool, foreground: bool, @@ -127,6 +142,10 @@ def serve( config.expected_pcr_hash = ( expected_pcr if expected_pcr.startswith("0x") else "0x" + expected_pcr ).lower() + if pii_scrub: + config.pii_scrub = True + if pii_all_dates: + config.pii_redact_all_dates = True # The detached child (--skip-setup) always runs the server in-process. _start_server(config, foreground=foreground or skip_setup) @@ -138,6 +157,10 @@ def _config_flags(config: ServerConfig) -> list[str]: flags += ["--tee-id", config.pinned_tee_id] if config.expected_pcr_hash: flags += ["--expected-pcr", config.expected_pcr_hash] + if config.pii_scrub: + flags += ["--pii-scrub"] + if config.pii_redact_all_dates: + flags += ["--pii-all-dates"] return flags diff --git a/veil/config.py b/veil/config.py index 9439fa7..b690fa7 100644 --- a/veil/config.py +++ b/veil/config.py @@ -57,6 +57,15 @@ class ServerConfig: # the registry. Useful for reproducible demos and debugging. pinned_tee_id: str | None = None + # Opt-in local PII redaction: scrub high-impact PII (email, SSN, bank numbers, + # DOB, and — with the [pii] extra — addresses) out of the agent's prompt + # *before* it leaves this process. Off by default; see veil.pii. + pii_scrub: bool = False + + # When PII scrubbing is on, also redact *every* date as a DOB (aggressive, + # safer). Off by default, so only birth-date-cued dates are redacted. + pii_redact_all_dates: bool = False + @classmethod def from_env(cls) -> "ServerConfig": return cls( @@ -64,6 +73,8 @@ def from_env(cls) -> "ServerConfig": port=int(os.getenv("OG_VEIL_PORT", "11434")), expected_pcr_hash=_norm_hex(os.getenv("OG_VEIL_EXPECTED_PCR_HASH")), pinned_tee_id=_norm_hex(os.getenv("OG_VEIL_TEE_ID")), + pii_scrub=_env_bool(os.getenv("OG_VEIL_PII_SCRUB")), + pii_redact_all_dates=_env_bool(os.getenv("OG_VEIL_PII_REDACT_ALL_DATES")), ) def advertised_base_url(self) -> str: @@ -71,6 +82,10 @@ def advertised_base_url(self) -> str: return f"http://{self.host}:{self.port}/v1" +def _env_bool(value: str | None) -> bool: + return value is not None and value.strip().lower() in ("1", "true", "yes", "on") + + def _norm_hex(value: str | None) -> str | None: if not value: return None diff --git a/veil/gateway.py b/veil/gateway.py index 42aa150..47ae18a 100644 --- a/veil/gateway.py +++ b/veil/gateway.py @@ -18,6 +18,7 @@ from opengradient.client.tee_registry import TEE_TYPE_LLM_PROXY, TEEEndpoint from veil.config import OHTTP_RELAY_PATH, ServerConfig +from veil.pii import build_redactor from veil.session import Session logger = logging.getLogger(__name__) @@ -36,6 +37,11 @@ def __init__(self, session: Session, config: ServerConfig): self._lock = threading.Lock() self._client: OhttpRelayClient | None = None self._tee: TEEEndpoint | None = None + # Optional local PII redaction, applied to the request before it is + # encrypted to the TEE. ``None`` when disabled (the default). + self._redactor = build_redactor( + enabled=config.pii_scrub, redact_all_dates=config.pii_redact_all_dates + ) cfg = session.config if not cfg.tee_registry_rpc_url or not cfg.tee_registry_address: @@ -112,6 +118,11 @@ def active_tee(self) -> TEEEndpoint | None: # --- inference --------------------------------------------------------- def chat(self, body: dict) -> VerifiedChatResponse: + # Redact PII locally before anything is sealed to the enclave. Done once, + # outside the retry loop, so a gateway re-selection resends the already- + # scrubbed body rather than re-scrubbing. + if self._redactor is not None: + body = self._redactor.scrub_request(body) try: return self._chat_once(body) except requests.exceptions.RequestException as exc: diff --git a/veil/pii.py b/veil/pii.py new file mode 100644 index 0000000..8d36897 --- /dev/null +++ b/veil/pii.py @@ -0,0 +1,275 @@ +"""Local, opt-in PII redaction applied *before* a prompt leaves this process. + +Veil already keeps prompts private end-to-end (Oblivious HTTP splits identity +from content; the enclave is attested and reproducible). This module is +*defense-in-depth* on top of that: when enabled, high-impact PII is irreversibly +replaced with ``[REDACTED_*]`` tags on the agent's request before it is HPKE- +sealed to the TEE, so the raw values never leave the machine at all — useful for +compliance, data-residency, and keeping PII out of any model-side logging. + +Two tiers, layered: + +* **Regex (always on when enabled, zero extra deps)** — structured PII that has a + recognizable shape: email, US SSN, and bank numbers (credit cards validated by + Luhn, IBANs by mod-97 checksum, plus routing/account numbers when labelled). + Dates of birth are caught by context ("DOB:", "born on …"). +* **NER (optional, ``pip install 'opengradient-veil[pii]'``)** — Microsoft + Presidio + a spaCy model adds *addresses/locations*, which are free-form prose + that regex fundamentally cannot see. Optionally redacts *all* dates too. + +This is risk-reduction, not a guarantee: NER misses a fraction of addresses every +run, and bare (unlabelled) account numbers can slip through. Redaction is +irreversible — there is deliberately no de-anonymization step, so the TEE's +signed ``output_hash`` covers exactly what it ran and nothing is restored locally. +""" + +from __future__ import annotations + +import logging +import re +from collections.abc import Callable + +logger = logging.getLogger(__name__) + +# Redaction tags. Kept human-readable so a redacted prompt still reads sensibly to +# the model (e.g. "wire it to [REDACTED_BANK_NUMBER]"). +EMAIL_TAG = "[REDACTED_EMAIL]" +SSN_TAG = "[REDACTED_SSN]" +BANK_TAG = "[REDACTED_BANK_NUMBER]" +DOB_TAG = "[REDACTED_DOB]" +ADDRESS_TAG = "[REDACTED_ADDRESS]" + + +class PiiSetupError(Exception): + """The NER engine was requested but its model/runtime isn't installed.""" + + +# --- regex tier ------------------------------------------------------------ + +_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") + +# Dashed/spaced SSN, e.g. 123-45-6789 or 123 45 6789. Bare 9-digit runs are too +# ambiguous (order IDs etc.) to redact unconditionally — those are handled by the +# context detector below. +_SSN_RE = re.compile(r"\b\d{3}[-\s]\d{2}[-\s]\d{4}\b") + +# Candidate card number: 13–19 digits, optionally split by spaces/hyphens. Only +# redacted if it passes the Luhn check, which kills almost all false positives. +_CC_RE = re.compile(r"\b(?:\d[ -]?){13,19}\b") + +# IBAN shape, allowing the common space-grouped form (GB82 WEST 1234 …); +# validated by mod-97 (spaces stripped) before redaction. +_IBAN_RE = re.compile(r"\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){11,30}\b") + +# Labelled SSN: "SSN: 123456789" / "social security number 123-45-6789". +_SSN_CTX_RE = re.compile( + r"(?i)(\b(?:ssn|social security(?:\s+number)?)\b\D{0,8})((?:\d[ -]?){8,9}\d)" +) + +# Labelled bank/routing/account numbers, including bare domestic account numbers +# that have no fixed shape and are only recognizable from their label. +_BANK_CTX_RE = re.compile( + r"(?i)(\b(?:account|acct|a/c|routing|aba|iban|swift|bic|sort\s?code|bank)\b" + r"(?:\s*(?:number|no\.?|#))?\s*[:#]?\s*)((?:\d[ -]?){5,17}\d)" +) + +# A date in common numeric or "Month DD, YYYY" forms — only redacted when it +# follows a birth-date cue, so ordinary dates in the prompt are left intact. +_DATE = ( + r"(?:\d{4}-\d{2}-\d{2}" + r"|\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}" + r"|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?" + r"|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" + r"\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4})" +) +_DOB_CTX_RE = re.compile( + r"(?i)(\b(?:d\.?o\.?b\.?|date\s+of\s+birth|born(?:\s+on)?|birth\s?date|birthday)\b\W{0,6})(" + + _DATE + + r")" +) + + +def _luhn_ok(digits: str) -> bool: + total = 0 + for i, ch in enumerate(reversed(digits)): + d = int(ch) + if i % 2 == 1: + d *= 2 + if d > 9: + d -= 9 + total += d + return total % 10 == 0 + + +def _iban_ok(candidate: str) -> bool: + s = candidate.upper().replace(" ", "") + rearranged = s[4:] + s[:4] + # Letters → 10..35; the whole thing mod 97 must equal 1 for a valid IBAN. + converted = "".join(str(int(c, 36)) if c.isalpha() else c for c in rearranged) + try: + return int(converted) % 97 == 1 + except ValueError: + return False + + +def _redact_cc(m: re.Match[str]) -> str: + digits = re.sub(r"\D", "", m.group(0)) + if 13 <= len(digits) <= 19 and _luhn_ok(digits): + return BANK_TAG + return m.group(0) + + +def _redact_iban(m: re.Match[str]) -> str: + return BANK_TAG if _iban_ok(m.group(0)) else m.group(0) + + +def _keep_label(tag: str) -> Callable[[re.Match[str]], str]: + """Replace only the value group, preserving the leading label (group 1).""" + + def repl(m: re.Match[str]) -> str: + return m.group(1) + tag + + return repl + + +# Order matters: validate-and-redact the strongly-shaped values first, then the +# label-anchored ones, so a labelled card number is still caught by the Luhn pass. +_Replacement = str | Callable[[re.Match[str]], str] +_REGEX_PASSES: list[tuple[re.Pattern[str], _Replacement]] = [ + (_EMAIL_RE, EMAIL_TAG), + (_IBAN_RE, _redact_iban), + (_CC_RE, _redact_cc), + (_SSN_RE, SSN_TAG), + (_SSN_CTX_RE, _keep_label(SSN_TAG)), + (_BANK_CTX_RE, _keep_label(BANK_TAG)), + (_DOB_CTX_RE, _keep_label(DOB_TAG)), +] + + +def _regex_scrub(text: str) -> str: + for pattern, repl in _REGEX_PASSES: + text = pattern.sub(repl, text) + return text + + +# --- NER tier (optional, Presidio) ----------------------------------------- + + +def _load_ner(redact_all_dates: bool) -> Callable[[str], str] | None: + """Build a Presidio-backed scrubber for addresses (and optionally all dates). + + Returns ``None`` if the optional ``[pii]`` extra isn't installed. Raises + :class:`PiiSetupError` if Presidio is present but its spaCy model isn't, so + the operator gets an actionable message instead of a cryptic stack trace. + """ + try: + from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found] + from presidio_analyzer.nlp_engine import ( # type: ignore[import-not-found] + NlpEngineProvider, + ) + from presidio_anonymizer import AnonymizerEngine # type: ignore[import-not-found] + from presidio_anonymizer.entities import ( # type: ignore[import-not-found] + OperatorConfig, + ) + except ImportError: + return None + + model = "en_core_web_sm" + provider = NlpEngineProvider( + nlp_configuration={ + "nlp_engine_name": "spacy", + "models": [{"lang_code": "en", "model_name": model}], + } + ) + try: + nlp_engine = provider.create_engine() + except Exception as exc: # noqa: BLE001 — spaCy model not downloaded yet + raise PiiSetupError( + f"the spaCy model '{model}' is not installed; run: " + f"python -m spacy download {model}" + ) from exc + + analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=["en"]) + anonymizer = AnonymizerEngine() + + entities = ["LOCATION"] + operators = {"LOCATION": OperatorConfig("replace", {"new_value": ADDRESS_TAG})} + if redact_all_dates: + entities.append("DATE_TIME") + operators["DATE_TIME"] = OperatorConfig("replace", {"new_value": DOB_TAG}) + + def run(text: str) -> str: + results = analyzer.analyze(text=text, entities=entities, language="en") + if not results: + return text + return anonymizer.anonymize(text=text, analyzer_results=results, operators=operators).text + + return run + + +# --- public surface --------------------------------------------------------- + + +class Redactor: + """Applies the regex tier (always) and the NER tier (when available).""" + + def __init__(self, *, redact_all_dates: bool = False): + self._ner: Callable[[str], str] | None = None + try: + self._ner = _load_ner(redact_all_dates) + except PiiSetupError as exc: + logger.warning( + "PII: %s — continuing with regex-only redaction (no address coverage)", exc + ) + if self._ner is None: + logger.warning( + "PII: address/NER redaction unavailable — install the extra for it: " + "pip install 'opengradient-veil[pii]'" + ) + + def scrub_text(self, text: str) -> str: + if not text: + return text + # Regex first so structured values become tags; the NER pass then runs on + # the partially-redacted text purely for free-form addresses/dates. + text = _regex_scrub(text) + if self._ner is not None: + text = self._ner(text) + return text + + def scrub_request(self, body: dict) -> dict: + """Return a copy of an OpenAI chat-completions body with message text redacted. + + Handles both string content and the multimodal ``[{"type": "text", ...}]`` + content-part form; non-text parts (images, audio) are passed through. + """ + messages = body.get("messages") + if not isinstance(messages, list): + return body + + scrubbed_messages = [] + for msg in messages: + if not isinstance(msg, dict): + scrubbed_messages.append(msg) + continue + content = msg.get("content") + if isinstance(content, str): + msg = {**msg, "content": self.scrub_text(content)} + elif isinstance(content, list): + parts = [ + {**p, "text": self.scrub_text(p["text"])} + if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str) + else p + for p in content + ] + msg = {**msg, "content": parts} + scrubbed_messages.append(msg) + + return {**body, "messages": scrubbed_messages} + + +def build_redactor(*, enabled: bool, redact_all_dates: bool = False) -> Redactor | None: + """Construct a :class:`Redactor` when PII scrubbing is enabled, else ``None``.""" + if not enabled: + return None + return Redactor(redact_all_dates=redact_all_dates) From 90f2553ce9f23ea001655b921496edda90425715 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 20:40:07 +0000 Subject: [PATCH 02/25] PII redaction: use Presidio as the single engine, drop handrolled regex Replace the handrolled regex detectors with Microsoft Presidio so the recognizers are community-maintained rather than ours to own. Presidio detects the structured types (email, US SSN, credit cards, IBANs, US bank/routing numbers) with its own regex/checksum recognizers and uses the spaCy model for free-form addresses/locations. Because detection now depends on Presidio + a spaCy model, PII scrubbing requires the optional [pii] extra; if --pii-scrub is set without it, the server refuses to start with an actionable message instead of silently passing PII through. Notable details: - Dates of birth: a custom Presidio PatternRecognizer anchored on a birth-date cue via variable-width lookbehind (the regex module Presidio uses supports it), giving true per-occurrence gating so an unrelated invoice date is left alone. --pii-all-dates switches to redacting every date via spaCy DATE_TIME. - Pin spaCy to stable 3.8.x in the extra: the repo allows prereleases (for an SDK dep), which otherwise pulls a spaCy 4.0 dev build whose compiled extensions break against numpy 2.x. - CI/docs install the spaCy model from the release wheel rather than `spacy download`, whose CLI breaks under the click version this project pins. Added a `make install-pii` helper and a model-install step to the tests workflow; PII tests importorskip when the extra is absent. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- .github/workflows/tests.yml | 10 +- Makefile | 9 +- README.md | 34 +++-- pyproject.toml | 5 +- tests/test_pii.py | 112 ++++++++------- uv.lock | 169 ++++++++++++++--------- veil/pii.py | 262 ++++++++++++++---------------------- veil/server.py | 7 +- 8 files changed, 309 insertions(+), 299 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 32454cd..8266851 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -21,8 +21,14 @@ jobs: - name: Set up Python run: uv python install 3.11 - - name: Install dependencies - run: uv sync --all-groups + - name: Install dependencies (incl. PII extra) + run: uv sync --all-groups --extra pii + + - name: Install spaCy model for PII tests + # Installed from the release wheel rather than `spacy download`, whose CLI + # breaks under the click version this project pins. Not needed at runtime, + # only to fetch the model. + run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl - name: Test run: make test diff --git a/Makefile b/Makefile index 358e6a6..43bad71 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,15 @@ -.PHONY: install build publish check test serve +.PHONY: install install-pii build publish check test serve install: uv sync --all-groups +# Dev install with the optional PII-redaction stack (Presidio + spaCy) and its +# model. The model is fetched from the release wheel rather than `spacy download`, +# whose CLI breaks under the click version this project pins. +install-pii: + uv sync --all-groups --extra pii + uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl + build: uv build diff --git a/README.md b/README.md index 9d84c52..00d75fc 100644 --- a/README.md +++ b/README.md @@ -185,25 +185,33 @@ irreversibly replaced with `[REDACTED_*]` tags *before* the prompt is encrypted the TEE, so the raw values never leave your machine. Handy for compliance, data-residency, and keeping PII out of any model-side logging. +Detection is delegated to **Microsoft Presidio** (community-maintained +recognizers) rather than handrolled patterns, so it ships as an optional extra. +Install it once: + ```sh -og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 +pip install 'opengradient-veil[pii]' +# fetch the spaCy model (from the release wheel — `spacy download` breaks under +# the click version this project pins): +pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl ``` -Two layered tiers: +Then enable it: + +```sh +og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 +``` -- **Regex (always on when enabled, zero extra deps)** — email, US SSN, and bank - numbers (credit cards Luhn-checked, IBANs mod-97-checked, plus routing/account - numbers when labelled). Dates of birth are caught by context (`DOB:`, `born on …`). -- **Addresses (optional)** — free-form street addresses are prose that regex can't - see, so they need a lightweight local NER model. Install the extra once: +What gets redacted (each replaced with a `[REDACTED_*]` tag): - ```sh - pip install 'opengradient-veil[pii]' - python -m spacy download en_core_web_sm - ``` +- **email, US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and + US bank/routing numbers, via Presidio's regex/checksum recognizers. +- **addresses / locations** — free-form prose, via the spaCy NER model. +- **dates of birth** — by default only *birth-date-cued* dates (`DOB:`, + `born on …`); pass `--pii-all-dates` to redact every date instead. - With the extra present, `--pii-scrub` also redacts addresses/locations. Without - it, scrubbing still runs (regex tier) and logs that address coverage is off. +If `--pii-scrub` is set but the extra/model isn't installed, the server refuses +to start with an actionable message rather than silently sending PII. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's signed `output_hash` covers exactly what it ran. This is risk-reduction, not a diff --git a/pyproject.toml b/pyproject.toml index 46bc1d9..6fd6cb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,10 @@ dependencies = [ pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", - "spacy>=3.7.0", + # Pin to stable spaCy 3.8.x: the repo allows prereleases (for an SDK dep), + # which otherwise pulls a spaCy 4.0 dev build whose compiled extensions break + # against numpy 2.x. 3.8 supports numpy 2 and has matching released models. + "spacy>=3.8.0,<4.0.0", ] [project.urls] diff --git a/tests/test_pii.py b/tests/test_pii.py index 4133730..6cfc287 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -1,76 +1,96 @@ -"""PII redaction tests — the always-on regex tier and request rewriting. +"""PII redaction tests. -The optional NER tier (Presidio + spaCy) is exercised only when the [pii] extra -is installed, so addresses are not asserted here; these cover the zero-dependency -behavior that ships in the base install. +Detection is delegated to Presidio + spaCy (the optional [pii] extra), so these +tests ``importorskip`` when it isn't installed — base installs and the release +CI skip them; the dedicated test workflow installs the extra + model and runs +them for real. ``build_redactor(enabled=False)`` is checked unconditionally. """ from __future__ import annotations +import pytest + from veil.pii import ( ADDRESS_TAG, BANK_TAG, DOB_TAG, EMAIL_TAG, SSN_TAG, - Redactor, + PiiSetupError, build_redactor, ) -# A Redactor with no NER engine loaded — pure regex tier. (If Presidio happens to -# be installed in the dev env it would only *add* address coverage, never change -# these structured-PII assertions.) -R = Redactor() + +def test_build_redactor_disabled_returns_none(): + # Works without the extra: disabled means no engine is constructed at all. + assert build_redactor(enabled=False) is None -def test_email_redacted(): - assert R.scrub_text("ping me at jane.doe+x@example.co.uk please") == ( - f"ping me at {EMAIL_TAG} please" - ) +def test_tags_are_distinct(): + assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, DOB_TAG, ADDRESS_TAG}) == 5 -def test_dashed_ssn_redacted(): - assert R.scrub_text("my ssn is 123-45-6789") == f"my ssn is {SSN_TAG}" +# --- everything below needs the [pii] extra + spaCy model ------------------ +pytest.importorskip("presidio_analyzer", reason="requires the [pii] extra") -def test_bare_ssn_only_with_label(): - # Bare 9-digit runs are left alone (too ambiguous)… - assert "987654321" in R.scrub_text("order 987654321 shipped") - # …but a labelled one is redacted, keeping the label. - out = R.scrub_text("SSN: 987654321") - assert "987654321" not in out and SSN_TAG in out and out.lower().startswith("ssn") +def _redactor(**kw): + try: + return build_redactor(enabled=True, **kw) + except PiiSetupError as exc: # presidio present but model missing + pytest.skip(str(exc)) -def test_valid_credit_card_redacted_invalid_kept(): - # 4111 1111 1111 1111 is the canonical Luhn-valid test number. - assert R.scrub_text("card 4111 1111 1111 1111") == f"card {BANK_TAG}" - # One digit off → fails Luhn → not redacted. - assert "4111 1111 1111 1112" in R.scrub_text("card 4111 1111 1111 1112") +@pytest.fixture(scope="module") +def R(): + return _redactor() -def test_valid_iban_redacted(): - out = R.scrub_text("send to GB82 WEST 1234 5698 7654 32 today") - assert BANK_TAG in out and "WEST" not in out +def test_email_redacted(R): + out = R.scrub_text("ping me at jane.doe+x@example.co.uk please") + assert "jane.doe" not in out and EMAIL_TAG in out -def test_invalid_iban_kept(): - assert "GB00WEST12345698765432" in R.scrub_text("ref GB00WEST12345698765432") +def test_ssn_redacted(R): + # A plausible SSN — Presidio deliberately rejects textbook fakes like + # 123-45-6789 / 078-05-1120 via its invalidate_result blacklist. + out = R.scrub_text("my SSN is 457-55-5462") + assert "457-55-5462" not in out and SSN_TAG in out + + +def test_credit_card_redacted(R): + # Luhn-valid canonical test number. + out = R.scrub_text("card 4111 1111 1111 1111 on file") + assert "4111" not in out and BANK_TAG in out + + +def test_iban_redacted(R): + out = R.scrub_text("send to GB82 WEST 1234 5698 7654 32 today") + assert "WEST" not in out and BANK_TAG in out -def test_labelled_account_number_redacted(): - out = R.scrub_text("account number: 0012345678") - assert "0012345678" not in out and BANK_TAG in out +def test_address_redacted(R): + # Free-form location — the thing only NER can see. + out = R.scrub_text("I live in San Francisco, California") + assert ADDRESS_TAG in out -def test_dob_context_redacts_only_cued_dates(): - out = R.scrub_text("DOB: 04/12/1990, meeting on 06/01/2026") + +def test_dob_context_only(R): + out = R.scrub_text("DOB: 04/12/1990. The invoice is dated 06/01/2026.") assert DOB_TAG in out assert "04/12/1990" not in out - # A non-birth date is untouched without the all-dates toggle. + # A non-birth date is left intact in the default (context-only) mode. assert "06/01/2026" in out -def test_scrub_request_string_content(): +def test_redact_all_dates_mode(): + R = _redactor(redact_all_dates=True) + out = R.scrub_text("the invoice is dated 06/01/2026") + assert "06/01/2026" not in out and DOB_TAG in out + + +def test_scrub_request_string_content(R): body = { "model": "gpt-4.1", "messages": [ @@ -79,12 +99,12 @@ def test_scrub_request_string_content(): ], } out = R.scrub_request(body) - assert out["messages"][1]["content"] == f"email me at {EMAIL_TAG}" + assert EMAIL_TAG in out["messages"][1]["content"] # Original body is not mutated. assert body["messages"][1]["content"] == "email me at a@b.com" -def test_scrub_request_multimodal_parts(): +def test_scrub_request_multimodal_parts(R): body = { "messages": [ { @@ -98,15 +118,5 @@ def test_scrub_request_multimodal_parts(): } out = R.scrub_request(body) parts = out["messages"][0]["content"] - assert parts[0]["text"] == f"reach me at {EMAIL_TAG}" + assert EMAIL_TAG in parts[0]["text"] assert parts[1] == {"type": "image_url", "image_url": {"url": "http://x/y.png"}} - - -def test_build_redactor_disabled_returns_none(): - assert build_redactor(enabled=False) is None - assert build_redactor(enabled=True) is not None - - -def test_address_tag_is_distinct(): - # Sanity: tags are unique strings so downstream tooling can grep them. - assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, DOB_TAG, ADDRESS_TAG}) == 5 diff --git a/uv.lock b/uv.lock index 4599bdf..6653f74 100644 --- a/uv.lock +++ b/uv.lock @@ -307,24 +307,42 @@ wheels = [ [[package]] name = "blis" -version = "0.7.11" +version = "1.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.0rc1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/8c/60c85350f2e1c9647df580083a0f6acc686ef32d1a91f4ab0c624b3ff867/blis-0.7.11.tar.gz", hash = "sha256:cec6d48f75f7ac328ae1b6fbb372dde8c8a57c89559172277f66e01ff08d4d42", size = 2897107, upload-time = "2023-09-22T06:28:25.103Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/59/c8010f380a16709e6d3ef5534845d1ca1e689079914ec67ab60f57edfc37/blis-0.7.11-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:1b68df4d01d62f9adaef3dad6f96418787265a6878891fc4e0fabafd6d02afba", size = 6123547, upload-time = "2023-09-22T06:27:28.47Z" }, - { url = "https://files.pythonhosted.org/packages/a8/73/0a9d4e7f6e78ef270e3a4532b17e060a02087590cf615ba9943fd1a283e9/blis-0.7.11-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:162e60d941a8151418d558a94ee5547cb1bbeed9f26b3b6f89ec9243f111a201", size = 1106895, upload-time = "2023-09-22T06:27:30.964Z" }, - { url = "https://files.pythonhosted.org/packages/51/f7/a5d9a0be0729f4172248dbae74d7e02b139b3a32cc29650d3ade7ab91fea/blis-0.7.11-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:686a7d0111d5ba727cd62f374748952fd6eb74701b18177f525b16209a253c01", size = 1707389, upload-time = "2023-09-22T06:27:32.321Z" }, - { url = "https://files.pythonhosted.org/packages/dc/23/eb01450dc284a7ea8ebc0e5296f1f8fdbe5299169f4c318f836b4284a119/blis-0.7.11-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0421d6e44cda202b113a34761f9a062b53f8c2ae8e4ec8325a76e709fca93b6e", size = 10172888, upload-time = "2023-09-22T06:27:34.529Z" }, - { url = "https://files.pythonhosted.org/packages/2f/09/da0592c74560cc33396504698122f7a56747c82a5e072ca7d2c3397898e1/blis-0.7.11-cp311-cp311-win_amd64.whl", hash = "sha256:0dc9dcb3843045b6b8b00432409fd5ee96b8344a324e031bfec7303838c41a1a", size = 6602835, upload-time = "2023-09-22T06:27:37.46Z" }, - { url = "https://files.pythonhosted.org/packages/e2/12/90897bc489626cb71e51ce8bb89e492fabe96a57811e53159c0f74ae90ec/blis-0.7.11-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dadf8713ea51d91444d14ad4104a5493fa7ecc401bbb5f4a203ff6448fadb113", size = 6121528, upload-time = "2023-09-22T06:27:39.451Z" }, - { url = "https://files.pythonhosted.org/packages/e2/5d/67a3f6b6108c39d3fd1cf55a7dca9267152190dad419c9de6d764b3708ca/blis-0.7.11-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:5bcdaf370f03adaf4171d6405a89fa66cb3c09399d75fc02e1230a78cd2759e4", size = 1105039, upload-time = "2023-09-22T06:27:41.143Z" }, - { url = "https://files.pythonhosted.org/packages/03/62/0d214dde0703863ed2d3dabb3f10606f7f55ac4eb07a52c3906601331b63/blis-0.7.11-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7de19264b1d49a178bf8035406d0ae77831f3bfaa3ce02942964a81a202abb03", size = 1701009, upload-time = "2023-09-22T06:27:42.672Z" }, - { url = "https://files.pythonhosted.org/packages/66/aa/bcbd1c6b1c7dfd717ff5c899a1c8adcc6b3e391fb7a0b00fdc64e4e54235/blis-0.7.11-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ea55c6a4a60fcbf6a0fdce40df6e254451ce636988323a34b9c94b583fc11e5", size = 10161187, upload-time = "2023-09-22T06:27:44.183Z" }, - { url = "https://files.pythonhosted.org/packages/9a/91/4aea63dccee6491a54c630d9817656a886e086ab97222e2d8101d8cdf894/blis-0.7.11-cp312-cp312-win_amd64.whl", hash = "sha256:5a305dbfc96d202a20d0edd6edf74a406b7e1404f4fa4397d24c68454e60b1b4", size = 6624079, upload-time = "2023-09-22T06:27:46.719Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/d0/d0/d8cc8c9a4488a787e7fa430f6055e5bd1ddb22c340a751d9e901b82e2efe/blis-1.3.3.tar.gz", hash = "sha256:034d4560ff3cc43e8aa37e188451b0440e3261d989bb8a42ceee865607715ecd", size = 2644873, upload-time = "2025-11-17T12:28:30.511Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a1/0a/a4c8736bc497d386b0ffc76d321f478c03f1a4725e52092f93b38beb3786/blis-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e10c8d3e892b1dbdff365b9d00e08291876fc336915bf1a5e9f188ed087e1a91", size = 6925522, upload-time = "2025-11-17T12:27:29.199Z" }, + { url = "https://files.pythonhosted.org/packages/83/5a/3437009282f23684ecd3963a8b034f9307cdd2bf4484972e5a6b096bf9ac/blis-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:66e6249564f1db22e8af1e0513ff64134041fa7e03c8dd73df74db3f4d8415a7", size = 1232787, upload-time = "2025-11-17T12:27:30.996Z" }, + { url = "https://files.pythonhosted.org/packages/d1/0e/82221910d16259ce3017c1442c468a3f206a4143a96fbba9f5b5b81d62e8/blis-1.3.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7260da065958b4e5475f62f44895ef9d673b0f47dcf61b672b22b7dae1a18505", size = 2844596, upload-time = "2025-11-17T12:27:32.601Z" }, + { url = "https://files.pythonhosted.org/packages/6c/93/ab547f1a5c23e20bca16fbcf04021c32aac3f969be737ea4980509a7ca90/blis-1.3.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9327a6ca67de8ae76fe071e8584cc7f3b2e8bfadece4961d40f2826e1cda2df", size = 11377746, upload-time = "2025-11-17T12:27:35.342Z" }, + { url = "https://files.pythonhosted.org/packages/6e/a6/7733820aa62da32526287a63cd85c103b2b323b186c8ee43b7772ff7017c/blis-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c4ae70629cf302035d268858a10ca4eb6242a01b2dc8d64422f8e6dcb8a8ee74", size = 3041954, upload-time = "2025-11-17T12:27:37.479Z" }, + { url = "https://files.pythonhosted.org/packages/87/53/e39d67fd3296b649772780ca6aab081412838ecb54e0b0c6432d01626a50/blis-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:45866a9027d43b93e8b59980a23c5d7358b6536fc04606286e39fdcfce1101c2", size = 14251222, upload-time = "2025-11-17T12:27:39.705Z" }, + { url = "https://files.pythonhosted.org/packages/ea/44/b749f8777b020b420bceaaf60f66432fc30cc904ca5b69640ec9cbef11ed/blis-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:27f82b8633030f8d095d2b412dffa7eb6dbc8ee43813139909a20012e54422ea", size = 6171233, upload-time = "2025-11-17T12:27:41.921Z" }, + { url = "https://files.pythonhosted.org/packages/16/d1/429cf0cf693d4c7dc2efed969bd474e315aab636e4a95f66c4ed7264912d/blis-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2a1c74e100665f8e918ebdbae2794576adf1f691680b5cdb8b29578432f623ef", size = 6929663, upload-time = "2025-11-17T12:27:44.482Z" }, + { url = "https://files.pythonhosted.org/packages/11/69/363c8df8d98b3cc97be19aad6aabb2c9c53f372490d79316bdee92d476e7/blis-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3f6c595185176ce021316263e1a1d636a3425b6c48366c1fd712d08d0b71849a", size = 1230939, upload-time = "2025-11-17T12:27:46.19Z" }, + { url = "https://files.pythonhosted.org/packages/96/2a/fbf65d906d823d839076c5150a6f8eb5ecbc5f9135e0b6510609bda1e6b7/blis-1.3.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d734b19fba0be7944f272dfa7b443b37c61f9476d9ab054a9ac53555ceadd2e0", size = 2818835, upload-time = "2025-11-17T12:27:48.167Z" }, + { url = "https://files.pythonhosted.org/packages/d5/ad/58deaa3ad856dd3cc96493e40ffd2ed043d18d4d304f85a65cde1ccbf644/blis-1.3.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1ef6d6e2b599a3a2788eb6d9b443533961265aa4ec49d574ed4bb846e548dcdb", size = 11366550, upload-time = "2025-11-17T12:27:49.958Z" }, + { url = "https://files.pythonhosted.org/packages/78/82/816a7adfe1f7acc8151f01ec86ef64467a3c833932d8f19f8e06613b8a4e/blis-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8c888438ae99c500422d50698e3028b65caa8ebb44e24204d87fda2df64058f7", size = 3023686, upload-time = "2025-11-17T12:27:52.062Z" }, + { url = "https://files.pythonhosted.org/packages/1e/e2/0e93b865f648b5519360846669a35f28ee8f4e1d93d054f6850d8afbabde/blis-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8177879fd3590b5eecdd377f9deafb5dc8af6d684f065bd01553302fb3fcf9a7", size = 14250939, upload-time = "2025-11-17T12:27:53.847Z" }, + { url = "https://files.pythonhosted.org/packages/20/07/fb43edc2ff0a6a367e4a94fc39eb3b85aa1e55e24cc857af2db145ce9f0d/blis-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:f20f7ad69aaffd1ce14fe77de557b6df9b61e0c9e582f75a843715d836b5c8af", size = 6192759, upload-time = "2025-11-17T12:27:56.176Z" }, + { url = "https://files.pythonhosted.org/packages/e6/f7/d26e62d9be3d70473a63e0a5d30bae49c2fe138bebac224adddcdef8a7ce/blis-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1e647341f958421a86b028a2efe16ce19c67dba2a05f79e8f7e80b1ff45328aa", size = 6928322, upload-time = "2025-11-17T12:27:57.965Z" }, + { url = "https://files.pythonhosted.org/packages/4a/78/750d12da388f714958eb2f2fd177652323bbe7ec528365c37129edd6eb84/blis-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d563160f874abb78a57e346f07312c5323f7ad67b6370052b6b17087ef234a8e", size = 1229635, upload-time = "2025-11-17T12:28:00.118Z" }, + { url = "https://files.pythonhosted.org/packages/e8/36/eac4199c5b200a5f3e93cad197da8d26d909f218eb444c4f552647c95240/blis-1.3.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:30b8a5b90cb6cb81d1ada9ae05aa55fb8e70d9a0ae9db40d2401bb9c1c8f14c4", size = 2815650, upload-time = "2025-11-17T12:28:02.544Z" }, + { url = "https://files.pythonhosted.org/packages/bf/51/472e7b36a6bedb5242a9757e7486f702c3619eff76e256735d0c8b1679c6/blis-1.3.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9f5c53b277f6ac5b3ca30bc12ebab7ea16c8f8c36b14428abb56924213dc127", size = 11359008, upload-time = "2025-11-17T12:28:04.589Z" }, + { url = "https://files.pythonhosted.org/packages/84/da/d0dfb6d6e6321ae44df0321384c32c322bd07b15740d7422727a1a49fc5d/blis-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6297e7616c158b305c9a8a4e47ca5fc9b0785194dd96c903b1a1591a7ca21ddf", size = 3011959, upload-time = "2025-11-17T12:28:06.862Z" }, + { url = "https://files.pythonhosted.org/packages/20/c5/2b0b5e556fa0364ed671051ea078a6d6d7b979b1cfef78d64ad3ca5f0c7f/blis-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3f966ca74f89f8a33e568b9a1d71992fc9a0d29a423e047f0a212643e21b5458", size = 14232456, upload-time = "2025-11-17T12:28:08.779Z" }, + { url = "https://files.pythonhosted.org/packages/31/07/4cdc81a47bf862c0b06d91f1bc6782064e8b69ac9b5d4ff51d97e4ff03da/blis-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:7a0fc4b237a3a453bdc3c7ab48d91439fcd2d013b665c46948d9eaf9c3e45a97", size = 6192624, upload-time = "2025-11-17T12:28:14.197Z" }, + { url = "https://files.pythonhosted.org/packages/5f/8a/80f7c68fbc24a76fc9c18522c46d6d69329c320abb18e26a707a5d874083/blis-1.3.3-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c3e33cfbf22a418373766816343fcfcd0556012aa3ffdf562c29cddec448a415", size = 6934081, upload-time = "2025-11-17T12:28:16.436Z" }, + { url = "https://files.pythonhosted.org/packages/e5/52/d1aa3a51a7fc299b0c89dcaa971922714f50b1202769eebbdaadd1b5cff7/blis-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6f165930e8d3a85c606d2003211497e28d528c7416fbfeafb6b15600963f7c9b", size = 1231486, upload-time = "2025-11-17T12:28:18.008Z" }, + { url = "https://files.pythonhosted.org/packages/99/4f/badc7bd7f74861b26c10123bba7b9d16f99cd9535ad0128780360713820f/blis-1.3.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:878d4d96d8f2c7a2459024f013f2e4e5f46d708b23437dae970d998e7bff14a0", size = 2814944, upload-time = "2025-11-17T12:28:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/72/a6/f62a3bd814ca19ec7e29ac889fd354adea1217df3183e10217de51e2eb8b/blis-1.3.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f36c0ca84a05ee5d3dbaa38056c4423c1fc29948b17a7923dd2fed8967375d74", size = 11345825, upload-time = "2025-11-17T12:28:21.354Z" }, + { url = "https://files.pythonhosted.org/packages/d4/6c/671af79ee42bc4c968cae35c091ac89e8721c795bfa4639100670dc59139/blis-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e5a662c48cd4aad5dae1a950345df23957524f071315837a4c6feb7d3b288990", size = 3008771, upload-time = "2025-11-17T12:28:23.637Z" }, + { url = "https://files.pythonhosted.org/packages/be/92/7cd7f8490da7c98ee01557f2105885cc597217b0e7fd2eeb9e22cdd4ef23/blis-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9de26fbd72bac900c273b76d46f0b45b77a28eace2e01f6ac6c2239531a413bb", size = 14219213, upload-time = "2025-11-17T12:28:26.143Z" }, + { url = "https://files.pythonhosted.org/packages/0a/de/acae8e9f9a1f4bb393d41c8265898b0f29772e38eac14e9f69d191e2c006/blis-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:9e5fdf4211b1972400f8ff6dafe87cb689c5d84f046b4a76b207c0bd2270faaf", size = 6324695, upload-time = "2025-11-17T12:28:28.401Z" }, ] [[package]] @@ -584,15 +602,11 @@ wheels = [ [[package]] name = "confection" -version = "0.1.5" +version = "1.3.3" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "srsly" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/51/d3/57c6631159a1b48d273b40865c315cf51f89df7a9d1101094ef12e3a37c2/confection-0.1.5.tar.gz", hash = "sha256:8e72dd3ca6bd4f48913cd220f10b8275978e740411654b6e8ca6d7008c590f0e", size = 38924, upload-time = "2024-05-31T16:17:01.559Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/65/efd0fe8a936fc8ca2978cb7b82581fb20d901c6039e746a808f746b7647b/confection-1.3.3.tar.gz", hash = "sha256:f0f6810d567ff73993fe74d218ca5e1ffb6a44fb03f391257fc5d033546cbfaa", size = 54895, upload-time = "2026-03-24T18:45:24.331Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0c/00/3106b1854b45bd0474ced037dfe6b73b90fe68a68968cef47c23de3d43d2/confection-0.1.5-py3-none-any.whl", hash = "sha256:e29d3c3f8eac06b3f77eb9dfb4bf2fc6bcc9622a98ca00a698e3d019c6430b14", size = 35451, upload-time = "2024-05-31T16:16:59.075Z" }, + { url = "https://files.pythonhosted.org/packages/8d/e4/d66708bdf0d92fb4d49b22cdff4b10cec38aca5dcd7e81d909bb55c65cd7/confection-1.3.3-py3-none-any.whl", hash = "sha256:b9fef9ee84b237ef4611ec3eb5797b70e13063e6310ad9f15536373f5e313c82", size = 35902, upload-time = "2026-03-24T18:45:22.664Z" }, ] [[package]] @@ -1612,15 +1626,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1f/9c/06dfcc88d02a6364e8d864c421ddd3736305cb0a6c853f75c302c80fe17c/langchain_protocol-0.0.16-py3-none-any.whl", hash = "sha256:3658c142c5d0fb3a023a4be442ce4c15c6d626aab6135eb79a76dc64ad19c3c3", size = 7037, upload-time = "2026-05-28T23:05:10.163Z" }, ] -[[package]] -name = "langcodes" -version = "3.5.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a9/75/f9edc5d72945019312f359e69ded9f82392a81d49c5051ed3209b100c0d2/langcodes-3.5.1.tar.gz", hash = "sha256:40bff315e01b01d11c2ae3928dd4f5cbd74dd38f9bd912c12b9a3606c143f731", size = 191084, upload-time = "2025-12-02T16:22:01.627Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dd/c1/d10b371bcba7abce05e2b33910e39c33cfa496a53f13640b7b8e10bb4d2b/langcodes-3.5.1-py3-none-any.whl", hash = "sha256:b6a9c25c603804e2d169165091d0cdb23934610524a21d226e4f463e8e958a72", size = 183050, upload-time = "2025-12-02T16:21:59.954Z" }, -] - [[package]] name = "langgraph" version = "1.2.4" @@ -2327,7 +2332,7 @@ requires-dist = [ { name = "presidio-analyzer", marker = "extra == 'pii'", specifier = ">=2.2.0" }, { name = "presidio-anonymizer", marker = "extra == 'pii'", specifier = ">=2.2.0" }, { name = "requests", specifier = ">=2.32.0" }, - { name = "spacy", marker = "extra == 'pii'", specifier = ">=3.7.0" }, + { name = "spacy", marker = "extra == 'pii'", specifier = ">=3.8.0,<4.0.0" }, ] provides-extras = ["pii"] @@ -3409,13 +3414,13 @@ wheels = [ [[package]] name = "spacy" -version = "4.0.0.dev3" +version = "3.8.14" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "catalogue" }, + { name = "confection" }, { name = "cymem" }, { name = "jinja2" }, - { name = "langcodes" }, { name = "murmurhash" }, { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.5.0rc1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, @@ -3424,7 +3429,6 @@ dependencies = [ { name = "pydantic" }, { name = "requests" }, { name = "setuptools" }, - { name = "smart-open" }, { name = "spacy-legacy" }, { name = "spacy-loggers" }, { name = "srsly" }, @@ -3434,30 +3438,39 @@ dependencies = [ { name = "wasabi" }, { name = "weasel" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/89/11f8e53b4517400be6f869286c5f45c7923a5887e18aac3a91b7841a82af/spacy-4.0.0.dev3.tar.gz", hash = "sha256:a528120ef136ad525654386173959167912076c92857ddd99628647e2d856a5e", size = 1287081, upload-time = "2024-04-22T09:56:42.929Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/0d/5ebc3da3e4983968f2da9dcadb6b6722124b0a50f5cee21af14aa96962d2/spacy-4.0.0.dev3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3ec399ae3166e86da282f5e1dce0105dc953488a01398c2016b81cb084ba4153", size = 6857652, upload-time = "2024-04-22T09:56:01.519Z" }, - { url = "https://files.pythonhosted.org/packages/41/e8/791aa1226870a569921879a419374471d05ff1897708a61707eac65a1da3/spacy-4.0.0.dev3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a3f2ec89f6ca7f6d4157f60843f3e9f0cecb6e7afb3bec54723a01e9e80f0f1", size = 6622584, upload-time = "2024-04-22T09:56:04.325Z" }, - { url = "https://files.pythonhosted.org/packages/19/5c/5a29034b10a113ae96abf4029050772aee6e1619ba85ef70e6fa9d39a2b6/spacy-4.0.0.dev3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da6901d541337bc7799849cc6990d8cb52a83a41f8790cdb3915abdfa9dd603", size = 6326685, upload-time = "2024-04-22T09:56:07.217Z" }, - { url = "https://files.pythonhosted.org/packages/23/73/4a8525bba6368e84877d1af39d1dac21767c5ca6c8c9b7a2f1f945835f94/spacy-4.0.0.dev3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68ca7d48dfb3a70bc5650565bf610f14332e88c0e9dad468dca63098e8776f04", size = 6680963, upload-time = "2024-04-22T09:56:09.519Z" }, - { url = "https://files.pythonhosted.org/packages/b5/25/3bdcc988f30e717c00e90e62795d90250b37e32781e1b812decfb13a374d/spacy-4.0.0.dev3-cp311-cp311-win_amd64.whl", hash = "sha256:24457f0b228f0bb7b6a0a1a9c64b19d6a24356527b364494bfc9f2bb2b07d330", size = 12252349, upload-time = "2024-04-22T09:56:11.676Z" }, - { url = "https://files.pythonhosted.org/packages/e3/18/7b5f3a9841cc14bdcee3a2192aa41a8a6b01ec83ad919581c4b74d70fc41/spacy-4.0.0.dev3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e221723b9194edc74f3b70e73d39531525fd7656b8ff6e9e652cefe45f93d119", size = 6417309, upload-time = "2024-04-22T09:56:14.735Z" }, - { url = "https://files.pythonhosted.org/packages/46/cd/b5a272123347ca4c00a0efb564257b666d19e924a1bbdec4035e09f305e6/spacy-4.0.0.dev3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:554f0818313b57ab72a7b801d19c49e52535019dede482a150656ad9b46553ca", size = 6207331, upload-time = "2024-04-22T09:56:17.397Z" }, - { url = "https://files.pythonhosted.org/packages/cb/fb/1120d9e3db87ead4ffa5e51691e4cbb4a3e9e3331b09a6585f39f2dcd13e/spacy-4.0.0.dev3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64e13f03eeef66168f883e285f6a6cbeba9452f2cae2f9cd14de8579ad3985b0", size = 6149396, upload-time = "2024-04-22T09:56:20.299Z" }, - { url = "https://files.pythonhosted.org/packages/31/ba/2053edb0670480530a688dacae72c1a3e76c51482e1ed1fc751861664b3e/spacy-4.0.0.dev3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5bfc7f789a696f9d4fddf405522e871c016a8397ed631f6a5b67d7d564ba4de1", size = 6543611, upload-time = "2024-04-22T09:56:23.315Z" }, - { url = "https://files.pythonhosted.org/packages/70/fd/39947ce507ca2724d2bdd219933144dbc5922a42f6455a57368ffbefdfdb/spacy-4.0.0.dev3-cp312-cp312-win_amd64.whl", hash = "sha256:f9ba698347e4fa46bb1ecdec1b5c942bf41a32868fac38a870907a5b72a3cf8d", size = 11836894, upload-time = "2024-04-22T09:56:25.847Z" }, + { url = "https://files.pythonhosted.org/packages/a9/8c/0ccf32d9a6b4fd8737bba33d599ddb98934399c1d523f825a4beb4bd1495/spacy-3.8.14-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:8c55cb123c3edfba8c252ce6ae27ffb3d7f60a53ba5e108c3534421586c5fdda", size = 6617470, upload-time = "2026-03-29T10:40:25.572Z" }, + { url = "https://files.pythonhosted.org/packages/e1/61/7f7d38e71daac7f91ffd362fb15645b6f9a68ad231e0ed6ff5c1dc6f6930/spacy-3.8.14-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ab6c1ace316338dac334fc93c849994bbd717f9ebf59d2bc4158e978b2f542ee", size = 6441524, upload-time = "2026-03-29T10:40:27.648Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ef/18385aa5aeb9bcb299e8074da162b24e5c8bea5aa4d1dfa3dbafb35e9d1f/spacy-3.8.14-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:7bece0450cd8ab841cfa8527fcc0ce18c4454f28e3b9fca42a450803a067355b", size = 32050591, upload-time = "2026-03-29T10:40:29.704Z" }, + { url = "https://files.pythonhosted.org/packages/fa/67/5c4a65ed2cedc598ad000a2b9f45afc76bb8d17a592cc01082dffa8bbc50/spacy-3.8.14-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cc5e5f2ed121d57d819d247bb59253dc320a58acbd237b85f86c2aa38cab6bd1", size = 32296467, upload-time = "2026-03-29T10:40:32.557Z" }, + { url = "https://files.pythonhosted.org/packages/03/a7/28c118879791b3a7ffa81796d22203daac428e6f75572f1b8da1539e1ac6/spacy-3.8.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3c4e5bc5cdefe39ea139985776a2e8eae05e7ff2bf51ca1bd65247dc45feeb8e", size = 32288404, upload-time = "2026-03-29T10:40:35.583Z" }, + { url = "https://files.pythonhosted.org/packages/72/1c/32aefcea2468782fcdb994f2f96cac93dc74f6589ce01047db42d9a299a2/spacy-3.8.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2c228f4c9ae618173334c17adb748d66b574b6594bc3575233e15cd5ad1cb26b", size = 33113476, upload-time = "2026-03-29T10:40:38.577Z" }, + { url = "https://files.pythonhosted.org/packages/86/32/fc00532eabeace451175dd9b152ddd636e8f6a42248b5d90141f98be2af5/spacy-3.8.14-cp311-cp311-win_amd64.whl", hash = "sha256:6f51d1ce8b1ba30123f6bef6e795c4bc5466608e6e8a015dc828bd21d399aa9c", size = 15359704, upload-time = "2026-03-29T10:40:41.25Z" }, + { url = "https://files.pythonhosted.org/packages/de/31/89ff6722ec91f328dc717932849c6f57249c8a9d429d8670a6c8f70e576b/spacy-3.8.14-cp311-cp311-win_arm64.whl", hash = "sha256:c0c6c9d8771cc3708e309b07310d330fc8443a6bca34f4ff20b0f22751d8faf9", size = 14717168, upload-time = "2026-03-29T10:40:43.916Z" }, + { url = "https://files.pythonhosted.org/packages/0c/78/e4f2ae19a791cae756cd0e801204953eaec4e9ab75a60ad39f671dbb8d5a/spacy-3.8.14-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:726f02c60a2c6b0029167370d22d51731172a053d29c7e2ea6190db6de3ab483", size = 6218335, upload-time = "2026-03-29T10:40:46.298Z" }, + { url = "https://files.pythonhosted.org/packages/06/df/178bbab47fa209c8baf2f1e609cbddc6b18a985200be1ceee22bd5b89beb/spacy-3.8.14-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e3ebe50b93f2d40e8ec3451255528bb622ccb12be39fd140bb87668ce8d1075b", size = 6033860, upload-time = "2026-03-29T10:40:47.861Z" }, + { url = "https://files.pythonhosted.org/packages/ff/e8/048d83b73b28686307bd9a60878a58de7b7b21b562ca4de8b5bd558031e9/spacy-3.8.14-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:daeb64b048f12c059997281aed53eb8776d26416dd313cf17ad6f63124b2b564", size = 32725099, upload-time = "2026-03-29T10:40:50.194Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3f/1799af5f4ccc8eb7500e4a20ca301488134429dba08cda5be68ce6ab2992/spacy-3.8.14-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d45715a24446f23b98ec3f09409a1d4111983d1d64613250ee38c3270e21853", size = 33205838, upload-time = "2026-03-29T10:40:53.029Z" }, + { url = "https://files.pythonhosted.org/packages/78/07/81ab9acd0ec64bfdd7339acfc4cf35f5fb74bbbb0b2be7e64d717c416bac/spacy-3.8.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:1069a8be34940809f8462eb69f09a3f0ce59bf8b9cb82475f2a8e3580f50ece0", size = 32090380, upload-time = "2026-03-29T10:40:56.115Z" }, + { url = "https://files.pythonhosted.org/packages/74/a5/b081b5bd3cedb2634c23eb470b5e24c65c894c57646567f47627291c2b3f/spacy-3.8.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2dfa77aec7fdebac0455d8afd4ce1d92d6f868b03d507ed1976179a63db7b374", size = 32991946, upload-time = "2026-03-29T10:40:58.852Z" }, + { url = "https://files.pythonhosted.org/packages/5f/55/4371413a6dfc1fa837282a365498165f828c2f3fe018dfb35336acc869e0/spacy-3.8.14-cp312-cp312-win_amd64.whl", hash = "sha256:9def18c76a4472b326cb91a195623c9ca38a2b86999ad2df9e00b49ba8c63734", size = 14226946, upload-time = "2026-03-29T10:41:01.63Z" }, + { url = "https://files.pythonhosted.org/packages/f3/5e/12ac876017da6c1e6b72afcc3c8b309996227fd3aa15382cd3311aee21b8/spacy-3.8.14-cp312-cp312-win_arm64.whl", hash = "sha256:d6257133357e4801c9c5d011925af5439b0a015aacf3c16528aa0009982431c7", size = 13628765, upload-time = "2026-03-29T10:41:03.806Z" }, + { url = "https://files.pythonhosted.org/packages/1b/e5/822bbdfa459fee863ef2e9879a34b0ae5db7cd1e3eb76d32c766f19222e9/spacy-3.8.14-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2b4f60fa8b9641a5e93e7a96db0cdd106d05d61756bf1d0ddcd1705ad347909a", size = 6202114, upload-time = "2026-03-29T10:41:06.119Z" }, + { url = "https://files.pythonhosted.org/packages/7e/de/0e512154113e1f341567f2b9341835775e4180c180221e60faedaebb2f65/spacy-3.8.14-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0860c57220c633ccb20468bcd64bfb0d28908990c371a8857951d093a148dc8e", size = 6015458, upload-time = "2026-03-29T10:41:07.79Z" }, + { url = "https://files.pythonhosted.org/packages/0c/4f/29c7e56afc7db07348a9e0efe0243b5eef465d5dc3d56433f164378c3fa6/spacy-3.8.14-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c24620b7dba879c69cebc51ef3b1107d4d4e44a1e0d4baa439372887d00c3fd9", size = 32510659, upload-time = "2026-03-29T10:41:09.88Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ce/cae678f664d5467016819253f5d6e52f8e68a12d8e799b651d73ec2a9a4b/spacy-3.8.14-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9699c1248d115d5825987c287a6f6acd66386ef3ebee7994ee67ba093e932c59", size = 32841057, upload-time = "2026-03-29T10:41:12.585Z" }, + { url = "https://files.pythonhosted.org/packages/04/d4/419868afd449bdd367df005932537eea66c71e97c899ba278f3124933f3c/spacy-3.8.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:042d799e342fdb6bb5b02a4213a95acc9116c40ed3c849bb0a8296fbe648ec22", size = 31763252, upload-time = "2026-03-29T10:41:15.569Z" }, + { url = "https://files.pythonhosted.org/packages/ec/53/df5c1fee45f200b749ba72eeb536fbb2c545fc56230324954263b2f3be00/spacy-3.8.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:69b2264294097336e86832e8663f1ab3a7215621184863c96c082ab17ee11937", size = 32717872, upload-time = "2026-03-29T10:41:18.193Z" }, + { url = "https://files.pythonhosted.org/packages/12/c2/f1882ec2f5cc9c4e73cf2132997a03c397d7ceeb5ee7f7bb878b51a16365/spacy-3.8.14-cp313-cp313-win_amd64.whl", hash = "sha256:4b6d4f20e291a7c70e37de2f246622b44a0ce82efaa710c9801c6bd599e75177", size = 14220335, upload-time = "2026-03-29T10:41:20.89Z" }, ] [[package]] name = "spacy-legacy" -version = "4.0.0.dev1" +version = "3.0.12" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "tqdm" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/95/c7/1f50e31b0db55eaa83fbfeaa219f68fad2fdc2b7262ae318d253e550630c/spacy-legacy-4.0.0.dev1.tar.gz", hash = "sha256:c3d778162cdf84f10cf681101ddc51bba322c4edf0ab49d09a09182a09603dce", size = 24285, upload-time = "2024-01-25T15:03:46.967Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/79/91f9d7cc8db5642acad830dcc4b49ba65a7790152832c4eceb305e46d681/spacy-legacy-3.0.12.tar.gz", hash = "sha256:b37d6e0c9b6e1d7ca1cf5bc7152ab64a4c4671f59c85adaf7a3fcb870357a774", size = 23806, upload-time = "2023-01-23T09:04:15.104Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/92/5e779b0334cae2231440e25a5f004315e69b75b37def4d2695a2407b081a/spacy_legacy-4.0.0.dev1-py2.py3-none-any.whl", hash = "sha256:1be48d206439a5288c14d90b2c357e3a06233ddb40e1d13c41286d2cb91ce601", size = 30313, upload-time = "2024-01-25T15:03:45.71Z" }, + { url = "https://files.pythonhosted.org/packages/c3/55/12e842c70ff8828e34e543a2c7176dac4da006ca6901c9e8b43efab8bc6b/spacy_legacy-3.0.12-py2.py3-none-any.whl", hash = "sha256:476e3bd0d05f8c339ed60f40986c07387c0a71479245d6d0f4298dbd52cda55f", size = 29971, upload-time = "2023-01-23T09:04:13.45Z" }, ] [[package]] @@ -3531,7 +3544,7 @@ wheels = [ [[package]] name = "thinc" -version = "9.0.0" +version = "8.3.13" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "blis" }, @@ -3548,18 +3561,40 @@ dependencies = [ { name = "srsly" }, { name = "wasabi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/20/7e/7706f5ca67e0f8f965985e59e2a359b9f5919621eb4e2cb73524ed4676f9/thinc-9.0.0.tar.gz", hash = "sha256:bbe7fc3dca4f2b89a2c1429e8bf1c9dc20b0faad13f0ac29053bbb3c2531ad57", size = 191333, upload-time = "2024-04-19T11:40:46.027Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/7b/5885c95c391d6c905630e4240f179faea8ae07774ea55d43d8672c254c34/thinc-9.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:68edaf4d75e6c4db8ba71c672d4735275ee9fab0784c2732f5c2517943f292e3", size = 874482, upload-time = "2024-04-19T11:40:05.35Z" }, - { url = "https://files.pythonhosted.org/packages/ac/9b/09ae5fc3f31d1f6facf191277254e3a42270643d803d1968db2131a12c67/thinc-9.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b1356e72055155b85644a436dd4967f209cc61faf0a4c0a515f867b107b01a2c", size = 789772, upload-time = "2024-04-19T11:40:08.003Z" }, - { url = "https://files.pythonhosted.org/packages/99/00/fd907206984e3067e0c82c2169921e99a825fb2877b388dc12ec2482a657/thinc-9.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5bd08923a591e7a0776518f2ff641d57af09cf28f0615f6d9f09932c17b62342", size = 786364, upload-time = "2024-04-19T11:40:10.626Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b6/ec656d330c505d3aad89c451fe9d29bd89d7b95fb8a4c8ae86e938239b6c/thinc-9.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e7f39a5c36aed1266aa883b48052cc520c7fd3296a3249bfd9a4bfc536f8fdb", size = 831750, upload-time = "2024-04-19T11:40:12.558Z" }, - { url = "https://files.pythonhosted.org/packages/28/aa/7b2f22eeff342eb30cd65e3e93e22cd69d56d53ca7f4505f0b4befa0f3c5/thinc-9.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:f37359839b4de41166d5c4e7b1535273484420b7708f94122822c52edebc1805", size = 1275633, upload-time = "2024-04-19T11:40:14.553Z" }, - { url = "https://files.pythonhosted.org/packages/f5/e3/f7649487a70b7a795c437c0e7efca6b2ea3f8aeeaed92c714a4fd2906f7b/thinc-9.0.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:99d6c1044a3c51cda7e913e56d19db455a690b23b0700e5b8e1c8f6d700ede6f", size = 834768, upload-time = "2024-04-19T11:40:16.642Z" }, - { url = "https://files.pythonhosted.org/packages/b7/08/5f0d84ae552939bc0fecf4b343c7722cb682a3f54ce25c7409aa3228205a/thinc-9.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2cb96d1267dd1bae0f13015c09ee187ddb67fbef04243bd841c0a4bf286a7c74", size = 766119, upload-time = "2024-04-19T11:40:19.641Z" }, - { url = "https://files.pythonhosted.org/packages/1f/44/f9c9713fa66b17fd22aa11ee82bfba885d684804abafd7057989b539fef6/thinc-9.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c020be88bb2c342cc10083ca2ec0db10ca9c730af540f65ae4061bf7a202950d", size = 740170, upload-time = "2024-04-19T11:40:22.601Z" }, - { url = "https://files.pythonhosted.org/packages/3d/98/df43d0625b1ed7923b6f079dd55657665c61bb229ef5f739490f483b1a45/thinc-9.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1e18f9e2dc7282d8885a9bb2269b6a3dc1b0036e4752aaf35ccdb6017baf83cc", size = 781873, upload-time = "2024-04-19T11:40:25.119Z" }, - { url = "https://files.pythonhosted.org/packages/bc/fe/31364ff4d9cc5fc9c308b365da2771e0da1cadb287c1781b2842326e17d7/thinc-9.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:67848dd4cbb2c269d044a41af3a3e3bd28aa068a0b679340c8753d5dac0c79fb", size = 1253617, upload-time = "2024-04-19T11:40:28.084Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/13/46/76df95f2c327f9a9cef30c1523bf285627897097163584dcf5f77b2ebce2/thinc-8.3.13.tar.gz", hash = "sha256:68e658549fc1eb3ff92aed5147fcbb9c15d6e9cc0e623b4d0998d16522ffb4f9", size = 194640, upload-time = "2026-03-23T07:22:36.41Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/72/ca06842a007e8c794e8c59462f242cdfd6167d7cc9d0155ad004b194b015/thinc-8.3.13-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4565102638038a01a2193c7f5d41ccbd6233fbdcb1f1b184322a06add4f51f18", size = 844359, upload-time = "2026-03-23T07:21:43.017Z" }, + { url = "https://files.pythonhosted.org/packages/48/44/e6aef092f478d263f72eb3933b55a6f37ba97c6a0ea0a61d13fbf9bf0c19/thinc-8.3.13-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:859fbd9d9b16af5278da23589b4afbe2ab6b0dd615df4d3229b7c4e67cd3107e", size = 812089, upload-time = "2026-03-23T07:21:44.618Z" }, + { url = "https://files.pythonhosted.org/packages/ff/8a/9ce0424d456cd3580cc3a855b23a7ff86b81d5299fceb496a2f56f06c1c0/thinc-8.3.13-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a518d5c761a0f2341e530e867de133dc3ed814558365b2a68ec53b89c482a43f", size = 4101388, upload-time = "2026-03-23T07:21:46.135Z" }, + { url = "https://files.pythonhosted.org/packages/ad/51/ec91c0434bd9a1096ab874bbd6dc110c5089d7fc513137e6af59bd051eec/thinc-8.3.13-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:81337dfbee37f58f36c0c70f9a819dce1b32cdc13d959181e10de079621f6ac6", size = 4131972, upload-time = "2026-03-23T07:21:48.403Z" }, + { url = "https://files.pythonhosted.org/packages/ff/67/e30dea753c90cff5cb9e5feb34948fdb89a6774b84d849585b49e16a730e/thinc-8.3.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:fbc0ee16edd260c6a4a9e365ff36d0a682c9e7ca6d7b985682659ef2e3e73826", size = 5101283, upload-time = "2026-03-23T07:21:49.991Z" }, + { url = "https://files.pythonhosted.org/packages/00/e9/b7544eddababa16e548b26a96fff29eeb307ce938df5fa4af9371fe8ed5d/thinc-8.3.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0355c37e40d1a9fc2a1b8e9c2e294d8586f6baa97bcac6b9002f2dddb4b82ae9", size = 5264488, upload-time = "2026-03-23T07:21:51.747Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a9/49391a40d703efc0f7a451310373261835f71fd3e6e2e8cfc08ee02f78ad/thinc-8.3.13-cp311-cp311-win_amd64.whl", hash = "sha256:0a0fa13dcfe4b319c3a396432c1dbff30d3de37dbbdee559e76600ee2b9486df", size = 1795058, upload-time = "2026-03-23T07:21:53.424Z" }, + { url = "https://files.pythonhosted.org/packages/c1/31/fd5348d44beda12a3ee415cbba9ed4fd0b17ce65db1d473c38a29a8d6153/thinc-8.3.13-cp311-cp311-win_arm64.whl", hash = "sha256:cd8a2b714c061969eee65802965167a6ada1fe708d82fe176d98dcb95ebe182a", size = 1721215, upload-time = "2026-03-23T07:21:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/3e/af/f7c1ebfe92eb5d27d7f2f3da67a11e2eb57bc30ab1553279af6dc65b65a8/thinc-8.3.13-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:77a41f66285321d20aaedaea1e87d7cd48dca6d2427bed1867ec7cba7109fc8d", size = 821097, upload-time = "2026-03-23T07:21:56.698Z" }, + { url = "https://files.pythonhosted.org/packages/45/8f/69d7338575d98df85d0b54c0f5fc277dba72587fe9ab846ecdd12a998bcb/thinc-8.3.13-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3710d318b4e5460cf366a6f7b5ddbefb5d39dbd4cfa408222750fdc6c27c4411", size = 791932, upload-time = "2026-03-23T07:21:58.38Z" }, + { url = "https://files.pythonhosted.org/packages/4b/a5/21d010c81e81e1589e5ccb4950e521804d13726e541e87f644c51815673b/thinc-8.3.13-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a08c87143a6d20177652dca1ec0dc815d88216d8fc62594a57e8bc45bf5ed49", size = 3854219, upload-time = "2026-03-23T07:21:59.819Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ff/6914bf370bd1d604d89e6dfb46b97d10cd9b00d42ff8c036283e92314a8c/thinc-8.3.13-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4b5ec9ff313819e7d8667794a3559463fa89ff45aaa73e3fd8d6273b1e0d7a7f", size = 3903307, upload-time = "2026-03-23T07:22:01.652Z" }, + { url = "https://files.pythonhosted.org/packages/f3/3d/5572b47fa155fb3388c071515b74024fa17a6efd1df9406da378f0aa84ef/thinc-8.3.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5c9a48f2bc1e04f138240ed5f9b815a9141a5de26accd0f08fa0137fcefed258", size = 4836882, upload-time = "2026-03-23T07:22:03.565Z" }, + { url = "https://files.pythonhosted.org/packages/f0/f0/a8d77c7bac089697c6df302cc3c936a1ab36a4720deae889e6f1dbcbd0eb/thinc-8.3.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:79a29a44d76bd02f5ac0624268c6e42b3576ae472c791a8ae9c2d813ae789b59", size = 5033398, upload-time = "2026-03-23T07:22:05.045Z" }, + { url = "https://files.pythonhosted.org/packages/21/82/5651bb1f904d04220fc7670035ada921bf0638e2cff6444d67c12887a968/thinc-8.3.13-cp312-cp312-win_amd64.whl", hash = "sha256:ed1dc709ac4f2f03b710457889e4e02f05de51bc8456980c241d0b28798bc7cb", size = 1721248, upload-time = "2026-03-23T07:22:06.749Z" }, + { url = "https://files.pythonhosted.org/packages/94/8d/683703de021ffbe46833d722b70f49ffbbca8e5bd6876256977555d92d7d/thinc-8.3.13-cp312-cp312-win_arm64.whl", hash = "sha256:c6a049703a6011c8fe26ee41af7e70272145594140d82f79bb23de619c6a6525", size = 1645777, upload-time = "2026-03-23T07:22:08.104Z" }, + { url = "https://files.pythonhosted.org/packages/af/b9/7b46942176df459d1804a9e77b0976f7c56f3abf3ec7485d0e5f836a0382/thinc-8.3.13-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c2811dfd8d46d8b5d3b39051b23e64006b2994a5143b1978b436938018792af8", size = 817337, upload-time = "2026-03-23T07:22:09.538Z" }, + { url = "https://files.pythonhosted.org/packages/a7/79/53085a72cd8f4fc4e6e313d05ea5aa98e870684f4a0fb318a9875fc0a964/thinc-8.3.13-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5593e6300cb1ebe0c0e546e9c9fb49e7c2627a0aa688795cd4f995a8b820d2ec", size = 788120, upload-time = "2026-03-23T07:22:11.215Z" }, + { url = "https://files.pythonhosted.org/packages/9e/3e/d61b462b16da95ac6885f95bb395e672040ee594833e571a6edcffd234f5/thinc-8.3.13-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f697174d3fb474966ce50b430bbafa101a6d2f7ffb559dac4b5c59389ef72d22", size = 3844666, upload-time = "2026-03-23T07:22:12.67Z" }, + { url = "https://files.pythonhosted.org/packages/78/4c/898cc654bb123734c71ec5a425c02ca34439517d01ce1c95a6563295580e/thinc-8.3.13-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e9c7c5c104737b414c8c4ec578e67d78b6c859afe25cbc0684402e721415bd7f", size = 3890658, upload-time = "2026-03-23T07:22:14.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/56/1abdbf0a4ad628e8a05d6516fe0745969649d805367a3dccad8ee872981b/thinc-8.3.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7a99d0e242d1ccd23f9ae6bea7cd502f8626efa65c156b91d84581d0356696c3", size = 4819933, upload-time = "2026-03-23T07:22:16.85Z" }, + { url = "https://files.pythonhosted.org/packages/f1/22/b84dbdc6be5055bbdb2a7352e2c393f67e8593c137f1b83c82bf1e062b6e/thinc-8.3.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e676edd21a747afbe3e6b9f3fca8b962e36d146ded03b070cb0c28e2dfbe9499", size = 5018099, upload-time = "2026-03-23T07:22:18.356Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a8/763cd7ba949334c9d2cddc92dadb68b344cb9546dc01b8d4a733dcaa16c1/thinc-8.3.13-cp313-cp313-win_amd64.whl", hash = "sha256:8ad40307f20e83f77af28ff5c6be0b86af7a8b251d1231c545508d2763157d8f", size = 1720309, upload-time = "2026-03-23T07:22:19.81Z" }, + { url = "https://files.pythonhosted.org/packages/f5/15/a11f7bb3cbc97dfecf32a90552f5a8f8a5c99316a99c6c17bdabf5baf256/thinc-8.3.13-cp313-cp313-win_arm64.whl", hash = "sha256:723949cab11d1925c15447928513a718276316cec6e0de28337cca0a62be0521", size = 1644606, upload-time = "2026-03-23T07:22:21.339Z" }, + { url = "https://files.pythonhosted.org/packages/80/40/f4937d113912c6d669ffe982356ab29dcb6c7fe3be926a15981dbbb6a91c/thinc-8.3.13-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7badb0be4825535e6362c19e8a41872b65409e9da46d3453a391b843a0720865", size = 817024, upload-time = "2026-03-23T07:22:23.005Z" }, + { url = "https://files.pythonhosted.org/packages/d2/00/4d4ed1a11ba2920b85a03a0683b16d97dc5beb2e78078dbf0e13e43bcea7/thinc-8.3.13-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:565300b7e13de799e5abff00d445f537e9256cf7da4dcb0d0f005fc16748a29e", size = 792096, upload-time = "2026-03-23T07:22:24.349Z" }, + { url = "https://files.pythonhosted.org/packages/44/5d/dc33d6932be8721af2ef76b4a3a6e8020648630eabae61fb916d2a861d1d/thinc-8.3.13-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c17cef1900a1aba7e1487493d16b8aa0a8633116f1b2a51c6649a4000697f17b", size = 3842215, upload-time = "2026-03-23T07:22:25.836Z" }, + { url = "https://files.pythonhosted.org/packages/af/bc/a6d37d8dadc2c5b524f51192413481160c42c9dd6105e8d5551531623225/thinc-8.3.13-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f4f26d1eec9b2a6a8f2e0298a5515d13eb06d70730d0d9e1040bb329e12bf3fb", size = 3849253, upload-time = "2026-03-23T07:22:27.845Z" }, + { url = "https://files.pythonhosted.org/packages/7a/59/ce9c7067f1dfe5985875927de9cf7a79f9dae3e69487fd650dfba558029d/thinc-8.3.13-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:a61a31fd0ce3c2771cf4901ba6df70e774ffe32febf1024c5b43d63575cd58fe", size = 4831163, upload-time = "2026-03-23T07:22:29.395Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a8/f57819347fc4d8bef2204d15fcbb9d7dff2d6cdd5f83d5ed91456ddacc55/thinc-8.3.13-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba8119daf84a12259ae4d251d36426417bafa0b34108890b4b7e2b50966bd990", size = 4986051, upload-time = "2026-03-23T07:22:30.933Z" }, + { url = "https://files.pythonhosted.org/packages/05/ef/a82214bb7c7c1e2d92b69e1a7654be90cfab180082c6108e45a98af2422c/thinc-8.3.13-cp314-cp314-win_amd64.whl", hash = "sha256:433e3826e018da489f1a8068e6de677f6eff3cc93991a599d90f12cd1bc26cdc", size = 1740382, upload-time = "2026-03-23T07:22:32.869Z" }, + { url = "https://files.pythonhosted.org/packages/9f/ef/1648fda54e9689058335ff54f650a7a314db2a42e21af1b83949b2dc748e/thinc-8.3.13-cp314-cp314-win_arm64.whl", hash = "sha256:11754fada9ad5ba2e02d5f3f234f940e24015b82333db58372f4a6aedad9b43f", size = 1667687, upload-time = "2026-03-23T07:22:34.967Z" }, ] [[package]] @@ -3767,22 +3802,22 @@ wheels = [ [[package]] name = "weasel" -version = "0.3.4" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cloudpathlib" }, { name = "confection" }, + { name = "httpx" }, { name = "packaging" }, { name = "pydantic" }, - { name = "requests" }, { name = "smart-open" }, { name = "srsly" }, { name = "typer" }, { name = "wasabi" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a3/15/5be22cbde3fa1704da51951bb0b3dcdf33a7930449e156440e518df11391/weasel-0.3.4.tar.gz", hash = "sha256:eb16f92dc9f1a3ffa89c165e3a9acd28018ebb656e0da4da02c0d7d8ae3f6178", size = 38252, upload-time = "2023-11-06T16:14:16.487Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/e5/e272bb9a045105a1fdf4b798d8086f5932a178f4d738f17a74f5c9e0ae9a/weasel-1.0.0.tar.gz", hash = "sha256:7b129b44c90cc543b760532974ca1e4eb30dad2aa2026f57bdce66354ae610fc", size = 38682, upload-time = "2026-03-20T08:10:25.266Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/e5/b63b8e255d89ba4155972990d42523251d4d1368c4906c646597f63870e2/weasel-0.3.4-py3-none-any.whl", hash = "sha256:ee48a944f051d007201c2ea1661d0c41035028c5d5a8bcb29a0b10f1100206ae", size = 50108, upload-time = "2023-11-06T16:14:15.198Z" }, + { url = "https://files.pythonhosted.org/packages/0a/07/57ebf7a6798b016c064bd0ca81b4c6a99daa4dc377b898bc7b41eb6b5af0/weasel-1.0.0-py3-none-any.whl", hash = "sha256:89518acee027f49d743126c3502d35e6dd14f5768be5c37c9af47c171b6005cc", size = 50713, upload-time = "2026-03-20T08:10:23.637Z" }, ] [[package]] diff --git a/veil/pii.py b/veil/pii.py index 8d36897..381363b 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -7,27 +7,33 @@ sealed to the TEE, so the raw values never leave the machine at all — useful for compliance, data-residency, and keeping PII out of any model-side logging. -Two tiers, layered: +Detection is delegated to **Microsoft Presidio** rather than handrolled patterns, +so the recognizers are community-maintained, versioned, and carry confidence +scores + context-word boosting. Note that Presidio detects the *structured* +types (email, SSN, cards, IBANs, bank numbers) with its own regex/checksum +recognizers — spaCy's statistical model is only used for free-form +*addresses/locations* (and, optionally, dates). Because of that, this feature +requires the optional extra: -* **Regex (always on when enabled, zero extra deps)** — structured PII that has a - recognizable shape: email, US SSN, and bank numbers (credit cards validated by - Luhn, IBANs by mod-97 checksum, plus routing/account numbers when labelled). - Dates of birth are caught by context ("DOB:", "born on …"). -* **NER (optional, ``pip install 'opengradient-veil[pii]'``)** — Microsoft - Presidio + a spaCy model adds *addresses/locations*, which are free-form prose - that regex fundamentally cannot see. Optionally redacts *all* dates too. + pip install 'opengradient-veil[pii]' + python -m spacy download en_core_web_sm + +What gets redacted (mapped to the tags below): + +* email, US SSN, credit cards (Luhn), IBANs (mod-97), US bank/routing numbers +* addresses / locations (spaCy NER) +* dates of birth — by default only *birth-date-cued* dates (``DOB:``, ``born on …``) + via a small custom recognizer; with ``redact_all_dates`` every date is redacted. This is risk-reduction, not a guarantee: NER misses a fraction of addresses every run, and bare (unlabelled) account numbers can slip through. Redaction is -irreversible — there is deliberately no de-anonymization step, so the TEE's -signed ``output_hash`` covers exactly what it ran and nothing is restored locally. +irreversible — there is deliberately no de-anonymization step, so the TEE's signed +``output_hash`` covers exactly what it ran and nothing is restored locally. """ from __future__ import annotations import logging -import re -from collections.abc import Callable logger = logging.getLogger(__name__) @@ -39,131 +45,53 @@ DOB_TAG = "[REDACTED_DOB]" ADDRESS_TAG = "[REDACTED_ADDRESS]" - -class PiiSetupError(Exception): - """The NER engine was requested but its model/runtime isn't installed.""" - - -# --- regex tier ------------------------------------------------------------ - -_EMAIL_RE = re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b") - -# Dashed/spaced SSN, e.g. 123-45-6789 or 123 45 6789. Bare 9-digit runs are too -# ambiguous (order IDs etc.) to redact unconditionally — those are handled by the -# context detector below. -_SSN_RE = re.compile(r"\b\d{3}[-\s]\d{2}[-\s]\d{4}\b") - -# Candidate card number: 13–19 digits, optionally split by spaces/hyphens. Only -# redacted if it passes the Luhn check, which kills almost all false positives. -_CC_RE = re.compile(r"\b(?:\d[ -]?){13,19}\b") - -# IBAN shape, allowing the common space-grouped form (GB82 WEST 1234 …); -# validated by mod-97 (spaces stripped) before redaction. -_IBAN_RE = re.compile(r"\b[A-Z]{2}\d{2}(?:[ ]?[A-Z0-9]){11,30}\b") - -# Labelled SSN: "SSN: 123456789" / "social security number 123-45-6789". -_SSN_CTX_RE = re.compile( - r"(?i)(\b(?:ssn|social security(?:\s+number)?)\b\D{0,8})((?:\d[ -]?){8,9}\d)" -) - -# Labelled bank/routing/account numbers, including bare domestic account numbers -# that have no fixed shape and are only recognizable from their label. -_BANK_CTX_RE = re.compile( - r"(?i)(\b(?:account|acct|a/c|routing|aba|iban|swift|bic|sort\s?code|bank)\b" - r"(?:\s*(?:number|no\.?|#))?\s*[:#]?\s*)((?:\d[ -]?){5,17}\d)" -) - -# A date in common numeric or "Month DD, YYYY" forms — only redacted when it -# follows a birth-date cue, so ordinary dates in the prompt are left intact. -_DATE = ( +# spaCy model backing the NER (address/date) detection. The small English model is +# the lightweight choice; recall climbs with the medium/large models at a large +# size cost. Override is intentionally not exposed — keep the install predictable. +_SPACY_MODEL = "en_core_web_sm" + +# The one pattern we own: a birth-date recognizer for an entity Presidio doesn't +# ship. Presidio compiles patterns with the `regex` module, which supports +# variable-width lookbehind — so we anchor on a birth-date *cue* ("DOB:", "born +# on", "date of birth") in the lookbehind and capture only the date itself. This +# gives true per-occurrence gating (an unrelated invoice date is left alone) +# without leaning on Presidio's context enhancer, which boosts dates text-wide. +_DOB_REGEX = ( + r"(?<=\b(?:d\.?o\.?b\.?|date\s+of\s+birth|born(?:\s+on)?|birth\s?date|birthday)\b\W{0,6})" r"(?:\d{4}-\d{2}-\d{2}" r"|\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}" r"|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?" r"|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" - r"\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4})" -) -_DOB_CTX_RE = re.compile( - r"(?i)(\b(?:d\.?o\.?b\.?|date\s+of\s+birth|born(?:\s+on)?|birth\s?date|birthday)\b\W{0,6})(" - + _DATE - + r")" + r"\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4})\b" ) +# Presidio's built-in structured recognizers we rely on, each mapped to a tag. +_STRUCTURED_OPERATORS = { + "EMAIL_ADDRESS": EMAIL_TAG, + "US_SSN": SSN_TAG, + "CREDIT_CARD": BANK_TAG, + "IBAN_CODE": BANK_TAG, + "US_BANK_NUMBER": BANK_TAG, + "LOCATION": ADDRESS_TAG, +} -def _luhn_ok(digits: str) -> bool: - total = 0 - for i, ch in enumerate(reversed(digits)): - d = int(ch) - if i % 2 == 1: - d *= 2 - if d > 9: - d -= 9 - total += d - return total % 10 == 0 - - -def _iban_ok(candidate: str) -> bool: - s = candidate.upper().replace(" ", "") - rearranged = s[4:] + s[:4] - # Letters → 10..35; the whole thing mod 97 must equal 1 for a valid IBAN. - converted = "".join(str(int(c, 36)) if c.isalpha() else c for c in rearranged) - try: - return int(converted) % 97 == 1 - except ValueError: - return False - - -def _redact_cc(m: re.Match[str]) -> str: - digits = re.sub(r"\D", "", m.group(0)) - if 13 <= len(digits) <= 19 and _luhn_ok(digits): - return BANK_TAG - return m.group(0) - - -def _redact_iban(m: re.Match[str]) -> str: - return BANK_TAG if _iban_ok(m.group(0)) else m.group(0) - - -def _keep_label(tag: str) -> Callable[[re.Match[str]], str]: - """Replace only the value group, preserving the leading label (group 1).""" - def repl(m: re.Match[str]) -> str: - return m.group(1) + tag - - return repl - - -# Order matters: validate-and-redact the strongly-shaped values first, then the -# label-anchored ones, so a labelled card number is still caught by the Luhn pass. -_Replacement = str | Callable[[re.Match[str]], str] -_REGEX_PASSES: list[tuple[re.Pattern[str], _Replacement]] = [ - (_EMAIL_RE, EMAIL_TAG), - (_IBAN_RE, _redact_iban), - (_CC_RE, _redact_cc), - (_SSN_RE, SSN_TAG), - (_SSN_CTX_RE, _keep_label(SSN_TAG)), - (_BANK_CTX_RE, _keep_label(BANK_TAG)), - (_DOB_CTX_RE, _keep_label(DOB_TAG)), -] - - -def _regex_scrub(text: str) -> str: - for pattern, repl in _REGEX_PASSES: - text = pattern.sub(repl, text) - return text - - -# --- NER tier (optional, Presidio) ----------------------------------------- +class PiiSetupError(Exception): + """PII scrubbing was requested but Presidio / the spaCy model isn't installed.""" -def _load_ner(redact_all_dates: bool) -> Callable[[str], str] | None: - """Build a Presidio-backed scrubber for addresses (and optionally all dates). +def _build_engine(redact_all_dates: bool): # noqa: ANN202 — Presidio types are untyped + """Construct the Presidio analyzer/anonymizer and the per-entity plan. - Returns ``None`` if the optional ``[pii]`` extra isn't installed. Raises - :class:`PiiSetupError` if Presidio is present but its spaCy model isn't, so - the operator gets an actionable message instead of a cryptic stack trace. + Raises :class:`PiiSetupError` with an actionable message if the optional + dependency or its spaCy model is missing. """ try: - from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found] + from presidio_analyzer import ( # type: ignore[import-not-found] + AnalyzerEngine, + Pattern, + PatternRecognizer, + ) from presidio_analyzer.nlp_engine import ( # type: ignore[import-not-found] NlpEngineProvider, ) @@ -171,71 +99,72 @@ def _load_ner(redact_all_dates: bool) -> Callable[[str], str] | None: from presidio_anonymizer.entities import ( # type: ignore[import-not-found] OperatorConfig, ) - except ImportError: - return None + except ImportError as exc: + raise PiiSetupError( + "PII scrubbing needs the optional extra. Install it with: " + "pip install 'opengradient-veil[pii]' && python -m spacy download " + f"{_SPACY_MODEL}" + ) from exc - model = "en_core_web_sm" provider = NlpEngineProvider( nlp_configuration={ "nlp_engine_name": "spacy", - "models": [{"lang_code": "en", "model_name": model}], + "models": [{"lang_code": "en", "model_name": _SPACY_MODEL}], } ) try: nlp_engine = provider.create_engine() except Exception as exc: # noqa: BLE001 — spaCy model not downloaded yet raise PiiSetupError( - f"the spaCy model '{model}' is not installed; run: " - f"python -m spacy download {model}" + f"the spaCy model '{_SPACY_MODEL}' is not installed; run: " + f"python -m spacy download {_SPACY_MODEL}" ) from exc analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=["en"]) anonymizer = AnonymizerEngine() - entities = ["LOCATION"] - operators = {"LOCATION": OperatorConfig("replace", {"new_value": ADDRESS_TAG})} + operators = { + entity: OperatorConfig("replace", {"new_value": tag}) + for entity, tag in _STRUCTURED_OPERATORS.items() + } + structured_entities = list(_STRUCTURED_OPERATORS) + if redact_all_dates: - entities.append("DATE_TIME") + # Aggressive: every date (spaCy DATE_TIME) is treated as a DOB. + structured_entities.append("DATE_TIME") operators["DATE_TIME"] = OperatorConfig("replace", {"new_value": DOB_TAG}) + else: + # Context-only: our cue-anchored recognizer matches just birth-dates. + analyzer.registry.add_recognizer( + PatternRecognizer( + supported_entity="DOB", + patterns=[Pattern(name="dob", regex=_DOB_REGEX, score=0.85)], + ) + ) + structured_entities.append("DOB") + operators["DOB"] = OperatorConfig("replace", {"new_value": DOB_TAG}) - def run(text: str) -> str: - results = analyzer.analyze(text=text, entities=entities, language="en") - if not results: - return text - return anonymizer.anonymize(text=text, analyzer_results=results, operators=operators).text - - return run - + def analyze(text: str): # noqa: ANN202 + return analyzer.analyze(text=text, entities=structured_entities, language="en") -# --- public surface --------------------------------------------------------- + return analyze, anonymizer, operators class Redactor: - """Applies the regex tier (always) and the NER tier (when available).""" + """Presidio-backed PII redactor. Built eagerly so misconfiguration fails fast.""" def __init__(self, *, redact_all_dates: bool = False): - self._ner: Callable[[str], str] | None = None - try: - self._ner = _load_ner(redact_all_dates) - except PiiSetupError as exc: - logger.warning( - "PII: %s — continuing with regex-only redaction (no address coverage)", exc - ) - if self._ner is None: - logger.warning( - "PII: address/NER redaction unavailable — install the extra for it: " - "pip install 'opengradient-veil[pii]'" - ) + self._analyze, self._anonymizer, self._operators = _build_engine(redact_all_dates) def scrub_text(self, text: str) -> str: if not text: return text - # Regex first so structured values become tags; the NER pass then runs on - # the partially-redacted text purely for free-form addresses/dates. - text = _regex_scrub(text) - if self._ner is not None: - text = self._ner(text) - return text + results = self._analyze(text) + if not results: + return text + return self._anonymizer.anonymize( + text=text, analyzer_results=results, operators=self._operators + ).text def scrub_request(self, body: dict) -> dict: """Return a copy of an OpenAI chat-completions body with message text redacted. @@ -258,7 +187,9 @@ def scrub_request(self, body: dict) -> dict: elif isinstance(content, list): parts = [ {**p, "text": self.scrub_text(p["text"])} - if isinstance(p, dict) and p.get("type") == "text" and isinstance(p.get("text"), str) + if isinstance(p, dict) + and p.get("type") == "text" + and isinstance(p.get("text"), str) else p for p in content ] @@ -269,7 +200,12 @@ def scrub_request(self, body: dict) -> dict: def build_redactor(*, enabled: bool, redact_all_dates: bool = False) -> Redactor | None: - """Construct a :class:`Redactor` when PII scrubbing is enabled, else ``None``.""" + """Construct a :class:`Redactor` when PII scrubbing is enabled, else ``None``. + + Raises :class:`PiiSetupError` if enabled but the ``[pii]`` extra / spaCy model + isn't installed, so the operator gets a clear message at startup rather than a + silent no-op. + """ if not enabled: return None return Redactor(redact_all_dates=redact_all_dates) diff --git a/veil/server.py b/veil/server.py index 34ce890..59e4b6b 100644 --- a/veil/server.py +++ b/veil/server.py @@ -21,6 +21,7 @@ from veil.config import ServerConfig from veil.gateway import Gateway, GatewayError +from veil.pii import PiiSetupError from veil.session import AuthError, Session logger = logging.getLogger(__name__) @@ -140,7 +141,11 @@ def _error(status: int, message: str): def serve(config: ServerConfig) -> None: """Load the session, resolve a TEE, and run the local server (blocking).""" session = Session.load() - gateway = Gateway(session, config) + try: + gateway = Gateway(session, config) + except PiiSetupError as exc: + # PII scrubbing was requested but the optional extra/model isn't present. + raise SystemExit(f"PII scrubbing is enabled but unavailable: {exc}") # Resolve a TEE eagerly so misconfiguration fails fast and /health is useful. try: gateway._get_client() # noqa: SLF001 — intentional eager warm-up From ac2f6f6e49864f590f88cf3a68733102e90ae9d5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 22:00:35 +0000 Subject: [PATCH 03/25] PII redaction: drop date/DOB scrubbing and the --pii-all-dates option Dates are too entangled with legitimate prompt content to redact without mangling it, so remove all date handling: the custom DOB recognizer, the DATE_TIME path, the --pii-all-dates flag, the OG_VEIL_PII_REDACT_ALL_DATES env var, and the DOB_TAG. PII scrubbing now covers email, US SSN, bank numbers, and addresses only. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- README.md | 6 ++--- tests/test_pii.py | 17 +++---------- veil/cli.py | 13 +--------- veil/config.py | 9 ++----- veil/gateway.py | 4 +-- veil/pii.py | 65 +++++++++++------------------------------------ 6 files changed, 26 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 00d75fc..75176b6 100644 --- a/README.md +++ b/README.md @@ -174,7 +174,6 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). | `OG_VEIL_EXPECTED_PCR_HASH` | `--expected-pcr` | — | Refuse any TEE whose `pcrHash` differs. | | `OG_VEIL_APP_URL` | `--app-url` | `https://chat.opengradient.ai` | Chat app origin for login. | | `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Redact high-impact PII from prompts locally before they leave the machine. | -| `OG_VEIL_PII_REDACT_ALL_DATES` | `--pii-all-dates` | off | With scrubbing on, redact *every* date as a DOB (aggressive) instead of only birth-date-cued ones. | ### Local PII redaction (opt-in) @@ -207,8 +206,9 @@ What gets redacted (each replaced with a `[REDACTED_*]` tag): - **email, US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and US bank/routing numbers, via Presidio's regex/checksum recognizers. - **addresses / locations** — free-form prose, via the spaCy NER model. -- **dates of birth** — by default only *birth-date-cued* dates (`DOB:`, - `born on …`); pass `--pii-all-dates` to redact every date instead. + +Dates are deliberately left alone — too entangled with legitimate prompt content +to redact without mangling it. If `--pii-scrub` is set but the extra/model isn't installed, the server refuses to start with an actionable message rather than silently sending PII. diff --git a/tests/test_pii.py b/tests/test_pii.py index 6cfc287..9477bb6 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -13,7 +13,6 @@ from veil.pii import ( ADDRESS_TAG, BANK_TAG, - DOB_TAG, EMAIL_TAG, SSN_TAG, PiiSetupError, @@ -27,7 +26,7 @@ def test_build_redactor_disabled_returns_none(): def test_tags_are_distinct(): - assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, DOB_TAG, ADDRESS_TAG}) == 5 + assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 4 # --- everything below needs the [pii] extra + spaCy model ------------------ @@ -76,18 +75,10 @@ def test_address_redacted(R): assert ADDRESS_TAG in out -def test_dob_context_only(R): +def test_dates_are_not_redacted(R): + # Dates are deliberately left intact. out = R.scrub_text("DOB: 04/12/1990. The invoice is dated 06/01/2026.") - assert DOB_TAG in out - assert "04/12/1990" not in out - # A non-birth date is left intact in the default (context-only) mode. - assert "06/01/2026" in out - - -def test_redact_all_dates_mode(): - R = _redactor(redact_all_dates=True) - out = R.scrub_text("the invoice is dated 06/01/2026") - assert "06/01/2026" not in out and DOB_TAG in out + assert "04/12/1990" in out and "06/01/2026" in out def test_scrub_request_string_content(R): diff --git a/veil/cli.py b/veil/cli.py index c232c11..16f6118 100644 --- a/veil/cli.py +++ b/veil/cli.py @@ -85,15 +85,9 @@ def setup(app_url: str, no_browser: bool, yes: bool) -> None: "--pii-scrub", is_flag=True, default=False, - help="Redact high-impact PII (email, SSN, bank numbers, DOB; addresses with the [pii] extra) " + help="Redact high-impact PII (email, SSN, bank numbers; addresses with the [pii] extra) " "from prompts locally before they leave this machine.", ) -@click.option( - "--pii-all-dates", - is_flag=True, - default=False, - help="With --pii-scrub, redact every date as a DOB (aggressive) instead of only birth-date-cued ones.", -) @click.option( "--app-url", default=DEFAULT_APP_URL, show_default=True, help="Chat app web origin for login." ) @@ -118,7 +112,6 @@ def serve( tee_id: str | None, expected_pcr: str | None, pii_scrub: bool, - pii_all_dates: bool, app_url: str, no_browser: bool, foreground: bool, @@ -144,8 +137,6 @@ def serve( ).lower() if pii_scrub: config.pii_scrub = True - if pii_all_dates: - config.pii_redact_all_dates = True # The detached child (--skip-setup) always runs the server in-process. _start_server(config, foreground=foreground or skip_setup) @@ -159,8 +150,6 @@ def _config_flags(config: ServerConfig) -> list[str]: flags += ["--expected-pcr", config.expected_pcr_hash] if config.pii_scrub: flags += ["--pii-scrub"] - if config.pii_redact_all_dates: - flags += ["--pii-all-dates"] return flags diff --git a/veil/config.py b/veil/config.py index b690fa7..8f02823 100644 --- a/veil/config.py +++ b/veil/config.py @@ -58,14 +58,10 @@ class ServerConfig: pinned_tee_id: str | None = None # Opt-in local PII redaction: scrub high-impact PII (email, SSN, bank numbers, - # DOB, and — with the [pii] extra — addresses) out of the agent's prompt - # *before* it leaves this process. Off by default; see veil.pii. + # and — with the [pii] extra — addresses) out of the agent's prompt *before* + # it leaves this process. Off by default; see veil.pii. pii_scrub: bool = False - # When PII scrubbing is on, also redact *every* date as a DOB (aggressive, - # safer). Off by default, so only birth-date-cued dates are redacted. - pii_redact_all_dates: bool = False - @classmethod def from_env(cls) -> "ServerConfig": return cls( @@ -74,7 +70,6 @@ def from_env(cls) -> "ServerConfig": expected_pcr_hash=_norm_hex(os.getenv("OG_VEIL_EXPECTED_PCR_HASH")), pinned_tee_id=_norm_hex(os.getenv("OG_VEIL_TEE_ID")), pii_scrub=_env_bool(os.getenv("OG_VEIL_PII_SCRUB")), - pii_redact_all_dates=_env_bool(os.getenv("OG_VEIL_PII_REDACT_ALL_DATES")), ) def advertised_base_url(self) -> str: diff --git a/veil/gateway.py b/veil/gateway.py index 47ae18a..dcc3841 100644 --- a/veil/gateway.py +++ b/veil/gateway.py @@ -39,9 +39,7 @@ def __init__(self, session: Session, config: ServerConfig): self._tee: TEEEndpoint | None = None # Optional local PII redaction, applied to the request before it is # encrypted to the TEE. ``None`` when disabled (the default). - self._redactor = build_redactor( - enabled=config.pii_scrub, redact_all_dates=config.pii_redact_all_dates - ) + self._redactor = build_redactor(enabled=config.pii_scrub) cfg = session.config if not cfg.tee_registry_rpc_url or not cfg.tee_registry_address: diff --git a/veil/pii.py b/veil/pii.py index 381363b..2455aff 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -12,8 +12,7 @@ scores + context-word boosting. Note that Presidio detects the *structured* types (email, SSN, cards, IBANs, bank numbers) with its own regex/checksum recognizers — spaCy's statistical model is only used for free-form -*addresses/locations* (and, optionally, dates). Because of that, this feature -requires the optional extra: +*addresses/locations*. Because of that, this feature requires the optional extra: pip install 'opengradient-veil[pii]' python -m spacy download en_core_web_sm @@ -22,8 +21,9 @@ * email, US SSN, credit cards (Luhn), IBANs (mod-97), US bank/routing numbers * addresses / locations (spaCy NER) -* dates of birth — by default only *birth-date-cued* dates (``DOB:``, ``born on …``) - via a small custom recognizer; with ``redact_all_dates`` every date is redacted. + +Dates are deliberately *not* redacted — they're too entangled with legitimate +prompt content to scrub without mangling it. This is risk-reduction, not a guarantee: NER misses a fraction of addresses every run, and bare (unlabelled) account numbers can slip through. Redaction is @@ -42,29 +42,13 @@ EMAIL_TAG = "[REDACTED_EMAIL]" SSN_TAG = "[REDACTED_SSN]" BANK_TAG = "[REDACTED_BANK_NUMBER]" -DOB_TAG = "[REDACTED_DOB]" ADDRESS_TAG = "[REDACTED_ADDRESS]" -# spaCy model backing the NER (address/date) detection. The small English model is -# the lightweight choice; recall climbs with the medium/large models at a large -# size cost. Override is intentionally not exposed — keep the install predictable. +# spaCy model backing the NER (address) detection. The small English model is the +# lightweight choice; recall climbs with the medium/large models at a large size +# cost. Override is intentionally not exposed — keep the install predictable. _SPACY_MODEL = "en_core_web_sm" -# The one pattern we own: a birth-date recognizer for an entity Presidio doesn't -# ship. Presidio compiles patterns with the `regex` module, which supports -# variable-width lookbehind — so we anchor on a birth-date *cue* ("DOB:", "born -# on", "date of birth") in the lookbehind and capture only the date itself. This -# gives true per-occurrence gating (an unrelated invoice date is left alone) -# without leaning on Presidio's context enhancer, which boosts dates text-wide. -_DOB_REGEX = ( - r"(?<=\b(?:d\.?o\.?b\.?|date\s+of\s+birth|born(?:\s+on)?|birth\s?date|birthday)\b\W{0,6})" - r"(?:\d{4}-\d{2}-\d{2}" - r"|\d{1,2}[/.\-]\d{1,2}[/.\-]\d{2,4}" - r"|(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?" - r"|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+" - r"\d{1,2}(?:st|nd|rd|th)?,?\s+\d{4})\b" -) - # Presidio's built-in structured recognizers we rely on, each mapped to a tag. _STRUCTURED_OPERATORS = { "EMAIL_ADDRESS": EMAIL_TAG, @@ -80,18 +64,14 @@ class PiiSetupError(Exception): """PII scrubbing was requested but Presidio / the spaCy model isn't installed.""" -def _build_engine(redact_all_dates: bool): # noqa: ANN202 — Presidio types are untyped +def _build_engine(): # noqa: ANN202 — Presidio types are untyped """Construct the Presidio analyzer/anonymizer and the per-entity plan. Raises :class:`PiiSetupError` with an actionable message if the optional dependency or its spaCy model is missing. """ try: - from presidio_analyzer import ( # type: ignore[import-not-found] - AnalyzerEngine, - Pattern, - PatternRecognizer, - ) + from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found] from presidio_analyzer.nlp_engine import ( # type: ignore[import-not-found] NlpEngineProvider, ) @@ -127,25 +107,10 @@ def _build_engine(redact_all_dates: bool): # noqa: ANN202 — Presidio types ar entity: OperatorConfig("replace", {"new_value": tag}) for entity, tag in _STRUCTURED_OPERATORS.items() } - structured_entities = list(_STRUCTURED_OPERATORS) - - if redact_all_dates: - # Aggressive: every date (spaCy DATE_TIME) is treated as a DOB. - structured_entities.append("DATE_TIME") - operators["DATE_TIME"] = OperatorConfig("replace", {"new_value": DOB_TAG}) - else: - # Context-only: our cue-anchored recognizer matches just birth-dates. - analyzer.registry.add_recognizer( - PatternRecognizer( - supported_entity="DOB", - patterns=[Pattern(name="dob", regex=_DOB_REGEX, score=0.85)], - ) - ) - structured_entities.append("DOB") - operators["DOB"] = OperatorConfig("replace", {"new_value": DOB_TAG}) + entities = list(_STRUCTURED_OPERATORS) def analyze(text: str): # noqa: ANN202 - return analyzer.analyze(text=text, entities=structured_entities, language="en") + return analyzer.analyze(text=text, entities=entities, language="en") return analyze, anonymizer, operators @@ -153,8 +118,8 @@ def analyze(text: str): # noqa: ANN202 class Redactor: """Presidio-backed PII redactor. Built eagerly so misconfiguration fails fast.""" - def __init__(self, *, redact_all_dates: bool = False): - self._analyze, self._anonymizer, self._operators = _build_engine(redact_all_dates) + def __init__(self): + self._analyze, self._anonymizer, self._operators = _build_engine() def scrub_text(self, text: str) -> str: if not text: @@ -199,7 +164,7 @@ def scrub_request(self, body: dict) -> dict: return {**body, "messages": scrubbed_messages} -def build_redactor(*, enabled: bool, redact_all_dates: bool = False) -> Redactor | None: +def build_redactor(*, enabled: bool) -> Redactor | None: """Construct a :class:`Redactor` when PII scrubbing is enabled, else ``None``. Raises :class:`PiiSetupError` if enabled but the ``[pii]`` extra / spaCy model @@ -208,4 +173,4 @@ def build_redactor(*, enabled: bool, redact_all_dates: bool = False) -> Redactor """ if not enabled: return None - return Redactor(redact_all_dates=redact_all_dates) + return Redactor() From 9d5def80eeae47079c49f0ec5b520e857aba79ee Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 22:08:13 +0000 Subject: [PATCH 04/25] PII redaction: make scrubbing configurable per request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The --pii-scrub flag / OG_VEIL_PII_SCRUB env var now set the *default*; clients can override scrubbing per request without restarting the proxy: - header `X-OpenGradient-PII-Scrub: true|false` — set once in the client/Hermes config, applies to every request (the easy path) - body field `pii_scrub` (via the OpenAI SDK's extra_body) — per call A per-request value overrides the server default; the body field wins over the header. The control field is stripped before the request reaches the TEE. The Presidio engine is now built lazily and cached, so the [pii] extra is only needed when scrubbing is actually used; if a request asks to scrub but the extra isn't installed, it fails closed with a 503 rather than forwarding PII raw. `og-veil test` gains a --scrub/--no-scrub flag. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- README.md | 31 +++++++++++++++-- tests/test_server.py | 83 +++++++++++++++++++++++++++++++++++++++++++- veil/cli.py | 14 ++++++-- veil/gateway.py | 54 ++++++++++++++++++++++------ veil/server.py | 43 ++++++++++++++++++++++- 5 files changed, 208 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 75176b6..10dd4eb 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). | `OG_VEIL_TEE_ID` | `--tee-id` | — | Pin a specific registry TEE. | | `OG_VEIL_EXPECTED_PCR_HASH` | `--expected-pcr` | — | Refuse any TEE whose `pcrHash` differs. | | `OG_VEIL_APP_URL` | `--app-url` | `https://chat.opengradient.ai` | Chat app origin for login. | -| `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Redact high-impact PII from prompts locally before they leave the machine. | +| `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Default for redacting high-impact PII from prompts locally before they leave the machine (clients can override per request). | ### Local PII redaction (opt-in) @@ -197,10 +197,34 @@ pip install https://github.com/explosion/spacy-models/releases/download/en_core_ Then enable it: +Set the proxy-wide default with the flag or env var: + ```sh og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 ``` +…or leave the default off and **toggle it per request from the client**, so you +can flip scrubbing on/off as you go without restarting the proxy. Either channel +overrides the server default (the body field wins if you set both): + +```python +from openai import OpenAI + +client = OpenAI() # base_url → og-veil +client.chat.completions.create( + model="claude-sonnet-4-6", + messages=[{"role": "user", "content": "…"}], + extra_body={"pii_scrub": True}, # per-call +) + +# or set it once for every request this client makes, via a default header: +client = OpenAI(default_headers={"X-OpenGradient-PII-Scrub": "true"}) +``` + +The header is the easy "set it once in your config" path — e.g. in your Hermes +config — while `extra_body` is handy for flipping it on a single call. +`og-veil test --scrub "…"` exercises the same path from the CLI. + What gets redacted (each replaced with a `[REDACTED_*]` tag): - **email, US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and @@ -210,8 +234,9 @@ What gets redacted (each replaced with a `[REDACTED_*]` tag): Dates are deliberately left alone — too entangled with legitimate prompt content to redact without mangling it. -If `--pii-scrub` is set but the extra/model isn't installed, the server refuses -to start with an actionable message rather than silently sending PII. +If scrubbing is requested (by default or per request) but the extra/model isn't +installed, the request **fails closed** with an actionable error rather than +silently sending PII — so a prompt you asked to scrub is never forwarded raw. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's signed `output_hash` covers exactly what it ran. This is risk-reduction, not a diff --git a/tests/test_server.py b/tests/test_server.py index cc2feb6..a9e4f92 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -11,6 +11,7 @@ from opengradient import RelayError, VerificationError, VerifiedChatResponse from opengradient.client.tee_verify import TeeProof +from veil.pii import PiiSetupError from veil.server import create_app @@ -31,8 +32,12 @@ def __init__(self, result=None, error=None): self._result = result self._error = error self.active_tee = type("T", (), {"tee_id": "0xabc", "endpoint": "https://gw.example"})() + self.last_body = None + self.last_scrub = "unset" - def chat(self, body): + def chat(self, body, *, scrub=None): + self.last_body = body + self.last_scrub = scrub if self._error: raise self._error return self._result @@ -133,6 +138,82 @@ def test_non_json_request_rejected(): assert resp.status_code == 415 +def _ok_result(): + body = { + "id": "chatcmpl-1", + "object": "chat.completion", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"} + ], + } + return VerifiedChatResponse(body=body, content="hi", proof=_proof()) + + +def test_pii_scrub_defaults_to_none_override(): + # No header / body field → no per-request override; gateway uses its default. + gw = _StubGateway(result=_ok_result()) + _client(gw).post( + "/v1/chat/completions", + json={"model": "gpt-4.1", "messages": [{"role": "user", "content": "hi"}]}, + ) + assert gw.last_scrub is None + + +def test_pii_scrub_body_field_overrides_and_is_stripped(): + gw = _StubGateway(result=_ok_result()) + _client(gw).post( + "/v1/chat/completions", + json={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "hi"}], + "pii_scrub": True, + }, + ) + assert gw.last_scrub is True + # The control field must not be forwarded to the TEE. + assert "pii_scrub" not in gw.last_body + + +def test_pii_scrub_header_override(): + gw = _StubGateway(result=_ok_result()) + _client(gw).post( + "/v1/chat/completions", + json={"model": "gpt-4.1", "messages": [{"role": "user", "content": "hi"}]}, + headers={"X-OpenGradient-PII-Scrub": "true"}, + ) + assert gw.last_scrub is True + + +def test_pii_scrub_body_field_beats_header(): + gw = _StubGateway(result=_ok_result()) + _client(gw).post( + "/v1/chat/completions", + json={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "hi"}], + "pii_scrub": False, + }, + headers={"X-OpenGradient-PII-Scrub": "true"}, + ) + assert gw.last_scrub is False + + +def test_pii_setup_error_fails_closed(): + # Scrubbing requested but the extra isn't installed → error, never a leak. + client = _client(_StubGateway(error=PiiSetupError("install the [pii] extra"))) + resp = client.post( + "/v1/chat/completions", + json={ + "model": "gpt-4.1", + "messages": [{"role": "user", "content": "x"}], + "pii_scrub": True, + }, + ) + assert resp.status_code == 503 + assert "pii" in resp.get_json()["error"]["message"].lower() + assert "choices" not in resp.get_json() + + def test_models_and_health(): client = _client(_StubGateway()) assert client.get("/v1/models").get_json()["object"] == "list" diff --git a/veil/cli.py b/veil/cli.py index 16f6118..8436644 100644 --- a/veil/cli.py +++ b/veil/cli.py @@ -245,7 +245,13 @@ def login_cmd(app_url: str, no_browser: bool, manual: bool) -> None: @main.command(name="test") @click.argument("prompt", nargs=-1) @click.option("--model", default="gpt-4.1", show_default=True, help="Model to send the prompt to.") -def test_cmd(prompt: tuple[str, ...], model: str) -> None: +@click.option( + "--scrub/--no-scrub", + "scrub", + default=None, + help="Override PII scrubbing for this request (defaults to the server's setting).", +) +def test_cmd(prompt: tuple[str, ...], model: str, scrub: bool | None) -> None: """Send a one-off PROMPT to the running local server and print the reply. Posts to the localhost OpenAI-compatible endpoint — the same one your agent @@ -254,11 +260,15 @@ def test_cmd(prompt: tuple[str, ...], model: str) -> None: """ import requests + from veil.server import _PII_BODY_FIELD + text = " ".join(prompt).strip() or "Say hello from a verified TEE in one short sentence." config = ServerConfig.from_env() base_url = config.advertised_base_url() - body = {"model": model, "messages": [{"role": "user", "content": text}]} + body: dict = {"model": model, "messages": [{"role": "user", "content": text}]} + if scrub is not None: + body[_PII_BODY_FIELD] = scrub try: resp = requests.post(f"{base_url}/chat/completions", json=body, timeout=120) except requests.exceptions.RequestException as exc: diff --git a/veil/gateway.py b/veil/gateway.py index dcc3841..3ce477e 100644 --- a/veil/gateway.py +++ b/veil/gateway.py @@ -18,7 +18,7 @@ from opengradient.client.tee_registry import TEE_TYPE_LLM_PROXY, TEEEndpoint from veil.config import OHTTP_RELAY_PATH, ServerConfig -from veil.pii import build_redactor +from veil.pii import PiiSetupError, Redactor from veil.session import Session logger = logging.getLogger(__name__) @@ -37,9 +37,17 @@ def __init__(self, session: Session, config: ServerConfig): self._lock = threading.Lock() self._client: OhttpRelayClient | None = None self._tee: TEEEndpoint | None = None - # Optional local PII redaction, applied to the request before it is - # encrypted to the TEE. ``None`` when disabled (the default). - self._redactor = build_redactor(enabled=config.pii_scrub) + + # Local PII redaction, applied to a request before it is encrypted to the + # TEE. ``config.pii_scrub`` is the *default*; individual requests can flip + # it on/off (see Gateway.chat / the server's header+body parsing). The + # Presidio engine is built lazily and cached: if scrubbing is the server + # default we build it eagerly below so misconfiguration fails fast at + # startup, otherwise it's built on the first request that asks for it. + self._pii_default = config.pii_scrub + self._redactor: Redactor | None = None + self._redactor_error: PiiSetupError | None = None + self._redactor_lock = threading.Lock() cfg = session.config if not cfg.tee_registry_rpc_url or not cfg.tee_registry_address: @@ -55,6 +63,30 @@ def __init__(self, session: Session, config: ServerConfig): else TEE_TYPE_LLM_PROXY ) + # Fail fast at startup if scrubbing is the server default but unavailable, + # rather than at the first request. + if self._pii_default: + self._ensure_redactor() + + # --- PII redaction ----------------------------------------------------- + def _ensure_redactor(self) -> Redactor: + """Build (once) and return the Presidio redactor, or raise PiiSetupError. + + The failure is cached so a missing ``[pii]`` extra doesn't pay the import + cost on every request — and so the proxy fails closed rather than ever + forwarding a prompt it was asked to scrub. + """ + with self._redactor_lock: + if self._redactor_error is not None: + raise self._redactor_error + if self._redactor is None: + try: + self._redactor = Redactor() + except PiiSetupError as exc: + self._redactor_error = exc + raise + return self._redactor + # --- TEE resolution ---------------------------------------------------- def _select_tee(self) -> TEEEndpoint: tees = [ @@ -115,12 +147,14 @@ def active_tee(self) -> TEEEndpoint | None: return self._tee # --- inference --------------------------------------------------------- - def chat(self, body: dict) -> VerifiedChatResponse: - # Redact PII locally before anything is sealed to the enclave. Done once, - # outside the retry loop, so a gateway re-selection resends the already- - # scrubbed body rather than re-scrubbing. - if self._redactor is not None: - body = self._redactor.scrub_request(body) + def chat(self, body: dict, *, scrub: bool | None = None) -> VerifiedChatResponse: + # Resolve the per-request preference over the server default, then redact + # PII locally before anything is sealed to the enclave. Done once, outside + # the retry loop, so a gateway re-selection resends the already-scrubbed + # body rather than re-scrubbing. Raises PiiSetupError if scrubbing was + # requested but the [pii] extra isn't installed (fail closed). + if scrub if scrub is not None else self._pii_default: + body = self._ensure_redactor().scrub_request(body) try: return self._chat_once(body) except requests.exceptions.RequestException as exc: diff --git a/veil/server.py b/veil/server.py index 59e4b6b..c343c0e 100644 --- a/veil/server.py +++ b/veil/server.py @@ -24,6 +24,15 @@ from veil.pii import PiiSetupError from veil.session import AuthError, Session +# Per-request PII-scrub control. Clients flip scrubbing on/off without restarting +# the proxy: a default header set once in the client config (the easy path), or a +# body field via the OpenAI SDK's ``extra_body`` (per call). Either overrides the +# server's ``--pii-scrub`` default; the body field wins if both are present. +_PII_HEADER = "X-OpenGradient-PII-Scrub" +_PII_BODY_FIELD = "pii_scrub" +_TRUTHY = {"1", "true", "yes", "on"} +_FALSY = {"0", "false", "no", "off"} + logger = logging.getLogger(__name__) # Model list for clients that probe /v1/models. Derived from the SDK's canonical @@ -66,8 +75,15 @@ def chat_completions(): if not request.is_json: return _error(415, "request must be application/json") body = request.get_json() + # Resolve the scrub preference before stripping our control field so it + # never reaches the TEE as an unknown OpenAI parameter. + scrub = _scrub_preference(body, request.headers) + if isinstance(body, dict): + body.pop(_PII_BODY_FIELD, None) try: - result = gateway.chat(body) + result = gateway.chat(body, scrub=scrub) + except PiiSetupError as exc: + return _error(503, f"PII scrubbing was requested but is unavailable: {exc}") except UnsupportedRequestError as exc: return _error(400, str(exc)) except AuthError as exc: @@ -131,6 +147,31 @@ def _verification_headers(resp, verification: dict) -> None: resp.headers["X-OpenGradient-TEE-Id"] = verification["tee_id"] +def _coerce_bool(value) -> bool | None: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in _TRUTHY: + return True + if lowered in _FALSY: + return False + return None + + +def _scrub_preference(body, headers) -> bool | None: + """Per-request PII-scrub override, or ``None`` to fall back to the server default. + + The body field takes precedence over the header; an unrecognized value is + ignored (treated as "no preference") rather than guessed at. + """ + if isinstance(body, dict) and _PII_BODY_FIELD in body: + decided = _coerce_bool(body.get(_PII_BODY_FIELD)) + if decided is not None: + return decided + return _coerce_bool(headers.get(_PII_HEADER)) + + def _error(status: int, message: str): # OpenAI-style error envelope so client SDKs surface a useful message. resp = jsonify({"error": {"message": message, "type": "veil_error"}}) From 79bf83b46215f99962066f4a5cb6bc127984fd8a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 22:20:24 +0000 Subject: [PATCH 05/25] Revert "PII redaction: make scrubbing configurable per request" This reverts commit 9d5def80eeae47079c49f0ec5b520e857aba79ee. --- README.md | 31 ++--------------- tests/test_server.py | 83 +------------------------------------------- veil/cli.py | 14 ++------ veil/gateway.py | 54 ++++++---------------------- veil/server.py | 43 +---------------------- 5 files changed, 17 insertions(+), 208 deletions(-) diff --git a/README.md b/README.md index 10dd4eb..75176b6 100644 --- a/README.md +++ b/README.md @@ -173,7 +173,7 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). | `OG_VEIL_TEE_ID` | `--tee-id` | — | Pin a specific registry TEE. | | `OG_VEIL_EXPECTED_PCR_HASH` | `--expected-pcr` | — | Refuse any TEE whose `pcrHash` differs. | | `OG_VEIL_APP_URL` | `--app-url` | `https://chat.opengradient.ai` | Chat app origin for login. | -| `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Default for redacting high-impact PII from prompts locally before they leave the machine (clients can override per request). | +| `OG_VEIL_PII_SCRUB` | `--pii-scrub` | off | Redact high-impact PII from prompts locally before they leave the machine. | ### Local PII redaction (opt-in) @@ -197,34 +197,10 @@ pip install https://github.com/explosion/spacy-models/releases/download/en_core_ Then enable it: -Set the proxy-wide default with the flag or env var: - ```sh og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 ``` -…or leave the default off and **toggle it per request from the client**, so you -can flip scrubbing on/off as you go without restarting the proxy. Either channel -overrides the server default (the body field wins if you set both): - -```python -from openai import OpenAI - -client = OpenAI() # base_url → og-veil -client.chat.completions.create( - model="claude-sonnet-4-6", - messages=[{"role": "user", "content": "…"}], - extra_body={"pii_scrub": True}, # per-call -) - -# or set it once for every request this client makes, via a default header: -client = OpenAI(default_headers={"X-OpenGradient-PII-Scrub": "true"}) -``` - -The header is the easy "set it once in your config" path — e.g. in your Hermes -config — while `extra_body` is handy for flipping it on a single call. -`og-veil test --scrub "…"` exercises the same path from the CLI. - What gets redacted (each replaced with a `[REDACTED_*]` tag): - **email, US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and @@ -234,9 +210,8 @@ What gets redacted (each replaced with a `[REDACTED_*]` tag): Dates are deliberately left alone — too entangled with legitimate prompt content to redact without mangling it. -If scrubbing is requested (by default or per request) but the extra/model isn't -installed, the request **fails closed** with an actionable error rather than -silently sending PII — so a prompt you asked to scrub is never forwarded raw. +If `--pii-scrub` is set but the extra/model isn't installed, the server refuses +to start with an actionable message rather than silently sending PII. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's signed `output_hash` covers exactly what it ran. This is risk-reduction, not a diff --git a/tests/test_server.py b/tests/test_server.py index a9e4f92..cc2feb6 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -11,7 +11,6 @@ from opengradient import RelayError, VerificationError, VerifiedChatResponse from opengradient.client.tee_verify import TeeProof -from veil.pii import PiiSetupError from veil.server import create_app @@ -32,12 +31,8 @@ def __init__(self, result=None, error=None): self._result = result self._error = error self.active_tee = type("T", (), {"tee_id": "0xabc", "endpoint": "https://gw.example"})() - self.last_body = None - self.last_scrub = "unset" - def chat(self, body, *, scrub=None): - self.last_body = body - self.last_scrub = scrub + def chat(self, body): if self._error: raise self._error return self._result @@ -138,82 +133,6 @@ def test_non_json_request_rejected(): assert resp.status_code == 415 -def _ok_result(): - body = { - "id": "chatcmpl-1", - "object": "chat.completion", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"} - ], - } - return VerifiedChatResponse(body=body, content="hi", proof=_proof()) - - -def test_pii_scrub_defaults_to_none_override(): - # No header / body field → no per-request override; gateway uses its default. - gw = _StubGateway(result=_ok_result()) - _client(gw).post( - "/v1/chat/completions", - json={"model": "gpt-4.1", "messages": [{"role": "user", "content": "hi"}]}, - ) - assert gw.last_scrub is None - - -def test_pii_scrub_body_field_overrides_and_is_stripped(): - gw = _StubGateway(result=_ok_result()) - _client(gw).post( - "/v1/chat/completions", - json={ - "model": "gpt-4.1", - "messages": [{"role": "user", "content": "hi"}], - "pii_scrub": True, - }, - ) - assert gw.last_scrub is True - # The control field must not be forwarded to the TEE. - assert "pii_scrub" not in gw.last_body - - -def test_pii_scrub_header_override(): - gw = _StubGateway(result=_ok_result()) - _client(gw).post( - "/v1/chat/completions", - json={"model": "gpt-4.1", "messages": [{"role": "user", "content": "hi"}]}, - headers={"X-OpenGradient-PII-Scrub": "true"}, - ) - assert gw.last_scrub is True - - -def test_pii_scrub_body_field_beats_header(): - gw = _StubGateway(result=_ok_result()) - _client(gw).post( - "/v1/chat/completions", - json={ - "model": "gpt-4.1", - "messages": [{"role": "user", "content": "hi"}], - "pii_scrub": False, - }, - headers={"X-OpenGradient-PII-Scrub": "true"}, - ) - assert gw.last_scrub is False - - -def test_pii_setup_error_fails_closed(): - # Scrubbing requested but the extra isn't installed → error, never a leak. - client = _client(_StubGateway(error=PiiSetupError("install the [pii] extra"))) - resp = client.post( - "/v1/chat/completions", - json={ - "model": "gpt-4.1", - "messages": [{"role": "user", "content": "x"}], - "pii_scrub": True, - }, - ) - assert resp.status_code == 503 - assert "pii" in resp.get_json()["error"]["message"].lower() - assert "choices" not in resp.get_json() - - def test_models_and_health(): client = _client(_StubGateway()) assert client.get("/v1/models").get_json()["object"] == "list" diff --git a/veil/cli.py b/veil/cli.py index 8436644..16f6118 100644 --- a/veil/cli.py +++ b/veil/cli.py @@ -245,13 +245,7 @@ def login_cmd(app_url: str, no_browser: bool, manual: bool) -> None: @main.command(name="test") @click.argument("prompt", nargs=-1) @click.option("--model", default="gpt-4.1", show_default=True, help="Model to send the prompt to.") -@click.option( - "--scrub/--no-scrub", - "scrub", - default=None, - help="Override PII scrubbing for this request (defaults to the server's setting).", -) -def test_cmd(prompt: tuple[str, ...], model: str, scrub: bool | None) -> None: +def test_cmd(prompt: tuple[str, ...], model: str) -> None: """Send a one-off PROMPT to the running local server and print the reply. Posts to the localhost OpenAI-compatible endpoint — the same one your agent @@ -260,15 +254,11 @@ def test_cmd(prompt: tuple[str, ...], model: str, scrub: bool | None) -> None: """ import requests - from veil.server import _PII_BODY_FIELD - text = " ".join(prompt).strip() or "Say hello from a verified TEE in one short sentence." config = ServerConfig.from_env() base_url = config.advertised_base_url() - body: dict = {"model": model, "messages": [{"role": "user", "content": text}]} - if scrub is not None: - body[_PII_BODY_FIELD] = scrub + body = {"model": model, "messages": [{"role": "user", "content": text}]} try: resp = requests.post(f"{base_url}/chat/completions", json=body, timeout=120) except requests.exceptions.RequestException as exc: diff --git a/veil/gateway.py b/veil/gateway.py index 3ce477e..dcc3841 100644 --- a/veil/gateway.py +++ b/veil/gateway.py @@ -18,7 +18,7 @@ from opengradient.client.tee_registry import TEE_TYPE_LLM_PROXY, TEEEndpoint from veil.config import OHTTP_RELAY_PATH, ServerConfig -from veil.pii import PiiSetupError, Redactor +from veil.pii import build_redactor from veil.session import Session logger = logging.getLogger(__name__) @@ -37,17 +37,9 @@ def __init__(self, session: Session, config: ServerConfig): self._lock = threading.Lock() self._client: OhttpRelayClient | None = None self._tee: TEEEndpoint | None = None - - # Local PII redaction, applied to a request before it is encrypted to the - # TEE. ``config.pii_scrub`` is the *default*; individual requests can flip - # it on/off (see Gateway.chat / the server's header+body parsing). The - # Presidio engine is built lazily and cached: if scrubbing is the server - # default we build it eagerly below so misconfiguration fails fast at - # startup, otherwise it's built on the first request that asks for it. - self._pii_default = config.pii_scrub - self._redactor: Redactor | None = None - self._redactor_error: PiiSetupError | None = None - self._redactor_lock = threading.Lock() + # Optional local PII redaction, applied to the request before it is + # encrypted to the TEE. ``None`` when disabled (the default). + self._redactor = build_redactor(enabled=config.pii_scrub) cfg = session.config if not cfg.tee_registry_rpc_url or not cfg.tee_registry_address: @@ -63,30 +55,6 @@ def __init__(self, session: Session, config: ServerConfig): else TEE_TYPE_LLM_PROXY ) - # Fail fast at startup if scrubbing is the server default but unavailable, - # rather than at the first request. - if self._pii_default: - self._ensure_redactor() - - # --- PII redaction ----------------------------------------------------- - def _ensure_redactor(self) -> Redactor: - """Build (once) and return the Presidio redactor, or raise PiiSetupError. - - The failure is cached so a missing ``[pii]`` extra doesn't pay the import - cost on every request — and so the proxy fails closed rather than ever - forwarding a prompt it was asked to scrub. - """ - with self._redactor_lock: - if self._redactor_error is not None: - raise self._redactor_error - if self._redactor is None: - try: - self._redactor = Redactor() - except PiiSetupError as exc: - self._redactor_error = exc - raise - return self._redactor - # --- TEE resolution ---------------------------------------------------- def _select_tee(self) -> TEEEndpoint: tees = [ @@ -147,14 +115,12 @@ def active_tee(self) -> TEEEndpoint | None: return self._tee # --- inference --------------------------------------------------------- - def chat(self, body: dict, *, scrub: bool | None = None) -> VerifiedChatResponse: - # Resolve the per-request preference over the server default, then redact - # PII locally before anything is sealed to the enclave. Done once, outside - # the retry loop, so a gateway re-selection resends the already-scrubbed - # body rather than re-scrubbing. Raises PiiSetupError if scrubbing was - # requested but the [pii] extra isn't installed (fail closed). - if scrub if scrub is not None else self._pii_default: - body = self._ensure_redactor().scrub_request(body) + def chat(self, body: dict) -> VerifiedChatResponse: + # Redact PII locally before anything is sealed to the enclave. Done once, + # outside the retry loop, so a gateway re-selection resends the already- + # scrubbed body rather than re-scrubbing. + if self._redactor is not None: + body = self._redactor.scrub_request(body) try: return self._chat_once(body) except requests.exceptions.RequestException as exc: diff --git a/veil/server.py b/veil/server.py index c343c0e..59e4b6b 100644 --- a/veil/server.py +++ b/veil/server.py @@ -24,15 +24,6 @@ from veil.pii import PiiSetupError from veil.session import AuthError, Session -# Per-request PII-scrub control. Clients flip scrubbing on/off without restarting -# the proxy: a default header set once in the client config (the easy path), or a -# body field via the OpenAI SDK's ``extra_body`` (per call). Either overrides the -# server's ``--pii-scrub`` default; the body field wins if both are present. -_PII_HEADER = "X-OpenGradient-PII-Scrub" -_PII_BODY_FIELD = "pii_scrub" -_TRUTHY = {"1", "true", "yes", "on"} -_FALSY = {"0", "false", "no", "off"} - logger = logging.getLogger(__name__) # Model list for clients that probe /v1/models. Derived from the SDK's canonical @@ -75,15 +66,8 @@ def chat_completions(): if not request.is_json: return _error(415, "request must be application/json") body = request.get_json() - # Resolve the scrub preference before stripping our control field so it - # never reaches the TEE as an unknown OpenAI parameter. - scrub = _scrub_preference(body, request.headers) - if isinstance(body, dict): - body.pop(_PII_BODY_FIELD, None) try: - result = gateway.chat(body, scrub=scrub) - except PiiSetupError as exc: - return _error(503, f"PII scrubbing was requested but is unavailable: {exc}") + result = gateway.chat(body) except UnsupportedRequestError as exc: return _error(400, str(exc)) except AuthError as exc: @@ -147,31 +131,6 @@ def _verification_headers(resp, verification: dict) -> None: resp.headers["X-OpenGradient-TEE-Id"] = verification["tee_id"] -def _coerce_bool(value) -> bool | None: - if isinstance(value, bool): - return value - if isinstance(value, str): - lowered = value.strip().lower() - if lowered in _TRUTHY: - return True - if lowered in _FALSY: - return False - return None - - -def _scrub_preference(body, headers) -> bool | None: - """Per-request PII-scrub override, or ``None`` to fall back to the server default. - - The body field takes precedence over the header; an unrecognized value is - ignored (treated as "no preference") rather than guessed at. - """ - if isinstance(body, dict) and _PII_BODY_FIELD in body: - decided = _coerce_bool(body.get(_PII_BODY_FIELD)) - if decided is not None: - return decided - return _coerce_bool(headers.get(_PII_HEADER)) - - def _error(status: int, message: str): # OpenAI-style error envelope so client SDKs surface a useful message. resp = jsonify({"error": {"message": message, "type": "veil_error"}}) From 421d2b50d14d805e092ce20430d645c253ffc14c Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:43:11 -0400 Subject: [PATCH 06/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- pyproject.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6fd6cb6..ecdae1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,10 +40,9 @@ dependencies = [ ] [project.optional-dependencies] -# Adds free-form address/location (and optional all-date) PII redaction on top of -# the always-available, zero-dependency regex tier. Microsoft Presidio drives a -# spaCy NER model; after installing this extra, download the model once with: -# python -m spacy download en_core_web_sm +# Enables local PII redaction via Microsoft Presidio + spaCy. +# After installing this extra, install the spaCy model once (e.g. `python -m spacy download en_core_web_sm` +# or by installing the `en_core_web_sm` wheel). pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", From dfd191c463c800b09061edeb624da821510e351d Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:43:43 -0400 Subject: [PATCH 07/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8266851..cf357a5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,9 +25,9 @@ jobs: run: uv sync --all-groups --extra pii - name: Install spaCy model for PII tests - # Installed from the release wheel rather than `spacy download`, whose CLI - # breaks under the click version this project pins. Not needed at runtime, - # only to fetch the model. + # Installed from the release wheel rather than `spacy download`, which can + # be unreliable in some environments. Not needed at runtime, only to fetch + # the model. run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl - name: Test From 1ae5b4b23a3b3795c8626a7032de32838cac2501 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:43:58 -0400 Subject: [PATCH 08/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 75176b6..e5ef6dc 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,8 @@ Install it once: ```sh pip install 'opengradient-veil[pii]' -# fetch the spaCy model (from the release wheel — `spacy download` breaks under -# the click version this project pins): +# fetch the spaCy model from the release wheel (avoids issues with `spacy download` +# in some environments): pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl ``` From 76fe0d77de5eb95307b36d2612eb48dfa7421576 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:44:06 -0400 Subject: [PATCH 09/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_pii.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_pii.py b/tests/test_pii.py index 9477bb6..2c4e649 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -1,9 +1,8 @@ """PII redaction tests. Detection is delegated to Presidio + spaCy (the optional [pii] extra), so these -tests ``importorskip`` when it isn't installed — base installs and the release -CI skip them; the dedicated test workflow installs the extra + model and runs -them for real. ``build_redactor(enabled=False)`` is checked unconditionally. +tests skip when the dependency or spaCy model isn't installed. ``build_redactor(enabled=False)`` +is checked unconditionally. """ from __future__ import annotations From 6da6ada62af9ae36f062661cb7e2bc11ae4c5bfa Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:44:20 -0400 Subject: [PATCH 10/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- veil/pii.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/veil/pii.py b/veil/pii.py index 2455aff..eabc096 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -82,8 +82,8 @@ def _build_engine(): # noqa: ANN202 — Presidio types are untyped except ImportError as exc: raise PiiSetupError( "PII scrubbing needs the optional extra. Install it with: " - "pip install 'opengradient-veil[pii]' && python -m spacy download " - f"{_SPACY_MODEL}" + "pip install 'opengradient-veil[pii]' and install the spaCy model " + f"'{_SPACY_MODEL}' (e.g. `python -m spacy download {_SPACY_MODEL}` or the model wheel)." ) from exc provider = NlpEngineProvider( From 90bde6d4c78cf0f6c086ee3635aad8319b1841f7 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 18:44:31 -0400 Subject: [PATCH 11/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43bad71..a4e6286 100644 --- a/Makefile +++ b/Makefile @@ -5,8 +5,7 @@ install: # Dev install with the optional PII-redaction stack (Presidio + spaCy) and its # model. The model is fetched from the release wheel rather than `spacy download`, -# whose CLI breaks under the click version this project pins. -install-pii: +# which can be unreliable in some environments. uv sync --all-groups --extra pii uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl From 672988bed1d97bc4194b1c5f72ff5967766853df Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 22:46:32 +0000 Subject: [PATCH 12/25] Add scripts/pii_repl.py for interactive PII-redaction testing Local-only helper that loads the same Redactor the proxy uses (no login, TEE, or network) so you can try the redaction on your own text interactively or via piped input. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- scripts/pii_repl.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 scripts/pii_repl.py diff --git a/scripts/pii_repl.py b/scripts/pii_repl.py new file mode 100644 index 0000000..ac1d481 --- /dev/null +++ b/scripts/pii_repl.py @@ -0,0 +1,39 @@ +"""Scratch helper: interactively try the PII redactor on your own text. + + uv run python scripts/pii_repl.py # interactive: type a line, see it scrubbed + echo "email me at a@b.com" | uv run python scripts/pii_repl.py # or pipe input + +Local-only — no login, no TEE, no network. Just the same Redactor the proxy uses. +Needs the extra: make install-pii (or: uv sync --extra pii + the model wheel) +""" + +from __future__ import annotations + +import sys + +from veil.pii import PiiSetupError, build_redactor + + +def main() -> None: + try: + redactor = build_redactor(enabled=True) + except PiiSetupError as exc: + sys.exit(f"PII redaction unavailable: {exc}") + assert redactor is not None + + if not sys.stdin.isatty(): # piped input: scrub each line and exit + for line in sys.stdin: + print(redactor.scrub_text(line.rstrip("\n"))) + return + + print("PII redactor — type text and press Enter (Ctrl-D / Ctrl-C to quit).\n") + try: + while True: + text = input("> ") + print(redactor.scrub_text(text)) + except (EOFError, KeyboardInterrupt): + print() + + +if __name__ == "__main__": + main() From 8677036cc7cdad47d4524f1777b31101d57b84be Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 22:47:34 +0000 Subject: [PATCH 13/25] Fix install-pii Makefile target dropped by autofix The automated fix commit removed the `install-pii:` target header, leaving its recipe lines attached to `install:`. Restore the header so `make install-pii` works again. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index a4e6286..329dc5e 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ install: # Dev install with the optional PII-redaction stack (Presidio + spaCy) and its # model. The model is fetched from the release wheel rather than `spacy download`, # which can be unreliable in some environments. +install-pii: uv sync --all-groups --extra pii uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl From c7f269a619a3f0e2144d397613ee25bccc52ff63 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 23:07:33 +0000 Subject: [PATCH 14/25] PII redaction: prioritize identity de-identification (names, contact, address) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reframe scrubbing around Veil's actual threat model: OHTTP unlinks identity from content at the network layer, but a prompt that names you re-identifies you to the model provider through the content. So the job is de-identifying the requester. Reprioritize accordingly: - Add person names (PERSON) and phone numbers — the primary identity linkers, previously not covered at all. - Add a custom street-address recognizer: spaCy NER catches named places (cities/states) but not free-form street lines, and Presidio ships nothing for them. No transformer needed. - Upgrade the spaCy model to en_core_web_lg for materially better PERSON recall across diverse names (a ~560 MB CPU CNN model, not a transformer). Structured IDs (SSN, cards, IBAN, bank) stay as a bonus. Docs/Makefile/CI updated to the lg model wheel; tests cover names, phone, and street addresses. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- .github/workflows/tests.yml | 2 +- Makefile | 2 +- README.md | 35 ++++++++------ pyproject.toml | 4 +- tests/test_pii.py | 21 +++++++- veil/pii.py | 95 +++++++++++++++++++++++++------------ 6 files changed, 110 insertions(+), 49 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cf357a5..6073b5d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: # Installed from the release wheel rather than `spacy download`, which can # be unreliable in some environments. Not needed at runtime, only to fetch # the model. - run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl + run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl - name: Test run: make test diff --git a/Makefile b/Makefile index 329dc5e..9b16731 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ install: # which can be unreliable in some environments. install-pii: uv sync --all-groups --extra pii - uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl + uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl build: uv build diff --git a/README.md b/README.md index e5ef6dc..1e700c9 100644 --- a/README.md +++ b/README.md @@ -177,12 +177,15 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). ### Local PII redaction (opt-in) -Veil already keeps prompts private end-to-end — OHTTP splits *who you are* from -*what you ask*, and the enclave is attested and reproducible. Local PII scrubbing -is **defense-in-depth** on top of that: when enabled, high-impact PII is -irreversibly replaced with `[REDACTED_*]` tags *before* the prompt is encrypted to -the TEE, so the raw values never leave your machine. Handy for compliance, -data-residency, and keeping PII out of any model-side logging. +Veil's privacy guarantee is *unlinkability* — OHTTP splits *who you are* from +*what you ask*, so the model provider sees your prompt but believes it came from +the enclave, not you. That holds **only if the prompt doesn't name you**: a +request containing your name, email, and address re-identifies you to the +provider through the content and undoes the split. Local PII redaction closes +that gap — when enabled, identity-revealing PII is irreversibly replaced with +`[REDACTED_*]` tags *before* the prompt is encrypted to the TEE, so it never +leaves your machine. Think of it as a peace-of-mind backstop to your own +discretion, not a replacement for it. Detection is delegated to **Microsoft Presidio** (community-maintained recognizers) rather than handrolled patterns, so it ships as an optional extra. @@ -192,10 +195,10 @@ Install it once: pip install 'opengradient-veil[pii]' # fetch the spaCy model from the release wheel (avoids issues with `spacy download` # in some environments): -pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl +pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl ``` -Then enable it: +(`make install-pii` does both.) Then enable it: ```sh og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 @@ -203,9 +206,12 @@ og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 What gets redacted (each replaced with a `[REDACTED_*]` tag): -- **email, US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and - US bank/routing numbers, via Presidio's regex/checksum recognizers. -- **addresses / locations** — free-form prose, via the spaCy NER model. +- **names of people, addresses** — the core identity linkers, via the spaCy NER + model (`en_core_web_lg`, a ~560 MB CPU model — not a transformer) plus a + street-address recognizer for the street lines NER misses. +- **email, phone numbers** — contact identity. +- **US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and US + bank/routing numbers, via Presidio's regex/checksum recognizers. Dates are deliberately left alone — too entangled with legitimate prompt content to redact without mangling it. @@ -214,9 +220,10 @@ If `--pii-scrub` is set but the extra/model isn't installed, the server refuses to start with an actionable message rather than silently sending PII. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's -signed `output_hash` covers exactly what it ran. This is risk-reduction, not a -guarantee: NER misses a fraction of addresses each run, and bare (unlabelled) -account numbers can slip through. +signed `output_hash` covers exactly what it ran. It's risk-reduction, not a +guarantee, and it's *blunt*: it can't tell your own name from a name you ask +about, so it over-redacts, and residual signals (writing style, niche topics) +can still re-identify you. ## Notes & limitations diff --git a/pyproject.toml b/pyproject.toml index ecdae1b..b09790b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ dependencies = [ [project.optional-dependencies] # Enables local PII redaction via Microsoft Presidio + spaCy. -# After installing this extra, install the spaCy model once (e.g. `python -m spacy download en_core_web_sm` -# or by installing the `en_core_web_sm` wheel). +# After installing this extra, install the spaCy model once (e.g. +# `python -m spacy download en_core_web_lg` or by installing the model wheel). pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", diff --git a/tests/test_pii.py b/tests/test_pii.py index 2c4e649..2b1b435 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -13,6 +13,8 @@ ADDRESS_TAG, BANK_TAG, EMAIL_TAG, + NAME_TAG, + PHONE_TAG, SSN_TAG, PiiSetupError, build_redactor, @@ -25,7 +27,7 @@ def test_build_redactor_disabled_returns_none(): def test_tags_are_distinct(): - assert len({EMAIL_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 4 + assert len({NAME_TAG, EMAIL_TAG, PHONE_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 6 # --- everything below needs the [pii] extra + spaCy model ------------------ @@ -45,6 +47,23 @@ def R(): return _redactor() +def test_name_redacted(R): + # Person names — the core identity linker. Needs NER. + out = R.scrub_text("Hi, I am Adam Balogh and I need help.") + assert "Adam Balogh" not in out and NAME_TAG in out + + +def test_phone_redacted(R): + out = R.scrub_text("call me at +1 (415) 555-0142 tomorrow") + assert "555-0142" not in out and PHONE_TAG in out + + +def test_street_address_redacted(R): + # Street lines need the custom recognizer — spaCy NER alone misses them. + out = R.scrub_text("ship it to 25 Park Lane South, Jersey City") + assert "25 Park Lane South" not in out and ADDRESS_TAG in out + + def test_email_redacted(R): out = R.scrub_text("ping me at jane.doe+x@example.co.uk please") assert "jane.doe" not in out and EMAIL_TAG in out diff --git a/veil/pii.py b/veil/pii.py index eabc096..9d0a664 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -1,34 +1,37 @@ """Local, opt-in PII redaction applied *before* a prompt leaves this process. -Veil already keeps prompts private end-to-end (Oblivious HTTP splits identity -from content; the enclave is attested and reproducible). This module is -*defense-in-depth* on top of that: when enabled, high-impact PII is irreversibly -replaced with ``[REDACTED_*]`` tags on the agent's request before it is HPKE- -sealed to the TEE, so the raw values never leave the machine at all — useful for -compliance, data-residency, and keeping PII out of any model-side logging. - -Detection is delegated to **Microsoft Presidio** rather than handrolled patterns, -so the recognizers are community-maintained, versioned, and carry confidence -scores + context-word boosting. Note that Presidio detects the *structured* -types (email, SSN, cards, IBANs, bank numbers) with its own regex/checksum -recognizers — spaCy's statistical model is only used for free-form -*addresses/locations*. Because of that, this feature requires the optional extra: +Veil's privacy guarantee is *unlinkability*: Oblivious HTTP splits who-you-are +from what-you-ask, so the relay sees your identity but only ciphertext, and the +model provider sees the prompt but believes it came from the enclave — not you. +That holds **only if the prompt content doesn't re-identify you**. A request that +says "I'm Adam Balogh, adam@example.com, 25 Park Lane" hands your identity to the +provider through the content and undoes the cryptographic split. So this module's +job is to de-identify the *requester*: strip the things that point back to a +specific person before the prompt is HPKE-sealed to the TEE. + +Detection is delegated to **Microsoft Presidio** so the recognizers are +community-maintained rather than handrolled. Presidio detects structured values +(email, phone, SSN, cards, IBANs, bank numbers) with its own regex/checksum +recognizers, and uses a spaCy NER model for the free-form identity linkers +(names, locations). This requires the optional extra plus a spaCy model: pip install 'opengradient-veil[pii]' - python -m spacy download en_core_web_sm + python -m spacy download en_core_web_lg What gets redacted (mapped to the tags below): -* email, US SSN, credit cards (Luhn), IBANs (mod-97), US bank/routing numbers -* addresses / locations (spaCy NER) +* names of people, and addresses / locations (spaCy NER + a street-line recognizer) +* email and phone numbers +* US SSN, credit cards (Luhn), IBANs (mod-97), US bank/routing numbers Dates are deliberately *not* redacted — they're too entangled with legitimate prompt content to scrub without mangling it. -This is risk-reduction, not a guarantee: NER misses a fraction of addresses every -run, and bare (unlabelled) account numbers can slip through. Redaction is -irreversible — there is deliberately no de-anonymization step, so the TEE's signed -``output_hash`` covers exactly what it ran and nothing is restored locally. +This is risk-reduction, not a guarantee. It's also *blunt*: it can't tell your +own name from a name you're asking about, so it over-redacts; and even perfect +PII removal leaves residual re-identification (writing style, niche topics). +Redaction is irreversible — there is deliberately no de-anonymization step, so +the TEE's signed ``output_hash`` covers exactly what it ran. """ from __future__ import annotations @@ -39,26 +42,48 @@ # Redaction tags. Kept human-readable so a redacted prompt still reads sensibly to # the model (e.g. "wire it to [REDACTED_BANK_NUMBER]"). +NAME_TAG = "[REDACTED_NAME]" EMAIL_TAG = "[REDACTED_EMAIL]" +PHONE_TAG = "[REDACTED_PHONE]" SSN_TAG = "[REDACTED_SSN]" BANK_TAG = "[REDACTED_BANK_NUMBER]" ADDRESS_TAG = "[REDACTED_ADDRESS]" -# spaCy model backing the NER (address) detection. The small English model is the -# lightweight choice; recall climbs with the medium/large models at a large size -# cost. Override is intentionally not exposed — keep the install predictable. -_SPACY_MODEL = "en_core_web_sm" - -# Presidio's built-in structured recognizers we rely on, each mapped to a tag. -_STRUCTURED_OPERATORS = { +# spaCy model backing the NER (name/location) detection. The large English model +# is used rather than the small one: names are the core identity linker here, and +# large meaningfully improves PERSON recall across diverse names. It's a ~560 MB +# CNN model (not a transformer) and runs on CPU. Override is intentionally not +# exposed — keep the install predictable. +_SPACY_MODEL = "en_core_web_lg" + +# Presidio entities we redact, each mapped to a tag. PERSON and LOCATION come from +# the spaCy NER model; the rest are Presidio's regex/checksum recognizers. +_ENTITY_TAGS = { + "PERSON": NAME_TAG, "EMAIL_ADDRESS": EMAIL_TAG, + "PHONE_NUMBER": PHONE_TAG, "US_SSN": SSN_TAG, "CREDIT_CARD": BANK_TAG, "IBAN_CODE": BANK_TAG, "US_BANK_NUMBER": BANK_TAG, "LOCATION": ADDRESS_TAG, + # Custom recognizer registered below; Presidio ships nothing for street lines. + "STREET_ADDRESS": ADDRESS_TAG, } +# Street-address line: a house number, a few name words, then a street-type +# suffix, optionally a direction. Presidio/spaCy NER catch named places (cities, +# states) but not free-form street lines, so this closes the most common gap +# without a transformer. Compiled by Presidio with IGNORECASE. +_STREET_ADDRESS_REGEX = ( + r"\b\d{1,6}\s+(?:[A-Za-z0-9.\-']+\s+){0,4}" + r"(?:street|st|avenue|ave|lane|ln|road|rd|boulevard|blvd|drive|dr|court|ct" + r"|place|pl|way|terrace|ter|circle|cir|highway|hwy|parkway|pkwy|square|sq" + r"|trail|trl|route|rte|crossing|xing|loop|row|alley|plaza|commons)\b\.?" + r"(?:\s+(?:northeast|northwest|southeast|southwest|north|south|east|west" + r"|ne|nw|se|sw|n|s|e|w)\b)?" +) + class PiiSetupError(Exception): """PII scrubbing was requested but Presidio / the spaCy model isn't installed.""" @@ -71,7 +96,11 @@ def _build_engine(): # noqa: ANN202 — Presidio types are untyped dependency or its spaCy model is missing. """ try: - from presidio_analyzer import AnalyzerEngine # type: ignore[import-not-found] + from presidio_analyzer import ( # type: ignore[import-not-found] + AnalyzerEngine, + Pattern, + PatternRecognizer, + ) from presidio_analyzer.nlp_engine import ( # type: ignore[import-not-found] NlpEngineProvider, ) @@ -101,13 +130,19 @@ def _build_engine(): # noqa: ANN202 — Presidio types are untyped ) from exc analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=["en"]) + analyzer.registry.add_recognizer( + PatternRecognizer( + supported_entity="STREET_ADDRESS", + patterns=[Pattern(name="street_address", regex=_STREET_ADDRESS_REGEX, score=0.6)], + ) + ) anonymizer = AnonymizerEngine() operators = { entity: OperatorConfig("replace", {"new_value": tag}) - for entity, tag in _STRUCTURED_OPERATORS.items() + for entity, tag in _ENTITY_TAGS.items() } - entities = list(_STRUCTURED_OPERATORS) + entities = list(_ENTITY_TAGS) def analyze(text: str): # noqa: ANN202 return analyzer.analyze(text=text, entities=entities, language="en") From 55a11f86145c0e22c3e5233f58875b03a5862d00 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 12 Jun 2026 23:22:15 +0000 Subject: [PATCH 15/25] PII redaction: redact only hard identifiers, keep names and free-form locations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drop the statistical-NER entities (PERSON, LOCATION). They over-redacted the third-party names real prompts are full of ("reply to Advait about Julia") and mislabeled uncommon names (e.g. tagging "Advait" as a location), wrecking common workflows for little gain. Keeping names out of a prompt stays the user's call; this layer now backstops only the concrete data. Now redacts only deterministic pattern/checksum identifiers: email, phone, SSN, credit cards, IBANs, US bank numbers, and street-address lines. No name or location guessing. Since no NER entities are redacted anymore, the spaCy model is only Presidio's tokenizer — downgrade en_core_web_lg back to en_core_web_sm (saves ~550 MB). Docs/Makefile/CI updated; tests assert names and cities pass through. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- .github/workflows/tests.yml | 2 +- Makefile | 2 +- README.md | 31 ++++++++--------- pyproject.toml | 2 +- tests/test_pii.py | 26 +++++++-------- veil/pii.py | 66 ++++++++++++++++++------------------- 6 files changed, 65 insertions(+), 64 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6073b5d..cf357a5 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -28,7 +28,7 @@ jobs: # Installed from the release wheel rather than `spacy download`, which can # be unreliable in some environments. Not needed at runtime, only to fetch # the model. - run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl + run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl - name: Test run: make test diff --git a/Makefile b/Makefile index 9b16731..329dc5e 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ install: # which can be unreliable in some environments. install-pii: uv sync --all-groups --extra pii - uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl + uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl build: uv build diff --git a/README.md b/README.md index 1e700c9..79f36bf 100644 --- a/README.md +++ b/README.md @@ -179,13 +179,12 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). Veil's privacy guarantee is *unlinkability* — OHTTP splits *who you are* from *what you ask*, so the model provider sees your prompt but believes it came from -the enclave, not you. That holds **only if the prompt doesn't name you**: a -request containing your name, email, and address re-identifies you to the -provider through the content and undoes the split. Local PII redaction closes -that gap — when enabled, identity-revealing PII is irreversibly replaced with -`[REDACTED_*]` tags *before* the prompt is encrypted to the TEE, so it never -leaves your machine. Think of it as a peace-of-mind backstop to your own -discretion, not a replacement for it. +the enclave, not you. That holds only if the prompt content doesn't re-identify +you. Local PII redaction strips the **concrete, unambiguous identifiers** — when +enabled, they're irreversibly replaced with `[REDACTED_*]` tags *before* the +prompt is encrypted to the TEE, so they never leave your machine. It's a +peace-of-mind backstop for the hard data, not a replacement for your own +discretion. Detection is delegated to **Microsoft Presidio** (community-maintained recognizers) rather than handrolled patterns, so it ships as an optional extra. @@ -195,7 +194,7 @@ Install it once: pip install 'opengradient-veil[pii]' # fetch the spaCy model from the release wheel (avoids issues with `spacy download` # in some environments): -pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_lg-3.8.0/en_core_web_lg-3.8.0-py3-none-any.whl +pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl ``` (`make install-pii` does both.) Then enable it: @@ -206,23 +205,25 @@ og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 What gets redacted (each replaced with a `[REDACTED_*]` tag): -- **names of people, addresses** — the core identity linkers, via the spaCy NER - model (`en_core_web_lg`, a ~560 MB CPU model — not a transformer) plus a - street-address recognizer for the street lines NER misses. - **email, phone numbers** — contact identity. - **US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and US bank/routing numbers, via Presidio's regex/checksum recognizers. +- **street addresses** — a deterministic street-line recognizer. -Dates are deliberately left alone — too entangled with legitimate prompt content -to redact without mangling it. +**Names, free-form locations (cities/countries), and dates are deliberately left +in.** Those rely on statistical NER that over-redacts the third-party names real +prompts are full of ("reply to *Advait* about *Julia*") and often mislabels +uncommon names — wrecking the prompt for little gain. Everything redacted here is +pattern/checksum-based: deterministic, no name guessing. (The small spaCy model +is just Presidio's tokenizer; no NER entities are redacted.) If `--pii-scrub` is set but the extra/model isn't installed, the server refuses to start with an actionable message rather than silently sending PII. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's signed `output_hash` covers exactly what it ran. It's risk-reduction, not a -guarantee, and it's *blunt*: it can't tell your own name from a name you ask -about, so it over-redacts, and residual signals (writing style, niche topics) +guarantee: because names and free-form text are left in, you stay responsible for +what you choose to disclose, and residual signals (writing style, niche topics) can still re-identify you. ## Notes & limitations diff --git a/pyproject.toml b/pyproject.toml index b09790b..0c42eb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ [project.optional-dependencies] # Enables local PII redaction via Microsoft Presidio + spaCy. # After installing this extra, install the spaCy model once (e.g. -# `python -m spacy download en_core_web_lg` or by installing the model wheel). +# `python -m spacy download en_core_web_sm` or by installing the model wheel). pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", diff --git a/tests/test_pii.py b/tests/test_pii.py index 2b1b435..8f70ecf 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -13,7 +13,6 @@ ADDRESS_TAG, BANK_TAG, EMAIL_TAG, - NAME_TAG, PHONE_TAG, SSN_TAG, PiiSetupError, @@ -27,7 +26,7 @@ def test_build_redactor_disabled_returns_none(): def test_tags_are_distinct(): - assert len({NAME_TAG, EMAIL_TAG, PHONE_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 6 + assert len({EMAIL_TAG, PHONE_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 5 # --- everything below needs the [pii] extra + spaCy model ------------------ @@ -47,10 +46,17 @@ def R(): return _redactor() -def test_name_redacted(R): - # Person names — the core identity linker. Needs NER. - out = R.scrub_text("Hi, I am Adam Balogh and I need help.") - assert "Adam Balogh" not in out and NAME_TAG in out +def test_names_are_not_redacted(R): + # Names are deliberately left in — they over-redact third parties and spaCy + # mislabels uncommon names. User discretion covers names. + out = R.scrub_text("Reply to Advait about our contractor Julia Smith.") + assert "Advait" in out and "Julia Smith" in out + + +def test_free_form_location_not_redacted(R): + # Cities/countries are not redacted (only deterministic street lines are). + out = R.scrub_text("I live in San Francisco") + assert "San Francisco" in out def test_phone_redacted(R): @@ -59,7 +65,7 @@ def test_phone_redacted(R): def test_street_address_redacted(R): - # Street lines need the custom recognizer — spaCy NER alone misses them. + # Street lines via the custom deterministic recognizer (no NER). out = R.scrub_text("ship it to 25 Park Lane South, Jersey City") assert "25 Park Lane South" not in out and ADDRESS_TAG in out @@ -87,12 +93,6 @@ def test_iban_redacted(R): assert "WEST" not in out and BANK_TAG in out -def test_address_redacted(R): - # Free-form location — the thing only NER can see. - out = R.scrub_text("I live in San Francisco, California") - assert ADDRESS_TAG in out - - def test_dates_are_not_redacted(R): # Dates are deliberately left intact. out = R.scrub_text("DOB: 04/12/1990. The invoice is dated 06/01/2026.") diff --git a/veil/pii.py b/veil/pii.py index 9d0a664..3b688af 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -3,35 +3,39 @@ Veil's privacy guarantee is *unlinkability*: Oblivious HTTP splits who-you-are from what-you-ask, so the relay sees your identity but only ciphertext, and the model provider sees the prompt but believes it came from the enclave — not you. -That holds **only if the prompt content doesn't re-identify you**. A request that -says "I'm Adam Balogh, adam@example.com, 25 Park Lane" hands your identity to the -provider through the content and undoes the cryptographic split. So this module's -job is to de-identify the *requester*: strip the things that point back to a -specific person before the prompt is HPKE-sealed to the TEE. +That holds only if the prompt content doesn't re-identify you. This module strips +the **concrete, unambiguous identifiers** that point back to you — contact info, +government/financial IDs, and street addresses — before the prompt is HPKE-sealed +to the TEE. + +It deliberately does *not* redact names or free-form locations (cities, +countries). Those are statistical-NER guesses that over-redact the third-party +names real prompts are full of ("reply to Advait about Julia") and frequently +mislabel uncommon names — wrecking the prompt for little gain. Keeping a name out +of a prompt you want private stays the user's call; this is a backstop for the +hard data, not a substitute for that discretion. Detection is delegated to **Microsoft Presidio** so the recognizers are -community-maintained rather than handrolled. Presidio detects structured values -(email, phone, SSN, cards, IBANs, bank numbers) with its own regex/checksum -recognizers, and uses a spaCy NER model for the free-form identity linkers -(names, locations). This requires the optional extra plus a spaCy model: +community-maintained rather than handrolled — its regex/checksum recognizers for +email, phone, SSN, cards, IBANs, and bank numbers, plus one custom recognizer for +street-address lines (which Presidio ships nothing for). This requires the +optional extra plus a small spaCy model (used only for tokenization now that no +NER entities are redacted): pip install 'opengradient-veil[pii]' - python -m spacy download en_core_web_lg + python -m spacy download en_core_web_sm What gets redacted (mapped to the tags below): -* names of people, and addresses / locations (spaCy NER + a street-line recognizer) * email and phone numbers * US SSN, credit cards (Luhn), IBANs (mod-97), US bank/routing numbers +* street-address lines -Dates are deliberately *not* redacted — they're too entangled with legitimate -prompt content to scrub without mangling it. - -This is risk-reduction, not a guarantee. It's also *blunt*: it can't tell your -own name from a name you're asking about, so it over-redacts; and even perfect -PII removal leaves residual re-identification (writing style, niche topics). -Redaction is irreversible — there is deliberately no de-anonymization step, so -the TEE's signed ``output_hash`` covers exactly what it ran. +Dates and names are deliberately *not* redacted. This is risk-reduction, not a +guarantee — and because names/free-form text are left in, you remain responsible +for what you choose to disclose. Redaction is irreversible: there is no +de-anonymization step, so the TEE's signed ``output_hash`` covers exactly what it +ran. """ from __future__ import annotations @@ -42,39 +46,35 @@ # Redaction tags. Kept human-readable so a redacted prompt still reads sensibly to # the model (e.g. "wire it to [REDACTED_BANK_NUMBER]"). -NAME_TAG = "[REDACTED_NAME]" EMAIL_TAG = "[REDACTED_EMAIL]" PHONE_TAG = "[REDACTED_PHONE]" SSN_TAG = "[REDACTED_SSN]" BANK_TAG = "[REDACTED_BANK_NUMBER]" ADDRESS_TAG = "[REDACTED_ADDRESS]" -# spaCy model backing the NER (name/location) detection. The large English model -# is used rather than the small one: names are the core identity linker here, and -# large meaningfully improves PERSON recall across diverse names. It's a ~560 MB -# CNN model (not a transformer) and runs on CPU. Override is intentionally not -# exposed — keep the install predictable. -_SPACY_MODEL = "en_core_web_lg" +# spaCy model required by Presidio's pipeline. We only redact pattern-based "hard" +# identifiers (no PERSON/LOCATION NER), so the model is used purely for +# tokenization/context — the small English model is all that's needed. Override is +# intentionally not exposed — keep the install predictable. +_SPACY_MODEL = "en_core_web_sm" -# Presidio entities we redact, each mapped to a tag. PERSON and LOCATION come from -# the spaCy NER model; the rest are Presidio's regex/checksum recognizers. +# Presidio entities we redact, each mapped to a tag. All are pattern/checksum +# recognizers (no statistical NER): deterministic, no name/location guessing. _ENTITY_TAGS = { - "PERSON": NAME_TAG, "EMAIL_ADDRESS": EMAIL_TAG, "PHONE_NUMBER": PHONE_TAG, "US_SSN": SSN_TAG, "CREDIT_CARD": BANK_TAG, "IBAN_CODE": BANK_TAG, "US_BANK_NUMBER": BANK_TAG, - "LOCATION": ADDRESS_TAG, # Custom recognizer registered below; Presidio ships nothing for street lines. "STREET_ADDRESS": ADDRESS_TAG, } # Street-address line: a house number, a few name words, then a street-type -# suffix, optionally a direction. Presidio/spaCy NER catch named places (cities, -# states) but not free-form street lines, so this closes the most common gap -# without a transformer. Compiled by Presidio with IGNORECASE. +# suffix, optionally a direction. Closes the most common address case +# deterministically (no NER, so it never mislabels a name as a place). Compiled by +# Presidio with IGNORECASE. _STREET_ADDRESS_REGEX = ( r"\b\d{1,6}\s+(?:[A-Za-z0-9.\-']+\s+){0,4}" r"(?:street|st|avenue|ave|lane|ln|road|rd|boulevard|blvd|drive|dr|court|ct" From f23c4b39592f3f5fc496f32f7bef6ed4dfcb566f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 00:44:17 +0000 Subject: [PATCH 16/25] PII redaction: drop the spaCy model, make install a single pip install MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The redactor only used pattern/checksum entities (no PERSON/LOCATION NER), so the spaCy model was dead weight — it was only there to satisfy Presidio's NLP engine. Run Presidio's predefined recognizers (email, phone, SSN, card, IBAN, US bank) plus the custom street-address recognizer directly with nlp_artifacts=None, and drop the AnalyzerEngine/NlpEngineProvider/model entirely. Result: installing PII redaction is now just `pip install 'opengradient-veil[pii]'` — no `spacy download`, no Makefile target, no pinned model wheel in docs/CI. Also faster startup (no model load) and lower RAM. A 0.4 score threshold drops Presidio's "very weak" context-dependent patterns (e.g. a bare 9-digit number as a maybe-SSN) so we don't over-redact ordinary numbers now that there's no NLP context. Dropped the now-unneeded typer pin; kept the spaCy<4 pin since presidio pulls spaCy transitively. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- .github/workflows/tests.yml | 6 --- Makefile | 5 +- README.md | 15 +++--- pyproject.toml | 13 +++--- uv.lock | 64 ++++++++++++++++++++++++-- veil/config.py | 7 +-- veil/pii.py | 91 ++++++++++++++++++------------------- 7 files changed, 121 insertions(+), 80 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cf357a5..d0a23b6 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,11 +24,5 @@ jobs: - name: Install dependencies (incl. PII extra) run: uv sync --all-groups --extra pii - - name: Install spaCy model for PII tests - # Installed from the release wheel rather than `spacy download`, which can - # be unreliable in some environments. Not needed at runtime, only to fetch - # the model. - run: uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl - - name: Test run: make test diff --git a/Makefile b/Makefile index 329dc5e..06da936 100644 --- a/Makefile +++ b/Makefile @@ -3,12 +3,9 @@ install: uv sync --all-groups -# Dev install with the optional PII-redaction stack (Presidio + spaCy) and its -# model. The model is fetched from the release wheel rather than `spacy download`, -# which can be unreliable in some environments. +# Dev install including the optional PII-redaction extra (Presidio). No model download. install-pii: uv sync --all-groups --extra pii - uv pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl build: uv build diff --git a/README.md b/README.md index 79f36bf..2887347 100644 --- a/README.md +++ b/README.md @@ -188,16 +188,14 @@ discretion. Detection is delegated to **Microsoft Presidio** (community-maintained recognizers) rather than handrolled patterns, so it ships as an optional extra. -Install it once: +It uses only pattern/checksum recognizers — no NER model to download — so install +is a single step: ```sh pip install 'opengradient-veil[pii]' -# fetch the spaCy model from the release wheel (avoids issues with `spacy download` -# in some environments): -pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl ``` -(`make install-pii` does both.) Then enable it: +Then enable it: ```sh og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 @@ -214,11 +212,10 @@ What gets redacted (each replaced with a `[REDACTED_*]` tag): in.** Those rely on statistical NER that over-redacts the third-party names real prompts are full of ("reply to *Advait* about *Julia*") and often mislabels uncommon names — wrecking the prompt for little gain. Everything redacted here is -pattern/checksum-based: deterministic, no name guessing. (The small spaCy model -is just Presidio's tokenizer; no NER entities are redacted.) +pattern/checksum-based: deterministic, no name guessing and no model to download. -If `--pii-scrub` is set but the extra/model isn't installed, the server refuses -to start with an actionable message rather than silently sending PII. +If `--pii-scrub` is set but the extra isn't installed, the server refuses to +start with an actionable message rather than silently sending PII. Redaction is **irreversible** — there's no de-anonymization step, so the TEE's signed `output_hash` covers exactly what it ran. It's risk-reduction, not a diff --git a/pyproject.toml b/pyproject.toml index 0c42eb5..edc932f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,15 +40,16 @@ dependencies = [ ] [project.optional-dependencies] -# Enables local PII redaction via Microsoft Presidio + spaCy. -# After installing this extra, install the spaCy model once (e.g. -# `python -m spacy download en_core_web_sm` or by installing the model wheel). +# Enables local PII redaction. Detection uses Microsoft Presidio's pattern/ +# checksum recognizers only (no NER), so there's no spaCy model to download — +# `pip install 'opengradient-veil[pii]'` is the whole install. pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", - # Pin to stable spaCy 3.8.x: the repo allows prereleases (for an SDK dep), - # which otherwise pulls a spaCy 4.0 dev build whose compiled extensions break - # against numpy 2.x. 3.8 supports numpy 2 and has matching released models. + # presidio-analyzer pulls spaCy transitively; pin to stable 3.8.x because the + # repo allows prereleases (for an SDK dep), which otherwise resolves a spaCy + # 4.0 dev build whose compiled extensions break against numpy 2.x. (No spaCy + # *model* is needed — we never load one.) "spacy>=3.8.0,<4.0.0", ] diff --git a/uv.lock b/uv.lock index 6653f74..b4967bf 100644 --- a/uv.lock +++ b/uv.lock @@ -152,6 +152,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/76/641ae371508676492379f16e2fa48f4e2c11741bd63c48be4b12a6b09cba/aiosignal-1.4.0-py3-none-any.whl", hash = "sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e", size = 7490, upload-time = "2025-07-03T22:54:42.156Z" }, ] +[[package]] +name = "annotated-doc" +version = "0.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, +] + [[package]] name = "annotated-types" version = "0.7.0" @@ -1779,6 +1788,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ce/62/b40b382fa0c66fee1478073eb8db352a4a6beda4a1adccf1df911d8c289c/librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", size = 102572, upload-time = "2026-05-10T18:17:06.809Z" }, ] +[[package]] +name = "markdown-it-py" +version = "4.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mdurl" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/ff/7841249c247aa650a76b9ee4bbaeae59370dc8bfd2f6c01f3630c35eb134/markdown_it_py-4.2.0.tar.gz", hash = "sha256:04a21681d6fbb623de53f6f364d352309d4094dd4194040a10fd51833e418d49", size = 82454, upload-time = "2026-05-07T12:08:28.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/81/4da04ced5a082363ecfa159c010d200ecbd959ae410c10c0264a38cac0f5/markdown_it_py-4.2.0-py3-none-any.whl", hash = "sha256:9f7ebbcd14fe59494226453aed97c1070d83f8d24b6fc3a3bcf9a38092641c4a", size = 91687, upload-time = "2026-05-07T12:08:27.182Z" }, +] + [[package]] name = "markupsafe" version = "3.0.3" @@ -1853,6 +1874,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "multidict" version = "6.7.1" @@ -3202,6 +3232,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, ] +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, +] + [[package]] name = "rlp" version = "5.0.0b1" @@ -3385,6 +3428,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, ] +[[package]] +name = "shellingham" +version = "1.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/15/8b3609fd3830ef7b27b655beb4b4e9c62313a4e8da8c676e142cc210d58e/shellingham-1.5.4.tar.gz", hash = "sha256:8dbca0739d487e5bd35ab3ca4b36e11c4078f3a234bfce294b0a0291363404de", size = 10310, upload-time = "2023-10-24T04:13:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -3635,15 +3687,17 @@ wheels = [ [[package]] name = "typer" -version = "0.9.4" +version = "0.26.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, - { name = "typing-extensions" }, + { name = "annotated-doc" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "rich" }, + { name = "shellingham" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e9/7d/b1e0399aa5e27071f0042784681d28417f3e526c61f62c8e3635ee5ad334/typer-0.9.4.tar.gz", hash = "sha256:f714c2d90afae3a7929fcd72a3abb08df305e1ff61719381384211c4070af57f", size = 276061, upload-time = "2024-03-23T17:07:55.568Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5e/ed/ef06584ccdd5c410df0837951ecd7e15d9a6144ea1bd4c73cecab1a89891/typer-0.26.7.tar.gz", hash = "sha256:e314a34c617e419c091b2830dda3ea1f257134ff593061a8f5b9717ab8dddb3a", size = 201709, upload-time = "2026-06-03T07:18:06.843Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/39/82c9d3e10979851847361d922a373bdfef4091020da7f893acfaf07c0225/typer-0.9.4-py3-none-any.whl", hash = "sha256:aa6c4a4e2329d868b80ecbaf16f807f2b54e192209d7ac9dd42691d63f7a54eb", size = 45973, upload-time = "2024-03-23T17:07:53.985Z" }, + { url = "https://files.pythonhosted.org/packages/24/25/2201973529af2c954de0bb725323c3aaed6d7f0ceee8f550dec9185df013/typer-0.26.7-py3-none-any.whl", hash = "sha256:5c87cfbc5d34491c5346ebf49c23e18d56ccb863268d3a8d592b26087c2f5e58", size = 122456, upload-time = "2026-06-03T07:18:05.732Z" }, ] [[package]] diff --git a/veil/config.py b/veil/config.py index 8f02823..5a152d0 100644 --- a/veil/config.py +++ b/veil/config.py @@ -57,9 +57,10 @@ class ServerConfig: # the registry. Useful for reproducible demos and debugging. pinned_tee_id: str | None = None - # Opt-in local PII redaction: scrub high-impact PII (email, SSN, bank numbers, - # and — with the [pii] extra — addresses) out of the agent's prompt *before* - # it leaves this process. Off by default; see veil.pii. + # Opt-in local PII redaction: scrub concrete identifiers (email, phone, SSN, + # bank numbers, street addresses) out of the agent's prompt *before* it leaves + # this process. Requires the optional ``[pii]`` extra; off by default. See + # :mod:`veil.pii`. pii_scrub: bool = False @classmethod diff --git a/veil/pii.py b/veil/pii.py index 3b688af..6007001 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -18,12 +18,11 @@ Detection is delegated to **Microsoft Presidio** so the recognizers are community-maintained rather than handrolled — its regex/checksum recognizers for email, phone, SSN, cards, IBANs, and bank numbers, plus one custom recognizer for -street-address lines (which Presidio ships nothing for). This requires the -optional extra plus a small spaCy model (used only for tokenization now that no -NER entities are redacted): +street-address lines (which Presidio ships nothing for). Because none of these +use statistical NER, they run without a spaCy model — so the whole feature is a +single ``pip install``, no model to download: pip install 'opengradient-veil[pii]' - python -m spacy download en_core_web_sm What gets redacted (mapped to the tags below): @@ -52,11 +51,14 @@ BANK_TAG = "[REDACTED_BANK_NUMBER]" ADDRESS_TAG = "[REDACTED_ADDRESS]" -# spaCy model required by Presidio's pipeline. We only redact pattern-based "hard" -# identifiers (no PERSON/LOCATION NER), so the model is used purely for -# tokenization/context — the small English model is all that's needed. Override is -# intentionally not exposed — keep the install predictable. -_SPACY_MODEL = "en_core_web_sm" +# Shown when scrubbing is requested but the optional stack isn't installed. +_INSTALL_HINT = "Install the optional PII extra:\n pip install 'opengradient-veil[pii]'" + +# Minimum recognizer confidence we act on. Presidio's recognizers emit "very weak" +# patterns (e.g. a bare 9-digit run as a maybe-SSN at 0.05) meant to be rescued by +# context words; with no NLP context we drop those to avoid redacting ordinary +# numbers, keeping only the strong pattern/checksum matches. +_SCORE_THRESHOLD = 0.4 # Presidio entities we redact, each mapped to a tag. All are pattern/checksum # recognizers (no statistical NER): deterministic, no name/location guessing. @@ -86,66 +88,61 @@ class PiiSetupError(Exception): - """PII scrubbing was requested but Presidio / the spaCy model isn't installed.""" + """PII scrubbing was requested but the optional ``[pii]`` extra isn't installed.""" def _build_engine(): # noqa: ANN202 — Presidio types are untyped - """Construct the Presidio analyzer/anonymizer and the per-entity plan. + """Build the recognizers, the anonymizer, and the per-entity replace plan. - Raises :class:`PiiSetupError` with an actionable message if the optional - dependency or its spaCy model is missing. + All recognizers are pattern/checksum based, so they run *without* an NLP + engine — no spaCy model to download. Raises :class:`PiiSetupError` with an + actionable message if the optional dependency isn't installed. """ try: - from presidio_analyzer import ( # type: ignore[import-not-found] - AnalyzerEngine, - Pattern, - PatternRecognizer, - ) - from presidio_analyzer.nlp_engine import ( # type: ignore[import-not-found] - NlpEngineProvider, + from presidio_analyzer import Pattern, PatternRecognizer # type: ignore[import-not-found] + from presidio_analyzer.predefined_recognizers import ( # type: ignore[import-not-found] + CreditCardRecognizer, + EmailRecognizer, + IbanRecognizer, + PhoneRecognizer, + UsBankRecognizer, + UsSsnRecognizer, ) from presidio_anonymizer import AnonymizerEngine # type: ignore[import-not-found] from presidio_anonymizer.entities import ( # type: ignore[import-not-found] OperatorConfig, ) except ImportError as exc: - raise PiiSetupError( - "PII scrubbing needs the optional extra. Install it with: " - "pip install 'opengradient-veil[pii]' and install the spaCy model " - f"'{_SPACY_MODEL}' (e.g. `python -m spacy download {_SPACY_MODEL}` or the model wheel)." - ) from exc - - provider = NlpEngineProvider( - nlp_configuration={ - "nlp_engine_name": "spacy", - "models": [{"lang_code": "en", "model_name": _SPACY_MODEL}], - } - ) - try: - nlp_engine = provider.create_engine() - except Exception as exc: # noqa: BLE001 — spaCy model not downloaded yet - raise PiiSetupError( - f"the spaCy model '{_SPACY_MODEL}' is not installed; run: " - f"python -m spacy download {_SPACY_MODEL}" - ) from exc - - analyzer = AnalyzerEngine(nlp_engine=nlp_engine, supported_languages=["en"]) - analyzer.registry.add_recognizer( + raise PiiSetupError(f"PII scrubbing requires Presidio.\n{_INSTALL_HINT}") from exc + + recognizers = [ + EmailRecognizer(), + PhoneRecognizer(), + UsSsnRecognizer(), + CreditCardRecognizer(), + IbanRecognizer(), + UsBankRecognizer(), PatternRecognizer( supported_entity="STREET_ADDRESS", patterns=[Pattern(name="street_address", regex=_STREET_ADDRESS_REGEX, score=0.6)], - ) - ) + ), + ] anonymizer = AnonymizerEngine() - operators = { entity: OperatorConfig("replace", {"new_value": tag}) for entity, tag in _ENTITY_TAGS.items() } - entities = list(_ENTITY_TAGS) def analyze(text: str): # noqa: ANN202 - return analyzer.analyze(text=text, entities=entities, language="en") + # Run each recognizer's pattern logic directly (no NLP artifacts → no + # context boosting), then keep only confident hits. The threshold drops + # Presidio's "very weak" patterns (e.g. a bare 9-digit number scoring 0.05 + # as a maybe-SSN) so we don't redact ordinary numbers. + results: list = [] + for rec in recognizers: + hits = rec.analyze(text, entities=rec.get_supported_entities(), nlp_artifacts=None) + results.extend(r for r in (hits or []) if r.score >= _SCORE_THRESHOLD) + return results return analyze, anonymizer, operators From 684fd9a769e17fc3308a75f016012efc4f2d7c90 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 00:45:26 +0000 Subject: [PATCH 17/25] README: tighten the PII redaction section MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Condense to the essentials — install, what's redacted vs left in, and the key caveats — cutting the verbose prose. https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- README.md | 52 +++++++++++++--------------------------------------- 1 file changed, 13 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 2887347..ef4cfa0 100644 --- a/README.md +++ b/README.md @@ -177,51 +177,25 @@ Session + prefs live in `~/.opengradient/local/` (override with `OG_VEIL_HOME`). ### Local PII redaction (opt-in) -Veil's privacy guarantee is *unlinkability* — OHTTP splits *who you are* from -*what you ask*, so the model provider sees your prompt but believes it came from -the enclave, not you. That holds only if the prompt content doesn't re-identify -you. Local PII redaction strips the **concrete, unambiguous identifiers** — when -enabled, they're irreversibly replaced with `[REDACTED_*]` tags *before* the -prompt is encrypted to the TEE, so they never leave your machine. It's a -peace-of-mind backstop for the hard data, not a replacement for your own -discretion. - -Detection is delegated to **Microsoft Presidio** (community-maintained -recognizers) rather than handrolled patterns, so it ships as an optional extra. -It uses only pattern/checksum recognizers — no NER model to download — so install -is a single step: +OHTTP unlinks *who you are* from *what you ask* — but only if the prompt itself +doesn't name you. With `--pii-scrub` on, concrete identifiers are replaced with +`[REDACTED_*]` tags locally *before* the prompt is encrypted, so they never leave +your machine. Install the optional extra (one step — no model download) and turn +it on: ```sh pip install 'opengradient-veil[pii]' -``` - -Then enable it: - -```sh og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 ``` -What gets redacted (each replaced with a `[REDACTED_*]` tag): - -- **email, phone numbers** — contact identity. -- **US SSN, bank numbers** — credit cards (Luhn), IBANs (mod-97), and US - bank/routing numbers, via Presidio's regex/checksum recognizers. -- **street addresses** — a deterministic street-line recognizer. - -**Names, free-form locations (cities/countries), and dates are deliberately left -in.** Those rely on statistical NER that over-redacts the third-party names real -prompts are full of ("reply to *Advait* about *Julia*") and often mislabels -uncommon names — wrecking the prompt for little gain. Everything redacted here is -pattern/checksum-based: deterministic, no name guessing and no model to download. - -If `--pii-scrub` is set but the extra isn't installed, the server refuses to -start with an actionable message rather than silently sending PII. - -Redaction is **irreversible** — there's no de-anonymization step, so the TEE's -signed `output_hash` covers exactly what it ran. It's risk-reduction, not a -guarantee: because names and free-form text are left in, you stay responsible for -what you choose to disclose, and residual signals (writing style, niche topics) -can still re-identify you. +Redacts **email, phone, US SSN, credit cards, IBANs, US bank numbers, and street +addresses** via [Microsoft Presidio](https://github.com/microsoft/presidio)'s +pattern/checksum recognizers. **Names, cities/countries, and dates are left in** — +detecting them needs statistical NER that over-redacts the third-party names real +prompts are full of and mislabels uncommon ones. So this is a backstop for the +hard data, not a substitute for your own discretion. Redaction is irreversible +(the TEE's signed `output_hash` covers exactly what it ran); if the extra isn't +installed, the server refuses to start rather than send PII. ## Notes & limitations From 81d0e0ffaa64439a8dbc6080a1d89ffddcef61e5 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:55:12 -0400 Subject: [PATCH 18/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- scripts/pii_repl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pii_repl.py b/scripts/pii_repl.py index ac1d481..d3184ae 100644 --- a/scripts/pii_repl.py +++ b/scripts/pii_repl.py @@ -4,7 +4,7 @@ echo "email me at a@b.com" | uv run python scripts/pii_repl.py # or pipe input Local-only — no login, no TEE, no network. Just the same Redactor the proxy uses. -Needs the extra: make install-pii (or: uv sync --extra pii + the model wheel) +Needs the extra: make install-pii (or: uv sync --extra pii) """ from __future__ import annotations From e94c6755296674b7938ca6fa0930e766ab8cb5e7 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:55:22 -0400 Subject: [PATCH 19/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_pii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pii.py b/tests/test_pii.py index 8f70ecf..11b8c28 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -29,7 +29,7 @@ def test_tags_are_distinct(): assert len({EMAIL_TAG, PHONE_TAG, SSN_TAG, BANK_TAG, ADDRESS_TAG}) == 5 -# --- everything below needs the [pii] extra + spaCy model ------------------ +# --- everything below needs the [pii] extra ------------------------------- pytest.importorskip("presidio_analyzer", reason="requires the [pii] extra") From 6137618753f07621af596e62ce58ed1470a7e701 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:55:37 -0400 Subject: [PATCH 20/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/test_pii.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_pii.py b/tests/test_pii.py index 11b8c28..040a705 100644 --- a/tests/test_pii.py +++ b/tests/test_pii.py @@ -1,7 +1,7 @@ """PII redaction tests. -Detection is delegated to Presidio + spaCy (the optional [pii] extra), so these -tests skip when the dependency or spaCy model isn't installed. ``build_redactor(enabled=False)`` +Detection is delegated to Presidio (the optional [pii] extra; it pulls in spaCy as a dependency), +so these tests skip when the extra isn't installed. ``build_redactor(enabled=False)`` is checked unconditionally. """ From 94dc16e7c9ea23fcb69c466bfbf9cdc06f0464de Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:55:56 -0400 Subject: [PATCH 21/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- veil/pii.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/veil/pii.py b/veil/pii.py index 6007001..5edd0a2 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -199,7 +199,7 @@ def scrub_request(self, body: dict) -> dict: def build_redactor(*, enabled: bool) -> Redactor | None: """Construct a :class:`Redactor` when PII scrubbing is enabled, else ``None``. - Raises :class:`PiiSetupError` if enabled but the ``[pii]`` extra / spaCy model + Raises :class:`PiiSetupError` if enabled but the optional ``[pii]`` extra isn't installed, so the operator gets a clear message at startup rather than a silent no-op. """ From 7e41eb2e1090ee38acf10450a196e62c552dcd83 Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:56:26 -0400 Subject: [PATCH 22/25] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- veil/server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/veil/server.py b/veil/server.py index 59e4b6b..62481f9 100644 --- a/veil/server.py +++ b/veil/server.py @@ -144,8 +144,7 @@ def serve(config: ServerConfig) -> None: try: gateway = Gateway(session, config) except PiiSetupError as exc: - # PII scrubbing was requested but the optional extra/model isn't present. - raise SystemExit(f"PII scrubbing is enabled but unavailable: {exc}") + # PII scrubbing was requested but the optional extra isn't present. # Resolve a TEE eagerly so misconfiguration fails fast and /health is useful. try: gateway._get_client() # noqa: SLF001 — intentional eager warm-up From bea3617fe22bd349e878c6f79aa2ff03b1fe2f7c Mon Sep 17 00:00:00 2001 From: kukac Date: Fri, 12 Jun 2026 20:57:33 -0400 Subject: [PATCH 23/25] Update server.py --- veil/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/veil/server.py b/veil/server.py index 62481f9..859725d 100644 --- a/veil/server.py +++ b/veil/server.py @@ -145,6 +145,7 @@ def serve(config: ServerConfig) -> None: gateway = Gateway(session, config) except PiiSetupError as exc: # PII scrubbing was requested but the optional extra isn't present. + raise SystemExit(f"PII scrubbing is enabled but unavailable: {exc}") # Resolve a TEE eagerly so misconfiguration fails fast and /health is useful. try: gateway._get_client() # noqa: SLF001 — intentional eager warm-up From ece103d33f562a3226504cff0b9b09b5c2215a6e Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 01:08:26 +0000 Subject: [PATCH 24/25] README: install the PII extra with uv to match the quickstart https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ef4cfa0..128cead 100644 --- a/README.md +++ b/README.md @@ -184,7 +184,7 @@ your machine. Install the optional extra (one step — no model download) and tu it on: ```sh -pip install 'opengradient-veil[pii]' +uv tool install 'opengradient-veil[pii]' # or: pipx install 'opengradient-veil[pii]' og-veil --pii-scrub # or: export OG_VEIL_PII_SCRUB=1 ``` From 9fd6a83bec939a89c8b004012e1ad6f9e5004d79 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 13 Jun 2026 01:13:09 +0000 Subject: [PATCH 25/25] Reference uv for the PII extra install in error hint, docstring, and pyproject https://claude.ai/code/session_014NfUqUiR5AaD3HcswJTA2a --- pyproject.toml | 2 +- veil/pii.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index edc932f..08c182e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dependencies = [ [project.optional-dependencies] # Enables local PII redaction. Detection uses Microsoft Presidio's pattern/ # checksum recognizers only (no NER), so there's no spaCy model to download — -# `pip install 'opengradient-veil[pii]'` is the whole install. +# `uv tool install 'opengradient-veil[pii]'` is the whole install. pii = [ "presidio-analyzer>=2.2.0", "presidio-anonymizer>=2.2.0", diff --git a/veil/pii.py b/veil/pii.py index 5edd0a2..bcdcbb6 100644 --- a/veil/pii.py +++ b/veil/pii.py @@ -20,9 +20,9 @@ email, phone, SSN, cards, IBANs, and bank numbers, plus one custom recognizer for street-address lines (which Presidio ships nothing for). Because none of these use statistical NER, they run without a spaCy model — so the whole feature is a -single ``pip install``, no model to download: +single install, no model to download: - pip install 'opengradient-veil[pii]' + uv tool install 'opengradient-veil[pii]' What gets redacted (mapped to the tags below): @@ -52,7 +52,7 @@ ADDRESS_TAG = "[REDACTED_ADDRESS]" # Shown when scrubbing is requested but the optional stack isn't installed. -_INSTALL_HINT = "Install the optional PII extra:\n pip install 'opengradient-veil[pii]'" +_INSTALL_HINT = "Install the optional PII extra:\n uv tool install 'opengradient-veil[pii]'" # Minimum recognizer confidence we act on. Presidio's recognizers emit "very weak" # patterns (e.g. a bare 9-digit run as a maybe-SSN at 0.05) meant to be rescued by